From bec32eeaccbd53473648962490de0065a337b94c Mon Sep 17 00:00:00 2001 From: Super User Date: Sun, 14 Jun 2026 19:17:44 +0600 Subject: [PATCH 01/13] Fix server model roles, RAG, image tasks, and GPU options --- app/src/main/assets/server_webui.html | 259 ++++++++++++------ .../dark/tool_neuron/data/AppPreferences.kt | 10 + .../tool_neuron/model/HuggingFaceModel.kt | 4 + .../com/dark/tool_neuron/model/NavScreens.kt | 3 +- .../dark/tool_neuron/repo/ImageGenManager.kt | 4 +- .../com/dark/tool_neuron/repo/ModelCatalog.kt | 26 +- .../dark/tool_neuron/repo/ModelRepository.kt | 13 + .../com/dark/tool_neuron/repo/RagManager.kt | 221 ++++++++++++++- .../tool_neuron/repo/RepositoryDataStore.kt | 8 + .../service/inference/InferenceService.kt | 2 + .../service/server/RemoteServerService.kt | 36 +-- .../service/server/ServerCatalog.kt | 9 + .../service/server/ServerController.kt | 126 +++++++-- .../service/server/ServerEngine.kt | 1 + .../service/server/ServerEngineRegistry.kt | 17 +- .../service/server/ServerModelRole.kt | 19 ++ .../service/server/ServerVlmEngine.kt | 1 + .../tool_neuron/ui/navigation/TNavigation.kt | 9 + .../ui/screens/home_screen/HomeScreenBody.kt | 11 +- .../home_screen/HomeScreenBottomBar.kt | 11 +- .../ui/screens/image_task/ImageTaskScreen.kt | 194 +++++++++++++ .../ui/screens/image_task/ImageViewerCard.kt | 52 +++- .../screens/model_config/ModelConfigScreen.kt | 12 +- .../model_manager/ModelManagerScreen.kt | 36 ++- .../screens/model_store/InstalledModelCard.kt | 71 +++-- .../ui/screens/model_store/ModelFilters.kt | 2 + .../model_store/ModelImportTypePicker.kt | 97 ++++++- .../ui/screens/server/ServerScreen.kt | 108 ++++---- .../ui/screens/settings/SettingsScreen.kt | 6 + .../ui/screens/system_ui/AppTopBar.kt | 5 + .../viewmodel/ImageTaskViewModel.kt | 229 +++++++++++++++- .../viewmodel/ModelStoreViewModel.kt | 82 +++++- .../tool_neuron/viewmodel/ServerViewModel.kt | 41 +-- .../viewmodel/SettingsViewModel.kt | 200 +++++++++++++- .../viewmodel/home_vm/InferenceCoordinator.kt | 34 ++- native-server/src/main/cpp/server_core.cpp | 28 +- native-server/src/main/cpp/server_models.cpp | 12 + native-server/src/main/cpp/server_models.h | 2 + 38 files changed, 1662 insertions(+), 339 deletions(-) create mode 100644 app/src/main/java/com/dark/tool_neuron/service/server/ServerModelRole.kt diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index 1cc4cb7e..cafc0224 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -10,25 +10,25 @@ @@ -824,17 +858,54 @@

Image

var av = document.createElement('div'); av.className = 'avatar'; av.textContent = m.role === 'user' ? 'You' : 'AI'; + var stack = document.createElement('div'); + stack.className = 'msg-stack'; var bub = document.createElement('div'); bub.className = 'bubble'; bub.dataset.id = m.id; bub.innerHTML = renderMarkdown(m.content || ''); if (m.streaming) bub.classList.add('cursor'); wireCopy(bub); + stack.appendChild(bub); + if (!m.streaming) stack.appendChild(renderMessageActions(m)); wrap.appendChild(av); - wrap.appendChild(bub); + wrap.appendChild(stack); return wrap; } + function renderMessageActions(m) { + var actions = document.createElement('div'); + actions.className = 'msg-actions'; + actions.appendChild(actionButton('Copy', copyIcon(), function() { copyMessage(m.id); })); + if (m.role === 'user') { + actions.appendChild(actionButton('Edit', editIcon(), function() { editMessage(m.id); })); + } else if (m.role === 'assistant') { + actions.appendChild(actionButton('Regenerate', regenIcon(), function() { regenerateMessage(m.id); })); + } + return actions; + } + + function actionButton(label, icon, handler) { + var btn = document.createElement('button'); + btn.className = 'msg-action'; + btn.type = 'button'; + btn.title = label; + btn.setAttribute('aria-label', label); + btn.innerHTML = icon + '' + label + ''; + btn.onclick = handler; + return btn; + } + + function copyIcon() { + return ''; + } + function editIcon() { + return ''; + } + function regenIcon() { + return ''; + } + function selectChat(id) { if (state.streaming) return; state.activeId = id; @@ -903,6 +974,74 @@

Image

document.title = (c && c.title && c.title !== 'New chat') ? c.title + ' · ToolNeuron' : 'ToolNeuron'; } + function findMessage(chat, id) { + if (!chat) return { index: -1, message: null }; + var idx = chat.messages.findIndex(function(m) { return m.id === id; }); + return { index: idx, message: idx >= 0 ? chat.messages[idx] : null }; + } + + function writeClipboard(text) { + if (navigator.clipboard && navigator.clipboard.writeText) { + return navigator.clipboard.writeText(text); + } + return new Promise(function(resolve, reject) { + try { + var ta = document.createElement('textarea'); + ta.value = text; + ta.setAttribute('readonly', ''); + ta.style.position = 'fixed'; + ta.style.opacity = '0'; + document.body.appendChild(ta); + ta.select(); + document.execCommand('copy'); + document.body.removeChild(ta); + resolve(); + } catch (e) { reject(e); } + }); + } + + function copyMessage(id) { + var found = findMessage(activeChat(), id); + if (!found.message) return; + writeClipboard(found.message.content || '').then(function() { + toast('Message copied'); + }).catch(function() { + toast('Could not copy'); + }); + } + + function editMessage(id) { + if (state.streaming) return; + var chat = activeChat(); + var found = findMessage(chat, id); + if (!found.message || found.message.role !== 'user') return; + els.input.value = found.message.content || ''; + chat.messages = chat.messages.slice(0, found.index); + if (chat.messages.length === 0) chat.title = 'New chat'; + save(); + refreshAll(); + els.input.dispatchEvent(new Event('input')); + els.input.focus(); + toast('Edit your message and send again'); + } + + function regenerateMessage(id) { + if (state.streaming) return; + var chat = activeChat(); + var found = findMessage(chat, id); + if (!chat || found.index < 0 || found.message.role !== 'assistant') return; + var userIdx = -1; + for (var i = found.index - 1; i >= 0; i--) { + if (chat.messages[i].role === 'user') { userIdx = i; break; } + } + if (userIdx < 0) return; + var prompt = chat.messages[userIdx].content || ''; + chat.messages = chat.messages.slice(0, userIdx); + save(); + refreshAll(); + sendMessage(prompt); + } + var scrollAnchored = true; els.messages.addEventListener('scroll', function() { var dist = els.messages.scrollHeight - els.messages.scrollTop - els.messages.clientHeight; @@ -916,7 +1055,7 @@

Image

els.input.addEventListener('input', function() { els.input.style.height = 'auto'; els.input.style.height = Math.min(els.input.scrollHeight, 240) + 'px'; - els.sendBtn.disabled = !state.streaming && els.input.value.trim().length === 0; + updateSendState(); }); els.input.addEventListener('keydown', function(e) { if (e.key === 'Enter' && !e.shiftKey && !e.isComposing) { @@ -931,17 +1070,19 @@

Image

return; } var text = els.input.value.trim(); - if (!text) return; + if (!text && studioPendingImages.length === 0) return; sendMessage(text); } async function sendMessage(text) { var chat = activeChat(); if (!chat) return; - var userMsg = { id: 'u' + Date.now(), role: 'user', content: text }; + var imagesToSend = (typeof window.consumePendingImages === 'function') ? window.consumePendingImages() : []; + var displayText = text || (imagesToSend.length > 0 ? '[Image attached]' : ''); + var userMsg = { id: 'u' + Date.now(), role: 'user', content: displayText }; chat.messages.push(userMsg); if (chat.title === 'New chat') { - chat.title = text.length > 40 ? text.slice(0, 40).trim() + '…' : text; + chat.title = displayText.length > 40 ? displayText.slice(0, 40).trim() + '…' : displayText; } els.input.value = ''; els.input.style.height = 'auto'; @@ -954,7 +1095,6 @@

Image

setStreaming(true); state.abort = new AbortController(); await ensureModelCache(); - var imagesToSend = (typeof window.consumePendingImages === 'function') ? window.consumePendingImages() : []; var model = resolveChatModel(imagesToSend.length > 0); if (!model) { finishStream(aiMsg, imagesToSend.length > 0 @@ -969,7 +1109,7 @@

Image

var last = messages[messages.length - 1]; if (last.role === 'user') { var parts = []; - if (last.content) parts.push({ type: 'text', text: last.content }); + if (text) parts.push({ type: 'text', text: text }); imagesToSend.forEach(function(img) { parts.push({ type: 'image_url', image_url: { url: img.dataUrl } }); }); @@ -1030,12 +1170,17 @@

Image

} aiMsg.streaming = false; setStreaming(false); + state.abort = null; save(); var bub = bubbleEl(aiMsg.id); if (bub) { bub.classList.remove('cursor'); bub.innerHTML = renderMarkdown(aiMsg.content); wireCopy(bub); + var stack = bub.parentElement; + if (stack && !stack.querySelector('.msg-actions')) { + stack.appendChild(renderMessageActions(aiMsg)); + } } } function appendMessage(m) { @@ -1063,12 +1208,17 @@

Image

function setStreaming(on) { state.streaming = on; els.sendBtn.classList.toggle('stop', on); - els.sendBtn.disabled = on ? false : els.input.value.trim().length === 0; + updateSendState(); els.sendBtn.setAttribute('aria-label', on ? 'Stop' : 'Send'); els.sendBtn.innerHTML = on ? '' : ''; } + function updateSendState() { + var hasText = els.input.value.trim().length > 0; + var hasImages = Array.isArray(studioPendingImages) && studioPendingImages.length > 0; + els.sendBtn.disabled = state.streaming ? false : !(hasText || hasImages); + } function authHeaders() { return state.token ? { 'Authorization': 'Bearer ' + state.token } : {}; } @@ -1159,7 +1309,7 @@

Image

bubble.querySelectorAll('.copy-code').forEach(function(btn) { btn.onclick = function() { var code = btn.parentElement.querySelector('code'); - navigator.clipboard.writeText(code.innerText).then(function() { + writeClipboard(code.innerText).then(function() { btn.classList.add('copied'); btn.textContent = 'Copied'; setTimeout(function() { @@ -1617,36 +1767,80 @@

Image

attachBtn.addEventListener('click', function() { attachInput.click(); }); attachInput.addEventListener('change', function() { var files = Array.prototype.slice.call(attachInput.files || []); - files.forEach(function(f) { - var reader = new FileReader(); - reader.onload = function(e) { - studioPendingImages.push({ name: f.name, dataUrl: e.target.result }); - renderAttachRow(); - }; - reader.readAsDataURL(f); - }); + addPendingImageFiles(files); attachInput.value = ''; }); } + function addPendingImageFiles(files) { + files.filter(function(f) { return f && /^image\//i.test(f.type || ''); }).forEach(function(f) { + var reader = new FileReader(); + reader.onload = function(e) { + studioPendingImages.push({ name: f.name || 'pasted image', dataUrl: e.target.result }); + renderAttachRow(); + updateSendState(); + }; + reader.readAsDataURL(f); + }); + } + els.input.addEventListener('paste', function(e) { + var items = e.clipboardData && e.clipboardData.items ? Array.prototype.slice.call(e.clipboardData.items) : []; + var files = items.map(function(item) { + return item.kind === 'file' ? item.getAsFile() : null; + }).filter(Boolean); + if (files.some(function(f) { return /^image\//i.test(f.type || ''); })) { + addPendingImageFiles(files); + toast('Image pasted'); + } + }); + ['dragenter', 'dragover'].forEach(function(evt) { + document.addEventListener(evt, function(e) { + if (e.dataTransfer && Array.prototype.some.call(e.dataTransfer.items || [], function(i) { return /^image\//i.test(i.type || ''); })) { + e.preventDefault(); + document.body.classList.add('drop-ready'); + } + }); + }); + ['dragleave', 'drop'].forEach(function(evt) { + document.addEventListener(evt, function() { + document.body.classList.remove('drop-ready'); + }); + }); + document.addEventListener('drop', function(e) { + var files = e.dataTransfer && e.dataTransfer.files ? Array.prototype.slice.call(e.dataTransfer.files) : []; + if (files.some(function(f) { return /^image\//i.test(f.type || ''); })) { + e.preventDefault(); + addPendingImageFiles(files); + toast('Image attached'); + window.switchTab('chat'); + els.input.focus(); + } + }); function renderAttachRow() { if (!attachRow) return; attachRow.innerHTML = ''; - if (studioPendingImages.length === 0) { attachRow.hidden = true; return; } + if (studioPendingImages.length === 0) { + attachRow.hidden = true; + updateSendState(); + return; + } attachRow.hidden = false; studioPendingImages.forEach(function(img, idx) { var chip = document.createElement('div'); chip.className = 'chip'; + var thumb = document.createElement('img'); thumb.src = img.dataUrl; thumb.alt = ''; var name = document.createElement('span'); name.textContent = img.name || 'image'; var rm = document.createElement('button'); rm.type = 'button'; rm.textContent = '×'; rm.onclick = function() { - studioPendingImages.splice(idx, 1); renderAttachRow(); + studioPendingImages.splice(idx, 1); renderAttachRow(); updateSendState(); }; - chip.appendChild(name); chip.appendChild(rm); + chip.appendChild(thumb); chip.appendChild(name); chip.appendChild(rm); attachRow.appendChild(chip); }); + updateSendState(); } window.consumePendingImages = function() { var images = studioPendingImages.slice(); studioPendingImages = []; renderAttachRow(); + updateSendState(); return images; }; From a7b09e264751193eb486196ac12690def42a043b Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Sun, 21 Jun 2026 14:28:17 +0600 Subject: [PATCH 04/13] Add model roles, idle unload, and backup support --- app/src/main/assets/server_webui.html | 50 ++++- .../dark/tool_neuron/data/AppPreferences.kt | 12 ++ .../tool_neuron/model/enums/ProviderType.kt | 11 +- .../tool_neuron/repo/ModelBackupManager.kt | 180 ++++++++++++++++++ .../dark/tool_neuron/repo/ModelRepository.kt | 2 +- .../service/server/ServerController.kt | 2 + .../screens/model_config/ModelConfigScreen.kt | 10 +- .../model_manager/ModelManagerScreen.kt | 66 +++++++ .../model_store/ModelImportTypePicker.kt | 6 + .../tool_neuron/viewmodel/HomeViewModel.kt | 18 +- .../viewmodel/ModelStoreViewModel.kt | 34 ++++ .../viewmodel/SettingsViewModel.kt | 41 +++- .../viewmodel/home_vm/ModelSessionManager.kt | 46 +++++ 13 files changed, 463 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/com/dark/tool_neuron/repo/ModelBackupManager.kt diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index 83f9e2ba..ac4ccfdf 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -33,6 +33,27 @@ --active-chat:#DCEBFF; --overlay:rgba(15,23,42,0.22); --shadow:rgba(15,23,42,0.18); } } +html[data-theme="light"] { + color-scheme: light; + --bg:#F7FAFE; --bg-soft:#ECF2FA; --surface:#FFFFFF; --surface-2:#EEF5FF; + --fg:#111827; --fg-dim:#334155; --muted:#64748B; + --border:#DDE7F2; --border-2:#C5D5E8; + --accent:#256CB8; --accent-2:#1D5EA8; --accent-on:#FFFFFF; + --accent-soft:rgba(37,108,184,0.10); --accent-edge:rgba(37,108,184,0.28); + --code-bg:#EEF3F9; + --active-chat:#DCEBFF; --overlay:rgba(15,23,42,0.22); --shadow:rgba(15,23,42,0.18); +} +html[data-theme="dark"] { + color-scheme: dark; + --bg:#10151B; --bg-soft:#151B23; --surface:#1D2530; --surface-2:#243040; + --fg:#F1F5F9; --fg-dim:#D9E2EE; --muted:#9BA8B8; + --border:#263241; --border-2:#35465A; + --accent:#9BC7FF; --accent-2:#6EABF7; --accent-on:#08111C; + --accent-soft:rgba(155,199,255,0.13); --accent-edge:rgba(155,199,255,0.32); + --danger:#FF7A7A; --warn:#F4B860; + --code-bg:#111923; + --active-chat:#1D3B5E; --overlay:rgba(0,0,0,0.42); --shadow:rgba(0,0,0,0.5); +} * { box-sizing: border-box; -webkit-tap-highlight-color: transparent; } *:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: var(--r-xs); } *:focus:not(:focus-visible) { outline: none; } @@ -682,6 +703,12 @@

Image

Loaded from /v1/models. Chat requests only use the model selected here.
+ +
@@ -711,9 +738,11 @@

Image

var STORAGE_KEY = 'tn.chats.v3'; var TOKEN_KEY = 'tn.token'; var CHAT_MODEL_KEY = 'tn.chatModel'; + var THEME_KEY = 'tn.theme'; var state = { chats: [], activeId: null, streaming: false, abort: null, model: localStorage.getItem(CHAT_MODEL_KEY) || '', token: localStorage.getItem(TOKEN_KEY) || '', + theme: localStorage.getItem(THEME_KEY) || 'system', }; var els = { chatList: document.getElementById('chatList'), @@ -730,6 +759,7 @@

Image

renameInput: document.getElementById('renameInput'), hostInfo: document.getElementById('hostInfo'), chatModelSelect: document.getElementById('chatModelSelect'), + themeSelect: document.getElementById('themeSelect'), modelSub: document.getElementById('modelSub'), toast: document.getElementById('toast'), conn: document.getElementById('conn'), @@ -1351,6 +1381,7 @@

Image

function openSettings() { els.tokenInput.value = state.token; els.hostInfo.value = location.host; + if (els.themeSelect) els.themeSelect.value = state.theme || 'system'; populateChatModelSelect(); ensureModelCache().then(populateChatModelSelect); els.settingsModal.classList.add('open'); @@ -1365,6 +1396,10 @@

Image

state.model = els.chatModelSelect.value || ''; if (state.model) localStorage.setItem(CHAT_MODEL_KEY, state.model); else localStorage.removeItem(CHAT_MODEL_KEY); + state.theme = els.themeSelect ? (els.themeSelect.value || 'system') : 'system'; + if (state.theme === 'system') localStorage.removeItem(THEME_KEY); + else localStorage.setItem(THEME_KEY, state.theme); + applyTheme(); closeSettings(); if (tokenChanged) modelCache = null; refreshModelCache(); @@ -1376,6 +1411,13 @@

Image

var suffix = m && m.type ? ' · ' + modelTypeLabel(m.type) : ''; els.modelSub.textContent = state.model ? state.model + suffix : 'No chat model selected'; } + function applyTheme() { + if (state.theme === 'dark' || state.theme === 'light') { + document.documentElement.dataset.theme = state.theme; + } else { + document.documentElement.removeAttribute('data-theme'); + } + } var toastTimer = null; function toast(msg) { els.toast.textContent = msg; @@ -1422,6 +1464,7 @@

Image

} } load(); + applyTheme(); refreshAll(); fetchModel(); checkHealth(); @@ -1495,9 +1538,10 @@

Image

function resolveChatModel(hasImages) { var selected = findModel(state.model); - if (selected && selected.type === (hasImages ? 'vlm' : 'gguf')) return selected.id; - if (hasImages) return defaultModelId('vlm'); - return defaultModelId('gguf'); + if (!selected) return defaultModelId(hasImages ? 'vlm' : 'gguf'); + if (hasImages && selected.type !== 'vlm') return ''; + if (!hasImages && (selected.type === 'gguf' || selected.type === 'vlm')) return selected.id; + return ''; } function populateChatModelSelect() { diff --git a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt index c558b435..a6212afe 100644 --- a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt +++ b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt @@ -208,6 +208,16 @@ class AppPreferences @Inject constructor( ?: DEFAULT_THREAD_MODE set(value) = putString(KEY_THREAD_MODE, value.coerceIn(0, 2).toString()) + var idleUnloadMinutes: Int + get() = getString(KEY_IDLE_UNLOAD_MINUTES, DEFAULT_IDLE_UNLOAD_MINUTES.toString()) + .toIntOrNull() + ?.let { if (it == 0) 0 else it.coerceIn(15, 30) } + ?: DEFAULT_IDLE_UNLOAD_MINUTES + set(value) { + val normalized = if (value == 0) 0 else value.coerceIn(15, 30) + putString(KEY_IDLE_UNLOAD_MINUTES, normalized.toString()) + } + var pluginOnnxEp: String get() = getString(KEY_PLUGIN_ONNX_EP, DEFAULT_PLUGIN_ONNX_EP) set(value) = putString(KEY_PLUGIN_ONNX_EP, value) @@ -292,6 +302,8 @@ class AppPreferences @Inject constructor( const val THREAD_MODE_POWER_SAVING = 0 const val THREAD_MODE_BALANCED = 1 const val THREAD_MODE_PERFORMANCE = 2 + const val KEY_IDLE_UNLOAD_MINUTES = "idle_unload_minutes" + const val DEFAULT_IDLE_UNLOAD_MINUTES = 20 const val KEY_PLUGIN_ONNX_EP = "plugin_onnx_ep" const val PLUGIN_ONNX_EP_CPU = "cpu" diff --git a/app/src/main/java/com/dark/tool_neuron/model/enums/ProviderType.kt b/app/src/main/java/com/dark/tool_neuron/model/enums/ProviderType.kt index 528be50a..c03f0565 100644 --- a/app/src/main/java/com/dark/tool_neuron/model/enums/ProviderType.kt +++ b/app/src/main/java/com/dark/tool_neuron/model/enums/ProviderType.kt @@ -1,3 +1,12 @@ package com.dark.tool_neuron.model.enums -enum class ProviderType { GGUF, TTS, STT, EMBEDDING, IMAGE_GEN, IMAGE_UPSCALER } +enum class ProviderType { + GGUF, + VISION_CHAT, + TOOL_SEARCH, + TTS, + STT, + EMBEDDING, + IMAGE_GEN, + IMAGE_UPSCALER, +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ModelBackupManager.kt b/app/src/main/java/com/dark/tool_neuron/repo/ModelBackupManager.kt new file mode 100644 index 00000000..0f865778 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/repo/ModelBackupManager.kt @@ -0,0 +1,180 @@ +package com.dark.tool_neuron.repo + +import android.content.Context +import android.net.Uri +import com.dark.tool_neuron.model.ModelConfig +import com.dark.tool_neuron.model.ModelInfo +import com.dark.tool_neuron.model.enums.PathType +import com.dark.tool_neuron.model.enums.ProviderType +import dagger.hilt.android.qualifiers.ApplicationContext +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.io.FileInputStream +import java.security.MessageDigest +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream +import java.util.zip.ZipOutputStream +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class ModelBackupManager @Inject constructor( + @ApplicationContext private val context: Context, + private val modelRepo: ModelRepository, +) { + fun exportTo(uri: Uri, models: List) { + context.contentResolver.openOutputStream(uri, "wt")?.use { raw -> + ZipOutputStream(raw.buffered()).use { zip -> + val manifestModels = JSONArray() + models.forEach { model -> + val root = File(model.path) + if (!root.exists()) return@forEach + val safeId = safeName(model.id) + val rootEntry = if (root.isDirectory) { + "models/$safeId/${root.name}" + } else { + "models/$safeId/${root.name}" + } + val files = JSONArray() + if (root.isDirectory) { + root.walkTopDown().filter { it.isFile }.forEach { file -> + val rel = file.relativeTo(root).invariantSeparatorsPath + val entry = "$rootEntry/$rel" + putFile(zip, file, entry) + files.put(fileJson(entry, file)) + } + } else { + putFile(zip, root, rootEntry) + files.put(fileJson(rootEntry, root)) + } + manifestModels.put( + JSONObject() + .put("id", model.id) + .put("name", model.name) + .put("providerType", model.providerType.name) + .put("pathType", PathType.FILE.name) + .put("rootEntry", rootEntry) + .put("fileSize", model.fileSize) + .put("isDirectory", root.isDirectory) + .put("config", configJson(modelRepo.getConfig(model.id))) + .put("files", files), + ) + } + val manifest = JSONObject() + .put("format", "tool-neuron-model-backup") + .put("version", 1) + .put("createdAt", System.currentTimeMillis()) + .put("models", manifestModels) + zip.putNextEntry(ZipEntry("manifest.json")) + zip.write(manifest.toString(2).toByteArray(Charsets.UTF_8)) + zip.closeEntry() + } + } ?: error("Could not open export destination") + } + + fun importFrom(uri: Uri): Int { + val root = File(context.filesDir, "imported_models/${System.currentTimeMillis()}").apply { mkdirs() } + var manifestText: String? = null + context.contentResolver.openInputStream(uri)?.use { raw -> + ZipInputStream(raw.buffered()).use { zip -> + while (true) { + val entry = zip.nextEntry ?: break + val name = entry.name + if (entry.isDirectory) continue + if (name == "manifest.json") { + manifestText = zip.readBytes().toString(Charsets.UTF_8) + continue + } + val outFile = safeOutputFile(root, name) + outFile.parentFile?.mkdirs() + outFile.outputStream().use { out -> zip.copyTo(out) } + } + } + } ?: error("Could not open import source") + val manifest = JSONObject(manifestText ?: error("Backup manifest missing")) + check(manifest.optString("format") == "tool-neuron-model-backup") { "Not a ToolNeuron model backup" } + val arr = manifest.getJSONArray("models") + var imported = 0 + for (i in 0 until arr.length()) { + val item = arr.getJSONObject(i) + val id = item.getString("id") + val restoredRoot = safeOutputFile(root, item.getString("rootEntry")) + if (!restoredRoot.exists()) continue + val provider = runCatching { ProviderType.valueOf(item.getString("providerType")) } + .getOrDefault(ProviderType.GGUF) + val folderSize = if (restoredRoot.isDirectory) { + restoredRoot.walkTopDown().filter { it.isFile }.sumOf { it.length() } + } else { + restoredRoot.length() + } + val config = item.optJSONObject("config")?.let { + ModelConfig( + id = it.optString("id", id), + modelId = id, + loadingParamsJson = it.optString("loadingParamsJson", "{}"), + inferenceParamsJson = it.optString("inferenceParamsJson", "{}"), + ) + } + modelRepo.insert( + ModelInfo( + id = id, + name = item.optString("name", id), + path = restoredRoot.absolutePath, + pathType = PathType.FILE, + providerType = provider, + fileSize = folderSize, + isActive = false, + ), + config, + ) + imported++ + } + return imported + } + + private fun putFile(zip: ZipOutputStream, file: File, entryName: String) { + zip.putNextEntry(ZipEntry(entryName)) + FileInputStream(file).use { it.copyTo(zip) } + zip.closeEntry() + } + + private fun fileJson(entry: String, file: File): JSONObject = + JSONObject() + .put("entry", entry) + .put("size", file.length()) + .put("sha256", sha256(file)) + + private fun configJson(config: ModelConfig?): JSONObject? = + config?.let { + JSONObject() + .put("id", it.id) + .put("modelId", it.modelId) + .put("loadingParamsJson", it.loadingParamsJson) + .put("inferenceParamsJson", it.inferenceParamsJson) + } + + private fun sha256(file: File): String { + val md = MessageDigest.getInstance("SHA-256") + FileInputStream(file).use { input -> + val buf = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val n = input.read(buf) + if (n <= 0) break + md.update(buf, 0, n) + } + } + return md.digest().joinToString("") { "%02x".format(it) } + } + + private fun safeName(value: String): String = + value.replace(Regex("[^A-Za-z0-9._-]"), "_").ifBlank { "model" } + + private fun safeOutputFile(root: File, entryName: String): File { + val out = File(root, entryName) + val rootPath = root.canonicalPath + File.separator + val outPath = out.canonicalPath + check(outPath.startsWith(rootPath)) { "Unsafe backup entry: $entryName" } + return out + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt b/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt index 64b2495d..91ddae72 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt @@ -103,7 +103,7 @@ class ModelRepository @Inject constructor( val records = storage.queryString(COL_MODELS, TAG_ID, modelId) records.forEach { record -> record.putString(TAG_PROVIDER_TYPE, providerType.name) - if (providerType != ProviderType.GGUF) { + if (providerType != ProviderType.GGUF && providerType != ProviderType.VISION_CHAT) { record.putBool(TAG_IS_ACTIVE, false) } storage.update(COL_MODELS, record) diff --git a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt index c4578ac5..b5604bc8 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt @@ -231,6 +231,8 @@ class ServerController @Inject constructor( ServerEngineKind.CHAT_GGUF } } + ProviderType.VISION_CHAT -> ServerEngineKind.VLM + ProviderType.TOOL_SEARCH -> ServerEngineKind.CHAT_GGUF ProviderType.EMBEDDING -> ServerEngineKind.EMBEDDING ProviderType.TTS -> ServerEngineKind.TTS ProviderType.STT -> ServerEngineKind.STT diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt index b1278b6c..4ded1ab8 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt @@ -142,7 +142,7 @@ fun ModelConfigScreen( val loading = JSONObject() val inference = JSONObject() when (selectedProviderType) { - ProviderType.GGUF -> { + ProviderType.GGUF, ProviderType.VISION_CHAT, ProviderType.TOOL_SEARCH -> { loading.put("contextSize", contextSize) loading.put("threadMode", threadMode) loading.put("flashAttn", flashAttn) @@ -298,7 +298,7 @@ fun ModelConfigScreen( } } } - ProviderType.GGUF -> { + ProviderType.GGUF, ProviderType.VISION_CHAT, ProviderType.TOOL_SEARCH -> { item { Spacer(Modifier.height(dimens.spacingSm)) StandardCard(title = "Loading") { @@ -777,6 +777,8 @@ private val CACHE_TYPES = listOf("f16", "q8_0", "q5_0", "q4_0") private val MIROSTAT_LABELS = listOf("Off", "v1", "v2") private val MODEL_TYPES = listOf( ProviderType.GGUF, + ProviderType.VISION_CHAT, + ProviderType.TOOL_SEARCH, ProviderType.EMBEDDING, ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER, @@ -822,6 +824,8 @@ private fun ModelIdentityOption( private fun modelTypeLabel(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat" + ProviderType.VISION_CHAT -> "Vision" + ProviderType.TOOL_SEARCH -> "Tool/Search" ProviderType.EMBEDDING -> "RAG" ProviderType.IMAGE_GEN -> "Image Gen" ProviderType.IMAGE_UPSCALER -> "Upscaler" @@ -831,6 +835,8 @@ private fun modelTypeLabel(type: ProviderType): String = when (type) { private fun modelTypeDescription(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat model: used by normal chat and OpenAI-compatible text completions." + ProviderType.VISION_CHAT -> "Vision chat model: used by normal chat and image-attached chat when a projector is available." + ProviderType.TOOL_SEARCH -> "Tool/Search model: intended for search query planning, source filtering, and structured tool-call output." ProviderType.EMBEDDING -> "Embedding model: used for document indexing and RAG retrieval." ProviderType.IMAGE_GEN -> "Image generation model: used by image create/inpaint tasks." ProviderType.IMAGE_UPSCALER -> "Image upscaler: used by image upscale tasks." diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt index d5d5c8b9..a552a106 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt @@ -1,5 +1,8 @@ package com.dark.tool_neuron.ui.screens.model_manager +import android.widget.Toast +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -26,12 +29,14 @@ import androidx.compose.material3.TextButton import androidx.compose.material3.TopAppBar import androidx.compose.material3.TopAppBarDefaults import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp @@ -61,6 +66,14 @@ fun ModelManagerScreen( val dimens = LocalDimens.current val installed by viewModel.installedModels.collectAsStateWithLifecycle() val deleteInProgress by viewModel.deleteInProgress.collectAsStateWithLifecycle() + val backupStatus by viewModel.backupStatus.collectAsStateWithLifecycle() + val context = LocalContext.current + val exportLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.CreateDocument("application/zip"), + ) { uri -> uri?.let { viewModel.exportModels(it) } } + val importLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.OpenDocument(), + ) { uri -> uri?.let { viewModel.importModels(it) } } var pendingDelete by remember { mutableStateOf(null) } var pendingTypeChange by remember { mutableStateOf(null) } var selectedType by remember { mutableStateOf(null) } @@ -113,6 +126,17 @@ fun ModelManagerScreen( ), verticalArrangement = Arrangement.spacedBy(dimens.spacingSm), ) { + item(key = "backup-actions") { + BackupActionsCard( + status = backupStatus, + onExport = { + exportLauncher.launch("Tool-Neuron-models.zip") + }, + onImport = { + importLauncher.launch(arrayOf("application/zip", "application/octet-stream")) + }, + ) + } item(key = "tabs") { ModelTypeTabs( groups = grouped.map { it.first }, @@ -138,6 +162,12 @@ fun ModelManagerScreen( } } + LaunchedEffect(backupStatus) { + val status = backupStatus ?: return@LaunchedEffect + Toast.makeText(context, status, Toast.LENGTH_SHORT).show() + viewModel.clearBackupStatus() + } + pendingDelete?.let { model -> AlertDialog( onDismissRequest = { pendingDelete = null }, @@ -169,6 +199,40 @@ fun ModelManagerScreen( } } +@Composable +private fun BackupActionsCard( + status: String?, + onExport: () -> Unit, + onImport: () -> Unit, +) { + val dimens = LocalDimens.current + val tnShapes = LocalTnShapes.current + Surface( + modifier = Modifier.fillMaxWidth(), + color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.3f), + shape = tnShapes.cardSmall, + ) { + Column( + modifier = Modifier.padding(dimens.cardPadding), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + Text( + text = "Model backup", + style = MaterialTheme.typography.titleSmall, + fontWeight = FontWeight.SemiBold, + ) + CaptionText( + text = "Export or restore installed models, roles, and per-model config as a ToolNeuron ZIP.", + ) + Row(horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { + Button(onClick = onExport) { Text("Export") } + Button(onClick = onImport) { Text("Import") } + } + if (!status.isNullOrBlank()) CaptionText(text = status) + } + } +} + @Composable private fun ModelManagerCard( model: ModelInfo, @@ -296,6 +360,8 @@ private data class Section( private val SECTIONS = listOf( Section(ProviderType.GGUF, "Chat models", "Used for the conversation in the chat screen.", "Chat"), + Section(ProviderType.VISION_CHAT, "Vision chat models", "Used for chat with image attachments.", "Vision"), + Section(ProviderType.TOOL_SEARCH, "Tool/Search models", "Used for query planning and structured tool calls.", "Tools"), Section(ProviderType.EMBEDDING, "Embedding models", "Used to index documents you attach to chats (RAG).", "RAG"), Section(ProviderType.IMAGE_GEN, "Image generation", "Used by the local image workspace.", "Image"), Section(ProviderType.IMAGE_UPSCALER, "Image upscalers", "Used by image upscale mode.", "Upscale"), diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt index a13b9798..1fd21d50 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt @@ -107,6 +107,8 @@ internal fun ModelTypePickerDialog( internal fun providerTypeLabel(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat (GGUF)" + ProviderType.VISION_CHAT -> "Vision chat" + ProviderType.TOOL_SEARCH -> "Tool/Search" ProviderType.EMBEDDING -> "Embedding (RAG)" ProviderType.IMAGE_GEN -> "Image generation" ProviderType.IMAGE_UPSCALER -> "Image upscaler" @@ -116,6 +118,8 @@ internal fun providerTypeLabel(type: ProviderType): String = when (type) { private fun providerTypeBlurb(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Conversation models used by chat and the remote server." + ProviderType.VISION_CHAT -> "Chat models that can accept image input when a projector is available." + ProviderType.TOOL_SEARCH -> "Small instruction models used for query planning and tool-call JSON." ProviderType.EMBEDDING -> "Vector models for document search and RAG." ProviderType.IMAGE_GEN -> "Diffusion models used by the image workspace." ProviderType.IMAGE_UPSCALER -> "Upscaling models shown in image upscale mode." @@ -125,6 +129,8 @@ private fun providerTypeBlurb(type: ProviderType): String = when (type) { private val PROVIDER_OPTIONS = listOf( ProviderType.GGUF, + ProviderType.VISION_CHAT, + ProviderType.TOOL_SEARCH, ProviderType.EMBEDDING, ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER, diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt index 964b2f09..b3b2c197 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt @@ -154,7 +154,12 @@ class HomeViewModel @Inject constructor( val installedModels: StateFlow> = modelRepo.models val chatModels: StateFlow> = modelRepo.models - .map { list -> list.filter { it.providerType == ProviderType.GGUF } } + .map { list -> + list.filter { + it.providerType == ProviderType.GGUF || + it.providerType == ProviderType.VISION_CHAT + } + } .stateIn(viewModelScope, SharingStarted.Eagerly, emptyList()) val embeddingModelInstalled: StateFlow = modelRepo.models @@ -499,6 +504,7 @@ class HomeViewModel @Inject constructor( viewModelScope.launch { _isIngestingDocument.value = true _documentError.value = null + modelSession.prepareForHeavyBackgroundWork() val result = ragManager.ingestDocument(chatId, uri, name, size, mimeType) _isIngestingDocument.value = false result.onSuccess { doc -> @@ -519,6 +525,7 @@ class HomeViewModel @Inject constructor( fun deepIndexDocument(docId: String) { viewModelScope.launch { _documentError.value = null + modelSession.prepareForHeavyBackgroundWork() val result = ragManager.deepIndex(docId) result.onSuccess { updated -> _chatDocuments.value = _chatDocuments.value.map { if (it.id == updated.id) updated else it } @@ -532,6 +539,7 @@ class HomeViewModel @Inject constructor( if (raptorBuilding.value.contains(docId)) return viewModelScope.launch { _documentError.value = null + modelSession.prepareForHeavyBackgroundWork() val result = ragManager.buildRaptorTree(docId) result.onSuccess { updated -> _chatDocuments.value = _chatDocuments.value.map { if (it.id == updated.id) updated else it } @@ -641,6 +649,7 @@ class HomeViewModel @Inject constructor( if (serverController.isBusy) return if (_isGenerating.value) return if (webSearchActive.value) return + modelSession.markActivity() viewModelScope.launch { modelSession.load(model) } } @@ -654,6 +663,7 @@ class HomeViewModel @Inject constructor( fun sendMessage(text: String) { if (serverController.isBusy) return if (webSearchActive.value) return + modelSession.markActivity() val trimmed = text.trim() val images = _pendingImages.value if (_isGenerating.value) return @@ -674,6 +684,10 @@ class HomeViewModel @Inject constructor( _loadModelWindow.value = true return } + if (images.isNotEmpty() && active.providerType != ProviderType.VISION_CHAT) { + _loadModelWindow.value = true + return + } val webSearchQuery = parseWebSearchInput(trimmed) if (webSearchQuery != null) { @@ -992,6 +1006,7 @@ class HomeViewModel @Inject constructor( } private fun runGeneration(chatId: String, isFirstTurn: Boolean, userText: String) { + modelSession.setGenerationBusy(true) _messages.value = chatRepo.getMessages(chatId) _streamingFragment.value = StreamingFragment(chatId, "", "") _isGenerating.value = true @@ -1042,6 +1057,7 @@ class HomeViewModel @Inject constructor( citations = outcome?.citations.orEmpty(), wasStopped = wasStopped, ) + modelSession.setGenerationBusy(false) } } } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt index 722fa75a..661a0951 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt @@ -20,6 +20,7 @@ import com.dark.tool_neuron.repo.DownloadCoordinator import com.dark.tool_neuron.repo.ExplorerRepo import com.dark.tool_neuron.repo.HuggingFaceExplorer import com.dark.tool_neuron.repo.InstallProgressTracker +import com.dark.tool_neuron.repo.ModelBackupManager import com.dark.tool_neuron.repo.ModelCatalog import com.dark.tool_neuron.repo.ModelRepository import com.dark.tool_neuron.repo.RagManager @@ -74,6 +75,7 @@ class ModelStoreViewModel @Inject constructor( private val installProgress: InstallProgressTracker, private val modelSession: ModelSessionManager, private val downloadCoordinator: DownloadCoordinator, + private val modelBackup: ModelBackupManager, ) : ViewModel() { val installedModels: StateFlow> = modelRepo.models @@ -106,6 +108,9 @@ class ModelStoreViewModel @Inject constructor( private val _deleteInProgress = MutableStateFlow(null) val deleteInProgress: StateFlow = _deleteInProgress.asStateFlow() + private val _backupStatus = MutableStateFlow(null) + val backupStatus: StateFlow = _backupStatus.asStateFlow() + private val _deviceInfo = MutableStateFlow>(emptyMap()) val deviceInfo: StateFlow> = _deviceInfo.asStateFlow() @@ -217,6 +222,35 @@ class ModelStoreViewModel @Inject constructor( } } + fun exportModels(uri: Uri) { + viewModelScope.launch(Dispatchers.IO) { + _backupStatus.value = "Exporting models..." + try { + val models = modelRepo.models.value + modelBackup.exportTo(uri, models) + _backupStatus.value = "Exported ${models.size} models" + } catch (t: Throwable) { + _backupStatus.value = "Export failed: ${t.message ?: t.javaClass.simpleName}" + } + } + } + + fun importModels(uri: Uri) { + viewModelScope.launch(Dispatchers.IO) { + _backupStatus.value = "Importing models..." + try { + val count = modelBackup.importFrom(uri) + _backupStatus.value = "Imported $count models" + } catch (t: Throwable) { + _backupStatus.value = "Import failed: ${t.message ?: t.javaClass.simpleName}" + } + } + } + + fun clearBackupStatus() { + _backupStatus.value = null + } + private fun loadDeviceInfo() { val soc = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MODEL else "Unknown" _deviceInfo.value = mapOf( diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt index 93298026..ecdfb865 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt @@ -67,6 +67,7 @@ class SettingsViewModel @Inject constructor( private val _ragDeepResearch = MutableStateFlow(prefs.ragDeepResearch) private val _vlmImageQuality = MutableStateFlow(prefs.vlmImageQuality) private val _threadMode = MutableStateFlow(prefs.threadMode) + private val _idleUnloadMinutes = MutableStateFlow(prefs.idleUnloadMinutes) private val _pluginOnnxEp = MutableStateFlow(prefs.pluginOnnxEp) private val _serverModelRolesJson = MutableStateFlow(prefs.serverModelRolesJson) private val _serverRoleDefaultsJson = MutableStateFlow(prefs.serverRoleDefaultsJson) @@ -87,6 +88,7 @@ class SettingsViewModel @Inject constructor( _ragDeepResearch, _vlmImageQuality, _threadMode, + _idleUnloadMinutes, _pluginOnnxEp, _serverModelRolesJson, _serverRoleDefaultsJson, @@ -106,11 +108,12 @@ class SettingsViewModel @Inject constructor( val deepResearch = values[10] as Boolean val vlmImageQuality = values[11] as String val threadMode = values[12] as Int - val pluginOnnxEp = values[13] as String - val serverModelRolesJson = values[14] as String - val serverRoleDefaultsJson = values[15] as String + val idleUnloadMinutes = values[13] as Int + val pluginOnnxEp = values[14] as String + val serverModelRolesJson = values[15] as String + val serverRoleDefaultsJson = values[16] as String @Suppress("UNCHECKED_CAST") - val installedPlugins = values[16] as List + val installedPlugins = values[17] as List val pluginCount = installedPlugins.size SettingsState( @@ -126,6 +129,7 @@ class SettingsViewModel @Inject constructor( ragDeepResearch = deepResearch, vlmImageQuality = vlmImageQuality, threadMode = threadMode, + idleUnloadMinutes = idleUnloadMinutes, pluginOnnxEp = pluginOnnxEp, serverModelRolesJson = serverModelRolesJson, serverRoleDefaultsJson = serverRoleDefaultsJson, @@ -168,6 +172,7 @@ class SettingsViewModel @Inject constructor( ragDeepResearch: Boolean, vlmImageQuality: String, threadMode: Int, + idleUnloadMinutes: Int, pluginOnnxEp: String, serverModelRolesJson: String, serverRoleDefaultsJson: String, @@ -177,7 +182,7 @@ class SettingsViewModel @Inject constructor( voiceSection(models, activeTts, activeStt), visionSection(vlmImageQuality), serverRolesSection(models, serverModelRolesJson, serverRoleDefaultsJson, activeTts, activeStt), - modelSection(), + modelSection(idleUnloadMinutes), performanceSection(threadMode), pluginsSection(pluginOnnxEp, pluginCount), privacySection(lockEnabled, panicPinSet), @@ -185,12 +190,31 @@ class SettingsViewModel @Inject constructor( aboutSection(), ) - private fun modelSection(): SettingsSection = SettingsSection( + private fun modelSection(idleUnloadMinutes: Int): SettingsSection = SettingsSection( id = SECTION_MODEL, title = "Model", - description = "Performance and per-model configuration.", + description = "Memory behavior and per-model configuration.", icon = TnIcons.Sliders, items = listOf( + SettingsItem.Choice( + id = ID_IDLE_UNLOAD_MINUTES, + title = "Idle unload", + subtitle = "Unload the active chat model after no chat activity to recover RAM.", + icon = TnIcons.Refresh, + selectedKey = idleUnloadMinutes.toString(), + options = listOf( + SettingsChoiceOption("0", "Off", "Keep the active chat model loaded."), + SettingsChoiceOption("15", "15 minutes", "Most aggressive RAM recovery."), + SettingsChoiceOption("20", "20 minutes", "Balanced default."), + SettingsChoiceOption("30", "30 minutes", "Less reloading, more RAM held."), + ), + onSelect = { key -> + val v = key?.toIntOrNull() ?: AppPreferences.DEFAULT_IDLE_UNLOAD_MINUTES + prefs.idleUnloadMinutes = v + _idleUnloadMinutes.value = prefs.idleUnloadMinutes + _dialog.value = null + }, + ), SettingsItem.Action( id = ID_OPEN_PERFORMANCE, title = "Performance", @@ -409,6 +433,8 @@ class SettingsViewModel @Inject constructor( ServerModelRole.CHAT } } + ProviderType.VISION_CHAT -> ServerModelRole.VLM + ProviderType.TOOL_SEARCH -> ServerModelRole.CHAT ProviderType.EMBEDDING -> ServerModelRole.EMBEDDING ProviderType.TTS -> ServerModelRole.TTS ProviderType.STT -> ServerModelRole.STT @@ -885,6 +911,7 @@ class SettingsViewModel @Inject constructor( private const val ID_SERVER_DEFAULT_PREFIX = "server_default_" private const val ID_SERVER_ROLE_PREFIX = "server_role_" private const val ID_THREAD_MODE = "thread_mode" + private const val ID_IDLE_UNLOAD_MINUTES = "idle_unload_minutes" private const val ID_OPEN_PERFORMANCE = "open_performance" private const val ID_OPEN_MODEL_EDITOR = "open_model_editor" private const val ID_OPEN_MODEL_EDITOR_FROM_SERVER = "open_model_editor_from_server" diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/ModelSessionManager.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/ModelSessionManager.kt index ebceffd7..9810e169 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/ModelSessionManager.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/ModelSessionManager.kt @@ -14,11 +14,14 @@ import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong import org.json.JSONObject import java.io.File import javax.inject.Inject @@ -70,6 +73,8 @@ class ModelSessionManager @Inject constructor( val userSystemPrompt: StateFlow = _userSystemPrompt.asStateFlow() private val watchdogScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val lastActivityMs = AtomicLong(System.currentTimeMillis()) + private val generationBusy = AtomicBoolean(false) init { watchdogScope.launch { @@ -82,9 +87,16 @@ class ModelSessionManager @Inject constructor( } } } + watchdogScope.launch { + while (true) { + delay(IDLE_CHECK_INTERVAL_MS) + maybeUnloadIdleModel() + } + } } suspend fun load(model: ModelInfo) { + markActivity() _loadState.value = ModelLoadState.Loading(model.id) _vlmAutoLoadError.value = null if (InferenceClient.isVlmLoaded.value) { @@ -151,6 +163,36 @@ class ModelSessionManager @Inject constructor( _supportsThinking.value = false _userSystemPrompt.value = "" _loadState.value = ModelLoadState.Idle + System.gc() + } + + fun markActivity() { + lastActivityMs.set(System.currentTimeMillis()) + } + + fun setGenerationBusy(busy: Boolean) { + generationBusy.set(busy) + markActivity() + } + + suspend fun prepareForHeavyBackgroundWork() { + if (generationBusy.get()) return + if (_loadState.value is ModelLoadState.Active || InferenceClient.isVlmLoaded.value) { + unload() + } + } + + private suspend fun maybeUnloadIdleModel() { + val minutes = appPrefs.idleUnloadMinutes + if (minutes <= 0) return + if (generationBusy.get()) return + if (_loadState.value !is ModelLoadState.Active) return + if (!InferenceClient.isModelLoaded.value && !InferenceClient.isVlmLoaded.value) return + val idleMs = System.currentTimeMillis() - lastActivityMs.get() + if (idleMs >= minutes * 60_000L) { + Log.i(TAG, "idle timeout reached (${minutes}m); unloading chat model") + unload() + } } fun setThinkingEnabled(enabled: Boolean) { @@ -223,4 +265,8 @@ class ModelSessionManager @Inject constructor( dst.put(key, src.get(key)) } } + + private companion object { + const val IDLE_CHECK_INTERVAL_MS = 60_000L + } } From 6ca2ebe0d67564d8b8b79a967eb2f33e8f162bd8 Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Sun, 21 Jun 2026 16:43:48 +0600 Subject: [PATCH 05/13] Polish model store, backups, and web UI --- app/src/main/assets/server_webui.html | 238 +++++++++++++-- .../com/dark/tool_neuron/TNApplication.kt | 3 + .../dark/tool_neuron/data/AppVisibility.kt | 28 ++ .../data/UserNotificationManager.kt | 99 ++++++ .../dark/tool_neuron/model/ModelTaxonomy.kt | 107 +++++++ .../tool_neuron/repo/ModelBackupManager.kt | 287 ++++++++++++++++-- .../tool_neuron/repo/RepositoryDataStore.kt | 9 + .../tool_neuron/ui/navigation/TNavigation.kt | 4 + .../model_manager/ModelManagerScreen.kt | 90 +++++- .../ui/screens/model_store/ModelFilters.kt | 46 +-- .../screens/setup_screen/ModelSetupScreen.kt | 32 ++ .../tool_neuron/viewmodel/HomeViewModel.kt | 30 ++ .../viewmodel/ModelStoreViewModel.kt | 152 +++++++++- 13 files changed, 1036 insertions(+), 89 deletions(-) create mode 100644 app/src/main/java/com/dark/tool_neuron/data/AppVisibility.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/data/UserNotificationManager.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/model/ModelTaxonomy.kt diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index ac4ccfdf..c41552b1 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -74,7 +74,11 @@ ::-webkit-scrollbar-thumb:hover { background: var(--muted); } button, input, textarea { font: inherit; color: inherit; } -.app { display: grid; grid-template-columns: 320px 1fr; height: 100dvh; } +.app { display: grid; grid-template-columns: 300px minmax(0,1fr) 320px; height: 100dvh; } +@media (max-width: 1180px) { + .app { grid-template-columns: 300px 1fr; } + .controls { display:none !important; } +} @media (max-width: 880px) { .app { grid-template-columns: 1fr; } .sidebar { @@ -192,6 +196,17 @@ } .menu-btn:hover { border-color: var(--border-2); background: var(--surface-2); } .topbar-title { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 800; font-size: 18px; } +.top-model { + min-width: min(340px, 34vw); + max-width: 420px; + border: 1px solid var(--border); + background: var(--surface); + color: var(--fg); + border-radius: 999px; + padding: 8px 12px; + font-size: 14px; +} +@media (max-width: 720px) { .top-model { min-width: 0; width: 46vw; } } .conn { display: inline-flex; align-items: center; gap: 6px; font-size: 11px; color: var(--muted); @@ -201,6 +216,48 @@ border: 1px solid var(--border); background: var(--surface); } +.controls { + background: var(--bg-soft); + border-left: 1px solid var(--border); + height: 100dvh; + overflow-y: auto; + padding: max(18px, var(--safe-top)) 16px 18px; + display: flex; + flex-direction: column; + gap: 14px; +} +.controls-head { display:flex; align-items:center; justify-content:space-between; gap:8px; } +.controls h2 { margin:0; font-size:16px; font-weight:750; letter-spacing:0; } +.control-card { + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--r-md); + padding: 12px; +} +.control-card summary { + cursor:pointer; + font-weight:650; + color:var(--fg); + list-style:none; + display:flex; + justify-content:space-between; + gap:8px; +} +.control-card summary::-webkit-details-marker { display:none; } +.control-card summary::after { content:'⌄'; color:var(--muted); } +.control-card[open] summary::after { content:'⌃'; } +.control-stack { margin-top:12px; display:flex; flex-direction:column; gap:10px; } +.control-stack label { + display:flex; flex-direction:column; gap:5px; + font-size:11px; text-transform:uppercase; letter-spacing:.06em; color:var(--muted); + font-family:ui-monospace, "SF Mono", Menlo, Consolas, monospace; +} +.control-stack input, .control-stack select, .control-stack textarea { + width:100%; border:1px solid var(--border); background:var(--bg); color:var(--fg); + border-radius:var(--r-sm); padding:8px 10px; font:inherit; font-size:13px; +} +.control-stack textarea { min-height:94px; resize:vertical; } +.control-note { color:var(--muted); font-size:12px; line-height:1.45; margin:0; } .conn .dot { width: 7px; height: 7px; border-radius: 50%; background: var(--danger); transition: background-color 200ms; } .conn.live .dot { background: var(--accent); box-shadow: 0 0 0 3px var(--accent-soft); } @@ -547,6 +604,7 @@ New chat + Checking… @@ -686,6 +744,37 @@

Image

+ -
+ +
Drop files to attach
-
-
-
-

Connect ToolNeuron

-

Enter the token from the Android Server screen, then choose the model this web chat should use.

+ +
+
+
+

Settings

+
-
Waiting for server settings.
- - - -
- - +
+
+
Connection
+
Paste the address and token shown on the app's Server screen, then connect.
+ + +
+ + +
+
+
+
Model
+ +
+ + +
+
Stream repliesShow text as it generates + +
+
+
+
Voice
+
Connect to see the speech models exposed by your server.
+ + +
+ + +
+
+
+
Appearance
+ +
+
+
Your data
+
+ + +
+
+ diff --git a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt index a6212afe..679fbcd5 100644 --- a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt +++ b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt @@ -120,6 +120,10 @@ class AppPreferences @Inject constructor( get() = getBoolean(KEY_MODEL_SETUP_DONE) set(value) = putBoolean(KEY_MODEL_SETUP_DONE, value) + var modelSetupInProgress: Boolean + get() = getBoolean(KEY_MODEL_SETUP_IN_PROGRESS) + set(value) = putBoolean(KEY_MODEL_SETUP_IN_PROGRESS, value) + var guideShown: Boolean get() = getBoolean(KEY_GUIDE_SHOWN) set(value) = putBoolean(KEY_GUIDE_SHOWN, value) @@ -187,6 +191,14 @@ class AppPreferences @Inject constructor( get() = getString(KEY_ACTIVE_STT_MODEL) set(value) = putString(KEY_ACTIVE_STT_MODEL, value) + var voiceTtsBackend: String + get() = getString(KEY_VOICE_TTS_BACKEND, VOICE_BACKEND_AUTO) + set(value) = putString(KEY_VOICE_TTS_BACKEND, value) + + var voiceSttBackend: String + get() = getString(KEY_VOICE_STT_BACKEND, VOICE_BACKEND_AUTO) + set(value) = putString(KEY_VOICE_STT_BACKEND, value) + var ragSmartRerank: Boolean get() = getBoolean(KEY_RAG_SMART_RERANK) set(value) = putBoolean(KEY_RAG_SMART_RERANK, value) @@ -277,6 +289,7 @@ class AppPreferences @Inject constructor( const val KEY_SETUP_DONE = "setup_done" const val KEY_SECURITY_SETUP_DONE = "security_setup_done" const val KEY_MODEL_SETUP_DONE = "model_setup_done" + const val KEY_MODEL_SETUP_IN_PROGRESS = "model_setup_in_progress" const val KEY_GUIDE_SHOWN = "guide_shown" const val KEY_ROOT_WARNING_SHOWN = "root_warning_shown" const val KEY_SERVER_TOKEN = "server_token" @@ -292,6 +305,11 @@ class AppPreferences @Inject constructor( const val KEY_HF_TAGS_CATALOG_AT = "hf_tags_catalog_v1_at" const val KEY_ACTIVE_TTS_MODEL = "active_tts_model" const val KEY_ACTIVE_STT_MODEL = "active_stt_model" + const val KEY_VOICE_TTS_BACKEND = "voice_tts_backend" + const val KEY_VOICE_STT_BACKEND = "voice_stt_backend" + const val VOICE_BACKEND_AUTO = "auto" + const val VOICE_BACKEND_OFFLINE = "offline" + const val VOICE_BACKEND_ANDROID_SYSTEM = "android_system" const val KEY_RAG_SMART_RERANK = "rag_smart_rerank" const val KEY_RAG_MULTI_QUERY = "rag_multi_query" const val KEY_RAG_DEEP_RESEARCH = "rag_deep_research" diff --git a/app/src/main/java/com/dark/tool_neuron/model/ModelInstallProgress.kt b/app/src/main/java/com/dark/tool_neuron/model/ModelInstallProgress.kt new file mode 100644 index 00000000..058e840a --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/model/ModelInstallProgress.kt @@ -0,0 +1,25 @@ +package com.dark.tool_neuron.model + +enum class ModelInstallPhase(val isActive: Boolean, val label: String) { + IDLE(false, "Idle"), + QUEUED(true, "Queued"), + DOWNLOADING(true, "Downloading"), + INSTALLING(true, "Installing"), + VERIFYING(true, "Verifying"), + INSTALLED(false, "Installed"), + FAILED(false, "Failed"), + CANCELLED(false, "Cancelled"), +} + +data class ModelInstallProgress( + val modelId: String, + val installKey: String, + val displayName: String, + val type: String, + val phase: ModelInstallPhase, + val message: String = "", + val hxdId: Int? = null, + val updatedAt: Long = System.currentTimeMillis(), +) { + val isActive: Boolean get() = phase.isActive +} diff --git a/app/src/main/java/com/dark/tool_neuron/service/server/ServerEngineRegistry.kt b/app/src/main/java/com/dark/tool_neuron/service/server/ServerEngineRegistry.kt index 06ff09e0..420b2890 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/server/ServerEngineRegistry.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/server/ServerEngineRegistry.kt @@ -29,7 +29,7 @@ class ServerEngineRegistry(private val app: Context) { suspend fun chatFor(modelId: String): ServerEngine? = chatLock.withLock { val entry = catalog.byId(modelId) ?: return@withLock null - if (entry.kind != ServerEngineKind.CHAT_GGUF) return@withLock null + if (entry.kind != ServerEngineKind.CHAT_GGUF && entry.kind != ServerEngineKind.VLM) return@withLock null if (chat.isLoaded && chat.loadedId() == entry.id) return@withLock chat if (chat.isLoaded) chat.unload() val ok = chat.load(entry.id, entry.path, entry.configJson) diff --git a/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt b/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt index 1c068138..68aa5b98 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt @@ -206,18 +206,19 @@ fun TNavigation( val backupStatus by storeVm.backupStatus.collectAsStateWithLifecycle() val backupProgress by storeVm.backupProgress.collectAsStateWithLifecycle() val importPreview by storeVm.backupImportPreview.collectAsStateWithLifecycle() - val activeDownloads by storeVm.activeDownloadCount.collectAsStateWithLifecycle() + val activeWorkCount by storeVm.activeWorkCount.collectAsStateWithLifecycle() val downloadLabels by storeVm.activeDownloadLabels.collectAsStateWithLifecycle() - var waitingForPack by remember { mutableStateOf(false) } + val installStates by storeVm.installStates.collectAsStateWithLifecycle() + val setupInProgress by storeVm.starterSetupActive.collectAsStateWithLifecycle() var downloadsStarted by remember { mutableStateOf(false) } var restoreStarted by remember { mutableStateOf(false) } - LaunchedEffect(waitingForPack, activeDownloads) { - if (!waitingForPack) return@LaunchedEffect - if (activeDownloads > 0) downloadsStarted = true - if (downloadsStarted && activeDownloads == 0) { - waitingForPack = false + LaunchedEffect(setupInProgress, activeWorkCount) { + if (!setupInProgress) return@LaunchedEffect + if (activeWorkCount > 0) downloadsStarted = true + if (downloadsStarted && activeWorkCount == 0) { downloadsStarted = false + storeVm.finishStarterPackSetup(markDone = true) onModelSetupComplete() } } @@ -234,10 +235,10 @@ fun TNavigation( ModelSetupScreen( innerPadding = innerPadding, onPackSelected = { packId -> - storeVm.downloadPack(packId) - waitingForPack = true + storeVm.beginStarterPackSetup(packId) downloadsStarted = false }, + setupBusy = setupInProgress, onOpenStore = { navController.navigate(NavScreens.ModelStore.route) }, onRestoreBackup = { uri -> storeVm.previewImport(uri) @@ -246,7 +247,10 @@ fun TNavigation( storeVm.importLocalModel(uri, name, size, type) onModelSetupComplete() }, - onSkip = { onModelSetupComplete() } + onSkip = { + storeVm.finishStarterPackSetup(markDone = false) + onModelSetupComplete() + } ) importPreview?.let { preview -> @@ -272,26 +276,30 @@ fun TNavigation( ) } - if (waitingForPack) { + if (setupInProgress) { AlertDialog( onDismissRequest = {}, title = { Text("Downloading starter pack") }, text = { Column(verticalArrangement = Arrangement.spacedBy(10.dp)) { LinearProgressIndicator(modifier = Modifier.fillMaxWidth()) - val names = downloadLabels.values.map { it.displayName }.distinct().take(4) + val activeInstalls = installStates.values + .filter { it.isActive } + .map { "${it.phase.label}: ${it.displayName}" } + val names = (downloadLabels.values.map { it.displayName } + activeInstalls) + .distinct() + .take(4) Text( - if (activeDownloads > 0 && names.isNotEmpty()) { - "Downloading ${names.joinToString(", ")}" + if (activeWorkCount > 0 && names.isNotEmpty()) { + names.joinToString("\n") } else { - "Preparing downloads..." + "Preparing setup..." }, ) } }, confirmButton = { Button(onClick = { - waitingForPack = false downloadsStarted = false onModelSetupComplete() }) { Text("Explore while downloading") } diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt index a2837bae..cf7688ac 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt @@ -35,6 +35,7 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.dark.download_manager.HxdState import com.dark.download_manager.HxdStatus import com.dark.tool_neuron.model.HuggingFaceModel +import com.dark.tool_neuron.model.ModelInstallProgress import com.dark.tool_neuron.ui.components.ActionButton import com.dark.tool_neuron.ui.components.CaptionText import com.dark.tool_neuron.ui.icons.TnIcons @@ -50,6 +51,7 @@ internal fun BrowseModelsTab( isLoading: Boolean, error: String?, downloadStates: Map, + installStates: Map, extractingIds: Set, extractingFile: Map, installedModelIds: Set, @@ -101,9 +103,9 @@ internal fun BrowseModelsTab( label = "repo_nav" ) { repoKey -> if (repoKey == null) { - RepoCardListView(viewModel, isLoading, downloadStates) + RepoCardListView(viewModel, isLoading, downloadStates, installStates) } else { - RepoDetailView(repoKey, viewModel, isLoading, downloadStates, extractingIds, extractingFile, installedModelIds, onDownload, onCancelDownload) + RepoDetailView(repoKey, viewModel, isLoading, downloadStates, installStates, extractingIds, extractingFile, installedModelIds, onDownload, onCancelDownload) } } } @@ -119,7 +121,8 @@ internal fun BrowseModelsTab( internal fun RepoCardListView( viewModel: ModelStoreViewModel, isLoading: Boolean, - downloadStates: Map + downloadStates: Map, + installStates: Map, ) { val dimens = LocalDimens.current val filteredModels by viewModel.filteredModels.collectAsStateWithLifecycle() @@ -134,7 +137,9 @@ internal fun RepoCardListView( val repoModels = remember(filteredModels, repoKey) { viewModel.getModelsForRepo(repoKey) } val hasActiveDownload = repoModels.any { model -> val state = downloadStates[model.id] - state != null && state.status in listOf(HxdStatus.QUEUED, HxdStatus.CONNECTING, HxdStatus.DOWNLOADING) + val install = installStates[model.id] + (state != null && state.status in listOf(HxdStatus.QUEUED, HxdStatus.CONNECTING, HxdStatus.DOWNLOADING)) || + install?.isActive == true } StoreRepoCard(info, hasActiveDownload) { viewModel.selectRepository(repoKey) } @@ -199,6 +204,7 @@ internal fun RepoDetailView( viewModel: ModelStoreViewModel, isLoading: Boolean, downloadStates: Map, + installStates: Map, extractingIds: Set, extractingFile: Map, installedModelIds: Set, @@ -253,6 +259,7 @@ internal fun RepoDetailView( model = model, isInstalled = model.id in installedModelIds, downloadState = downloadStates[model.id], + installProgress = installStates[model.id], isExtracting = model.id in extractingIds, extractingEntryName = extractingFile[model.id], onDownload = { onDownload(model) }, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt index 28b3f689..d151e64f 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt @@ -29,6 +29,8 @@ import androidx.compose.ui.unit.dp import com.dark.download_manager.HxdState import com.dark.download_manager.HxdStatus import com.dark.tool_neuron.model.HuggingFaceModel +import com.dark.tool_neuron.model.ModelInstallPhase +import com.dark.tool_neuron.model.ModelInstallProgress import com.dark.tool_neuron.ui.components.ActionButton import com.dark.tool_neuron.ui.components.ActionProgressButton import com.dark.tool_neuron.ui.components.TnIndeterminateProgressBar @@ -44,6 +46,7 @@ fun CatalogModelCard( model: HuggingFaceModel, isInstalled: Boolean, downloadState: HxdState?, + installProgress: ModelInstallProgress? = null, isExtracting: Boolean = false, extractingEntryName: String? = null, onDownload: () -> Unit, @@ -53,7 +56,12 @@ fun CatalogModelCard( val shapes = LocalTnShapes.current val isActive = downloadState != null && downloadState.status in listOf(HxdStatus.QUEUED, HxdStatus.CONNECTING, HxdStatus.DOWNLOADING) - val isFailed = downloadState != null && downloadState.status == HxdStatus.FAILED + val isInstallActive = installProgress?.isActive == true + val isInstalling = isExtracting || + installProgress?.phase == ModelInstallPhase.INSTALLING || + installProgress?.phase == ModelInstallPhase.VERIFYING + val isFailed = downloadState?.status == HxdStatus.FAILED || + installProgress?.phase == ModelInstallPhase.FAILED Surface( shape = shapes.cardSmall, @@ -114,14 +122,14 @@ fun CatalogModelCard( modifier = Modifier.size(24.dp) ) } - isExtracting -> { + isInstalling -> { CircularProgressIndicator( modifier = Modifier.size(24.dp), strokeWidth = 2.dp, color = MaterialTheme.colorScheme.primary, ) } - isActive -> { + isActive || isInstallActive -> { ActionProgressButton( onClickListener = onCancel, contentDescription = "Cancel" @@ -196,7 +204,7 @@ fun CatalogModelCard( } AnimatedVisibility( - visible = isActive, + visible = isActive || installProgress?.phase == ModelInstallPhase.DOWNLOADING || installProgress?.phase == ModelInstallPhase.QUEUED, enter = Motion.Enter, exit = Motion.Exit ) { @@ -206,7 +214,7 @@ fun CatalogModelCard( } AnimatedVisibility( - visible = isExtracting, + visible = isInstalling, enter = Motion.Enter, exit = Motion.Exit, ) { @@ -220,7 +228,10 @@ fun CatalogModelCard( strokeWidth = 2.dp, color = MaterialTheme.colorScheme.primary, ) - val label = extractingEntryName?.let { "Extracting $it" } ?: "Extracting…" + val label = installProgress?.message?.takeIf { it.isNotBlank() } + ?: extractingEntryName?.let { "Extracting $it" } + ?: installProgress?.phase?.label + ?: "Installing..." Text( text = label, style = MaterialTheme.typography.labelSmall, @@ -234,7 +245,7 @@ fun CatalogModelCard( AnimatedVisibility(visible = isFailed) { Text( - text = "Download failed: ${downloadState?.error ?: "Unknown error"}", + text = "Install failed: ${installProgress?.message?.takeIf { it.isNotBlank() } ?: downloadState?.error ?: "Unknown error"}", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.error, modifier = Modifier.padding(top = dimens.spacingXs) diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt index d28bee2d..14d07fc7 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt @@ -62,11 +62,12 @@ fun ModelStoreScreen( val downloadStates by viewModel.downloadStates.collectAsStateWithLifecycle() val extractingIds by viewModel.extractingIds.collectAsStateWithLifecycle() val extractingFile by viewModel.extractingFile.collectAsStateWithLifecycle() + val installStates by viewModel.installStates.collectAsStateWithLifecycle() val installedModels by viewModel.installedModels.collectAsStateWithLifecycle() val installedMmprojIds by viewModel.installedMmprojIds.collectAsStateWithLifecycle() val deviceInfo by viewModel.deviceInfo.collectAsStateWithLifecycle() val deleteInProgress by viewModel.deleteInProgress.collectAsStateWithLifecycle() - val activeDownloadCount by viewModel.activeDownloadCount.collectAsStateWithLifecycle() + val activeWorkCount by viewModel.activeWorkCount.collectAsStateWithLifecycle() var searchQuery by remember { mutableStateOf("") } var showSearch by remember { mutableStateOf(false) } @@ -127,7 +128,7 @@ fun ModelStoreScreen( icon = TnIcons.Download, contentDescription = "Downloads", ) - if (activeDownloadCount > 0) { + if (activeWorkCount > 0) { Box( modifier = Modifier .offset(x = (-4).dp, y = 4.dp) @@ -198,6 +199,7 @@ fun ModelStoreScreen( isLoading = isLoading, error = error, downloadStates = downloadStates, + installStates = installStates, extractingIds = extractingIds, extractingFile = extractingFile, installedModelIds = installedModels.map { it.id }.toSet() + installedMmprojIds, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/setup_screen/ModelSetupScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/setup_screen/ModelSetupScreen.kt index 8a4522ff..8cdff676 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/setup_screen/ModelSetupScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/setup_screen/ModelSetupScreen.kt @@ -92,6 +92,7 @@ private val SETUP_PACKS = listOf( fun ModelSetupScreen( innerPadding: PaddingValues, onPackSelected: (packId: String) -> Unit, + setupBusy: Boolean = false, onOpenStore: () -> Unit, onRestoreBackup: (uri: Uri) -> Unit, onLocalImport: (uri: Uri, name: String, size: Long, type: ProviderType) -> Unit, @@ -198,6 +199,7 @@ fun ModelSetupScreen( selectedPack = selectedPack, onPackChange = { selectedPack = it }, onContinue = { selectedPack?.let(onPackSelected) }, + enabled = !setupBusy, ) SetupPath.Restore -> RestoreSection( @@ -271,6 +273,7 @@ private fun PacksSection( selectedPack: String?, onPackChange: (String) -> Unit, onContinue: () -> Unit, + enabled: Boolean, ) { val dimens = LocalDimens.current Column { @@ -281,6 +284,7 @@ private fun PacksSection( subtitle = pack.subtitle, selected = selectedPack == pack.id, onClick = { onPackChange(pack.id) }, + enabled = enabled, ) if (index != SETUP_PACKS.lastIndex) { Spacer(Modifier.height(dimens.spacingSm)) @@ -293,7 +297,7 @@ private fun PacksSection( onClickListener = onContinue, icon = TnIcons.ArrowRight, text = "Continue", - enabled = selectedPack != null, + enabled = selectedPack != null && enabled, ) } } @@ -335,7 +339,8 @@ private fun PackCard( title: String, subtitle: String, selected: Boolean, - onClick: () -> Unit + onClick: () -> Unit, + enabled: Boolean = true, ) { val tnShapes = LocalTnShapes.current val dimens = LocalDimens.current @@ -355,7 +360,7 @@ private fun PackCard( ) Surface( - onClick = onClick, + onClick = { if (enabled) onClick() }, modifier = Modifier.fillMaxWidth(), shape = tnShapes.card, color = containerColor, diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt index 93044f09..511874f4 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt @@ -13,6 +13,8 @@ import com.dark.tool_neuron.model.HuggingFaceModel import com.dark.tool_neuron.model.ModelCategory import com.dark.tool_neuron.model.ModelConfig import com.dark.tool_neuron.model.ModelInfo +import com.dark.tool_neuron.model.ModelInstallPhase +import com.dark.tool_neuron.model.ModelInstallProgress import com.dark.tool_neuron.model.ModelTaxonomy import com.dark.tool_neuron.model.SizeCategory import com.dark.tool_neuron.model.enums.PathType @@ -47,10 +49,13 @@ import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import java.security.MessageDigest import javax.inject.Inject @@ -138,6 +143,16 @@ class ModelStoreViewModel @Inject constructor( private val _extractingFile = MutableStateFlow>(emptyMap()) val extractingFile: StateFlow> = _extractingFile.asStateFlow() + private val installMutex = Mutex() + private val downloadGuard = Any() + private val _installStates = MutableStateFlow>(emptyMap()) + val installStates: StateFlow> = _installStates.asStateFlow() + val activeWorkCount: StateFlow = combine(activeDownloadCount, installStates) { downloads, installs -> + downloads + installs.values.count { it.isActive } + }.stateIn(viewModelScope, SharingStarted.WhileSubscribed(5000), 0) + private val _starterSetupActive = MutableStateFlow(prefs.modelSetupInProgress) + val starterSetupActive: StateFlow = _starterSetupActive.asStateFlow() + private var starterSetupObservedWork = false val isModelLoaded = InferenceClient.isModelLoaded .stateIn(viewModelScope, SharingStarted.WhileSubscribed(5000), false) @@ -200,6 +215,15 @@ class ModelStoreViewModel @Inject constructor( init { loadDeviceInfo() loadModels() + viewModelScope.launch { + activeWorkCount.collect { count -> + if (!_starterSetupActive.value) return@collect + if (count > 0) starterSetupObservedWork = true + if (starterSetupObservedWork && count == 0) { + finishStarterPackSetup(markDone = true) + } + } + } } fun selectTab(tab: StoreTab) { _selectedTab.value = tab } @@ -511,6 +535,21 @@ class ModelStoreViewModel @Inject constructor( } } + fun beginStarterPackSetup(packId: String) { + if (_starterSetupActive.value) return + prefs.modelSetupInProgress = true + _starterSetupActive.value = true + starterSetupObservedWork = activeWorkCount.value > 0 + downloadPack(packId) + } + + fun finishStarterPackSetup(markDone: Boolean = false) { + prefs.modelSetupInProgress = false + if (markDone) prefs.modelSetupDone = true + _starterSetupActive.value = false + starterSetupObservedWork = false + } + private fun enqueueChatModel(pool: List, modelId: String) { val candidates = pool.filter { it.repoId == modelId || it.id.startsWith(modelId) } val preferred = QUICK_START_QUANT_PRIORITY.firstNotNullOfOrNull { q -> @@ -555,39 +594,62 @@ class ModelStoreViewModel @Inject constructor( private data class PackEntry(val id: String, val kind: PackEntryKind) fun downloadModel(model: HuggingFaceModel) { - if (_downloadIds.value.containsKey(model.id)) return - - val destFile = when { - model.isMmproj && model.repoPath.isNotBlank() -> - modelRepo.vlmMmprojFile(model.repoPath, model.fileName) - model.isVlm && model.repoPath.isNotBlank() -> - modelRepo.vlmModelFile(model.repoPath, model.fileName) - model.modelType == "tts" -> - modelRepo.voiceArchiveFile("tts", model.id, model.fileName) - model.modelType == "stt" -> - modelRepo.voiceArchiveFile("stt", model.id, model.fileName) - model.modelType == "image_gen" -> - modelRepo.imageModelArchive(model.id, model.fileName) - model.modelType == "image_upscaler" -> - modelRepo.imageUpscalerFile(model.id, model.fileName) - else -> - modelRepo.modelFile(model.id, model.fileName) + if (modelRepo.models.value.any { it.id == model.id }) { + setInstallState(model, destinationFileFor(model), ModelInstallPhase.INSTALLED) + return } + + val destFile = destinationFileFor(model) destFile.parentFile?.mkdirs() + synchronized(downloadGuard) { + val existingInstall = _installStates.value[model.id] + if (existingInstall?.isActive == true && existingInstall.installKey == installKey(model, destFile)) return + if (_downloadIds.value.containsKey(model.id)) return + + HxdManager.activeFor(model.fileUri, destFile.absolutePath)?.let { existing -> + downloadCoordinator.registerLabel(existing.id, model.name, downloadTypeOf(model)) + _downloadIds.value = _downloadIds.value + (model.id to existing.id) + _downloadStates.value = _downloadStates.value + (model.id to existing) + setInstallState(model, destFile, phaseForDownload(existing.status), hxdId = existing.id) + observeDownload(model, destFile, existing.id) + return + } + + setInstallState(model, destFile, ModelInstallPhase.QUEUED) + } + val hxdId = HxdManager.enqueue(context, model.fileUri, destFile.absolutePath) downloadCoordinator.registerLabel(hxdId, model.name, downloadTypeOf(model)) _downloadIds.value = _downloadIds.value + (model.id to hxdId) _downloadStates.value = _downloadStates.value + (model.id to HxdState(hxdId, model.fileUri, destFile.absolutePath, 0L, -1L, 0L, HxdStatus.QUEUED)) + setInstallState(model, destFile, ModelInstallPhase.QUEUED, hxdId = hxdId) + observeDownload(model, destFile, hxdId) + } + private fun observeDownload(model: HuggingFaceModel, destFile: java.io.File, hxdId: Int) { viewModelScope.launch(Dispatchers.IO) { + var terminalHandled = false HxdManager.observe(hxdId).collect { state -> if (state == null) return@collect _downloadStates.value = _downloadStates.value + (model.id to state) + if (!terminalHandled) setInstallState( + model = model, + destFile = destFile, + phase = phaseForDownload(state.status), + message = state.error.orEmpty(), + hxdId = hxdId, + ) + + if (state.status in setOf(HxdStatus.COMPLETED, HxdStatus.FAILED, HxdStatus.CANCELLED) && terminalHandled) { + return@collect + } when (state.status) { HxdStatus.COMPLETED -> { + terminalHandled = true + setInstallState(model, destFile, ModelInstallPhase.VERIFYING, hxdId = hxdId) if (model.isMmproj) { finalizeMmprojDownload(model, destFile) } else if (model.isVlm && model.mmprojFileUri.isNotBlank()) { @@ -608,9 +670,13 @@ class ModelStoreViewModel @Inject constructor( } } HxdStatus.FAILED -> { + terminalHandled = true + setInstallState(model, destFile, ModelInstallPhase.FAILED, state.error ?: "Download failed", hxdId) _downloadIds.value = _downloadIds.value - model.id } HxdStatus.CANCELLED -> { + terminalHandled = true + setInstallState(model, destFile, ModelInstallPhase.CANCELLED, hxdId = hxdId) _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id } @@ -620,9 +686,58 @@ class ModelStoreViewModel @Inject constructor( } } + private fun destinationFileFor(model: HuggingFaceModel): java.io.File = when { + model.isMmproj && model.repoPath.isNotBlank() -> + modelRepo.vlmMmprojFile(model.repoPath, model.fileName) + model.isVlm && model.repoPath.isNotBlank() -> + modelRepo.vlmModelFile(model.repoPath, model.fileName) + model.modelType == "tts" -> + modelRepo.voiceArchiveFile("tts", model.id, model.fileName) + model.modelType == "stt" -> + modelRepo.voiceArchiveFile("stt", model.id, model.fileName) + model.modelType == "image_gen" -> + modelRepo.imageModelArchive(model.id, model.fileName) + model.modelType == "image_upscaler" -> + modelRepo.imageUpscalerFile(model.id, model.fileName) + else -> + modelRepo.modelFile(model.id, model.fileName) + } + + private fun installKey(model: HuggingFaceModel, destFile: java.io.File): String = + "${model.id}|${destFile.absolutePath}" + + private fun phaseForDownload(status: HxdStatus): ModelInstallPhase = when (status) { + HxdStatus.QUEUED, HxdStatus.CONNECTING -> ModelInstallPhase.QUEUED + HxdStatus.DOWNLOADING, HxdStatus.PAUSED -> ModelInstallPhase.DOWNLOADING + HxdStatus.COMPLETED -> ModelInstallPhase.VERIFYING + HxdStatus.FAILED -> ModelInstallPhase.FAILED + HxdStatus.CANCELLED -> ModelInstallPhase.CANCELLED + } + + private fun setInstallState( + model: HuggingFaceModel, + destFile: java.io.File, + phase: ModelInstallPhase, + message: String = "", + hxdId: Int? = null, + ) { + _installStates.value = _installStates.value + ( + model.id to ModelInstallProgress( + modelId = model.id, + installKey = installKey(model, destFile), + displayName = model.name, + type = downloadTypeOf(model), + phase = phase, + message = message, + hxdId = hxdId, + ) + ) + } + private fun finalizeMmprojDownload(model: HuggingFaceModel, destFile: java.io.File) { _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id + setInstallState(model, destFile, ModelInstallPhase.INSTALLED) _installedMmprojIds.value = _installedMmprojIds.value + model.id } @@ -643,16 +758,29 @@ class ModelStoreViewModel @Inject constructor( HxdState(hxdId, model.mmprojFileUri, mmprojFile.absolutePath, 0L, -1L, 0L, HxdStatus.QUEUED)) viewModelScope.launch(Dispatchers.IO) { + var terminalHandled = false HxdManager.observe(hxdId).collect { state -> if (state == null) return@collect _downloadStates.value = _downloadStates.value + (model.id to state) + if (!terminalHandled) setInstallState(model, mmprojFile, phaseForDownload(state.status), state.error.orEmpty(), hxdId) + if (state.status in setOf(HxdStatus.COMPLETED, HxdStatus.FAILED, HxdStatus.CANCELLED) && terminalHandled) { + return@collect + } when (state.status) { - HxdStatus.COMPLETED -> finalizeVlmDownload(model, baseFile, mmprojFile) + HxdStatus.COMPLETED -> { + terminalHandled = true + setInstallState(model, mmprojFile, ModelInstallPhase.VERIFYING, hxdId = hxdId) + finalizeVlmDownload(model, baseFile, mmprojFile) + } HxdStatus.FAILED -> { + terminalHandled = true _error.value = "Projector download failed for ${model.name}" + setInstallState(model, mmprojFile, ModelInstallPhase.FAILED, state.error ?: "Projector download failed", hxdId) _downloadIds.value = _downloadIds.value - model.id } HxdStatus.CANCELLED -> { + terminalHandled = true + setInstallState(model, mmprojFile, ModelInstallPhase.CANCELLED, hxdId = hxdId) _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id } @@ -674,20 +802,30 @@ class ModelStoreViewModel @Inject constructor( private fun finalizeVoiceDownload(model: HuggingFaceModel, archive: java.io.File) { viewModelScope.launch(Dispatchers.IO) { + installMutex.withLock { _extractingIds.value = _extractingIds.value + model.id installProgress.extractStarted(model.id) try { if (!archive.exists()) { _error.value = "Downloaded archive missing for ${model.name}" - return@launch + setInstallState(model, archive, ModelInstallPhase.FAILED, "Downloaded archive missing") + return@withLock } + setInstallState(model, archive, ModelInstallPhase.INSTALLING, "Preparing voice archive") val kind = if (model.modelType == "tts") VoiceKind.TTS else VoiceKind.STT val parent = modelRepo.voiceDir(if (kind == VoiceKind.TTS) "tts" else "stt") + var lastProgressAt = 0L val result = VoiceArchive.extractAndBuildConfig(archive, parent, kind) { entryName -> - _extractingFile.update { it + (model.id to entryName) } + val now = android.os.SystemClock.elapsedRealtime() + if (now - lastProgressAt >= 250L) { + lastProgressAt = now + _extractingFile.update { it + (model.id to entryName) } + setInstallState(model, archive, ModelInstallPhase.INSTALLING, "Extracting $entryName") + } } when (result) { is VoiceArchive.ExtractResult.Success -> { + setInstallState(model, archive, ModelInstallPhase.VERIFYING, "Verifying voice model") archive.delete() val provider = if (kind == VoiceKind.TTS) ProviderType.TTS else ProviderType.STT val folderSize = result.folder.walkTopDown() @@ -712,15 +850,18 @@ class ModelStoreViewModel @Inject constructor( if (kind == VoiceKind.STT && prefs.activeSttModelId.isBlank()) { prefs.activeSttModelId = model.id } + setInstallState(model, archive, ModelInstallPhase.INSTALLED) } is VoiceArchive.ExtractResult.Failure -> { android.util.Log.e("ModelStoreVM", "Voice extract failed: ${result.reason}") _error.value = "Extraction failed: ${result.reason}" + setInstallState(model, archive, ModelInstallPhase.FAILED, result.reason) } } } catch (t: Throwable) { android.util.Log.e("ModelStoreVM", "finalizeVoiceDownload threw", t) _error.value = "Extraction error: ${t.message}" + setInstallState(model, archive, ModelInstallPhase.FAILED, t.message ?: "Extraction error") } finally { _extractingIds.value = _extractingIds.value - model.id _extractingFile.value = _extractingFile.value - model.id @@ -728,28 +869,40 @@ class ModelStoreViewModel @Inject constructor( _downloadStates.value = _downloadStates.value - model.id installProgress.extractFinished(model.id) } + } } } private fun finalizeImageGenDownload(model: HuggingFaceModel, archive: java.io.File) { viewModelScope.launch(Dispatchers.IO) { + installMutex.withLock { _extractingIds.value = _extractingIds.value + model.id installProgress.extractStarted(model.id) try { if (!archive.exists()) { _error.value = "Downloaded archive missing for ${model.name}" - return@launch + setInstallState(model, archive, ModelInstallPhase.FAILED, "Downloaded archive missing") + return@withLock } + setInstallState(model, archive, ModelInstallPhase.INSTALLING, "Preparing image model archive") val targetDir = modelRepo.imageModelDir(model.id) targetDir.listFiles()?.forEach { it.deleteRecursively() } + var lastProgressAt = 0L val ok = unzipInto(archive, targetDir) { entryName -> - _extractingFile.update { it + (model.id to entryName) } + val now = android.os.SystemClock.elapsedRealtime() + if (now - lastProgressAt >= 250L) { + lastProgressAt = now + _extractingFile.update { it + (model.id to entryName) } + setInstallState(model, archive, ModelInstallPhase.INSTALLING, "Extracting $entryName") + } } if (!ok) { _error.value = "Extraction failed for ${model.name}" targetDir.deleteRecursively() - return@launch + setInstallState(model, archive, ModelInstallPhase.FAILED, "Extraction failed") + return@withLock } + setInstallState(model, archive, ModelInstallPhase.VERIFYING, "Verifying image model") archive.delete() // Walk down through any chain of single-subdir wrappers (xororz/sd-qnn ZIPs // wrap their contents in `output_/qnn_models_/`) until we find @@ -767,9 +920,11 @@ class ModelStoreViewModel @Inject constructor( ), ) refreshServerCatalog() + setInstallState(model, archive, ModelInstallPhase.INSTALLED) } catch (t: Throwable) { android.util.Log.e("ModelStoreVM", "finalizeImageGenDownload threw", t) _error.value = "Extraction error: ${t.message}" + setInstallState(model, archive, ModelInstallPhase.FAILED, t.message ?: "Extraction error") } finally { _extractingIds.value = _extractingIds.value - model.id _extractingFile.value = _extractingFile.value - model.id @@ -777,6 +932,7 @@ class ModelStoreViewModel @Inject constructor( _downloadStates.value = _downloadStates.value - model.id installProgress.extractFinished(model.id) } + } } } @@ -785,8 +941,10 @@ class ModelStoreViewModel @Inject constructor( try { if (!destFile.exists()) { _error.value = "Downloaded file missing for ${model.name}" + setInstallState(model, destFile, ModelInstallPhase.FAILED, "Downloaded file missing") return@launch } + setInstallState(model, destFile, ModelInstallPhase.VERIFYING) modelRepo.insert( ModelInfo( id = model.id, name = model.name, @@ -796,6 +954,7 @@ class ModelStoreViewModel @Inject constructor( ), ) refreshServerCatalog() + setInstallState(model, destFile, ModelInstallPhase.INSTALLED) } finally { _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id @@ -878,6 +1037,13 @@ class ModelStoreViewModel @Inject constructor( } private fun finalizeNonVlmDownload(model: HuggingFaceModel, destFile: java.io.File) { + if (!destFile.exists()) { + _error.value = "Downloaded file missing for ${model.name}" + setInstallState(model, destFile, ModelInstallPhase.FAILED, "Downloaded file missing") + _downloadIds.value = _downloadIds.value - model.id + return + } + setInstallState(model, destFile, ModelInstallPhase.VERIFYING) val provider = when (model.modelType) { "tts" -> ProviderType.TTS "stt" -> ProviderType.STT @@ -899,6 +1065,7 @@ class ModelStoreViewModel @Inject constructor( } _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id + setInstallState(model, destFile, ModelInstallPhase.INSTALLED) } private fun finalizeVlmDownload( @@ -908,10 +1075,12 @@ class ModelStoreViewModel @Inject constructor( ) { if (!mmprojFile.exists() || mmprojFile.length() <= 0L) { _error.value = "Vision projector missing for ${model.name}" + setInstallState(model, baseFile, ModelInstallPhase.FAILED, "Vision projector missing") _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id return } + setInstallState(model, baseFile, ModelInstallPhase.VERIFYING) modelRepo.insert(ModelInfo( id = model.id, name = model.name, path = baseFile.absolutePath, pathType = PathType.FILE, @@ -922,6 +1091,7 @@ class ModelStoreViewModel @Inject constructor( _downloadIds.value = _downloadIds.value - model.id _downloadStates.value = _downloadStates.value - model.id _installedMmprojIds.value = _installedMmprojIds.value + model.id + setInstallState(model, baseFile, ModelInstallPhase.INSTALLED) } private fun downloadTypeOf(model: HuggingFaceModel): String = when { diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt index ecdfb865..c734e9cb 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt @@ -18,6 +18,7 @@ import com.dark.tool_neuron.service.server.ServerModelRole import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.screens.crash_report.CrashReportActivity import com.dark.tool_neuron.voice.VoiceModelManager +import com.dark.tool_neuron.voice.VoiceBackend import com.dark.tool_neuron.util.VlmPaths import com.dark.tool_neuron.ui.screens.settings.model.SettingsChoiceOption import com.dark.tool_neuron.ui.screens.settings.model.SettingsDialog @@ -62,6 +63,8 @@ class SettingsViewModel @Inject constructor( private val _panicPinSet = MutableStateFlow(security.hasPanicPin) private val _activeTts = MutableStateFlow(prefs.activeTtsModelId) private val _activeStt = MutableStateFlow(prefs.activeSttModelId) + private val _voiceTtsBackend = MutableStateFlow(prefs.voiceTtsBackend) + private val _voiceSttBackend = MutableStateFlow(prefs.voiceSttBackend) private val _ragSmartRerank = MutableStateFlow(prefs.ragSmartRerank) private val _ragMultiQuery = MutableStateFlow(prefs.ragMultiQuery) private val _ragDeepResearch = MutableStateFlow(prefs.ragDeepResearch) @@ -82,6 +85,8 @@ class SettingsViewModel @Inject constructor( _snackbar, _activeTts, _activeStt, + _voiceTtsBackend, + _voiceSttBackend, _panicPinSet, _ragSmartRerank, _ragMultiQuery, @@ -102,18 +107,20 @@ class SettingsViewModel @Inject constructor( val snackbar = values[4] as String? val activeTts = values[5] as String val activeStt = values[6] as String - val panicSet = values[7] as Boolean - val rerank = values[8] as Boolean - val multiQuery = values[9] as Boolean - val deepResearch = values[10] as Boolean - val vlmImageQuality = values[11] as String - val threadMode = values[12] as Int - val idleUnloadMinutes = values[13] as Int - val pluginOnnxEp = values[14] as String - val serverModelRolesJson = values[15] as String - val serverRoleDefaultsJson = values[16] as String + val voiceTtsBackend = values[7] as String + val voiceSttBackend = values[8] as String + val panicSet = values[9] as Boolean + val rerank = values[10] as Boolean + val multiQuery = values[11] as Boolean + val deepResearch = values[12] as Boolean + val vlmImageQuality = values[13] as String + val threadMode = values[14] as Int + val idleUnloadMinutes = values[15] as Int + val pluginOnnxEp = values[16] as String + val serverModelRolesJson = values[17] as String + val serverRoleDefaultsJson = values[18] as String @Suppress("UNCHECKED_CAST") - val installedPlugins = values[17] as List + val installedPlugins = values[19] as List val pluginCount = installedPlugins.size SettingsState( @@ -123,6 +130,8 @@ class SettingsViewModel @Inject constructor( lockEnabled = lockOn, activeTts = activeTts, activeStt = activeStt, + voiceTtsBackend = voiceTtsBackend, + voiceSttBackend = voiceSttBackend, panicPinSet = panicSet, ragSmartRerank = rerank, ragMultiQuery = multiQuery, @@ -166,6 +175,8 @@ class SettingsViewModel @Inject constructor( lockEnabled: Boolean, activeTts: String, activeStt: String, + voiceTtsBackend: String, + voiceSttBackend: String, panicPinSet: Boolean, ragSmartRerank: Boolean, ragMultiQuery: Boolean, @@ -179,7 +190,7 @@ class SettingsViewModel @Inject constructor( pluginCount: Int, ): List = listOf( chatAndRagSection(models, defaultEmbedding, ragSmartRerank, ragMultiQuery, ragDeepResearch), - voiceSection(models, activeTts, activeStt), + voiceSection(models, activeTts, activeStt, voiceTtsBackend, voiceSttBackend), visionSection(vlmImageQuality), serverRolesSection(models, serverModelRolesJson, serverRoleDefaultsJson, activeTts, activeStt), modelSection(idleUnloadMinutes), @@ -602,6 +613,8 @@ class SettingsViewModel @Inject constructor( models: List, activeTts: String, activeStt: String, + voiceTtsBackend: String, + voiceSttBackend: String, ): SettingsSection { val ttsModels = models.filter { it.providerType == ProviderType.TTS } val sttModels = models.filter { it.providerType == ProviderType.STT } @@ -613,12 +626,38 @@ class SettingsViewModel @Inject constructor( } val resolvedTts = activeTts.takeIf { it.isNotBlank() && ttsModels.any { m -> m.id == it } } val resolvedStt = activeStt.takeIf { it.isNotBlank() && sttModels.any { m -> m.id == it } } + val backendOptions = listOf( + SettingsChoiceOption( + key = VoiceBackend.AUTO.key, + label = VoiceBackend.AUTO.label, + description = "Offline model first; Android system fallback if no local voice model is installed.", + ), + SettingsChoiceOption( + key = VoiceBackend.OFFLINE.key, + label = VoiceBackend.OFFLINE.label, + description = "Use installed local Whisper/Piper models only.", + ), + SettingsChoiceOption( + key = VoiceBackend.ANDROID_SYSTEM.key, + label = VoiceBackend.ANDROID_SYSTEM.label, + description = "Uses the device/OEM speech service; it may use online Google/OEM services.", + ), + ) return SettingsSection( id = "voice", title = "Voice", description = "Defaults for text-to-speech and speech-to-text.", icon = TnIcons.Volume, items = listOf( + SettingsItem.Choice( + id = ID_TTS_BACKEND, + title = "Text-to-speech backend", + subtitle = "Auto keeps ToolNeuron local-first, then falls back to Android system TTS.", + icon = TnIcons.Volume, + selectedKey = VoiceBackend.fromKey(voiceTtsBackend).key, + options = backendOptions, + onSelect = { key -> applyVoiceTtsBackend(key) }, + ), SettingsItem.Choice( id = ID_DEFAULT_TTS, title = "Default text-to-speech model", @@ -629,6 +668,15 @@ class SettingsViewModel @Inject constructor( emptyMessage = if (ttsOptions.isEmpty()) "Install one from the Store" else "Auto-pick first", onSelect = { key -> applyActiveTts(key) }, ), + SettingsItem.Choice( + id = ID_STT_BACKEND, + title = "Speech-to-text backend", + subtitle = "Android system STT may use Google/OEM services depending on the device.", + icon = TnIcons.Mic, + selectedKey = VoiceBackend.fromKey(voiceSttBackend).key, + options = backendOptions, + onSelect = { key -> applyVoiceSttBackend(key) }, + ), SettingsItem.Choice( id = ID_DEFAULT_STT, title = "Default speech-to-text model", @@ -643,6 +691,22 @@ class SettingsViewModel @Inject constructor( ) } + private fun applyVoiceTtsBackend(key: String?) { + val next = VoiceBackend.fromKey(key.orEmpty()).key + prefs.voiceTtsBackend = next + _voiceTtsBackend.value = next + _dialog.value = null + viewModelScope.launch { voiceManager.unloadTts() } + } + + private fun applyVoiceSttBackend(key: String?) { + val next = VoiceBackend.fromKey(key.orEmpty()).key + prefs.voiceSttBackend = next + _voiceSttBackend.value = next + _dialog.value = null + viewModelScope.launch { voiceManager.unloadStt() } + } + private fun applyActiveTts(key: String?) { val next = key.orEmpty() prefs.activeTtsModelId = next @@ -901,7 +965,9 @@ class SettingsViewModel @Inject constructor( const val SECTION_ABOUT = "about" private const val ID_DEFAULT_EMBEDDING = "default_embedding_model" + private const val ID_TTS_BACKEND = "tts_backend" private const val ID_DEFAULT_TTS = "default_tts_model" + private const val ID_STT_BACKEND = "stt_backend" private const val ID_DEFAULT_STT = "default_stt_model" private const val ID_RAG_DEBUG = "rag_debug" private const val ID_RAG_RERANK = "rag_smart_rerank" diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/InferenceCoordinator.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/InferenceCoordinator.kt index b1177fde..b38ac5fc 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/InferenceCoordinator.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/home_vm/InferenceCoordinator.kt @@ -461,6 +461,17 @@ class InferenceCoordinator @Inject constructor( .filter { it.first >= 0 } .minByOrNull { it.first } if (next == null) { + val closeOnly = THINK_TAGS + .map { (_, close) -> raw.indexOf(close, i) to close } + .filter { it.first >= 0 } + .minByOrNull { it.first } + if (closeOnly != null) { + val (end, closeTag) = closeOnly + if (thinking.isNotEmpty()) thinking.append("\n\n") + thinking.append(raw, i, end) + i = end + closeTag.length + continue + } content.append(raw, i, raw.length) break } diff --git a/app/src/main/java/com/dark/tool_neuron/voice/SystemSttRecognizer.kt b/app/src/main/java/com/dark/tool_neuron/voice/SystemSttRecognizer.kt new file mode 100644 index 00000000..cd99a881 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/voice/SystemSttRecognizer.kt @@ -0,0 +1,133 @@ +package com.dark.tool_neuron.voice + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import androidx.core.content.ContextCompat +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeout +import java.util.Locale +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SystemSttRecognizer @Inject constructor( + @param:ApplicationContext private val context: Context, +) { + private var recognizer: SpeechRecognizer? = null + private var pendingResult: CompletableDeferred? = null + + private val _isRecording = MutableStateFlow(false) + val isRecording: StateFlow = _isRecording.asStateFlow() + + private val _amplitude = MutableStateFlow(0f) + val amplitude: StateFlow = _amplitude.asStateFlow() + + fun hasPermission(): Boolean = ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + + fun isAvailable(): Boolean = SpeechRecognizer.isRecognitionAvailable(context) + + fun start(): Boolean { + if (_isRecording.value) return true + if (!hasPermission() || !isAvailable()) return false + pendingResult = CompletableDeferred() + android.os.Handler(android.os.Looper.getMainLooper()).post { + try { + val sr = SpeechRecognizer.createSpeechRecognizer(context.applicationContext) + recognizer = sr + sr.setRecognitionListener(listener()) + sr.startListening(intent()) + _isRecording.value = true + } catch (t: Throwable) { + _isRecording.value = false + pendingResult?.complete(null) + } + } + return true + } + + suspend fun stopAndRecognize(): String? { + val result = pendingResult ?: return null + withContext(Dispatchers.Main) { + runCatching { recognizer?.stopListening() } + } + return try { + withTimeout(RECOGNITION_TIMEOUT_MS) { result.await() } + } catch (_: TimeoutCancellationException) { + cancel() + null + } finally { + cleanup() + } + } + + fun cancel() { + android.os.Handler(android.os.Looper.getMainLooper()).post { + runCatching { recognizer?.cancel() } + cleanup() + } + pendingResult?.complete(null) + } + + private fun listener(): RecognitionListener = object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) = Unit + override fun onBeginningOfSpeech() = Unit + override fun onRmsChanged(rmsdB: Float) { + _amplitude.value = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f) + } + override fun onBufferReceived(buffer: ByteArray?) = Unit + override fun onEndOfSpeech() { + _isRecording.value = false + } + override fun onError(error: Int) { + pendingResult?.complete(null) + cleanup() + } + override fun onResults(results: Bundle?) { + pendingResult?.complete(firstResult(results)) + cleanup() + } + override fun onPartialResults(partialResults: Bundle?) = Unit + override fun onEvent(eventType: Int, params: Bundle?) = Unit + } + + private fun intent(): Intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault()) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1) + } + + private fun firstResult(bundle: Bundle?): String? = + bundle?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + ?.firstOrNull() + ?.trim() + ?.takeIf { it.isNotBlank() } + + private fun cleanup() { + _isRecording.value = false + _amplitude.value = 0f + runCatching { recognizer?.destroy() } + recognizer = null + pendingResult = null + } + + private companion object { + const val RECOGNITION_TIMEOUT_MS = 20_000L + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/voice/SystemTtsPlayer.kt b/app/src/main/java/com/dark/tool_neuron/voice/SystemTtsPlayer.kt new file mode 100644 index 00000000..52d21418 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/voice/SystemTtsPlayer.kt @@ -0,0 +1,108 @@ +package com.dark.tool_neuron.voice + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import android.util.Log +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.withContext +import java.util.Locale +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +class SystemTtsPlayer @Inject constructor( + @param:ApplicationContext private val context: Context, +) { + private var engine: TextToSpeech? = null + private var ready: CompletableDeferred? = null + + private val _speakingId = MutableStateFlow(null) + val speakingId: StateFlow = _speakingId.asStateFlow() + + suspend fun speak(messageId: String, text: String): Boolean { + if (!ensureReady()) return false + val clean = sanitize(text) + if (clean.isBlank()) return false + val tts = engine ?: return false + _speakingId.value = messageId + return withContext(Dispatchers.Main) { + val params = Bundle().apply { + putString(TextToSpeech.Engine.KEY_PARAM_UTTERANCE_ID, messageId) + } + val rc = tts.speak(clean.take(MAX_TTS_CHARS), TextToSpeech.QUEUE_FLUSH, params, messageId) + rc == TextToSpeech.SUCCESS + } + } + + fun stop() { + _speakingId.value = null + runCatching { engine?.stop() } + } + + fun release() { + stop() + runCatching { engine?.shutdown() } + engine = null + ready = null + } + + private suspend fun ensureReady(): Boolean { + ready?.let { return it.await() } + val created = CompletableDeferred() + ready = created + withContext(Dispatchers.Main) { + engine = TextToSpeech(context.applicationContext) { status -> + val ok = status == TextToSpeech.SUCCESS + if (ok) { + runCatching { engine?.language = Locale.getDefault() } + engine?.setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) = Unit + override fun onDone(utteranceId: String?) { + if (_speakingId.value == utteranceId) _speakingId.value = null + } + @Deprecated("Deprecated in Java") + override fun onError(utteranceId: String?) { + if (_speakingId.value == utteranceId) _speakingId.value = null + } + override fun onError(utteranceId: String?, errorCode: Int) { + if (_speakingId.value == utteranceId) _speakingId.value = null + } + }) + } else { + Log.w(TAG, "Android TextToSpeech init failed: $status") + } + created.complete(ok) + } + } + return created.await() + } + + private fun sanitize(text: String): String = + text.replace(CODE_FENCE, " ") + .replace(INLINE_CODE, " ") + .replace(LINK) { it.groupValues[1] } + .replace(HEADER, "") + .replace(EMPHASIS, "") + .replace(EMOJI, "") + .replace(WHITESPACE, " ") + .trim() + + private companion object { + const val TAG = "SystemTtsPlayer" + const val MAX_TTS_CHARS = 4000 + val CODE_FENCE = Regex("```[\\s\\S]*?```") + val INLINE_CODE = Regex("`[^`]*`") + val LINK = Regex("\\[([^]]+)]\\([^)]+\\)") + val HEADER = Regex("(?m)^#+\\s*") + val EMPHASIS = Regex("[*_]{1,3}") + val EMOJI = Regex("[\\x{1F000}-\\x{1FAFF}\\x{2600}-\\x{27BF}\\x{FE0F}\\x{200D}]") + val WHITESPACE = Regex("\\s+") + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/voice/TtsPlayer.kt b/app/src/main/java/com/dark/tool_neuron/voice/TtsPlayer.kt index ec296789..1885ba2a 100644 --- a/app/src/main/java/com/dark/tool_neuron/voice/TtsPlayer.kt +++ b/app/src/main/java/com/dark/tool_neuron/voice/TtsPlayer.kt @@ -131,7 +131,8 @@ class TtsPlayer @Inject constructor() { val noEmphasis = noInlineCode.replace(EMPHASIS, "") val noLinks = noEmphasis.replace(LINK) { it.groupValues[1] } val noHeaders = noLinks.replace(HEADER, "") - return noHeaders.replace(WHITESPACE, " ").trim() + val noEmoji = noHeaders.replace(EMOJI, "") + return noEmoji.replace(WHITESPACE, " ").trim() } private fun splitIntoSentences(text: String): List { @@ -161,6 +162,7 @@ class TtsPlayer @Inject constructor() { private val EMPHASIS = Regex("[*_]{1,3}") private val LINK = Regex("\\[([^]]+)]\\([^)]+\\)") private val HEADER = Regex("(?m)^#+\\s*") + private val EMOJI = Regex("[\\x{1F000}-\\x{1FAFF}\\x{2600}-\\x{27BF}\\x{FE0F}\\x{200D}]") private val WHITESPACE = Regex("\\s+") } } diff --git a/app/src/main/java/com/dark/tool_neuron/voice/VoiceArchive.kt b/app/src/main/java/com/dark/tool_neuron/voice/VoiceArchive.kt index 52f2830a..72172428 100644 --- a/app/src/main/java/com/dark/tool_neuron/voice/VoiceArchive.kt +++ b/app/src/main/java/com/dark/tool_neuron/voice/VoiceArchive.kt @@ -38,9 +38,10 @@ object VoiceArchive { .removeSuffix(".tar.bz2") .removeSuffix(".tbz2") .removeSuffix(".tar") - val dest = File(destParent, folderName) - if (dest.exists()) dest.deleteRecursively() - dest.mkdirs() + val finalDest = File(destParent, folderName) + val tempDest = File(destParent, "$folderName.installing-${System.currentTimeMillis()}") + tempDest.deleteRecursively() + tempDest.mkdirs() try { runBlocking { @@ -77,7 +78,7 @@ object VoiceArchive { val entry = tarIn.nextEntry ?: break if (!tarIn.canReadEntryData(entry)) continue val relative = entry.name.trimStart('/') - val target = safeResolve(dest, relative) ?: continue + val target = safeResolve(tempDest, relative) ?: continue if (entry.isDirectory) { target.mkdirs() } else { @@ -104,18 +105,18 @@ object VoiceArchive { writer.join() } } catch (t: Throwable) { - dest.deleteRecursively() + tempDest.deleteRecursively() return ExtractResult.Failure(t.message ?: "Extraction failed") } - val actualRoot = pickRoot(dest) - val (files, subdirs) = collectTree(actualRoot) + val extractedRoot = pickRoot(tempDest) + val (files, subdirs) = collectTree(extractedRoot) - val config = when (kind) { + val valid = when (kind) { VoiceKind.TTS -> buildTtsConfig(files, subdirs) VoiceKind.STT -> buildSttConfig(files) } ?: run { - dest.deleteRecursively() + tempDest.deleteRecursively() return ExtractResult.Failure( when (kind) { VoiceKind.TTS -> "Archive missing model.onnx + tokens.txt" @@ -123,7 +124,31 @@ object VoiceArchive { } ) } - return ExtractResult.Success(actualRoot, config.toString()) + if (valid.length() <= 0) { + tempDest.deleteRecursively() + return ExtractResult.Failure("Archive validation failed") + } + + try { + finalDest.deleteRecursively() + finalDest.parentFile?.mkdirs() + val moved = extractedRoot.renameTo(finalDest) + if (!moved) { + extractedRoot.copyRecursively(finalDest, overwrite = true) + } + tempDest.deleteRecursively() + } catch (t: Throwable) { + finalDest.deleteRecursively() + tempDest.deleteRecursively() + return ExtractResult.Failure(t.message ?: "Failed to finalize install") + } + + val (finalFiles, finalSubdirs) = collectTree(finalDest) + val config = when (kind) { + VoiceKind.TTS -> buildTtsConfig(finalFiles, finalSubdirs) + VoiceKind.STT -> buildSttConfig(finalFiles) + } ?: return ExtractResult.Failure("Installed voice model could not be verified") + return ExtractResult.Success(finalDest, config.toString()) } private fun pickRoot(dest: File): File { diff --git a/app/src/main/java/com/dark/tool_neuron/voice/VoiceBackend.kt b/app/src/main/java/com/dark/tool_neuron/voice/VoiceBackend.kt new file mode 100644 index 00000000..f719abc0 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/voice/VoiceBackend.kt @@ -0,0 +1,13 @@ +package com.dark.tool_neuron.voice + +enum class VoiceBackend(val key: String, val label: String) { + AUTO("auto", "Auto"), + OFFLINE("offline", "Offline model"), + ANDROID_SYSTEM("android_system", "Android system"), + ; + + companion object { + fun fromKey(key: String): VoiceBackend = + entries.firstOrNull { it.key == key } ?: AUTO + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/voice/VoiceModelManager.kt b/app/src/main/java/com/dark/tool_neuron/voice/VoiceModelManager.kt index 66c18216..b1c8663b 100644 --- a/app/src/main/java/com/dark/tool_neuron/voice/VoiceModelManager.kt +++ b/app/src/main/java/com/dark/tool_neuron/voice/VoiceModelManager.kt @@ -11,8 +11,11 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext @@ -27,33 +30,57 @@ class VoiceModelManager @Inject constructor( private val prefs: Lazy, private val ttsPlayer: TtsPlayer, private val sttRecorder: SttRecorder, + private val systemTts: SystemTtsPlayer, + private val systemStt: SystemSttRecognizer, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val ttsLock = Mutex() private val sttLock = Mutex() + @Volatile private var activeRecordingBackend: VoiceBackend = VoiceBackend.OFFLINE private val _error = MutableStateFlow(null) val error: StateFlow = _error.asStateFlow() - val speakingId: StateFlow = ttsPlayer.speakingId - val isRecording: StateFlow = sttRecorder.isRecording - val recordingAmplitude: StateFlow = sttRecorder.amplitude + val speakingId: StateFlow = combine(ttsPlayer.speakingId, systemTts.speakingId) { offline, system -> + offline ?: system + }.stateIn(scope, SharingStarted.Eagerly, null) + val isRecording: StateFlow = combine(sttRecorder.isRecording, systemStt.isRecording) { offline, system -> + offline || system + }.stateIn(scope, SharingStarted.Eagerly, false) + val recordingAmplitude: StateFlow = combine(sttRecorder.amplitude, systemStt.amplitude) { offline, system -> + maxOf(offline, system) + }.stateIn(scope, SharingStarted.Eagerly, 0f) fun clearError() { _error.value = null } - fun hasTts(): Boolean = findActiveTts() != null - fun hasStt(): Boolean = findActiveStt() != null + fun hasTts(): Boolean = when (ttsBackend()) { + VoiceBackend.OFFLINE -> findActiveTts() != null + VoiceBackend.ANDROID_SYSTEM -> true + VoiceBackend.AUTO -> findActiveTts() != null || true + } + + fun hasStt(): Boolean = when (sttBackend()) { + VoiceBackend.OFFLINE -> findActiveStt() != null + VoiceBackend.ANDROID_SYSTEM -> systemStt.isAvailable() + VoiceBackend.AUTO -> findActiveStt() != null || systemStt.isAvailable() + } suspend fun unloadStt() { + systemStt.cancel() try { InferenceClient.unloadSttModel() } catch (_: Exception) {} } suspend fun unloadTts() { ttsPlayer.stop() + systemTts.stop() try { InferenceClient.unloadTtsModel() } catch (_: Exception) {} } - fun sttPermissionGranted(): Boolean = sttRecorder.hasPermission() + fun sttPermissionGranted(): Boolean = sttRecorder.hasPermission() || systemStt.hasPermission() + + private fun ttsBackend(): VoiceBackend = VoiceBackend.fromKey(prefs.get().voiceTtsBackend) + + private fun sttBackend(): VoiceBackend = VoiceBackend.fromKey(prefs.get().voiceSttBackend) private fun findActiveTts(): ModelInfo? { val models = modelRepo.models.value.filter { it.providerType == ProviderType.TTS } @@ -70,31 +97,100 @@ class VoiceModelManager @Inject constructor( } suspend fun speak(messageId: String, text: String): Boolean { - val ok = ensureTtsLoaded() ?: return false - if (!ok) return false + return when (ttsBackend()) { + VoiceBackend.ANDROID_SYSTEM -> speakWithSystem(messageId, text) + VoiceBackend.OFFLINE -> speakWithOffline(messageId, text, allowSystemFallback = false) + VoiceBackend.AUTO -> speakWithOffline(messageId, text, allowSystemFallback = true) + } + } + + fun stopSpeaking() { + ttsPlayer.stop() + systemTts.stop() + } + + private suspend fun speakWithOffline( + messageId: String, + text: String, + allowSystemFallback: Boolean, + ): Boolean { + val model = findActiveTts() + if (model == null) { + return if (allowSystemFallback) { + speakWithSystem(messageId, text) + } else { + _error.value = "No TTS model installed. Import one in Voice settings." + false + } + } + val ok = ensureTtsLoaded() ?: return if (allowSystemFallback) speakWithSystem(messageId, text) else false + if (!ok) return if (allowSystemFallback) speakWithSystem(messageId, text) else false ttsPlayer.speak(messageId, text) return true } - fun stopSpeaking() { ttsPlayer.stop() } + private suspend fun speakWithSystem(messageId: String, text: String): Boolean { + val ok = systemTts.speak(messageId, text) + if (!ok) _error.value = "Android system TTS is unavailable on this device" + return ok + } fun startRecording(): Boolean { - if (!sttRecorder.hasPermission()) { + if (!sttRecorder.hasPermission() && !systemStt.hasPermission()) { _error.value = "Microphone permission required" return false } - if (findActiveStt() == null) { + val backend = resolveSttBackendForStart() + if (backend != VoiceBackend.ANDROID_SYSTEM && findActiveStt() == null) { _error.value = "No STT model installed. Import one in Voice settings." return false } - val started = sttRecorder.start() - if (!started) _error.value = "Failed to start recording" + if (backend == VoiceBackend.ANDROID_SYSTEM && !systemStt.isAvailable()) { + _error.value = "Android system STT is unavailable on this device" + return false + } + activeRecordingBackend = backend + val started = when (backend) { + VoiceBackend.ANDROID_SYSTEM -> systemStt.start() + VoiceBackend.OFFLINE, VoiceBackend.AUTO -> sttRecorder.start() + } + if (!started) { + _error.value = when (backend) { + VoiceBackend.ANDROID_SYSTEM -> "Android system STT is unavailable on this device" + else -> "Failed to start recording" + } + } return started } - fun cancelRecording() { sttRecorder.cancel() } + private fun resolveSttBackendForStart(): VoiceBackend = when (sttBackend()) { + VoiceBackend.ANDROID_SYSTEM -> VoiceBackend.ANDROID_SYSTEM + VoiceBackend.OFFLINE -> { + if (findActiveStt() == null) { + _error.value = "No STT model installed. Import one in Voice settings." + } + VoiceBackend.OFFLINE + } + VoiceBackend.AUTO -> { + if (findActiveStt() != null) VoiceBackend.OFFLINE else VoiceBackend.ANDROID_SYSTEM + } + } + + fun cancelRecording() { + sttRecorder.cancel() + systemStt.cancel() + } suspend fun stopRecordingAndRecognize(): String? = withContext(Dispatchers.IO) { + if (activeRecordingBackend == VoiceBackend.ANDROID_SYSTEM) { + val text = systemStt.stopAndRecognize() + if (text == null) { + _error.value = "Transcription failed" + } else if (text.isBlank()) { + _error.value = "No speech detected" + } + return@withContext text + } val samples = sttRecorder.stop() if (samples.isEmpty()) { _error.value = "No audio captured" diff --git a/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt b/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt index f404fda1..b17b15fb 100644 --- a/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt +++ b/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt @@ -77,6 +77,22 @@ object HxdManager { return id } + fun activeFor(url: String, destPath: String): HxdState? { + val activeStatuses = setOf( + HxdStatus.QUEUED, + HxdStatus.CONNECTING, + HxdStatus.DOWNLOADING, + HxdStatus.PAUSED, + ) + return _tasks.value.values.firstOrNull { state -> + state.destPath == destPath && + state.url == url && + state.status in activeStatuses + } + } + + fun snapshot(): List = _tasks.value.values.toList() + fun pause(id: Int) { HxdNative.nativePause(id) updateStatus(id, HxdStatus.PAUSED) diff --git a/gradle.properties b/gradle.properties index 09f32dfc..77191a0b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,5 +2,5 @@ org.gradle.jvmargs=-Xmx4048m -Dfile.encoding=UTF-8 kotlin.code.style=official org.gradle.configuration-cache=true -tn.versionCode=31 -tn.versionName=3.0 \ No newline at end of file +tn.versionCode=32 +tn.versionName=3.1 diff --git a/native-server/src/main/cpp/server_core.cpp b/native-server/src/main/cpp/server_core.cpp index 7676e54f..d1356afb 100644 --- a/native-server/src/main/cpp/server_core.cpp +++ b/native-server/src/main/cpp/server_core.cpp @@ -100,6 +100,25 @@ namespace tn::server { return true; } + bool ensure_text_chat_model(httplib::Response& res, std::string& model_id) { + if (model_id.empty()) model_id = resolve_model_id(model_id, m::Kind::ChatGguf); + if (model_id.empty()) model_id = resolve_model_id(model_id, m::Kind::Vlm); + if (model_id.empty()) { + std::string msg = (m::has_any_of_kind(m::Kind::ChatGguf) || m::has_any_of_kind(m::Kind::Vlm)) + ? "no default chat model configured" + : "no chat model installed"; + respond_error(res, 404, "model_not_found", msg, "invalid_request_error"); + return false; + } + if (!m::has_id_of_kind(model_id, m::Kind::ChatGguf) && + !m::has_id_of_kind(model_id, m::Kind::Vlm)) { + std::string msg = std::string("model not available for chat: ") + model_id; + respond_error(res, 404, "model_not_found", msg, "invalid_request_error"); + return false; + } + return true; + } + struct PreparedImages { std::vector tmp_paths; ~PreparedImages() { @@ -479,7 +498,7 @@ namespace tn::server { if (route_vlm) { if (!ensure_model_for(res, model_id, m::Kind::Vlm, "VLM")) return; } else { - if (!ensure_model_for(res, model_id, m::Kind::ChatGguf, "chat")) return; + if (!ensure_text_chat_model(res, model_id)) return; } parsed.request.model = model_id; From 6086e829dbd7645ba9bf87d79a93f0e537f39183 Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Tue, 23 Jun 2026 07:26:16 +0600 Subject: [PATCH 09/13] feat: refine store downloads and image workflows --- AGENTS.md | 1148 +++++++++++++++++ CLAUDE.md | 82 +- app/build.gradle.kts | 1 + app/src/main/assets/server_webui.css | 419 ++++++ app/src/main/assets/server_webui.html | 358 +---- .../dark/tool_neuron/data/AppPreferences.kt | 7 + .../dark/tool_neuron/model/ModelTaxonomy.kt | 44 +- .../com/dark/tool_neuron/model/NavScreens.kt | 1 + .../dark/tool_neuron/model/VlmFileGroup.kt | 120 ++ .../dark/tool_neuron/model/WebSearchEvent.kt | 14 + .../tool_neuron/model/WebSearchUiState.kt | 31 +- .../dark/tool_neuron/repo/ImageGenManager.kt | 11 + .../tool_neuron/repo/ImageUpscalePipeline.kt | 438 +++++++ .../com/dark/tool_neuron/repo/ModelCatalog.kt | 112 +- .../dark/tool_neuron/repo/ModelRepository.kt | 28 +- .../dark/tool_neuron/repo/StorageInspector.kt | 156 +++ .../tool_neuron/repo/web_search/HtmlText.kt | 63 + .../repo/web_search/PageFetcher.kt | 42 + .../repo/web_search/WebSearchMode.kt | 91 ++ .../repo/web_search/WebSearchPrompts.kt | 93 +- .../service/inference/InferenceClient.kt | 4 + .../service/server/RemoteServerService.kt | 4 + .../service/server/ServerController.kt | 1 + .../tool_neuron/ui/navigation/TNavigation.kt | 9 + .../ui/screens/downloads/DownloadsScreen.kt | 44 +- .../screens/hf_explorer/HfRepoDetailScreen.kt | 85 +- .../home_screen/HomeScreenBottomBar.kt | 3 + .../screens/home_screen/ToolsPickerWindow.kt | 23 +- .../ui/screens/image_task/ImageTaskScreen.kt | 363 +++++- .../ui/screens/image_task/ImageViewerCard.kt | 43 +- .../screens/model_config/ModelConfigScreen.kt | 11 +- .../model_manager/ModelManagerScreen.kt | 1 - .../ui/screens/model_store/BrowseModelsTab.kt | 11 +- .../ui/screens/model_store/ModelCard.kt | 24 +- .../ui/screens/model_store/ModelFilters.kt | 38 + .../model_store/ModelImportTypePicker.kt | 5 +- .../screens/model_store/ModelStoreScreen.kt | 1 + .../ui/screens/settings/SettingsScreen.kt | 198 ++- .../ui/screens/storage/StorageScreen.kt | 133 ++ .../ui/screens/system_ui/AppTopBar.kt | 5 + .../ui/screens/web_search/WebSearchCard.kt | 59 +- .../com/dark/tool_neuron/util/ImageExport.kt | 34 + .../viewmodel/DownloadsViewModel.kt | 15 +- .../viewmodel/HfExplorerViewModel.kt | 10 +- .../tool_neuron/viewmodel/HomeViewModel.kt | 45 +- .../viewmodel/ImageTaskViewModel.kt | 218 +++- .../viewmodel/ModelStoreViewModel.kt | 49 +- .../viewmodel/SettingsViewModel.kt | 52 +- .../tool_neuron/viewmodel/StorageViewModel.kt | 35 + .../viewmodel/WebSearchCoordinator.kt | 225 +++- .../com/dark/download_manager/HxdManager.kt | 36 +- .../com/dark/download_manager/HxdService.kt | 56 +- native-server/src/main/cpp/native_server.cpp | 9 + native-server/src/main/cpp/server_auth.cpp | 1 + native-server/src/main/cpp/server_core.cpp | 8 + native-server/src/main/cpp/server_webui.cpp | 21 + native-server/src/main/cpp/server_webui.h | 4 + .../com/dark/native_server/NativeServer.kt | 1 + networking/src/main/cpp/ddg_client.cpp | 108 +- networking/src/main/cpp/html_extract.cpp | 117 +- networking/src/main/cpp/html_extract.h | 4 + networking/src/main/cpp/url_util.cpp | 47 + networking/src/main/cpp/url_util.h | 2 + 63 files changed, 4609 insertions(+), 812 deletions(-) create mode 100644 AGENTS.md create mode 100644 app/src/main/assets/server_webui.css create mode 100644 app/src/main/java/com/dark/tool_neuron/model/VlmFileGroup.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/repo/ImageUpscalePipeline.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/repo/web_search/HtmlText.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/repo/web_search/PageFetcher.kt create mode 100644 app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchMode.kt diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..e71835fa --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,1148 @@ +# ToolNeuron — Repo Guide + +Project memory for this repo. **When you change anything that affects future work — architecture, security behavior, new features, deprecated paths, public APIs, native JNI contracts, new scope — update this file as part of the same change.** A future session reads this to reconstruct intent; if it drifts, work breaks. + +--- + +## Project scope + +Privacy-first, offline-only on-device AI assistant. No Google Play services, no network telemetry, no analytics. In-scope pillars: on-device LLM chat, RAG over user documents, vision-language models (VLM), voice (TTS+STT), Remote Server with bundled web UI, HF Explorer, **on-device image generation / img2img / inpaint / adaptive upscale via the `:ai_sd` AAR (re-pivoted in 2026-05-08; upscale expanded 2026-06-23)**, **first-party plugin system with ONNX inference + capability-gated APIs (re-pivoted in 2026-05-11)**. Out of scope: tool calling, Termux integration, background-removal / segmentation / object-erase image operations. (Image generation was originally cut on 2026-04-20 and re-added on 2026-05-08. Plugin marketplace was also originally cut on 2026-04-20 and re-added on 2026-05-11 as a first-party plugin runtime — DexClassLoader, Plugin contract with @Composable Content(), capability-gated OnnxApi/HxsApi/NetworkApi, floating plugin dock with smooth switch transitions.) + +Package: `com.dark.tool_neuron` · minSdk 29 · targetSdk 36 · abiFilters `arm64-v8a`, `x86_64`. + +Modules: +- `:app` — UI (Compose), viewmodels, DI graph, activities, services. +- `:hxs` — encrypted key-value store (Kotlin wrapper + C++ core). +- `:hxs_encryptor` — crypto + integrity primitives: Argon2id, AES-GCM/ChaCha20-Poly1305, BoringSSL, ML-KEM-768, ML-DSA-65, Ed25519, HKDF, mmap+mlock `SecureBuffer`, plus the native security policy / auth / boot-integrity stack. +- `:native-server` — embedded OpenAI-compatible HTTP server (cpp-httplib + nlohmann/json header-only via FetchContent, no BoringSSL / OpenSSL / zlib dep). Powers Remote Server mode. +- `:plugin-api` — pure-Kotlin contract plugin authors compile against. `Plugin` interface with lifecycle + `@Composable Content()`, `PluginContext`, `PluginCapability` enum, `PluginManifest`, `OnnxApi`, `HxsApi`, `NetworkApi`. Compose deps are `compileOnly`; host provides them at runtime via classloader-parent. +- `:plugin-exc` — plugin host runtime. `PluginExecutor`, `PluginLoader` (DexClassLoader from `/plugins//classes.dex` + .so detection across SUPPORTED_ABIS), `PluginRegistry`, `PluginInstance`, `CapabilityGate` (PolicyEngine.hasSession + manifest cap check), `PluginContainerActivity` (hosts plugin's `Content()` with AnimatedContent fade+scale plugin-switch transitions), `PluginDock` (floating M3-expressive surface with circle chips per open plugin, tertiary-corner native-code badge), `OnnxApiImpl` (wraps onnxruntime-android 1.20.0, gated by AI_ONNX), `HxsApiImpl` (per-plugin collection `plugin_`, indexed string keys, gated by HXS_READ/WRITE), `NetworkApiImpl` (WebNative.fetchBytes GET, gated by INTERNET). +- `:download_manager`, `:networking` — ancillary modules. + +Prebuilt AARs in `libs/`: +- `gguf_lib-release.aar` — chat + VLM + embedding inference engine. +- `ai_sherpa-release.aar` — TTS / STT. +- `ai_sd-release.aar` — Stable Diffusion (text-to-image / img2img / inpaint / 4× upscale) via QNN on Snapdragon NPU and MNN on CPU. Currently the **debug** AAR is shipped because the release AAR's R8 minified the `StableDiffusionManager.Companion.getInstance` accessor; consumer-rules in `:ai_sd` need a `-keep class com.dark.ai_sd.StableDiffusionManager$Companion { *; }` before we can switch to release. + +--- + +## Build & Release + +- **Dev (debug):** `./gradlew :app:compileDebugKotlin` to verify compilation. `./gradlew :app:installDebug` to install. Never `assemble + adb install`. +- **Release:** built **from Android Studio**, signed via `local.properties` keys `TN_KEYSTORE_PATH / TN_KEYSTORE_PASSWORD / TN_KEY_ALIAS / TN_KEY_PASSWORD`. If any are missing, release falls back to unsigned so dev flow isn't blocked. +- **Native:** `./gradlew :hxs_encryptor:externalNativeBuildDebug` — BoringSSL + liboqs fetched via CMake FetchContent. The LSP flags `'openssl/mem.h' not found` etc. as false positives — build-green is the source of truth, not clangd. +- **Instrumented tests:** `./gradlew :app:connectedDebugAndroidTest`. + +--- + +## Hard rules + +1. **HXS-only persisted storage.** No `SharedPreferences`, Room, DataStore, or raw files. The only exception is `app_bootstrap/k.bin` (XOR-masked raw blob holding the Keystore-wrapped DEK) — it has to live outside the encrypted vault by construction. +2. **Security logic lives in C++/JNI.** Kotlin wraps native; every auth / trust / policy decision is made native and crosses JNI as opaque token or bool. No boolean-trust-through-JNI, no Kotlin `if (verify)` gating. +3. **No comments in source** except one-liner `//` for non-obvious WHY. No decorative banners, no block comments, no docstrings on internal/private. Names and structure must be self-documenting. +4. **Never write fully-qualified class names inline.** `import` at the top; short name in the body. +5. **ViewModels live under `com.dark.tool_neuron.viewmodel`.** Never co-locate a VM with its screen. +6. **Commit hygiene:** conventional commits, no `Co-Authored-By` trailer, never push without explicit ask, never skip hooks. Don't commit unless the user explicitly asks. +7. **Research / exploration subagents run on Sonnet at low effort** — not Opus — unless the user overrides. +8. **No TODOs, stubs, or half-implementations.** Every task is coded end-to-end. +9. **When you change security-affecting state, update AGENTS.md in the same change.** +10. **One Scaffold only** — the root `AppScaffold`. Screens take `innerPadding: PaddingValues` and render plain `Column`/`LazyColumn`/`Box`. Per-route top bars go in `AppTopBar.kt`'s `when`, bottom bars in `AppBottomBar.kt`'s `when`. +11. **Library modules must NOT minify.** Only `:app` minifies. R8 collides on `Type a.a is defined multiple times` against pre-minified prebuilt jars (e.g. `gguf_lib-release-runtime.jar`) if libraries also pre-minify. Library rules go in each module's `consumer-rules.pro`. +12. **No spec/plan/research docs in the repo.** Project memory belongs here. Implementation roadmaps belong in conversation context, not in `*.md` files at the repo root. + +--- + +## Security architecture + +### Layered model + +- **UI:** PasswordScreen / SetupPasswordScreen wrapped in `SecureScreen` (FLAG_SECURE). `AppScaffold` watches `shouldLock` and re-routes to PasswordScreen. +- **App Kotlin:** `SecurityManager` (only auth API the app consumes), `SessionHolder` (opaque 32-byte token), `AppLockObserver` (ProcessLifecycleOwner; clears on ON_STOP), `NativeIntegrity` (TOFU .so hashes + APK signer capture), `AccessibilityGuard`, `RootGuard`, `PinStrength`, `AppPreferences` (encrypted HXS + sealed AuthState), `AppKeyStore` (Android Keystore wrap/unwrap of DEK). +- **Native (libhxs_encryptor.so):** `PolicyEngine` — single `is_allowed(feature_id, token)` gate; `AuthNative` (Argon2id setup/verify, emits session token); `BootIntegrity` (JNI_OnLoad hooks, hook-baseline capture); `IntegrityGuard` (debugger / frida / xposed / sig / hash); `CryptoEngine` (AEAD, HKDF, Argon2id, Ed25519, X25519); `HybridKem` / `HybridSign` (X25519+ML-KEM-768, Ed25519+ML-DSA-65). + +### Keystore DEK flow + +1. `AppKeyStore` reads / writes `filesDir/app_bootstrap/k.bin`. Layout: `[magic "TNDK"(4)][version(1)][iv_len(2)][iv][ct_len(2)][ct]` masked byte-wise with a 32-byte hardcoded XOR key. XOR is obfuscation; the cryptographic protection is the Keystore-wrapped ciphertext inside. +2. Keystore alias `toolneuron_vault_dek_v1`: AES-256-GCM, `setIsStrongBoxBacked(true)` with `StrongBoxUnavailableException` fallback to TEE. NOT `setUserAuthenticationRequired(true)` (chicken-and-egg with setup flow). +3. First launch: generate 32-byte DEK via `SecureRandom`, wrap, write XOR-masked blob. +4. Every launch: read, unmask, parse, unwrap. +5. `AppKeyStore.backing()` classifies via `KeyInfo.securityLevel` (API 31+) or `KeyInfo.isInsideSecureHardware` → STRONGBOX / TEE / SOFTWARE_FALLBACK / UNKNOWN. +6. `AppKeyStore.wipe()` deletes `k.bin` + Keystore alias. +7. **Legacy migration:** if `k.bin` is missing but `app_bootstrap/` has any other file, wipe `app_bootstrap/*` + `app_prefs/*` + Keystore alias, then re-bootstrap. + +### Signer-bound user-keys (v2) + +Every per-vault user-key is derived as `HKDF(ikm = DEK, salt = installSignerHash, info = "tn..user_key.v2")`. The signer hash is `SHA-256(packageInfo.signingInfo.firstSigner.toByteArray())`, computed once per process via `AppKeyStore.installSignerHash()` and cached in `cachedSignerHash` (cleared on `wipe()`). On API < 28 it falls back to `GET_SIGNATURES`. + +Why salt-bind to the signer: Keystore-wrapped DEK is already device-bound (a different device cannot unwrap `k.bin`). Signer-binding closes the **same-device, replaced-APK** attack — root + repack ToolNeuron with an attacker cert + boot it on the legit device. The Keystore alias is uid-scoped, so the patched APK *can* unwrap the DEK; but its signing certificate hashes to a different value, so every user-key derived under the attacker build is wrong. AEAD records fail to decrypt. The repo's `openOrRebuild` helper detects the open failure and wipes the vault, so the attacker gets a fresh empty vault — the original encrypted bytes on disk stay sealed under the legitimate signer's user-key forever. + +If `getPackageInfo(... GET_SIGNING_CERTIFICATES)` returns null/empty (some weird OEM, broken install), `installSignerHash()` throws `SecurityException` and the app refuses to bootstrap. Don't add a fallback that returns zeros — that would defeat the binding. + +### Vault inventory + +| Vault dir | Sealed under | Notes | +|---|---|---| +| `app_bootstrap/k.bin` | Android Keystore alias `toolneuron_vault_dek_v1` (StrongBox/TEE), wrapped in XOR-masked envelope | Format: `[magic "TNDK"(4)][version(1)][iv_len(2)][iv][ct_len(2)][ct]`, byte-XOR with hardcoded 32-byte key. The XOR is obfuscation; the cryptographic protection is the Keystore-wrapped `ct`. | +| `app_prefs/` | `tn.app_prefs.user_key.v2` | All preferences. AuthState rides a second AEAD layer keyed `tn.app_prefs.auth_key.v2`, AAD `"tn.auth_state.v1"`. | +| `chat_store_v2/` | `tn.chats.user_key.v2` | Chats + messages. Replaces the legacy plaintext `chat_store/` (deleted on first v2 boot). | +| `chat_documents_meta_v1/` | `tn.chat_documents.user_key.v2` | RAG document metadata (id, name, mime, chunk count, sourceId). Dir name is historical — the user-key is v2. | +| `chat_documents/sources_v2/` | per-file AEAD via `SourceFileVault` | Each `.bin` is `[iv(12)][ct][tag(16)]` AEAD blob. Per-file key is `HKDF(DEK, salt=signerHash, info="tn.chat_doc_source.user_key.v2@")`. AAD = sourceId UTF-8 bytes (rename → decrypt fails). Replaces the legacy plaintext `chat_documents/sources/` (deleted on first v2 boot). | +| `rag_keyword_v1/` | `tn.rag_keyword.user_key.v2` | BM25 inverted-index records. | +| `download_history_v1/` | `tn.download_history.user_key.v2` | Download history (id, displayName, type, status, totalBytes, completedAt, error). Capped at 50 newest; oldest pruned on insert. Fresh-created on first launch of the Downloads-screen build; no migration. | + +**v1 → v2 migration is destructive.** Each repo's `openOrRebuild` tries `openEncrypted` with the v2 key. If that fails (existing v1 data sealed under the old non-signer-bound key), it wipes the vault dir and re-creates fresh. On first launch with a v2 build, an existing user loses their PIN, chat history, and RAG attachments — one time. The Keystore alias is preserved (so the DEK is still the same), only the per-vault user-keys change. + +### AppPreferences — encrypted HXS + sealed AuthState + +`HexStorage.openEncrypted(path, appKey=DEK, userKey=HKDF(DEK, salt=signerHash, info="tn.app_prefs.user_key.v2"), encryptor)`. Auth-critical state rides a second AEAD layer: `writeAuthState`/`readAuthState` use key `HKDF(DEK, salt=signerHash, info="tn.app_prefs.auth_key.v2")` and AAD `"tn.auth_state.v1"`. Ordinary flags (`onboarding_complete`, `tc_accepted`, `setup_done`, server settings, etc.) are plain encrypted records. + +### AuthState v4 + +``` +version(1) = 4 +security_mode(1) — 0=NONE, 1=APP_PASSWORD +salt_len(2) + salt +hash_len(2) + hash +failed_attempts(2) +next_attempt_at_ms(8) +has_panic(1) + if has_panic: + panic_salt_len(2) + panic_salt + panic_hash_len(2) + panic_hash +last_seen_now_ms(8) — monotonic wall-clock anchor +``` + +Decoder accepts v1/v2/v3/v4 and zero-fills missing fields. Bump `AuthState.VERSION` and the decoder when extending. + +### Native auth path + +- `hxs::auth::setup(pin) → {salt[16], hash[32]}` — Argon2id `t=4 / m=128 MiB (131072 KiB) / p=1 / outLen=32`. +- `hxs::auth::verify(pin, salt, stored_hash) → 32-byte session_token | null`. Constant-time `CRYPTO_memcmp`, then `policy::register_session(token)`. +- `hxs::auth::invalidate()` → `policy::invalidate_session()`. + +### PolicyEngine + +Every gated call: `PolicyEngine.isAllowed(Feature, sessionToken)`. Native logic order: +1. `tampered` → false (latched one-way). +2. `is_pro_feature(fid)` (fid ≥ 1000) → **false**. *This is the flip-point for the future license system.* +3. `is_unauth_feature(fid)` (APP_LAUNCH, OPEN_VAULT, AUTH_SETUP, AUTH_VERIFY, UI_PASSWORD_SCREEN, UI_SETUP_SCREEN, UI_INTRO) → true. +4. `passthrough` is on (set when `security_mode == NONE`) → true. +5. Else require `session_active && CRYPTO_memcmp(stored_token, given_token, 32) == 0`. + +State mutations: `register_session`, `invalidate_session`, `set_passthrough`, `mark_tampered`, `reset_for_testing` (test-only). Feature IDs in `policy_engine.h` mirrored as `PolicyEngine.Feature` in Kotlin — keep in sync. + +### Lockout / backoff / wipe / clock rollback + +`LockoutPolicy.backoffMillis(failed)` — first 3 free, then 1m → 5m → 15m → 1h → 4h → 12h → 24h. `WIPE_THRESHOLD = 10` triggers `SecurityManager.hardWipe()`. + +Clock-rollback defense: `AuthState.lastSeenNowMs` updated on every verify; if `nowMs + CLOCK_SKEW_GRACE_MS (5 min) < lastSeenNowMs`, the attempt is double-penalized and backoff extends from `max(nowMs, lastSeenNowMs)`. + +`hardWipe()`: `session.clear()` → `PolicyEngine.invalidateSession()` → `PolicyEngine.markTampered()` → `prefs.clearAuthState()` → `keyStore.wipe()`. `keyStore.wipe()` is scorched-earth: clears the cached DEK reference, recursively deletes everything under `filesDir` (models, voice, chat_store, chat_documents, model_store, plugin_store, rag_prefs, app_prefs, app_bootstrap, config, cache subtree) and `cacheDir`, then removes the Keystore alias. After hardWipe the app is in an unrecoverable in-process state (`markTampered` latches, files are gone), so the user must hit Restart on `WipedScreen` to bootstrap fresh. + +Panic PIN: `SecurityManager.setPanicPin(pin)` writes a second Argon2id hash. The gate is `securityMode == APP_PASSWORD` only — the live-session check was removed because `ProcessLifecycleOwner.ON_STOP` (notification panel pull, brief focus loss) clears the session via `AppLockObserver` while a Compose dialog stays visible above the locked screen, producing a non-deterministic "Couldn't set panic PIN" failure when the user submits. The panic-PIN UI is reachable only from Settings, which is itself gated by `shouldLock → PasswordScreen` re-routing, so the persistent "lock is set" fact is sufficient. `verifyPassword` tries real first; on mismatch, if `hasPanic`, tries panic. Panic match → `hardWipe()` + `VerifyResult.Wiped` (UX-indistinguishable from "attempts exceeded"). `clearPanicPin` and `disableLock` use the same `securityMode == APP_PASSWORD` gate for the same reason. + +PIN rules: 6 digits exactly. Weak PINs (all-same, monotonic ±1, top-20 commons) rejected at setup via `PinStrength.evaluate`. + +### Boot integrity + +`JNI_OnLoad` in `hxs_encryptor.cpp`: +1. `boot::install_ptrace_self_trace()` — PTRACE_TRACEME. +2. `boot::capture_hook_baselines()` — first 32 bytes of `auth::verify`, `policy::is_allowed`, `boot::hard_fail`. + +`TNApplication.onCreate()` in main process: +1. `integrity.scanProcessEnvironment()` — debugger / frida / xposed. Only `FAIL_DEBUGGER | FAIL_FRIDA` → `BootIntegrity.hardFail(reasons)`. `FAIL_XPOSED` is *not* a hard fail — it's recorded into `TNApplication.softEnvReasons` and surfaced as `tamperEvidence` in the one-time `RootWarningDialog` later. +2. `integrity.bootVerify()` — TOFU .so hash walk rebound to install identity. Manifest layout v2: `version(1) + signerHashLen(2)+signerHash + versionCode(8) + lastUpdateTime(8) + count(4) + (nameLen(2)+filename + hashLen(2)+hash)*`. If `{signerHash, longVersionCode, lastUpdateTime}` differs (or no manifest), re-TOFU and store. Within the same install identity, filename-set + hash mismatch → `FAIL_LIB_HASH` → hard fail. Filenames are stored (not absolute paths) since Android reshuffles `/data/app/~~…/`. `apk_signer_hash_v1` is also written for the future license-binding path. +3. `BootIntegrity.verifyHookBaselines()` — re-reads the prologue and compares; catches inline hooks. +4. `accessibilityGuard.scan()` — does NOT hard-fail any longer. Suspicious packages flow through `ScaffoldViewModel` into `RootWarning.a11yPackages` for the same one-time warning dialog. +5. `appLockObserver.register()`. + +Root detection was removed from the boot path. `RootGuard.scan()` runs from `ScaffoldViewModel` on first launch; if rooted and `rootWarningShown == false`, `AppScaffold` shows a one-time `RootWarningDialog`. Acknowledging flips the flag. + +The same one-time-warning treatment now also covers two adjacent "rooted-user" tamper signals that previously hard-failed the app: + +- **Xposed / LSPosed / Riru** — `scan_process_environment` still detects the `/proc/self/maps` substrings, but `TNApplication.onCreate` only hard-fails on `FAIL_DEBUGGER | FAIL_FRIDA`. A standalone `FAIL_XPOSED` bit is recorded into `TNApplication.softEnvReasons` (process-singleton, `internal set`) and surfaced as `tamperEvidence` in `RootWarningDialog`. `FAIL_DEBUGGER` and `FAIL_FRIDA` remain hard fails — those are active attack tools, not just a user-installed framework. +- **Suspicious accessibility services** — `accessibilityGuard.scan()` no longer hard-fails in release on `SuspiciousAttached`. The packages flow into `RootWarning.a11yPackages` and render as a third paragraph in the same dialog. + +`ScaffoldViewModel.RootWarning` is `(rootEvidence, tamperEvidence, a11yPackages)` — three independent `Set`. The dialog renders a paragraph per non-empty section with a single "I understand" button. `acknowledgeRootWarning()` flips `rootWarningShown` once and silences all three sources for that install. + +`BootIntegrity.hardFail(reason)` → `PolicyEngine.markTampered()` + `_exit(1)`, unless `setRelaxedForTesting(true)` (tests). + +### Tamper / hook obfuscation + +All detection strings in `integrity.cpp` use `HXS_OBF(var, "literal")` (compile-time XOR). Verified clean: `strings libhxs_encryptor.so | grep -iE '^(frida|gadget|linjector|xposed|TracerPid)$'` returns nothing. + +### Session lifecycle + UI lock + +- `SessionHolder.active: StateFlow` flips on `AuthNative.verify` success, off on `clear()`. +- `AppLockObserver` (a `DefaultLifecycleObserver` on `ProcessLifecycleOwner`) calls `session.clear()` on `ON_STOP` when `security.isLockEnabled`. Clear also calls `AuthNative.invalidate()`. +- `ScaffoldViewModel.shouldLock = security.isLockEnabled && !session.active`. `AppScaffold` re-routes to PasswordScreen with `popUpTo(0) { inclusive = true }`, except when on a non-interruptible route (`PasswordScreen`, `SetupScreen`, `IntroScreen`). + +### FLAG_SECURE + SecureClipboard + +`ui/components/SecureScreen.kt` adds FLAG_SECURE on enter, clears on dispose. Applied to PIN entry. Not global (so users can screenshot their own chats). `util/SecureClipboard.kt` sets `EXTRA_IS_SENSITIVE` (TIRAMISU+) and auto-clears after 30 s if the clip still matches. + +### Release hardening + +`app/build.gradle.kts`: `isMinifyEnabled = true`, `isShrinkResources = true`, `isDebuggable = false`, `isJniDebuggable = false`. ProGuard (`app/proguard-rules.pro`) strips `Log.d/v/i` via `assumenosideeffects` (keeps `w`/`e`), `-repackageclasses ''`, `-allowaccessmodification`. Manifest: `allowBackup=false`; `TempActivity` + `ColorShowcaseActivity` un-exported; `InferenceService` runs in `:inference` process. + +--- + +## Future pro license system — the hook + +The license plumbing is live: every gated feature routes through `PolicyEngine.isAllowed(Feature, sessionToken)`. Feature IDs `≥ 1000` are `PRO_*` and currently return false. To enable monetization: + +1. Flip the `is_pro_feature` branch in `policy_engine.cpp` from `return false` to "verify signed license blob". +2. License blob layout (planned): `{device_id_hash, features_bitmap, expiry_unix, nonce, signature}` — Ed25519 or ML-DSA-65 signed; public key XOR-baked into native at build time. +3. APK signer SHA-256 is **already captured** on first launch and stored as `apk_signer_hash_v1`. That's the anchor. +4. Device id must be an attested Keystore key fingerprint (hardware-rooted), **NOT** `Settings.Secure.ANDROID_ID`. +5. License blob lives in a separate HXS collection sealed under the same DEK. +6. On tamper detection, invalidate any loaded license too — single fail-closed path. + +--- + +## File map + +### Native (`hxs_encryptor/src/main/cpp/`) +- `policy_engine.{h,cpp}` — `is_allowed`, session registry, tamper latch, `reset_for_testing`. +- `auth.{h,cpp}` — `setup(pin)`, `verify(pin, salt, hash)`, hardened Argon2id. +- `boot_integrity.{h,cpp}` — env scan, lib-hash verify, hook-baseline capture/verify, `hard_fail`, `setRelaxedForTesting`. +- `integrity.{h,cpp}` — debugger / frida / xposed checks, file hashing, APK sig compare. +- `xor_str.h` — `HXS_OBF(var, "literal")` compile-time XOR. +- `crypto_engine.{h,cpp}` — AEAD, HKDF, PBKDF2, Argon2id, Ed25519, X25519, SHA-256. +- `memory_guard.{h,cpp}` — `SecureBuffer` (mmap+mlock+mprotect), `secure_zero`, `secure_compare`. +- `pq_kem.{h,cpp}` — X25519 + ML-KEM-768 hybrid KEM. +- `pq_sign.{h,cpp}` — Ed25519 + ML-DSA-65 hybrid signatures. +- `hxs_encryptor.cpp` — JNI bindings + `JNI_OnLoad`. +- `CMakeLists.txt` — fetches BoringSSL + liboqs; `-march=armv8-a+crypto+sha2` on arm64; LTO/gc-sections/icf on release; `-Wl,-z,max-page-size=16384` on every owned native CMake target. + +### Encryptor module Kotlin (`hxs_encryptor/src/main/java/com/dark/hxs_encryptor/`) +`HxsEncryptor.kt`, `PolicyEngine.kt`, `AuthNative.kt`, `BootIntegrity.kt`. + +### App-side security (`app/src/main/java/com/dark/tool_neuron/data/`) +`AppKeyStore`, `SessionHolder`, `SecurityManager`, `SecurityModule`, `AppPreferences`, `AuthState`, `VerifyResult`, `LockoutPolicy`, `NativeIntegrity`, `AppLockObserver`, `AccessibilityGuard`, `RootGuard`, `PinStrength`, `KeyFingerprint`. + +### UI +- `ui/components/SecureScreen.kt` — FLAG_SECURE wrapper. +- `ui/screens/password_screen/PasswordScreen.kt` + `setup_screen/SetupPasswordScreen.kt` — wrapped in SecureScreen. +- `ui/screens/setup_screen/SetupThemeScreen.kt` — first-run theme + palette. +- `ui/screens/system_ui/AppScaffold.kt` — single Scaffold; auto-lock + server-lockdown re-routing. +- `util/SecureClipboard.kt`, `util/VlmPaths.kt`. +- `ui/screens/guide/` — hub + 7 detail screens via `GuideDetailLayout` + `GuideTopBar`. +- `ui/screens/home_screen/PlusMenu.kt` — Documents / Thinking / Attach image (image disabled until `isVlmLoaded`). +- `ui/screens/home_screen/HomeScreenBottomBar.kt` — image-attach button + mic button + transcribe equalizer. +- `ui/screens/server/ServerScreen.kt` + `ServerTopBar.kt` — Remote Server config + token + status + request log. +- `ui/screens/hf_explorer/{HfExplorerScreen,HfRepoDetailScreen}.kt` — search / filter / repo browser. + +### Build +`app/build.gradle.kts`, `app/proguard-rules.pro`, `hxs_encryptor/build.gradle.kts`, `gradle/libs.versions.toml`, `app/src/main/AndroidManifest.xml`. + +--- + +## VLM (vision-language models) + +Vision rides on top of an active GGUF chat model via a separate mmproj projector file. Image data crosses AIDL via `ParcelFileDescriptor[]` (1 MB binder limit forbids `byte[]`). + +### Folder layout + +VLM models live as `/vlm//{base.gguf, mmproj.gguf}`. A HuggingFace repo is detected as VLM if any `.gguf` file in its tree has `mmproj` (case-insensitive) in its name. Downloads pull both files into the per-repo folder; loading the base auto-loads the colocated mmproj. There is no manual "load projector" UI. + +### AIDL surface (used) + +``` +boolean loadVlmProjector(String path, int threads, int imageMinTokens, int imageMaxTokens); +boolean loadVlmProjectorFromFd(in ParcelFileDescriptor pfd, int threads, int imageMinTokens, int imageMaxTokens); +void releaseVlmProjector(); +boolean isVlmLoaded(); +String getVlmInfo(); +String getVlmDefaultMarker(); +void generateVlm(String messagesJson, in ParcelFileDescriptor[] imageFds, int maxTokens, IGenerationCallback callback); +``` + +### Service flow + +`InferenceService.generateVlm(messagesJson, imageFds, maxTokens, cb)` reads each PFD via `AutoCloseInputStream.readBytes()` on Dispatchers.IO, hands `List` to `engine.generateVlmFlow`, and bridges `GenerationEvent` → `IGenerationCallback`. Read failures → `callback.onError`. + +### Client / coordinator + +`InferenceClient.isVlmLoaded: StateFlow` mirrors service-side state. `loadVlmProjector(path, threads=2)` is the path-based load used by auto-load. `generateVlm(context, messagesJson, imageUris, maxTokens): Flow` opens PFDs, hands the array to the service, closes after the call. `InferenceCoordinator.run()` per-iteration: if iteration==0 AND last user has non-empty `imageUris` AND `isVlmLoaded.value` → VLM route. `buildMessagesJson(messages, vlmLastUserId=lastUser.id)` prepends `getVlmDefaultMarker()` to the last user's content. + +### Auto-load + +`ModelSessionManager.load(model)`: +1. `releaseVlmProjector()` if currently loaded. +2. Load base. +3. On success, if `pathType == FILE` and the path is inside `/vlm/`, call `VlmPaths.colocatedMmproj(baseFile)`. Present → load. Missing → surface `vlmAutoLoadError` via `StateFlow`; UI shows `VlmErrorBanner`. + +`ModelSessionManager.unload()` releases projector first. + +### Persistence + +`ChatMessage.imageUris: List` persisted via `ChatRepository.TAG_MSG_IMAGES = 8` (JSON array of URI strings). Image bytes never land on disk. + +`Chat.forkedFromChatId: String?` persisted via `ChatRepository.TAG_FORKED_FROM = 9` (chats collection). Set by `ChatRepository.forkChat(sourceChatId, atMessageId)` — clones every message up to and including the cut point into a new chat with title `" (fork)"`. Drawer renders `TnIcons.Fork` + "Forked" label next to the title when the field is non-null. Forking is gated on `!isGenerating` in `HomeViewModel.forkFromMessage`. + +### Catalog + downloads + +`ModelCatalog.fetchRepo` flags any repo whose tree contains a `*mmproj*.gguf` file; non-mmproj `.gguf` rows get `isVlm=true`, `repoPath`, `mmprojFileName`, `mmprojFileUri`, `mmprojSizeBytes`. Tag list adds "VLM". `ModelStoreViewModel.downloadModel` routes VLM base into `vlmModelFile(repoPath, fileName)`; on completion, enqueues mmproj into the same folder under the same `modelId`. Finalize inserts a single `ModelInfo` whose path is the base `.gguf`. + +Store VLM browsing is provider-only: Qwen, LFM, Gemma, SmolVLM, MiniCPM, LLaVA, and Other vision. Do not bring back expandable VLM file-group cards in the Store; users pick the base model row and the Store auto-downloads the matching projector internally. HF Explorer may still use grouped file display for raw repo inspection, but the Store must stay provider-first and simple. + +### UI + +PlusMenu Attach-image is disabled when `!isVlmLoaded`, with a "Attach image · VLM required" badge. PendingImageRow renders thumbnails via `BitmapFactory.decodeStream`. `MessageBubble` renders `UserImageThumbnails` when `message.imageUris.isNotEmpty()`. `InstalledModelCard` shows a "VLM" tag for paths under `models/vlm/`. + +### DI + +`HomeViewModel` and `InferenceCoordinator` take `Application` (for `contentResolver` + `generateVlm(app, ...)`). + +--- + +## Voice (TTS + STT) + +Streaming TTS playback of assistant messages and tap-to-toggle STT input via the sherpa-onnx AAR. The AAR exposes VITS + Kokoro TTS and Whisper STT only; SupertonicTTS is not supported. + +**Install path: Store only.** BYOM / SAF directory import was removed (2026-04-24). The Store downloads `.tar.bz2` from sherpa-onnx GitHub releases, extracts into `/voice///`, builds the sherpa-onnx config JSON, inserts `ModelInfo` + `ModelConfig`. Archive deleted after extraction. First TTS/STT download of each kind is auto-selected as active. + +### AIDL surface + +``` +boolean loadTtsModel(String configJson); void unloadTtsModel(); boolean isTtsLoaded(); +float[] synthesize(String text, int speakerId, float speed); int getTtsSampleRate(); +boolean loadSttModel(String configJson); void unloadSttModel(); boolean isSttLoaded(); +String recognize(in float[] samples, int sampleRate); +String recognizeFromFd(in ParcelFileDescriptor pfd, int sampleCount, int sampleRate); +``` + +`synthesize` and `recognize` are batch — there is no streaming callback. Streaming TTS is faked by sentence-chunking at the **text** layer. + +### Layout + +`app/src/main/java/com/dark/tool_neuron/voice/`: +- `TtsPlayer` — sentence-chunk streaming via `AudioTrack.MODE_STREAM + WRITE_BLOCKING`. Cancellable per-chunk via `_speakingId.value == messageId` check. +- `SttRecorder` — `AudioRecord` 16 kHz mono `ENCODING_PCM_FLOAT`, source `MediaRecorder.AudioSource.VOICE_RECOGNITION`. Exposes `isRecording`, `amplitude` flows for UI. +- `VoiceModelManager` — `@Singleton`. Auto-loads active TTS/STT on first use by reading `AppPreferences.activeTtsModelId` / `activeSttModelId`. Uses `Mutex` to serialize loads. Injects `Lazy` to avoid eager construction in non-main processes. +- `VoiceArchive` — extraction. Streams `.tar.bz2` through `BZip2CompressorInputStream` → `TarArchiveInputStream`, writes each entry into a per-archive folder, builds the sherpa-onnx config JSON. Per-entry `safeResolve` rejects path-traversal. Calls back `onEntry(name)` for per-file UI progress. + +### Model distinction + +No `modelType` field on `ModelInfo`. `ProviderType` is canonical (`GGUF` / `TTS` / `STT` / `EMBEDDING`). `HuggingFaceModel.modelType: String` is the pre-install hint; `ModelStoreViewModel.finalizeNonVlmDownload` maps it to `ProviderType` at insert time. `HomeViewModel.chatModels` filters to `ProviderType.GGUF`. + +### Persistence + +`AppPreferences` keys `active_tts_model` and `active_stt_model` (encrypted HXS records). Empty → fallback to first installed model of that type. Voice models live under `/voice///`. Voice deletes need `deleteRecursively()` since the folder is non-empty (current limitation: store delete uses `File.delete`). + +### Streaming TTS approximation + +`TtsPlayer.sanitize(text)` strips code fences / inline code / markdown emphasis / links / headers. `splitIntoSentences(text)` breaks at `.`/`!`/`?`/`…`/`;`/`\n` after ≥20 chars or at comma/space if ≥180 chars. Each chunk synth'd on Dispatchers.IO and written into the `AudioTrack` with `WRITE_BLOCKING`. `AudioTrack` is lazy at `getTtsSampleRate()`; recreated on rate change. + +### STT + +`SttRecorder.start()` reads 1024-sample chunks in a tight loop, snapshots max abs into `_amplitude`, appends to a synchronized buffer. `stop()` snapshots into `FloatArray`, releases. The array is passed to `InferenceClient.recognize(samples, 16000)` from `HomeViewModel.stopRecordingAndTranscribe`, which pushes recognized text into `_transcribedText: StateFlow`. `HomeScreenBottomBar` observes and appends to its local text state, then calls `consumeTranscribedText()`. STT is unloaded after each transcription to free memory. + +### Permissions + +`RECORD_AUDIO` requested at first mic tap via `ActivityResultContracts.RequestPermission`; on grant, immediately `startRecording()`. No `FOREGROUND_SERVICE_MICROPHONE` (UI is held while recording). + +### UI + +- Speak / Stop button on assistant bubbles (`MessageActions`) when `voiceTtsAvailable`. Icon flips between `TnIcons.Volume` and `TnIcons.PlayerStop`, and shows a CircularProgressIndicator with stop-icon overlay while the TTS model is loading (`isSpeakLoading`). +- Mic `ActionButton` always rendered. No STT installed → navigate to ModelStore. Permission missing → request. Else `startRecording()`. +- Recording crossfades the input bar to `RecordingEqualizer` (`[X cancel] [waveform] [✓ stop]`). Stop calls `stopRecordingAndTranscribe`. +- Image-attach button moved out of PlusMenu into the input bar. +- Voice errors surface through the same `VlmErrorBanner` component. +- No dedicated Voice Settings screen — Store manages downloads + first-install becomes active. Default TTS / STT swap surfaces in Settings → Voice section (`SettingsViewModel.voiceSection`); selecting a different model writes `active_tts_model` / `active_stt_model` and calls `VoiceModelManager.unloadTts/Stt()` so the next request reloads the new pick. + +### DI + +`VoiceModelManager`, `TtsPlayer`, `SttRecorder` are all `@Singleton`. `HomeViewModel` injects `VoiceModelManager`. `ModelStoreViewModel` injects `AppPreferences` to flag first install of each kind as active. + +### Catalog (2026-04-24) + +`ModelCatalog.BUILT_IN_MODELS` carries four sherpa-onnx releases: `vits-piper-en_US-amy-low` (TTS, ~30 MB), `vits-piper-en_US-libritts-high` (TTS, ~124 MB), `sherpa-onnx-whisper-tiny-en` (STT, ~75 MB), `sherpa-onnx-whisper-tiny` (STT, ~82 MB). URLs hit `sherpa-onnx/releases/download/{tts,asr}-models/…`. If sherpa-onnx restructures their releases, the Store surfaces the failure. + +--- + +## Remote Server + +Embedded HTTP server exposing every installed engine over an OpenAI-compatible API on the local network. Standalone replacement for the rejected Ktor PR. As of 2026-05-11 the server is multi-engine: chat GGUF, VLM (chat + images), embeddings, TTS, STT, and image generation (txt2img / img2img / inpaint / 4x upscale). No TLS, no mDNS, no outbound calls. + +### Process model + +Three processes: + +- `:app` — UI, chat ViewModels, `ServerController` (AIDL client), `InferenceClient` (AIDL client of `:inference`). HXS / Keystore live here; nothing crosses out. +- `:inference` — `InferenceService` (chat-side llama.cpp + sherpa-onnx). Untouched by the server. +- `:server` — `RemoteServerService`, its own per-type engine instances (`ServerEngine` chat, `ServerVlmEngine`, `ServerEmbeddingEngine`, `ServerTtsEngine`, `ServerSttEngine`, `ServerImageEngine`), the embedded native HTTP server, the bearer token in native memory. Foreground (`dataSync`, `stopWithTask="false"`). Independent: app crash doesn't kill it, server crash doesn't kill the app. + +`:server` doesn't open HXS. The bearer token, full engines catalog (per-model id / display name / file path / mmproj path / per-model config JSON / kind), web-UI HTML, and docs HTML are all handed across via AIDL `start(configJson)`. When the user rotates the token in the UI, `:app` regenerates + persists, then pushes the new token to `:server` via `IRemoteServerService.rotateToken(newToken)`. + +When the user swipes the app away, `:app` and `:inference` both die. `:server` keeps running because it's foreground **and** because `handleStart` self-calls `startService(Intent(this, RemoteServerService::class.java))` immediately before `startForeground`. That transitions it from bind-only to started lifecycle — without it, every binder client dying (which happens on swipe-to-kill) makes the service eligible for destruction even with a foreground notification. Reopening the app re-binds; `ServerController` calls `currentSnapshotJson()` and `recentRequestEventsJson(100)` to rehydrate the Server Screen with whatever's running right now. + +Tapping the notification body opens `MainActivity` with `EXTRA_OPEN_SERVER_SCREEN=true`, which routes straight to the Server Screen. The Stop button on the notification fires `startService(action=ACTION_STOP)` against `:server`, which tears down in-process. + +### Engine model — lazy per-type, primary preload + +`ServerEngineRegistry` (in `:server`) holds one instance per kind. Each kind is loaded lazily on first request and cached: + +| Kind | Wrapper | Library backing | +|------------------|--------------------------|------------------------------------------------------| +| `gguf` | `ServerEngine` | `com.dark.gguf_lib.GGMLEngine` | +| `vlm` | `ServerVlmEngine` | `GGMLEngine` + mmproj projector | +| `embedding` | `ServerEmbeddingEngine` | `com.dark.gguf_lib.EmbeddingEngine` | +| `tts` | `ServerTtsEngine` | `com.dark.ai_sherpa.OfflineTts` (VITS) | +| `stt` | `ServerSttEngine` | `com.dark.ai_sherpa.OfflineRecognizer` (Whisper) | +| `image_gen` | `ServerImageEngine` | `com.dark.ai_sd.StableDiffusionManager` (QNN/MNN) | +| `image_upscaler` | `ServerImageEngine` | same SDK, separate `loadUpscaler` path | + +Why lazy-load: a 4 GB device cannot hold every engine simultaneously. On `start(configJson)` the primary chat GGUF (or first VLM if no chat) is preloaded so first /v1/chat/completions returns fast; everything else materialises on first request to that endpoint. Per-kind `Mutex`/`synchronized` lock prevents two requests racing the load. Engine instances are NOT reaped on idle in this iteration — only `shutdownAll()` on server stop. If RAM becomes a problem on smaller devices, the natural extension is a per-kind TTL cache or a `POST /v1/admin/unload?kind=image_gen` route. + +Cross-process gotchas: +- `:server` loads `StableDiffusionManager.getInstance(context)` independently from `:app`. They each have their own native pipeline. The `qnnlibs.tar.xz` runtime extraction is uid-shared at `/ai_sd_runtime/`; whichever process extracts first wins, the other sees the existing files and skips. Image gen via the server therefore requires the user to have downloaded the SD runtime through the Image Task screen at least once. +- `:server` does NOT open HXS to read `prefs.activeTtsModelId` / `activeSttModelId`. Voice "default" is whatever the catalog ranks first per kind — `ServerController.buildEnginesCatalog` walks the `ModelRepository` in install order, so the active voice model from `:app` doesn't influence which voice the server uses as fallback. Clients pick explicitly via the `model` field. If the request `model` is unknown, the server falls back to `first_of_kind`. +- `:inference` (the chat-side llama.cpp service) is untouched by the server — no AIDL hops between `:server` and `:inference`. Server-side load and chat-side load are independent. This is intentional: it keeps the request path on one process and prevents the server lockdown from also blocking chat-side voice/image flows on the same engine. + +### Routes + +| Method | Path | Auth | Stream | Purpose | +|--------|-------------------------------|------|--------|---------------------------------------------------------| +| GET | `/`, `/index.html`, `/webui` | public | - | Bundled Material-3 web UI | +| GET | `/docs`, `/docs/` | public | - | API documentation | +| GET | `/health` | public | - | Liveness ping `{status:ok}` | +| GET | `/v1/models` | auth | - | Full enabled-engine catalog (id, type, owned_by) | +| POST | `/v1/chat/completions` | auth | yes | Chat GGUF; auto-routes to VLM when message contains `image_url` parts | +| POST | `/v1/embeddings` | auth | - | Dense embeddings — `{input: string \| string[]}` | +| POST | `/v1/audio/speech` | auth | - | TTS — body `{model, input, voice, speed}`, returns `audio/wav` | +| POST | `/v1/audio/transcriptions` | auth | - | STT — multipart `file=` + `model=` | +| POST | `/v1/images/generations` | auth | - | txt2img — body `{model, prompt, negative_prompt?, steps?, cfg?, width?, height?, seed?}`, returns `{data:[{b64_json}]}` | +| POST | `/v1/images/edits` | auth | - | img2img / inpaint — multipart `image`, optional `mask`, `model`, `prompt`, sampling params | +| POST | `/v1/images/upscale` | auth | - | 4x upscale — multipart `image` + `model` | + +Every non-public route is gated by the same `pre_routing_handler`: rate-limit token bucket → ban list → bearer auth (constant-time compare, 20 consecutive fails → 1 h ban). The same `post_routing_handler` records every request into the 128-entry audit ring buffer + pushes it across to `:app` via `InferenceBridge.onRequestEvent`. No exceptions, no per-route auth bypass. + +### Native (`:native-server`) + +``` +native-server/src/main/cpp/ + server_core.{h,cpp} — httplib::Server lifecycle, pre/post-routing, every route registration + server_auth.{h,cpp} — bearer token store + constant-time compare + 401/403 + server_crypto.{h,cpp} — getrandom(2) RNG, const_time_eq, base64url, base64 std, base64 decoder, secure_zero + server_models.{h,cpp} — typed catalog: id + display_name + path + mmproj_path + config_json + Kind + created + + has_id_of_kind / first_of_kind / has_any_of_kind / build_list_response + server_audit.{h,cpp} — 128-entry ring buffer of request events + server_rate_limit.{h,cpp} — per-client token bucket (cap=30, refill=1/s) + auth-fail ban (20 fails → 1 h) + server_webui.{h,cpp} — set/clear/get/has HTML (mutex-protected std::string) + server_docs.{h,cpp} — same for /docs + server_staging.{h,cpp} — tmpfile dir for large binary payloads; staged paths handed across JNI (no byte[] copies) + wav_codec.{h,cpp} — minimal RIFF/PCM16 + IEEE-float decode (STT input); encode is on the Kotlin side + gen_session.{h,cpp} — chat / VLM streaming session: token queue + cancellation + reply_session.{h,cpp} — single-shot reply session for embeddings / TTS / STT / image; carries text or staged binary path + openai_schema.{h,cpp} — ChatRequest parser (detects has_images), VLM image part extractor (base64 data URLs ONLY), + error envelope, embedding response, transcription response, image response builders + jvm_bridge.{h,cpp} — JavaVM pin, JNI upcalls (startGeneration + startEmbedding + startTts + startStt + + startImageGen + startImageUpscale + cancelGeneration + onRequestEvent) + native_server.cpp — JNI entry points (start/stop/token/catalog/bridge/feeders/staging/audit/rl/webui/docs) + JNI_OnLoad +``` + +CMake fetches `cpp-httplib v0.18.5` and `nlohmann/json v3.11.3`, both header-only (`HTTPLIB_COMPILE=OFF`, `HTTPLIB_REQUIRE_OPENSSL=OFF`, `HTTPLIB_REQUIRE_ZLIB=OFF`, `JSON_BuildTests=OFF`). Same flags as `:hxs_encryptor`: c++17, `-fvisibility=hidden`, `-fstack-protector-strong`, LTO/gc-sections/icf release, `-march=armv8-a+crypto+sha2` on arm64, `-Wl,-z,max-page-size=16384`. Read timeout was lifted from 15s → 60s and payload max from 1 MB → 64 MB to accommodate base64-encoded VLM images and multipart audio/image uploads. + +Payload mechanics: +- **Streaming** (chat / VLM): `gen_session` queue with `nativeFeedToken / nativeFeedDone / nativeFeedError`; the httplib chunked content provider drains it onto an SSE stream. +- **Single-response** (embeddings / TTS / STT / image): `reply_session` — bridge calls `nativeFeedReplyText(replyId, body, mime)` for JSON/text or `nativeFeedReplyBinary(replyId, path, mime)` for staged binary; the route handler blocks on `session->wait(timeout)`. +- **Big binary upload** (multipart image, mask, wav): cpp-httplib decodes multipart natively; the route writes each part to `/server-staging/tn__` via `server_staging::write_bytes`, hands the path to Java via JNI string (avoids byte[] JNI copies), and unlinks on response. +- **Big binary download** (TTS wav, generated PNG): Kotlin writes the bytes to the staged path, hands the path back via `nativeFeedReplyBinary(path, mime)`, the C++ side reads + sends + unlinks. PNG responses are base64-encoded into JSON `b64_json` per OpenAI; WAV is sent as raw `audio/wav`. +- **VLM image_url parts**: only `data:image/...;base64,...` URLs are accepted. Network URLs return 400 (offline-only scope). Decoded bytes are staged to tmpfiles and the paths passed to `InferenceBridge.startGeneration(..., imagePaths=[...])`. Sanitised messages (image parts collapsed into text-only `content`) are forwarded to the engine alongside the path list — the Kotlin bridge reads each tmpfile and feeds the bytes to `GGMLEngine.generateVlmFlow(imageData = [...])`. + +### Web UI + +Bundled at `app/src/main/assets/server_webui.html`. Single Material-3 SPA with a sidebar tab strip that swaps the main panel between four workspaces: + +- **Chat** — preserved from the prior build: localStorage history, markdown rendering, streaming with blinking cursor, settings dialog, connection indicator. Adds an attach-image button (📎) that converts the uploaded image to a `data:image/...;base64,...` URL and appends it as an OpenAI multi-part `image_url` content entry on the next send. Server auto-detects and routes to the VLM engine. +- **Embeddings** — model select, multi-line input (one row per line), runs `/v1/embeddings`, shows vector count + first 8 dims of each row. +- **Voice** — two cards. TTS: model + text + voice id + speed, plays the returned WAV inline. STT: model + WAV upload, shows transcribed text. +- **Image** — segmented switch (Generate / Edit / Inpaint / Upscale). Prompt + negative + steps/CFG/width/height for diffusion modes. Input image file for Edit/Inpaint/Upscale. Mask file for Inpaint. Result is rendered inline from `b64_json`. + +`refreshModelCache()` hits `/v1/models` once per tab activation and filters per-kind for the model dropdowns. JNI: `nativeSetWebUiHtml(html)` pushes the bundled file at server start; `nativeClearWebUi()` clears on stop. Same applies to `/docs` via `nativeSetDocsHtml` + `app/src/main/assets/server_docs.html`. The docs file documents every endpoint with copy-pasteable curl examples. + +### Start config (`configJson`) schema + +JSON object passed to `IRemoteServerService.start(configJson)`. Built by `ServerController.start()` from `:app`: + +```json +{ + "token": "tn_sk_", + "port": 11434, + "bindMode": "ALL_INTERFACES | LOOPBACK_ONLY | WIFI_ONLY", + "webUiHtml": "", + "docsHtml": "", + "engines": [ + { "id":"...", "name":"...", "path":"", "type":"gguf|vlm|embedding|tts|stt|image_gen|image_upscaler", + "mmproj_path":"", "config_json":"{...}", "created":1715000000, "primary":true|false } + ] +} +``` + +`engines` is built by walking the entire installed `ModelRepository` and mapping `ProviderType` → engine kind. GGUF chat models living under `/vlm/` with a colocated `*mmproj*.gguf` are auto-classified as `vlm`. `config_json` merges the per-model `loadingParamsJson` + `inferenceParamsJson` from `ModelConfig`. URI-pathType models are skipped — the server only supports `FILE` paths because there's no clean way to trampoline content URIs across the `:server` process boundary. + +### `:server`-side Kotlin (in `app/src/main/java/com/dark/tool_neuron/service/server/`, runs in `:server` process) + +- `RemoteServerService.kt` — plain `Service`, NOT `@AndroidEntryPoint`. Holds the `ServerEngineRegistry`, the `ServerInferenceBridge`, and a `RemoteCallbackList`. Implements `IRemoteServerService.Stub` inline. Foreground promotion happens *inside* the AIDL `start(configJson)` call — parses the catalog, sets the staging dir, calls `registry.setCatalog`, preloads the primary chat (or first VLM if no chat), configures + starts the native HTTP server, publishes a `ServerSnapshot` to all callbacks. `onStartCommand` only handles `ACTION_STOP` (the notification's Stop button). `onCreate` calls `nativeSetStagingDir(/server-staging/)` so the cleanup path is wired even before a `start`. +- `ServerEngine.kt` — wraps `GGMLEngine` for chat GGUF. `load(modelId, path, configJson)` (carries the id so the registry can decide reload vs. reuse), `unload()`, `generateMultiTurnFlow(...)`, `setSampling`, `setSystemPrompt`, `stopGeneration`. Same JSON shape `InferenceService` parses for chat (contextSize, threadMode, flashAttn, cacheTypeK/V, sampling, kvSink/Window/Evict). +- `ServerVlmEngine.kt` — separate `GGMLEngine` instance. `ensureLoaded(modelId, basePath, mmprojPath, configJson)` releases any prior projector, loads the base GGUF, then auto-loads the mmproj (preferring the explicit path; falling back to colocated `*mmproj*.gguf`). `generateFlow(messagesJson, imageBytes, maxTokens)` dispatches to `GGMLEngine.generateVlmFlow(imageData=..., imageQuality=HIGH)`. +- `ServerEmbeddingEngine.kt` — wraps `EmbeddingEngine`. `ensureLoaded(modelId, path, configJson)` calls `engine.load(path, threads, contextSize)`. Exposes `embedBatch(texts, normalize=true)` — the bridge JSON-encodes the result for `nativeFeedReplyText`. +- `ServerTtsEngine.kt` — wraps sherpa-onnx `OfflineTts` (VITS only). `ensureLoaded` builds `OfflineTtsConfig` from the model config JSON; `synthesize(text, speakerId, speed)` returns mono float samples; `sampleRate()` exposes the codec rate so the WAV encoder writes the right header. +- `ServerSttEngine.kt` — wraps sherpa-onnx `OfflineRecognizer` (Whisper only). `recognize(samples, sampleRate)` returns the transcribed text or null on failure. +- `ServerImageEngine.kt` — wraps `StableDiffusionManager`. `ensureRuntime()` is gated on `/ai_sd_runtime/qnnlibs.tar.xz` existing (downloaded by `:app`-side `ImageGenManager`). `loadDiffusion(id, name, path, width, height)` walks the model dir, builds `DiffusionModelConfig`, and calls `sdk.loadModel`. `loadUpscaler(id, path)` toggles MNN vs. OpenCL based on filename. `generate(params)` is `sdk.generateImageSync(...)` (blocks until result). `upscale(bitmap)` posts the bitmap and `.first {}`s on `upscaleState` for Complete/Error. PNG encoding/decoding lives here too. +- `ServerEngineRegistry.kt` — single source of truth for catalog + lazy loading. `chatFor`, `vlmFor`, `embedFor`, `ttsFor`, `sttFor`, `imageGenFor(width, height)`, `upscalerFor`. Each method picks the entry by id or falls back to `firstOf(kind)`. Per-kind locks (`Mutex` for suspend, plain `Object` for sherpa's synchronous load) serialise concurrent loads. +- `ServerInferenceBridge.kt` — extends `InferenceBridge`. Each upcall (`startGeneration` / `startEmbedding` / `startTts` / `startStt` / `startImageGen` / `startImageUpscale`) launches a coroutine on a `SupervisorJob` IO scope, calls the right registry method, dispatches to the engine, and feeds the result back via `NativeServer.nativeFeed*`. Chat + VLM streaming uses the existing `nativeFeedToken / nativeFeedDone / nativeFeedError` triplet; everything else uses the single-shot `nativeFeedReplyText / nativeFeedReplyBinary / nativeFeedReplyError`. VLM marker is prefixed onto the last user message via `engine.defaultMarker()` (= `engine.getVlmDefaultMarker()` in the Kotlin call site). +- `ServerCatalog.kt` — typed catalog model + `ServerEngineKind` enum + JSON serializer matching the C++ `set_catalog_json` parser. +- `ServerWavCodec.kt` — Kotlin-side RIFF reader/writer used by TTS (encode floats → wav) and STT (decode wav → floats). Mirrors the native helper. +- `ServerSnapshot` (in `RemoteServerService.kt`, internal) — phase / modelId / modelName / host / displayHost / lanHost / port / bindModeName / wifiActive / reason. Serialised to JSON for cross-process shipping. `modelId / modelName` reflect the **primary** engine (chat GGUF or first VLM) — the snapshot doesn't enumerate every loaded engine. +- `BindResolver.kt`, `ServerTypes.kt` — unchanged. + +### `:app`-side Kotlin + +- `ServerController.kt` — `@Singleton`, AIDL client. `start()` walks `ModelRepository.models.value`, builds the full multi-engine catalog (one JSON entry per installed model), packages with token / port / bind mode / web-UI HTML / docs HTML, calls `IRemoteServerService.start(configJson)`. URI-pathType models are silently skipped. The "selected chat model" pref still exists but is now only used to mark a `"primary": true` flag inside the engines array — if it's unset or invalid the first installed chat GGUF wins. Start enables as long as **any** engine is installed (was: required a chat model). `stop()` and `rotateToken()` forward via AIDL. +- `viewmodel/ServerViewModel.kt` — adds `anyEngineInstalled: StateFlow`. Chat-model selector card stays; selecting only seeds the `primary` flag. +- `ui/screens/server/ServerScreen.kt` — Start button enabled if any engine is installed OR a chat is selected. + +### State sync / process-survival semantics + +Unchanged from the single-engine build. `:app` calls `bindService(intent, conn, BIND_AUTO_CREATE)`. Android starts `:server`. AIDL stub returned. App registers callback, reads `currentSnapshotJson()` to rehydrate. App-killed-but-server-running: re-launching the app re-binds; `currentSnapshotJson()` returns `phase=running` with all live fields, plus `recentRequestEventsJson(100)` for the log card. Server foregrounds *only* during AIDL `start`, not on `bindService` alone — so a brief "exists, idle, no notification" state is impossible (we never enter it). + +### Lockdown + +`ScaffoldViewModel.serverRunning: StateFlow` derived from `ServerController.state`. `AppScaffold` `LaunchedEffect(serverRunning, currentRoute)` re-routes to `ServerScreen` with `popUpTo(0) { inclusive = true }` when running and not already there. `BackHandler(running) {}` inside ServerScreen absorbs back. Drawer gesture hidden via `showDrawer = currentRoute == HomeScreen.route && !serverRunning`. Chat-side load/unload + sendMessage in `HomeViewModel` are gated on `!serverController.isBusy`; same for `ModelStoreViewModel.downloadPack / downloadModel / setActive`. Because the lockdown is UI-routing rather than per-VM gating, the Image Task / Voice screens are inherently unreachable while server is running — no per-VM gate needed there. The server owns whatever model state it has loaded for its current request; chat-side reload would have nothing to clobber anyway because they're separate engine instances in separate processes. + +### Auth + token lifecycle + +- `ensureToken()` calls `nativeGenerateToken()` → `tn_sk_` + 32 random bytes base64url-encoded (getrandom(2)). +- Stored plaintext in encrypted HXS vault (`AppPreferences.serverToken`). +- Handed to native at start (`nativeSetToken`); zeroed on stop (`nativeClearToken`). +- 20 consecutive auth-fails from same client_addr → 1 h ban. +- Reveal in UI gates on `session.isAllowed(AUTH_VERIFY)`. Rotate generates a new token + invalidates the old. + +### HXS server keys + +`server_token` (String), `server_port` (String, validated [1024..65535], default `11434`), `server_bind_mode` (String, default `ALL_INTERFACES`), `server_auto_start` (Boolean, reserved), `server_configured` (Boolean), `server_selected_model` (String — primary chat hint only as of multi-engine). All ride the same encrypted `app_prefs` vault. + +### Deliberate omissions + +HTTPS / TLS, mDNS / Bonjour, QR-pairing, dynamic-model-load over the wire, streaming usage metrics, request-log persistence to HXS, network-URL image fetching (offline-only). Audio transcoding — `/v1/audio/transcriptions` accepts WAV PCM (16-bit or 32-bit float) only; MP3/AAC etc. return a generic decode failure. Per-engine RAM accounting — engines stay loaded until server stop; no idle TTL. Image-gen progress streaming — `/v1/images/generations` is single-response (uses `generateImageSync`); the live diffusion intermediate previews available in the in-app Image Task screen are not exposed. + +--- + +## HF Explorer + +Rewritten 2026-04-29. All HF traffic flows through `:networking` (curl-impersonate Chrome116 + bundled CA bundle); the previous `HttpURLConnection` path is gone. Filter chips are populated dynamically from `/api/models-tags-by-type`; the README on the detail screen renders client-side from `/{author}/{repo}/raw/main/README.md`. + +### Layers + +- `repo/HuggingFaceApi.kt` (Hilt `@Singleton class`) — URL builders + thin HTTP layer. Methods: `fetchJson(url): Result`, `fetchJsonArray(url): Result`, `fetchRaw(url): Result`, `probe(url): Result`. All go through `WebNative.fetch` with `Accept: application/json` and `Accept-Encoding: gzip`. URL builders: `modelInfoUrl`, `modelTreeUrl`, `resolveFileUrl`, `rawFileUrl`, `searchUrl`, `quickSearchUrl`, `trendingUrl`, `tagsByTypeUrl`. **Failures are typed via `HfApiError`** (`RateLimited(retryAfterSeconds)`, `NotFound`, `Forbidden`, `Network`, `Parse`, `Http`). +- `repo/hf/HfClient.kt` (Hilt `@Singleton`) — typed explorer endpoints over `HuggingFaceApi`. `searchModels`, `quickSearch`, `trending`, `modelDetail`, `readme`, `tagsCatalog` (cached 24h in encrypted `app_prefs` under keys `hf_tags_catalog_v1` + `hf_tags_catalog_v1_at`). +- `repo/hf/HfModels.kt` — `HfModelSummary`, `HfModelDetail` (with `HfSibling`/`HfGgufMeta`/`HfCardData`), `HfTrendingItem`, `HfQuickResult`, `HfTagsCatalog`/`HfTagEntry`, `HfGated` enum (OPEN/GATED/AUTO). +- `repo/hf/HfJsonParse.kt` — internal `org.json` parsers for each shape. +- `repo/HuggingFaceExplorer.kt` — kept as a thin compat wrapper exposing `searchModels` / `searchGgufRepos` / `fetchRepoDetail` mapped to legacy `ExplorerRepo` / `HfRepoDetail` types for `ModelStoreViewModel`. + +`ModelCatalog` and `RepositoryValidator` inject `HuggingFaceApi` directly. They no longer touch `HttpURLConnection`. + +### VM (`viewmodel/HfExplorerViewModel.kt`) + +State flows: `query`, `filters: HfFilters`, `results: List`, `isSearching`, `searchError: HfApiError?`, `tagsCatalog`, `trending`, `history`, `hideAdded`, `detailState`, `fileFilter`, `fileSizeBucket`, `existingRepoPaths`. On VM init: kicks off `tagsCatalog()` and `trending(12)` once each (cached). + +**Search trigger policy** (intentional, rate-limit conservative): +- IME action / Search button → fires `client.searchModels`. +- Any chip toggle / sort change / param-range slider release → fires fresh search via `updateAndSearch`. +- Per-keystroke quicksearch is **deliberately not wired**. + +### Filter set (intentionally minimal, 2026-04-29) + +`HfFilters` carries only fields that map to documented HF list params: `libraries: Set` (default `{"gguf"}`, multiple `filter=…`), `author: String` (`author=…`), `gated: GatedFilter` (`gated=true|false`), `paramsMinMillions/Max` (`num_parameters=min:7B,max:13B`), `sort: HfSort`. The previous "kitchen sink" filters (apps, inference_provider, languages, licenses, regions, other-tags, quant chips, trained-dataset, pipeline-tag, inference-warm) and post-filter sliders (min-downloads, min-likes, recent-days) were dropped because (a) HF rejected the speculative URL params with HTTP 400, and (b) the heavy filter UI added clutter without unlocking working searches. Tags catalog still fetched + cached for future use; just not wired to chip rows yet. + +### Screens + +- `ui/screens/hf_explorer/HfExplorerScreen.kt` — search hero (TnTextField + ActionButton submit), history strip when empty, sort row, gated/hide-added quick toggles, collapsible Filters card with: param-range slider, library chips (GGUF/Transformers/Safetensors/ONNX/MLX/Diffusers), author text. Trending strip when results empty + query empty. Result cards with author-initials avatar, downloads/likes/pipeline pills, tag chips, Gated badge variants (Gated / Gated · auto), Add/Added trailing icon. Errors render via `ErrorBanner` with rate-limit aware copy. +- `ui/screens/hf_explorer/HfRepoDetailScreen.kt` — `HeaderCard` with stats + gated badge; `GatedNotice` block when gated (license prompt preview + sign-in CTA); `GgufCard` (architecture, context, total bytes, BOS/EOS) when GGUF; `CardDataView` (license, base model, languages, task, tag chips); file filter pills + file rows; **README rendered via `lazyMarkdownItems`** from raw markdown; failure view distinguishes rate-limit / not-found / forbidden / network / parse / http. + +### Tags catalog cache + +Sealed in encrypted `app_prefs` (under existing `tn.app_prefs.user_key.v2` HKDF key). Plaintext JSON never lands on disk. 24h TTL; `forceRefresh = true` bypasses. On every cold start the first explorer open hydrates from the cache; if expired or empty, hits `/api/models-tags-by-type` once and re-persists. + +--- + +## RAG attachments + +The Action Window's third tab is **Attach** (formerly Tools). It shows the current chat's attachments and a single full-width "Add attachment" button. Tapping it opens `AttachmentPickerDialog` with two paths: + +- **Pick from previous chats** — opens `PrevChatsPickerDialog`, a full-screen `Dialog` with a list grouped by source chat title. Tapping a row re-attaches the document to the active chat. +- **Pick from storage** — launches `ActivityResultContracts.OpenDocument` with the existing MIME filter (text/*, pdf, json, xml, rtf, epub, odt, docx, pptx, xlsx). + +### Persistence model + +Every attached document is stored content-addressed by SHA-256 of its bytes: + +- `/chat_documents/sources/.bin` — raw bytes, written once per unique content; multiple chats sharing the same content share the file. +- `/chat_documents_meta_v1/` — **encrypted** HXS collection holding `(id, chatId, sourceId, name, mimeType, chunkCount, sizeBytes, addedAt)`. Sealed under `HKDF(DEK, "tn.chat_documents.user_key.v1")`. `DocumentRepository.init` migrates legacy plaintext at `chat_documents/` (top-level files) into the encrypted vault on first launch. `sources/` subdirectory is preserved during migration. +- `/rag_keyword_v1/` — **native HXS-encrypted** keyword index for hybrid retrieval. Sealed under `HKDF(DEK, "tn.rag_keyword.user_key.v1")`. Tokenization, inverted index construction, BM25 scoring all live in C++ (`hxs/src/main/cpp/rag_keyword.{h,cpp}`); only the wrapper class is in Kotlin. Inverted index is rebuilt in-RAM on every process start by scanning the HXS records (bounded by # of chunks). +- `chat_documents` HXS collection — same TAG layout as before: `(1=id, 2=chatId, 3=name, 4=mimeType, 5=chunkCount, 6=sizeBytes, 7=addedAt, 8=sourceId)`. Persisted across restarts. **Do not** call `documentRepo.clearAll()` from `RagManager.init` — that's the previous (wrong) behavior that wiped doc history every boot. +- `id` is the compound `:`. Same content attached to two chats produces two records sharing one `sourceId.bin` blob. + +`RagManager.hydrateChat(chatId)` re-ingests persisted records into the live RAG engine on chat-open (the engine itself is rebuilt fresh per process). It tracks `ingestedDocIds: MutableSet` to avoid duplicate ingests; the set clears on `engine.close()`. Hydration also re-populates the FTS5 BM25 index for text-format documents (idempotent — `keywordIndex.docCount(docId) > 0` check skips already-indexed). + +`RagManager.attachExisting(currentChatId, source)` is the prev-chat re-attach: builds the new compound docId, re-reads `.bin`, calls `engine.ingestBytes(...)`, persists the new record. Idempotent — if the chat already has the same `sourceId`, returns the existing record. + +`RagManager.removeDocument(docId)` removes the chunks from the engine + the FTS5 keyword rows + record from HXS, and deletes `.bin` only when no other record references that sourceId (`documentRepo.countWithSource(sourceId) == 0`). + +### Hybrid retrieval (dense + BM25 + RRF) + +`RagManager.augment(chatId, query, originalPrompt, maxContextTokens)` returns `RagAugmentation(augmentedPrompt, chunks)`: + +1. **Optional multi-query** — if `appPrefs.ragMultiQuery`, `RagQueryRewriter` asks the loaded chat model to generate 3 alternative phrasings of the user's query. Falls back to single-query if the model isn't loaded or the rewriter times out (8s). +2. **Per-query retrieval** — for each query (original + variants), runs the dense engine `engine.query(q)` (capped at `topN = DENSE_CANDIDATES = 20`) AND `RagKeywordIndex.query(q, chatId, KEYWORD_CANDIDATES = 20)` BM25 lookup against the FTS5 index. +3. **RRF fusion** — `rrfFuseMany` over all 2-N rankings (`k = 60`, identity = `(docId, chunkIndex)` pair). Items appearing in multiple rankings get summed RRF scores; items in only one ranking still score. Returns `FUSED_POOL_SIZE = 12` candidates. +4. **Optional LLM rerank** — if `appPrefs.ragSmartRerank`, `RagReranker` asks the loaded chat model to score each pooled chunk 1–5 against the query (single LLM call, 15s timeout, 256 max tokens). Returns reordered list. Falls back to RRF order if the model isn't loaded or scoring fails. +5. **Token budget** — `InferenceCoordinator.computeRagBudget(messages)` derives `contextSize - maxTokens - approxHistoryTokens - 256` (clamped to 256–4096). `RagManager.buildAugmentedPrompt` walks ranked chunks in order, summing approx tokens (chars/4), keeping until budget exhausted. Truncates the first chunk if it alone exceeds budget. Caps at `FINAL_TOP_N = 8` chunks. +6. **Citation contract** — the prompt instructs the LLM to cite chunks inline as `[1]`, `[2]`, etc. After generation, `RagCitationMatcher.match(response, chunks)` parses explicit `[N]` markers AND runs a 4-gram overlap check (≥3 hits = cited). Resulting `List` is stored on the assistant `ChatMessage` via `ChatRepository.TAG_MSG_CITATIONS = 13` (JSON array). UI: `CitationStrip` renders chip per citation below the message bubble; tap opens an `AlertDialog` with the snippet, doc name, score, and cited/possibly-used label. + +`RagKeywordIndex` is now native — backed by `hxs::RagKeywordIndex` in `hxs/src/main/cpp/rag_keyword.{h,cpp}`. Per-chunk records are stored in HXS-encrypted collection `rag_chunks` with TAG layout `(1=docId, 2=chatId, 3=sourceId, 4=chunkIndex, 5=text)`. The C++ side maintains an in-memory `unordered_map>` rebuilt at construction by scanning the encrypted records. Tokenizer is ASCII alphanumeric + underscore + UTF-8 bytes ≥0x80 passthrough, lowercased, length 2-64. BM25 params k1=1.2, b=0.75. JNI surface: `nativeRagIngest`, `nativeRagQuery`, `nativeRagRemoveDocument`, `nativeRagClear`, `nativeRagDocCount`. Replaces the prior SQLite FTS5 implementation, which broke on devices with stripped SQLite (no `fts5` module) and lived plaintext at rest. + +**FTS5 limitation:** only text-format documents are indexed. The native engine doesn't expose extracted text back to Kotlin (#329 is blocked-native), so binary formats (PDF/DOCX/EPUB/etc.) bypass BM25 — they only get dense retrieval. `RagManager.isTextLike(mime, name)` decides via mime-prefix `text/`, `application/{json,xml,rtf,javascript,yaml}`, or extensions `txt|md|markdown|json|xml|csv|tsv|html|htm|rtf|yaml|yml|log|ini|toml|properties|kt|java|py|js|ts`. + +`RagChunker` does Kotlin-side recursive splitting for the FTS5 path (target 1024 chars, min 200, separators in priority `\n\n / \n / . / ! / ? / ; / , / space`). The native engine's chunking is independent — chunk indices from FTS5 do not align with native engine indices. RRF treats them as separate items by `(docId, chunkIndex)` identity, which is fine. + +### Deep Index (contextual retrieval, simplified) + +Per attached document, a "Deep Index" sparkles-icon affordance in the Attach tab triggers `RagManager.deepIndex(docId)`. The flow: +1. Read source bytes from `chat_documents/sources/.bin` (text-format docs only — `RagManager.isTextLike` mime/extension gate). +2. `RagDocSummarizer` asks the loaded chat model to write a one-sentence document summary (≤320 chars, 30 s timeout, 200 max tokens). One LLM call per document. +3. `RagChunker` splits the source text into ~1024-char Kotlin-side chunks. +4. For each chunk, prepend `[Document context: — ]` and re-ingest into the dense engine + BM25 index using compound docId `${origDocId}::ctx`. The native engine internally re-chunks the (context + chunk) blob into multiple sub-chunks, each carrying the doc context. The original doc remains untouched. +5. `ChatDocument.isDeepIndexed = true` is persisted (TAG 9 on the chat_documents collection); the UI shows a "Deep" badge next to the filename. +6. `RagManager.deepIndexing: StateFlow>` exposes the in-flight set so the UI can show a spinner per row. + +Inflation factor: ~Nx storage per deep-indexed doc, where N = (Kotlin chunk size + summary length) / native chunk size. For 1024-char Kotlin chunks + native chunk_size=256, ~5x more native chunks per doc. + +Augment-side change: `RagManager.augment` strips `::ctx` suffixes when looking up the parent ChatDocument so citations group under the original doc, not its context-pseudo-children. + +Cleanup: `RagManager.removeDocument` recurses through `ingestedDocIds` removing every `${origDocId}::ctx*` from the engine + BM25 index before deleting the parent record. + +Limitations: text-format docs only (PDFs/DOCX blocked by no native extract API). One doc-level summary per doc, NOT per-chunk Anthropic-style — simpler v1, marginally lower quality than per-chunk contextual retrieval but ~1 LLM call vs. N. Idempotent: skipped if already deep-indexed. + +### Retrieval debug screen + +`Settings → Chat & RAG → Retrieval debug` opens `RagDebugScreen` (route `NavScreens.RagDebug`). VM is `RagDebugViewModel` (injects `RagManager` + `ChatRepository`). Renders: + +- Status pill (ready + active embedding name). +- Chat dropdown to scope the test query. +- Query text field + Run button. +- Tabs: Fused (RRF result), Dense (raw native), BM25 (raw FTS5), Context (final assembled `` block + token count), Engine (raw `engine.info()` JSON). +- Each hit card shows chunkIndex, score, docId, first 600 chars of text. + +Backed by `RagManager.debugQuery(chatId, query, budget)` which returns `RagDebugResult`. Multi-query is NOT applied in the debug path (single-query for clarity). + +### Extension badge + +`model/DocExtension.kt` enum maps mime + filename to a `(label, tint)` pair (PDF/DOCX/XLSX/PPTX/ODT/EPUB/RTF/MD/HTML/JSON/XML/CSV/TXT/OTHER). `ExtensionBadge` in `ui/components/action_window/Attachments.kt` renders a rounded card with the label centered, tinted from the entry's color. Used in the Attach tab and the prev-chats picker. + +### PlusMenu cleanup + +The PlusMenu's old "Documents" button is gone — attachments live entirely in the Attach tab now. PlusMenu shows only Thinking when `supportsThinking`; if not supported, `PlusMenuCard` returns null. + +--- + +## Web search + +Replaces the prior Research pipeline (2026-05-15). Single-shot LLM-driven web search. User flips the Web Search toggle on the bottom action bar (or types `/search `); next chat send becomes a web-search run. + +Flow (`viewmodel/WebSearchCoordinator.kt`): + +1. **Plan** — coordinator emits `WebSearchEvent.Plan(userQuery)` so the card renders immediately. +2. **GenerateQueries** — one LLM call (`WebSearchPrompts.generateQueries`) asking for exactly 3 numbered queries. Regex-parsed via `QUERY_LINE_REGEX = ^\s*(?:\d+[.)\-:]|[-*•])\s+(.+)$`. Failures fall through to `WebSearchEvent.Failed`. +3. **Search** — for each of the 3 queries, `WebSearcher.search(query, maxResults=5, idx)` via `WebNative.search` (DDG HTML). Per-query results are deduped against a session-wide `seenUrls` set so cross-query overlap doesn't double-feed the synthesizer. Total cap: 3 queries × 5 results = 15 unique snippets. +4. **Synthesize** — one LLM call (`WebSearchPrompts.synthesize`) with the user query + numbered `[i]` snippet list. Output is markdown with inline `[1]/[2]/[3]` citations and a trailing Sources section. +5. **Done** — emits `WebSearchEvent.Done(answer, sources)`; card renders the markdown answer + collapsible tappable source list (chip → `LocalUriHandler.openUri`). + +No URL fetching. No document extraction. No iteration loop. The user-visible difference vs. old Research: seconds instead of minutes, single inline result instead of a "research document" archive screen. + +### Persistence + +State rides entirely on the chat message via `webSearchRunId: String?` (`TAG_MSG_WEBSEARCH_RUN = 14`) and `webSearchState: String` (`TAG_MSG_WEBSEARCH_STATE = 15`, JSON-serialized `WebSearchUiState`). `ChatMessageList` renders `WebSearchCard` instead of `MessageBubble` when `webSearchRunId != null`. Done runs survive process restart because the terminal state is on the message. No separate vault, no Documents archive, no DocumentViewer screen. + +`HomeViewModel.handleWebSearchEvent` looks up `(chatId, messageId)` via `webSearchMessages[runId]`, applies the event to the persisted state, and writes the updated message back. The map evicts on Done/Cancelled/Failed. + +### LLM context inclusion + +The card message stores the user's original query in `msg.content` (read by the card Header), and the synthesized answer in `msg.webSearchState`. `InferenceCoordinator.buildMessagesJson` is the single point that prepares chat history for the LLM — when it encounters a `webSearchRunId != null` message, it swaps `content` for `WebSearchUiState.fromJson(webSearchState).answer.trim()` so the model sees the synthesized markdown as the prior assistant turn (not the echoed user query). Cards with an empty answer (in-flight, cancelled, failed) are skipped entirely so the LLM doesn't get a blank assistant message. + +### Lockdown + +Same pattern as the old research lockdown — `webSearchActive: StateFlow` is derived from `webSearchCoordinator.activeRuns.isNotEmpty()`. `sendMessage`, `loadModel`, and `unloadModel` all early-return while a run is active because the chat LLM is borrowed for both the GenerateQueries and Synthesize calls. + +### Card UI + +`ui/screens/web_search/WebSearchCard.kt` is a single Surface with: +- Header (Globe icon, "Web search", user query) +- Queries strip (3 rows with per-query progress indicators — Circle / spinner / Check + hit count) +- AnimatedContent for current phase (Plan / Queries / Search / Synthesize / Done / Cancelled / Failed) +- Stop button while in flight +- For Done: markdown answer + collapsible `N sources` accordion with `[i]` chips opening URLs externally + +### File map + +- `model/WebSearchEvent.kt` — sealed event class + `WebSearchHit` data class. +- `model/WebSearchUiState.kt` — phase machine + JSON serde. +- `repo/web_search/WebSearcher.kt` — thin wrapper over `WebNative.search`. +- `repo/web_search/WebSearchPrompts.kt` — prompt templates + `QUERY_LINE_REGEX`. +- `viewmodel/WebSearchCoordinator.kt` — single coordinator (no repository, stateless across runs). +- `ui/screens/web_search/WebSearchCard.kt` — the only UI. +- Modified: `ChatMessage` (+ webSearchRunId, webSearchState), `ChatRepository` (TAG 14/15 renamed), `HomeViewModel` (toggle, slash parse, coordinator wiring, event mirror), `HomeScreen{Body,BottomBar}` + `ToolsPickerWindow` (Web search toggle), `ChatMessageList` (WebSearchCard render). + +--- + +## Image generation (`:ai_sd` AAR) + +Re-pivoted into scope on 2026-05-08. Drop-in port of LocalDream's catalog (xororz/sd-qnn + xororz/sd-mnn + xororz/sdxl-qnn + xororz/upscaler) onto the existing TN model store. SD tasks stay under `:ai_sd`: **Generate (txt2img)**, **Edit (img2img)**, **Inpaint**, **Upscale**. Background removal, segmentation/matting, and object erase were removed from current scope on 2026-06-23. + +### SoC bucket policy (mirrors LocalDream) + +`data/SocBucket.kt` reads `Build.SOC_MODEL` (API ≥ 31; pre-S falls back to `"CPU"` and the user only sees CPU-bucket models). `chipsetModelSuffixes` maps known Snapdragons to one of three buckets: + +``` +SM8475, SM8450 → "8gen1" +SM8550, SM8550P, QCS8550, QCM8550, SM8650, SM8650P, SM8750, SM8750P, +SM8850, SM8850P, SM8735, SM8845 → "8gen2" (also covers 8 Gen 3 / Elite / Elite Gen 5) +any other SM* → "min" +non-Qualcomm → null (CPU-only) +``` + +`isSdxlCapable(soc)` is a stricter predicate: only `{SM8650, SM8845, SM8750, SM8750P, SM8850, SM8850P}` get the SDXL rows. SDXL contexts are baked at a single `_8gen3.zip` variant (no per-bucket file). + +### Catalog wiring + +`ModelCatalog.imageModels()` is computed per-call (not in `BUILT_IN_MODELS` const) so the `Build.SOC_MODEL` read picks up cleanly. When a Snapdragon bucket is available it emits 5 SD 1.5 NPU rows (AnythingV5, QteaMix, AbsoluteReality, CuteYukiMix, ChilloutMix), the 2 SDXL rows (gated on `isSdxlCapable`), and QNN upscaler rows. On non-Snapdragon devices it instead emits 5 SD 1.5 CPU/MNN rows from `xororz/sd-mnn`. The built-in Store also carries MNN-compatible upscalers from `tumuyan2/realsr-models`, tagged by use case (`Use: General`, `Photo`, `Portrait`, `Anime`, `Sharp cleanup`, `Other`), `runtime=MNN`, and `scale=...`; do not add unsupported `.pth` / ONNX upscalers as installable rows. `qnn2.28` is baked into the URL as the SDK version token; if the AAR ever upgrades to `qnn2.30` both the URL constant and the `v3` upgrade marker need to bump together. + +`HuggingFaceModel` carries new image-gen fields (`isSdxl`, `requiresNpu`, `isUpscaler`, `featureLabel`, `defaultPrompt`, `defaultNegativePrompt`, `generationSize`); `modelType ∈ {"image_gen", "image_upscaler"}` switches the download finalize path. `ProviderType.IMAGE_GEN` and `ProviderType.IMAGE_UPSCALER` are the canonical categories on `ModelInfo` after install. + +### Download + extraction + +`ModelStoreViewModel.finalizeImageGenDownload` extracts the QNN/MNN ZIP into `/sd_models//` via `java.util.zip.ZipFile` with a hardened path-traversal check (entry's canonical path must start with the target's canonical path + `File.separator`). Archive deleted after extraction, `ModelInfo` inserted with `path` = the dir. `finalizeImageUpscalerDownload` installs a single `.bin` or `.mnn` file under `/sd_upscalers//`. Old ONNX image-operation model records are deleted during `ModelRepository.refresh()` and `/image_onnx/` is removed. Legacy `TOOL_SEARCH` provider records are migrated to `GGUF` during refresh; Tool/Search is not a visible model class unless a real dedicated routing path is added later. + +### Runtime singleton + +`repo/ImageGenManager.kt` is the Hilt `@Singleton` wrapper around `StableDiffusionManager.getInstance(context)`. `ensureRuntime()` is mutex-guarded and fires `StableDiffusionManager.initialize()` on first use, which extracts `qnnlibs.tar.xz` from the AAR's bundled assets into `/ai_sd_runtime/`. Subsequent `loadDiffusionModel(model, w, h)` calls run model-specific load on the engine. The active model id is cached so re-entering Image Task screen with the same model is a no-op. + +### Image Task screen + +`ui/screens/image_task/ImageTaskScreen.kt` + `ImageTaskTopBar.kt` + `viewmodel/ImageTaskViewModel.kt`. Route: `NavScreens.ImageTask` (`"image_task"`). Reachable from the chat drawer's "Images" quick-link. The screen is one LazyColumn of cards: + +- **Task** — mode switch for Generate / Inpaint / Upscale. +- **Image model / Upscaler** — list of installed models for the picked task; tapping a row triggers `loadDiffusionModel` or `loadUpscaler`. +- **Prompt** — TnTextField for prompt + negative prompt (hidden in Upscale mode). +- **Settings** — `ActionToggleGroup` rows for Steps, CFG, Scheduler, Resolution, Denoise (img2img / inpaint only). +- **Input image** — SAF `OpenDocument` picker, shown for Edit / Inpaint / Upscale. +- **Run** — Generate / Edit / Inpaint / Upscale 4× button, with Stop appearing during generation. LinearProgressIndicator binds to `DiffusionGenerationState.Progress.progress`. +- **Output** — renders the final `Bitmap` (or live intermediate if `showDiffusionProcess` is on). + +### Things to know + +- The `:ai_sd` library declares `commons-compress` and `xz` as `api` deps in its module build. When consuming as a path AAR (`implementation(files(...))`) Gradle does NOT pull transitive deps from a POM-less file dependency, so `app/build.gradle.kts` must declare both directly. `xz` was added to `gradle/libs.versions.toml` as `org.tukaani:xz:1.12`. +- `app/proguard-rules.pro` adds `-keep class com.dark.ai_sd.** { *; }` and `-dontwarn com.dark.ai_sd.**` alongside the existing gguf_lib / ai_sherpa rules. +- The AAR ships `qnnlibs.tar.xz` (~200 MB compressed) inside `assets/qnnlibs/`. First-run setup extracts it into `filesDir/ai_sd_runtime/` and is observable through `RuntimeSetupState`. Don't move the runtime dir without bumping the SDK version token in catalog URLs. +- Currently the **debug** AAR is shipped (release AAR's R8 mangled `StableDiffusionManager.Companion.getInstance`). Once `:ai_sd`'s `consumer-rules.pro` adds `-keep class com.dark.ai_sd.StableDiffusionManager$Companion { *; }` and the AAR is rebuilt, swap to release. + +### Future expansion + +To add Object Removal (LaMa), Segmentation (MobileSAM), Depth (MiDaS / Depth Anything V2), or Style Transfer (AdaIN) — the C++ already implements all four; needs a small JNI surface on `SDNativeLib` + matching state flows on `StableDiffusionManager` + new `ImageTaskMode` values + per-feature catalog rows pointing at the matching HF repos. + +To add SDXL on devices that aren't in `isSdxlCapable` — pipeline-level, not gating-level. SDK has `textEmbeddingSize=768` hardcoded across the C++ pipeline + SDK keeps 4-channel latents + 77-token CLIP + LDM-style weight names. SDXL needs 2048-dim, dual CLIP, additional UNet conditioning inputs (`text_embeds`, `time_ids`), and matching `sd_structure.h` / `lora_mapping.h` entries. Out of scope for this pivot. + +--- + +## Downloads screen + +Dedicated screen aggregating every queued / downloading download plus a persistent history of completed / failed / cancelled ones. Reachable from a Download icon in the Model Store top bar (badge dot when active count > 0). Route: `NavScreens.Downloads("downloads")`. + +### Architecture + +Two singletons sit between `HxdManager` and the UI: + +- **`DownloadCoordinator`** (`@Singleton`, in `repo/`) — subscribes to `HxdManager.tasks` once at first construction. Holds an in-memory `ConcurrentHashMap` (hxdId → displayName + type). On every emission, recomputes `activeCount: StateFlow` (count of QUEUED/CONNECTING/DOWNLOADING/PAUSED tasks) and detects per-task terminal transitions (COMPLETED/FAILED/CANCELLED). On a first-time terminal transition it removes the label, then writes a `DownloadHistoryEntry` via the repo. `recordedTerminals: Set` dedupes so a task that bounces between flow emissions only gets one history row. `DownloadLabel.fromUrl(url)` provides a fallback when the label was never registered (e.g. a download that survived process restart). `HxdService` retries transient HTTP/network failures up to 3 attempts with short backoff and preserves `.hxd_tmp` for resume; explicit cancel/clear deletes the temp file. +- **`DownloadHistoryRepository`** (`@Singleton`, in `repo/`) — HXS-backed at `/download_history_v1/`. Sealed under `HKDF(DEK, salt=signerHash, info="tn.download_history.user_key.v2")` — same v2-signer-bound pattern as every other vault. Collection name `download_history`. TAG layout `1=id (UUID), 2=displayName, 3=type, 4=status (ord), 5=totalBytes, 6=completedAt, 7=error`. `insert()` writes + caps to MAX_ENTRIES=50 newest + flushes + refreshes the StateFlow. `clearAll()` deletes every record. Capping happens inside `insert()` so the vault can't grow unbounded. + +Every enqueue site MUST call `coordinator.registerLabel(hxdId, displayName, type)` immediately after `HxdManager.enqueue(...)`. Current sites: +- `ModelStoreViewModel.downloadModel(model)` — label = `model.name`, type from `downloadTypeOf(model)` mapping (`mmproj` / `vlm` / `gguf` / per-model-type strings). +- `ImageGenManager.downloadRuntime()` — label = `"AI Image Runtime"`, type = `"runtime"`. + +Any new download path that calls `HxdManager.enqueue` MUST register a label too, otherwise the row falls back to `url.substringAfterLast('/')` and the history entry shows the raw filename instead of a human-readable name. + +### Process isolation + +`DownloadCoordinator` + `DownloadHistoryRepository` are `:app`-process-only by construction. Verified at build time: neither `:server` (`service/server/`) nor `:inference` (`service/inference/`) imports them. The coordinator is reached only via `ModelStoreViewModel` and `ImageGenManager`, both of which are constructed only in `:app`. If a future service-process class needs to know about downloads (e.g. a notification surfacing history in `:server`), it MUST go through AIDL — never inject the coordinator directly, because that would attempt HXS unwrap from a non-main process and crash. + +### ViewModel + screen + +- **`DownloadsViewModel`** (`@HiltViewModel`) — exposes `activeDownloads: StateFlow>` (joined HxdManager.tasks + coordinator labels, filtered to active or failed attention-needed rows, sorted by hxdId) and `history: StateFlow>` (passthrough from repo). Methods: `cancel(hxdId)`, `retry(hxdId)`, `clear(hxdId)`, `clearHistory()`. `ActiveDownloadItem(hxdId, displayName, type, state)`. +- **`DownloadsScreen`** at `ui/screens/downloads/DownloadsScreen.kt` — single `LazyColumn` with section headers (`Active · N` then `History · N` with trash icon to clear). Failed active rows expose Retry and Clear; Retry resumes the preserved partial file and Clear deletes the temp state. Uses `LocalDimens` / `LocalTnShapes` / `TnIcons` throughout; no inline `dp` constants for theme tokens. Empty state when both are empty (Download icon + "No downloads yet"). +- **`DownloadsTopBar`** — minimal GuideTopBar-style top bar; dispatched from `AppTopBar.kt`'s `when` block on `NavScreens.Downloads.route`. + +### Active row + +For each `ActiveDownloadItem`: +- Header row: type-icon badge (mapped from `type` string) + display name + cancel icon (X), or Retry/Clear for failed rows. +- Body switches on `HxdStatus`: + - QUEUED → "Queued" text + - CONNECTING → `TnIndeterminateProgressBar` + "Connecting…" + - PAUSED → `TnProgressBar` (if totalBytes > 0) + "Paused" + - FAILED → error text + Retry/Clear actions + - DOWNLOADING → `TnProgressBar` (or indeterminate if totalBytes ≤ 0) + `"% · / "` left, `state.speedFormatted` right. + +### History row + +For each `DownloadHistoryEntry`: +- Status icon (CircleCheck primary / AlertTriangle error / X muted) + display name. +- Subtitle: `" · · "` (size omitted if ≤ 0). Failed/cancelled rows append the first 48 chars of the error string when present. +- Relative time via `android.text.format.DateUtils.getRelativeTimeSpanString(..., MINUTE_IN_MILLIS, FORMAT_ABBREV_RELATIVE)`. + +## Storage maintenance + +`StorageScreen` includes a maintenance card above the category list. `StorageInspector.maintenance(mode)` owns the scan logic: + +- `QUICK_CLEAN` removes `.hxd_tmp`, `_archive_*`, and cache leftovers, then recreates `cacheDir`. +- `DETAILED_CHECK` verifies installed model records point to existing non-empty files/folders and reports likely orphan model files. +- `DEEP_MODEL_TEST` adds lightweight local validation: GGUF magic header, VLM colocated mmproj, image model layout signals, voice config presence, and model config JSON validity. It must not load large models into inference just to run a storage check. + +Maintenance reports return checked / issue / fixed / skipped counts plus capped issue rows. Destructive broad category clearing still uses the existing confirmation dialog; quick-clean is limited to temp/archive/cache leftovers. + +### Top-bar badge + +`ModelStoreScreen` adds a Download `ActionButton` in the `TopAppBar.actions` slot, always visible (not gated on tab). When `viewModel.activeDownloadCount > 0`, a small `MaterialTheme.colorScheme.primary` circle (8.dp) is drawn at top-end via `Box(contentAlignment = TopEnd)` + `Modifier.offset(x=-4.dp, y=4.dp)`. Count comes from `ModelStoreViewModel.activeDownloadCount` which is a direct passthrough of `DownloadCoordinator.activeCount`. + +### What's deliberately NOT in v1 + +- No retry button on failed history rows. To retry, the user re-initiates from the Store. +- No per-row delete in history. The "Clear all" trash icon in the section header is the only deletion path. +- No pause/resume controls (HxdManager exposes them, but the existing Store UI doesn't surface them either — keeping parity). +- No filtering or search in history (50-entry cap keeps the list short enough). +- No grouping by type. Single chronological list ordered newest-first. + +--- + +## Dynamic Island overlay + +Floating black pill that morphs into a card, drawn over every app via `TYPE_APPLICATION_OVERLAY`. Lives in `app/src/main/java/com/dark/tool_neuron/service/island/`. Foreground service (`IslandOverlayService`, `foregroundServiceType="dataSync"`, `stopWithTask="false"`) keeps the overlay alive across recents-swipe. Window params are `WRAP_CONTENT + FLAG_NOT_FOCUSABLE + FLAG_LAYOUT_NO_LIMITS + gravity TOP|CENTER_HORIZONTAL + x=0`; the pill is **always horizontally centered** at the top of the screen. WRAP_CONTENT means the window resizes in lockstep with the morph animation (pill grows into card symmetrically both sides) so unrelated taps miss the window entirely in pill mode. The user calibrates only Y (`offsetYDp`) via the prototype screen slider — there is no X calibration because the pill is always centered. + +**Compose surface is a single `updateTransition`-driven Surface with iOS-style squircle.** Shape comes from `islandShape(cornerRadius)` in `IslandShapes.kt` — a `ContinuousRoundedRectangle` built on a dedicated `G2Continuity` profile (`extendedFraction = 0.6, arcFraction = 0.4, bezierCurvatureScale = 1.2, arcCurvatureScale = 1.2`) tuned for the iOS Dynamic-Island squircle look. The profile is intentionally separate from the global `TnContinuity` in `Shapes.kt` so the island's aesthetic can drift without bleeding into the rest of the app. Other spacing/dimensions route through the existing `LocalDimens` / `LocalTnShapes`: outer padding = `dimens.spacingSm`, card content padding = `dimens.spacingLg`, action-button background shape = `LocalTnShapes.current.full`. Pixel-sized identity constants (PILL_W/H, CARD_W/H, CARD_CORNER_DP=32, PRESS_SCALE=0.92, SWIPE_THRESHOLD_DP=48) live in `IslandGeometry` because they aren't generic theme tokens. + +**Single-progress morph for perfect frame-sync.** `updateTransition(expanded)` drives ONE `animateFloat` (`progress: 0f → 1f`). Width, height, cornerRadius, pill-icon-alpha, and card-content-alpha are all derived via `lerp(...)` of that one progress value — they CANNOT desync, drift, or arrive at the target on different frames. The morph spec is a custom `spring(dampingRatio = 0.85f, stiffness = StiffnessMediumLow, visibilityThreshold = 0.0005f)` — chosen instead of `motionScheme.defaultSpatialSpec` because M3 Expressive's default has a noticeable overshoot that, when applied to corner radius / size simultaneously, reads as juttery for a Dynamic-Island-style morph. `0.85f` damping gives a tiny bit of bounce on settle without overshoot; `StiffnessMediumLow` runs the morph at ~400ms which feels fluid. Other motion specs still come from `motionScheme`: press scale uses `fastSpatialSpec()`, mode-swap slide uses `defaultSpatialSpec()`, all cross-fades use `fastEffectsSpec()`. + +**Cross-fade timing:** `pillAlpha = (1f - progress * 2f).coerceIn(0f, 1f)` and `cardAlpha = ((progress - 0.5f) * 2f).coerceIn(0f, 1f)`. Pill content fades out across the first half of the morph, card content fades in across the second half. At progress=0.5 both are 0 → clean handoff with no visual overlap. + +**Shape allocation guard:** `cornerRadius.roundToInt()` keys the `remember { islandShape(...) }` block, so the kyant `ContinuousRoundedRectangle` is reconstructed only at 1dp granularity (~15 allocations across a full morph instead of ~60). 1dp jumps in corner radius are visually imperceptible; preventing per-frame Shape allocation is what keeps the morph smooth on mid-tier devices. + +**Modes + gestures.** `IslandMode` enum has `ASSISTANT` and `CONTROL`. State lives locally in `IslandSurface` since modes don't need to outlive the composable. The current mode is visible in BOTH pill and card states: +- **Pill state** → centered glyph icon (Sparkles for ASSISTANT, Sliders for CONTROL) rendered at `dimens.iconSm`, animated via `AnimatedContent` with horizontal slide on mode change. +- **Card state** → full layout with glyph + title + action buttons, same `AnimatedContent` slide. +- **Tap** → `onToggle()` + `HapticFeedbackConstants.CONFIRM` (pill ↔ card). +- **Long-press** → `onToggle()` + `HapticFeedbackConstants.LONG_PRESS` (same logical action, stronger haptic to confirm a deliberate gesture). +- **Press-and-hold animation** → the `onPress` lambda flips a `pressed` boolean; `pressScale` animatable shrinks the surface to `PRESS_SCALE = 0.92` via `graphicsLayer { scaleX = pressScale; scaleY = pressScale }` and snaps back on release. +- **Horizontal swipe (works in BOTH pill and card state)** → `detectHorizontalDragGestures` accumulates `dragAmount`. Crossing `SWIPE_THRESHOLD_DP = 48` swaps `mode` (ASSISTANT ↔ CONTROL) with `HapticFeedbackConstants.GESTURE_END` haptic. Swiping on the pill changes the badge icon and pre-selects the mode for next expansion. Tap and drag are in separate `pointerInput` modifiers — Compose's per-modifier gesture isolation lets them coexist (tap claims on no-slop release, drag claims on slop-exceeded movement). +- **Action button taps** (mic/send/volume/settings inside the card) fire `HapticFeedbackConstants.CONFIRM`; the actions themselves aren't wired to anything yet — this is still the prototype surface. + +**Icons via `TnIcons`.** Assistant mode shows `Sparkles` + Mic/Send. Control mode shows `Sliders` + Volume/Settings. All icons are TnIcons (stroke-based, 24x24 viewport, tinted via Compose `Icon(tint = Color.White)`). + +**Placement is via `WindowManager.LayoutParams.y`, NOT Compose `Modifier.offset`.** The service holds a single `Animatable` (`animY`) and feeds its value into `windowManager.updateViewLayout(islandView, params)` via a `snapshotFlow` collector. User-position slider changes `snapTo` instantly; smart-dodge changes animate with a bouncy spring (`dampingRatio = 0.55f, stiffness = StiffnessMediumLow`). Compose-side only renders the pill at its natural `padding(8dp)` position; it does NOT know about position or dodge. Reason: if you offset via Compose, the WindowManager window stays at its original rect — touches keep landing on the original location (so a menu button under the old pill stays untappable) AND the pill render gets clipped at the window's WRAP_CONTENT bounds (the pill visually disappears past the wrapped edge). Moving the window itself solves both. + +### Smart dodge (AccessibilityService) — vertical only + +`IslandAccessibilityService` watches `TYPE_WINDOW_STATE_CHANGED` + `TYPE_WINDOW_CONTENT_CHANGED`, coalesces 150 ms, walks `rootInActiveWindow` for `isVisibleToUser && (isClickable || isLongClickable)` nodes, and checks whether any clickable rect overlaps the pill's natural screen rect. The dodge output is a single `Float` Y nudge (dp) published to `IslandPositionStore.dodgeY`. Since the pill is always horizontally centered, the only thing the service can move is Y — there is no horizontal escape direction to choose. + +The pill rect is computed manually from `IslandGeometry` constants + `IslandPositionStore.position.value.offsetYDp` + `statusBarTopInsetPx()` — NOT from the `windows` API. Manual computation is stable across animation frames; using `windows` would create an oscillation loop (dodge → pill moves → no overlap → dodge=0 → pill snaps back → overlap → …). Geometry: `pillLeft = (screenWidth - pillWidth) / 2`, `pillTop = statusBar + outerPadding + offsetY`. The accessibility service expands this by `DODGE_MARGIN_DP = 8` for the search zone (proactive dodge before strict overlap) but uses the un-expanded rect for the actual push-down math. + +Dodge math: for each clickable obstacle that strictly intersects `pillRect`, compute `pushDown = obstacle.bottom + margin - pillRect.top`. Take MAX across overlapping obstacles, clamp to `[0, MAX_DODGE_DP = 96]`, convert to dp, publish to `dodgeY`. No obstacles → `dodgeY = 0`. + +`IslandOverlayService` observes both `position` and `dodgeY`: +- Slider Y changes → `animY.snapTo(position + dodge)` (instant; calibration tool must feel responsive) +- `dodgeY` changes → `animY.animateTo(position + dodge, dodgeSpring)` (bouncy 0.55 / StiffnessMediumLow) +- `snapshotFlow { animY.value }` collector pushes each frame's value to `LayoutParams.y` via `updateViewLayout`. Gravity stays `TOP|CENTER_HORIZONTAL` + `x=0` always. + +On `TYPE_WINDOW_STATE_CHANGED` (app switch) `dodgeY` is reset to `0f` first so a stale dodge from app A doesn't leak into app B before the re-scan completes. On `onUnbind` (accessibility service disabled) `dodgeY` also resets to `0f`. + +The overlay window is tagged with `LayoutParams.title = IslandGeometry.OVERLAY_WINDOW_TITLE` (`"TnIsland"`) for `adb shell dumpsys accessibility` debuggability — the accessibility service no longer uses it for pill-rect lookup (manual computation replaces that), but the title is kept because it costs nothing and helps inspect the overlay's bounds during troubleshooting. + +The service is opt-in: requires `BIND_ACCESSIBILITY_SERVICE` (system-granted) which the user enables via Settings → Accessibility → Tool Neuron Island. The prototype screen surfaces a button that deep-links to `Settings.ACTION_ACCESSIBILITY_SETTINGS`. Detection of enabled state uses both `IslandPositionStore.accessibilityActive` (live, set in `onServiceConnected`/`onUnbind`) AND `Settings.Secure.ENABLED_ACCESSIBILITY_SERVICES` parsing (initial render before the service connects). + +`AccessibilityGuard` now self-excludes `context.packageName` — our own service does NOT trip our own one-time `RootWarningDialog`. The existing allowlist (Google / Samsung / OEM accessibility services) remains intact for OTHER vendors. + +### File map (island) + +- `IslandOverlayService.kt` — foreground service, owns the WindowManager view + Compose host. +- `IslandComposeView.kt` — FrameLayout host for the inner ComposeView with one-time ViewTreeOwner attach. +- `IslandOverlayRoot.kt` — `IslandSurface(expanded, position, nudge, onToggle)` morph composable. +- `IslandGeometry.kt` — pill width/height/padding/dodge constants shared between composable and accessibility service. +- `IslandPosition.kt` — `IslandPosition(offsetXDp, offsetYDp)` + `IslandNudge(dxDp, dyDp)`. +- `IslandPositionStore.kt` — Kotlin `object` singleton; `position`, `nudge`, `running`, `accessibilityActive` StateFlows; `setOffset` (persisted), `setNudge` / `clearNudge` (transient), `setRunning`, `setAccessibilityActive`. +- `IslandAccessibilityService.kt` — window-event coalesce + clickable-node walk + dodge math. +- `OverlayLifecycleOwner.kt` — minimal LifecycleOwner / ViewModelStoreOwner / SavedStateRegistryOwner for the overlay window. +- `app/src/main/res/xml/island_accessibility_service.xml` — accessibility service config (events, flags, `canRetrieveWindowContent="true"`). +- `app/src/main/res/values/strings.xml` — `island_accessibility_label` + `island_accessibility_description`. + +--- + +## Setup flow + +Sequence: Intro → **TermsConditions** (if !tcAccepted) → **DevNotes** (if !onboardingComplete) → SetupScreen (lock mode) → (SetupPassword if password chosen) → SetupTheme → ModelSetup → SetupRag → Home. + +`ScaffoldViewModel.resolveStartDestination()` ordering: tcAccepted, then onboardingComplete, then securitySetupDone, then modelSetupDone, then `isLockEnabled` → PasswordScreen, else HomeScreen. + +### TermsConditions + +- Route: `NavScreens.TermsConditions("terms_conditions")`. +- Screen: `ui/screens/terms_conditions/TermsConditionsScreen.kt`. Top bar: `TermsConditionsTopBar.kt`. Bottom bar: `TermsConditionsBottomBar.kt`. VM: `viewmodel/TermsConditionsViewModel.kt` writes `prefs.tcAccepted = true` on accept. `BackHandler(true) {}` absorbs back so users can't escape to Intro. +- AppScaffold handoff: `onTermsAccepted` calls `markTermsAccepted()` then navigates DevNotes (popping T&C). The same callback is reusable from Settings later — accept becomes a no-op + popBackStack when `tcAccepted` is already true. +- The screen body is plain-English use-at-your-own-risk language, not legalese. No "decline" button — close the app or accept. + +### DevNotes + +The first interactive welcome screen for new users (NOT release notes for engineers). Compose-native section cards with icons covering: data stays here, chat models, voice, document attachments, vision, web search, local-network server, and a short "rough edges" honesty section. Lives at `ui/screens/dev_notes/DevNotesScreenBody.kt`. Replaces the previous markdown-blob version. `fun DevNotesScreen(innerPadding: PaddingValues)` signature is stable so `TNavigation` keeps working. All copy is plain-language; no jargon, no em dashes, no rule-of-three patterns. + +### SetupTheme + +- Route: `NavScreens.SetupTheme("setup_theme")`. +- Screen: `ui/screens/setup_screen/SetupThemeScreen.kt`. VM: `viewmodel/SetupThemeViewModel.kt` (injects `ThemeController`). Selection commits immediately. +- Continue button: `SetupThemeBottomBar.kt`, dispatched from `AppBottomBar.kt`. AppScaffold handoff: `onSetupComplete → SetupTheme` (TNavigation); `SetupTheme → ModelSetup` (AppBottomBar callback wires the navigation). +- Top bar: `SetupScreenTopBar()` dispatched from `AppTopBar` on `SetupTheme.route`. +- No "themeSetupDone" pref — defaults are valid on first launch. + +### ModelSetup — Packs + +`ModelSetupScreen.kt` shows three feature packs plus a "Custom" toggle for power users. Packs are bundles of catalog ids downloaded sequentially: + +| Pack id | Includes | Approx size | +|---|---|---| +| `chat_only` | LFM2 350M | 200 MB | +| `chat_voice` | LFM2 350M + sherpa-onnx-whisper-tiny-en + vits-piper-en_US-amy-low | 310 MB | +| `chat_voice_large` | Qwen3 0.6B + same STT + same TTS | 530 MB | + +Pack content is defined as `PACK_CONTENTS: Map>` in `ModelStoreViewModel`. `downloadPack(packId: String)` resolves each catalog id and enqueues via `downloadModel`. Reuses the existing `downloadByQuickStartId` for chat-model resolution (preferred quant priority: `Q4_K_M → Q4_K_S → Q4_0 → Q5_K_M → Q5_K_S → Q8_0` then smallest-by-size). The Custom side keeps "Browse all models" (opens ModelStore) and "Pick a local file" (SAF picker → `ModelImportTypePicker`). + +--- + +## App Guide + +Hub + 7 detail screens, all **single-Scaffold** (accept `innerPadding: PaddingValues`): + +- Hub `AppGuideScreen.kt` — three categories ("Getting started" / "Advanced AI" / "Your phone, your data"). Cards dispatch via `onOpenEntry(key)`. +- `GuideDetailLayout(innerPadding, lede, icon, steps: List, tips)`. Steps numbered, optional `visual` composable. +- `GuideTopBar(title, onBack)` dispatched from `AppTopBar.kt` for each guide route. +- Detail screens: `GuideChatScreen`, `GuideModelsScreen`, `GuideRagScreen`, `GuideVlmScreen`, `GuideVoiceScreen`, `GuideSecurityScreen`, `GuideThemesScreen` (and optionally `GuideServerScreen` for Remote Server). +- Adding a feature: add `GuideEntry` in `AppGuideScreen.guideCategories()`, key in `GuideEntryKeys`, route in `NavScreens`, detail screen, `composable(...)` registration in `TNavigation`, and a `when` case in `AppTopBar.kt`. + +--- + +## Test layout + +49+ instrumented tests across 7 classes (PhaseOne/Two/Three/Four, ExtraHardening, Resilience, ExampleInstrumentedTest). All green on Pixel_Tablet AVD API 35. Any test mutating native global state must call `PolicyEngine.resetForTesting()` in `@Before` AND `BootIntegrity.setRelaxedForTesting(true)` BEFORE any `hardFail` path (otherwise the process `_exit(1)`s mid-test). + +--- + +## Things still deferred + +- **Encrypt WAL** (`hxs/src/main/cpp/wal.cpp`) — plaintext even in encrypted mode. Real audit finding; needs HXS WAL format work. +- **Native cert pinning** — low priority; offline-only scope. +- **Play Integrity opt-in** — conflicts with privacy-first. + +--- + +## Things NOT to regress + +- Don't re-introduce `Settings.Secure.ANDROID_ID` — Keystore-attested identities only. +- Don't take Argon2 below `t=4 / m=131072 / p=1`. Constants in `auth.h`. +- Don't re-add `OPENSSL_NO_ASM=1` — ARM crypto matters for performance. +- Don't expand the unauth feature set in `policy_engine.cpp::is_unauth_feature` without explicit threat-model review. +- Don't remove `setRelaxedForTesting` wiring; tests rely on it. +- Don't emit plaintext detection strings in native code — wrap them in `HXS_OBF`. +- Don't switch back to `verifyPassword(): Boolean`. The contract is `VerifyResult`. +- Don't route any gated feature around `PolicyEngine.isAllowed`. +- Don't collapse the Quick-Start quant preference list in `TNavigation.kt`. The priority `Q4_K_M → Q4_K_S → Q4_0 → Q5_K_M → Q5_K_S → Q8_0` then smallest-by-size keeps the "Tiny & Fast" download tiny. +- Don't send VLM image bytes as `byte[]` over AIDL — `ParcelFileDescriptor[]` only (1 MB binder limit). +- Don't read images on the main thread in `InferenceService.generateVlm` — PFD reads happen in the `scope.launch` Dispatchers.IO collector. +- Don't drop the VLM marker prefix when `isVlmLoaded`. `buildMessagesJson(messages, vlmLastUserId)` must prepend `getVlmDefaultMarker()`. +- Don't key VLM repo detection off anything other than the `mmproj` substring (case-insensitive). Repos use `mmproj--F16.gguf`, `*-mmproj-*.gguf`, etc. +- Don't re-add a manual "Load projector" UI. Auto-load is the contract. +- Don't flatten the VLM folder layout. Base + mmproj as siblings under `models/vlm//`. +- Don't register the mmproj as its own `ModelInfo`. Mmproj is a sibling on disk. +- Don't skip `releaseVlmProjector()` at the top of `ModelSessionManager.load` and `.unload`. +- Don't break the setup-flow handoff. Order is Intro → TermsConditions (if !tcAccepted) → DevNotes (if !onboardingComplete) → SetupScreen → SetupTheme → ModelSetup → SetupRag → Home. `ScaffoldViewModel.resolveStartDestination()` checks `tcAccepted` FIRST, before `onboardingComplete`. AppScaffold callback chain: `onTermsAccepted → DevNotes`, `onSetupComplete → SetupTheme`, `onThemeSetupComplete → ModelSetup`, `onModelSetupComplete → SetupRag`, `onRagSetupComplete → Home`. T&C must come before DevNotes — DevNotes is informational; T&C is the user's legal acknowledgment. Re-ordering them is a regression. +- Don't fold the ModelSetup "Packs" toggle back to a single-model picker. The packs flow is the default for non-technical users; `ModelStoreViewModel.downloadPack(packId)` enqueues every catalog id in `PACK_CONTENTS` for that pack id and the Custom toggle covers Browse-all + Pick-local for power users. Pack ids: `chat_only`, `chat_voice`, `chat_voice_large`. Voice catalog ids are pulled from `ModelCatalog.BUILT_IN_MODELS` (`sherpa-onnx-whisper-tiny-en`, `vits-piper-en_US-amy-low`); chat repos resolve via `downloadByQuickStartId` so the existing quant-priority is preserved. +- Don't drop the auto-active + auto-load contract on chat send. Three coupled rules: + (a) `ModelStoreViewModel.finalizeNonVlmDownload` MUST mark a freshly-installed GGUF model as `isActive=true` when no other GGUF is currently active. Without this, pack-based setup leaves the user with no active chat model and the next chat-send opens the manual model picker instead of generating. Voice models use the separate `prefs.activeTtsModelId` / `prefs.activeSttModelId` first-install pattern in `finalizeVoiceDownload` and that path stays intact. + (b) `HomeViewModel.sendMessage` MUST (1) fall back to `chatModels.value.firstOrNull()?.also { modelRepo.setActive(it.id) }` when `activeModel.value` is null but a chat model is installed, and (2) call `modelSession.load(active)` then check `loadState.value is ModelLoadState.Active` before invoking `runGeneration` when the engine is not yet loaded. The user's typed message must already be persisted to chat history BEFORE the load coroutine kicks off so the input bar clears and nothing is lost on slow loads. Only fall back to opening `_loadModelWindow` when there are zero chat models installed at all. + (c) `HomeScreenBottomBar.canSend` MUST be `text.isNotBlank() && !isGenerating && (isModelLoaded || installedModels.isNotEmpty())`. Tying `canSend` to `isModelLoaded` alone re-introduces the original bug where the load-model composable pops up on send instead of generating, because pack-based setup completes with the engine still unloaded. + Together these three put the manual load-model composable on the rare-empty path only. The common path is: type, hit Send, the inline pill flips to Loading then Generating, the response streams. The native `Model loaded (ctx=...)` log appearing seconds after Send is normal and expected on the first send post-launch. +- Don't re-restrict `canEdit` in `ChatMessageList` to user messages only, and don't re-hardcode `canEdit = false` on `AssistantBubble`. Both user AND assistant messages are editable; the VM is `HomeViewModel.editMessage(messageId, newContent)` which branches by role — user message: save + delete every later message + regenerate from there (existing semantics); assistant message: save in place, do NOT touch subsequent messages and do NOT regenerate. Editing a message whose `archivedByCompactId != null` (folded into a compact summary) is rejected at the VM layer. The `EditMessageDialog`'s confirm button uses `"Save & regenerate"` for user edits and `"Save"` for AI edits — those labels are deliberate UX signaling that the two paths behave differently. +- Don't re-add `&& !archived` to `canFork` in `MessageBubble`. Archived (folded-into-compact) messages AND `CompactSummary` cards themselves must remain forkable — that's the whole point of the fork action for compacted history: lets the user spin off a new chat from a summary point or from any pre-summary turn. `ChatRepository.forkChat(sourceChatId, atMessageId)` is role/kind-agnostic, so the VM imposes no restriction either. The `CompactSummaryCard` carries its own `MessageActions` row with `canFork = true` and everything else off (no edit, no delete, no regen) — adding delete to compact summaries would orphan their archived predecessors from any visible history. +- Don't put back `documentRepo.clearAll()` in `RagManager.init`. Doc records persist across restarts; the engine re-ingests lazily through `hydrateChat(chatId)`. Wiping breaks the prev-chats picker. +- Don't generate a UUID-based docId for chat documents. `id = "$chatId:$sourceId"` so re-attach is idempotent and `removeDocument` can reference-count the source blob. +- Don't ingest a chat document without first writing its bytes to `/chat_documents/sources/.bin`. The picker re-ingests from that file on demand. +- Don't downgrade `DocumentRepository` back to `openPlaintext`. Metadata is sealed under HKDF(DEK, "tn.chat_documents.user_key.v1") at `chat_documents_meta_v1/`. The init's legacy migration is one-shot — re-running on already-migrated installs only deletes top-level files in `chat_documents/` (preserving `sources/`), so it's safe to leave in place. +- Don't drop the BM25 `RagKeywordIndex` from the augment path. Pure dense retrieval is the regression we already fixed. The `RagManager.augment` flow is: optional multi-query (LLM rewriter) → per-query (dense + BM25) → `rrfFuseMany` → optional LLM rerank → token-budget trim → top-N. Keep the order; flipping rerank before fusion gives the rerank LLM nothing useful to look at. +- Don't move the BM25 index back to Kotlin / SQLite. The tokenizer + inverted index + scoring all live in `hxs::RagKeywordIndex` (C++) and the records are encrypted via the existing HXS AEAD path. Reasons: (1) tamper resistance — index ranking logic is harder to manipulate when it's behind libhxs.so; (2) on-device portability — some Android OEMs strip the FTS5 module from system SQLite, breaking SQLite-based BM25 entirely; (3) privacy — chunk text was plaintext on disk in `databases/rag_keyword_v1.db` previously. The new vault at `/rag_keyword_v1/` is sealed under `HKDF(DEK, "tn.rag_keyword.user_key.v1")` like the rest of the app's data. +- Don't bypass `appPrefs.ragSmartRerank` / `appPrefs.ragMultiQuery` toggles. Both Phase 2 features are user-opt-in (off by default) because they each cost an LLM call per query. The rerank prompt is in `RagReranker.buildPrompt`; the variants prompt is in `RagQueryRewriter.buildPrompt`. Don't strip the `withTimeoutOrNull(15s/8s)` either — the chat model can hang on bad input. +- Don't change the Citation TAG byte. `ChatRepository.TAG_MSG_CITATIONS = 13` for assistant messages; older messages without the TAG decode with `citations = emptyList()`. JSON shape: `{sourceId, docId, chunkIndex, score, name, mimeType, snippet, cited}`. `RagCitationMatcher.match` writes them on every assistant turn that ran through RAG augmentation; `MessageBubble` renders them via `CitationStrip` (chip per citation, tap → AlertDialog). +- Don't pass binary-format documents through the FTS5 indexer. Native engine is the only thing that can extract their text. `RagManager.isTextLike(mime, name)` is the gate — text/* mimes, structured-text mimes (json/xml/rtf/yaml/javascript), and known text extensions (txt/md/html/csv/log/code). Binary docs (pdf/docx/epub/etc.) get dense-only retrieval until a Kotlin extractor or a native API addition lands (see #329). +- Don't change the RRF identity key. Items in the fused pool are keyed by `(docId, chunkIndex)`. Native chunks and FTS5 chunks use independent indices, so identical text from both rankers is treated as two separate hits — that's intentional, since they come from different chunk boundaries. RRF still works correctly. +- Don't enable algorithmic darkening / unguarded toggles in the Settings → Chat & RAG section without a `prefs.` write paired with a `_.value = value` update. The `combine(...)` flow has 13 inputs now; if the toggle's StateFlow doesn't update, the UI stays stale until process restart. +- Don't drop `-Wl,-z,max-page-size=16384` in any owned native CMake target. Android 15+ / Play Store requires 0x4000 LOAD alignment on arm64 + x86_64. Verify: `unzip -p libs/ai_sherpa-release.aar jni/arm64-v8a/libai_sherpa.so > /tmp/s.so && readelf -l /tmp/s.so | awk '/LOAD/{getline;print $NF}'` → `0x4000`. +- Don't `secureWipe` the userKey passed to `HexStorage.openEncrypted`/`createEncrypted`. `hxs.cpp` keeps a `NewGlobalRef`; zeroing turns every later AEAD op into a zero-key op (silent decrypt failure on next launch). +- Don't zero `AppKeyStore.cachedDek` in-place inside `wipe()`. The same ByteArray is held by HXS via `NewGlobalRef` (it's the `appKey` passed into `openEncrypted`). Filling it with zeros breaks every in-flight HXS op in the running process and crashes the app immediately after panic-PIN wipe. Just null the reference and let process-kill on the WipedScreen Restart button handle physical memory clearing. +- Don't tighten `migrateLegacyIfNeeded()` back to bootstrap-only. After `hardWipe()`, `app_bootstrap/` is empty (k.bin deleted) but `app_prefs/*` still has records sealed under the now-gone DEK. The migration must fire when EITHER the bootstrap dir has stale files OR `app_prefs/` has files but `k.bin` is missing — otherwise next-launch tries to decrypt records under a fresh DEK and `SecurityException`s. +- Don't downgrade `keyStore.wipe()` back to deleting only `k.bin` + the Keystore alias. The panic-PIN/wipe contract is "delete everything the user owns" — models, voice files, chat history, RAG documents, plugin state, repo config, the lot. Implementation: `context.filesDir.listFiles().forEach { deleteRecursively() }` + `context.cacheDir.listFiles().forEach { deleteRecursively() }` + alias delete. Anything held mmap'd by `:inference` or `:server` survives via POSIX inode-after-unlink until those processes die, but the data is gone after the user taps Restart. +- Don't run Argon2id on the main thread. `SecurityManager.setPassword` / `verifyPassword` / `setPanicPin` are all ~1.5 s on a Pixel 8; calling them from a Compose `onSubmit` lambda freezes the UI. Every call site in viewmodels must wrap with `viewModelScope.launch { withContext(Dispatchers.Default) { … } }`. `PasswordViewModel.submit` already does this; `SettingsViewModel.openPinDialog / openDisableLockDialog / openPanicPinDialog` and `SetupViewModel.submitPassword(onSuccess)` were main-thread until 2026-04-27. +- Try the panic PIN BEFORE the lockout gate in `SecurityManager.verifyPassword`. A duress-PIN must work even when the user is locked out — that's the whole point. Order: (1) try panic against `panicSalt/panicHash`, hardWipe + return Wiped on match; (2) check `nextAttemptAtMs > nowMs` and return LockedOut; (3) try real PIN; (4) bump counter on miss. +- Prime `PasswordViewModel._lockedUntilMs` from `security.snapshotLockoutState().nextAttemptAtMs` at construction. If you initialize it to `0L`, an app restart while locked shows the password input again (the persisted lockout only kicks in after a SUBMIT, letting the user freely retry inputs). The countdown screen must come up immediately on launch when the lock is still active. +- Don't eagerly construct `AppKeyStore` or `AppPreferences` from any process other than main. `InferenceService` runs in `:inference`. `TNApplication.isMainProcess()` early-returns; integrity / pref Hilt fields must be `dagger.Lazy`. +- Don't wrap individual screens in their own `Scaffold`. One Scaffold = `AppScaffold`. Per-route bars go in `AppTopBar.kt` / `AppBottomBar.kt` `when` blocks. +- Don't set `isMinifyEnabled = true` on any library module. Only `:app` minifies. Library minification collides on `Type a.a is defined multiple times` against pre-minified prebuilts. +- Don't remove the per-step `visual` composables in guide detail screens. Update them if the real UI changes. +- Don't key the TOFU `.so` manifest by absolute path. Filenames + `nativeLibraryDir` resolve. +- Don't verify the `.so` manifest across app updates without rebinding to install identity. Mismatched `{signerHash, longVersionCode, lastUpdateTime}` triggers re-TOFU, not hard-fail. +- Don't re-add a root hard-fail. One-time `RootWarningDialog`, gated on `rootWarningShown`. +- Don't re-add a hard-fail for `FAIL_XPOSED` or `AccessibilityGuard.SuspiciousAttached`. `scan_process_environment` returns a bitmask; `TNApplication.onCreate` only hard-fails on `FAIL_DEBUGGER | FAIL_FRIDA`. The `FAIL_XPOSED` bit lands in `TNApplication.softEnvReasons` and `accessibilityGuard.scan()` results land in `ScaffoldViewModel.resolveInitialRootWarning`; both surface as additional paragraphs in the existing `RootWarningDialog`. Reason: rooted users almost universally also have LSPosed and/or one third-party a11y service installed (banking-overlay-blocker, password manager, Tasker, Shizuku-helper). Hard-failing the whole app on first launch is a worse user experience than warning once and letting them opt in. The active-attack tools (`debugger`, `frida-server`) still hard-fail because those are not "device customisation", they're "someone is poking at the running process right now". +- Don't bind `:inference` from `:app` with `BIND_IMPORTANT`. That flag elevates `:inference` into the same OOM-priority bucket as `:app` (or higher), so on low-RAM devices like Snapdragon 662 / 4 GB phones the kernel's lowmemorykiller picks `:app` to evict instead of the process actually holding the multi-GB model mmap. Plain `BIND_AUTO_CREATE` keeps `:inference` on the foreground-service path (it calls `startForeground` in `onCreate`), so it's still well-protected — but `:app` is no longer demoted relative to it. `InferenceClient.performBind` uses `BIND_AUTO_CREATE` only. +- Don't leave `_service.first { it != null }` un-timed in the `generate` / `generateMultiTurn` / `generateVlm` callbackFlows. Wrap with `withTimeoutOrNull(BIND_TIMEOUT_MS)` and emit `InferenceEvent.Error("Inference service unavailable")` on null — otherwise a permanently failed rebind hangs the UI in `_isGenerating = true` forever with no surfacing. +- Don't drop the `failPendingLoads` call from `onNullBinding`. All four `ServiceConnection` death paths (`onServiceDisconnected`, `onBindingDied`, `onNullBinding`, `unbind`) must drain `pendingLoads` and resume them with `Result.failure`, otherwise a load coroutine that started right before service-process death suspends forever. +- Don't wire the foreground notification's Stop button as `PendingIntent.getService(... InferenceService, ACTION_STOP)`. That kills `:inference`, but `:app`'s `BIND_AUTO_CREATE` binding is still alive — `onServiceDisconnected` fires, our handler calls `rebind()`, and Android respawns `:inference` immediately. The user's Stop appears to do nothing. The correct wiring is `PendingIntent.getBroadcast(... InferenceStopReceiver, NOTIFICATION_STOP)` → receiver runs in `:app` → calls `InferenceClient.requestUserStop(context)` → sets `userStopRequested = true` AND `unbind()`s first (removes the BIND_AUTO_CREATE anchor so respawn can't happen) → THEN `startService(ACTION_STOP)` so `:inference` runs its `unloadEverything + stopForeground + stopSelf + killSelfProcess` teardown. The `userStopRequested` flag also short-circuits any `onServiceDisconnected` / `onBindingDied` that races the unbind, so even the race window can't trigger a respawn. Both belts and the suspenders are required — the flag alone doesn't help if the binding is still alive; the unbind alone has a tiny window between the kill and the unbind delivering on the binder thread. +- Don't drop the `try { modelSession.load(active) } catch (Exception)` wrapper in `HomeViewModel.sendMessage`'s auto-load branch. `viewModelScope` is `SupervisorJob`-backed, but an unhandled throw from a child coroutine still reaches the thread's default uncaught handler and crashes `:app`. The existing `Result` contract from `InferenceClient.loadModel` keeps the happy path safe; the wrapper is defense-in-depth for any future code that throws here. +- Don't drop the pre-load RAM check in `InferenceService.loadModel`. `File(path).length()` vs `MemAvailable:` from `/proc/meminfo`: if `memAvail in 1 until (modelSize * 6 / 5)`, surface a clear "Not enough free memory" error to the client immediately. Without this guard, low-RAM devices try to mmap a model larger than physical RAM, the kernel page-faults loop, and `:inference` gets killed mid-load — visible to the user as a generic "service died" with no actionable message. Skip the check when `memAvail <= 0` (read failed) so the gate doesn't accidentally block valid loads. +- Don't strip the `logDeviceProfile()` call at the top of `InferenceService.loadModel`. Logs `Build.MODEL`, `Build.SOC_MODEL`, `Build.SUPPORTED_ABIS`, `MemTotal`, and the `Features:` line from `/proc/cpuinfo` exactly once per service-process lifetime. This is the only telemetry that lets remote debugging triage a "model load crash on $unknown_device" report — specifically whether `dotprod`/`i8mm`/`fp16` are present (Snapdragon 662 / Cortex-A53 lacks all three; many llama.cpp dispatch paths assume at least asimd-dotprod). +- Don't re-add a plaintext HXS container for the bootstrap DEK. Raw XOR-masked `k.bin` only. +- Don't skip `migrateLegacyIfNeeded()` in `AppKeyStore.init`. +- Don't drop the signer-binding salt from any user-key derivation. Every `encryptor.deriveKey(ikm = dek, salt = ?, info = ?)` call in app code MUST use `salt = keyStore.installSignerHash()` (not `salt = dek`, not `salt = null`). Sites: `AppPreferences.init`, `AppPreferences.deriveAuthKey`, `DocumentRepository.init`, `RagKeywordIndex.init`, `ChatRepository.init`, `SourceFileVault.keyFor`. Without this salt, a same-device replaced-APK attack (root + repack with attacker's cert) can unwrap the DEK via the inherited uid-scoped Keystore alias and decrypt every vault. Salting with the signer hash means the patched APK derives a different user-key and AEAD fails — the data stays sealed. +- Don't add a "fallback to all zeros" or "return empty bytes on failure" path in `AppKeyStore.computeSignerHash()` / `installSignerHash()`. If the platform can't read the install signer, throw `SecurityException` and let the app refuse to bootstrap. A zero-fallback collapses the binding for any device with a broken signature lookup, which is exactly the path an attacker would try to engineer. +- Don't bump user-key info strings without bumping the version suffix (`v2` → `v3`). Bumping invalidates existing v2 vaults — the open-or-rebuild helper detects the AEAD failure on `openEncrypted` and wipes the dir. Documented loss is intentional; a silent loss because the info string was edited inline is not. +- Don't downgrade `ChatRepository` back to `openPlaintext`. Chats and messages are sealed under `tn.chats.user_key.v2`. Plaintext on disk was the cross-device readability hole — closed in this build. +- Don't bypass `SourceFileVault` for `chat_documents/sources_v2/` reads or writes. Every byte of every attached RAG document is AEAD-sealed per-file under a key bound to (DEK, signerHash, sourceId), with AAD = sourceId. This means: (a) cross-device read fails (no DEK unwrap), (b) cross-build read fails (different signer), (c) renaming a file breaks decryption (AAD mismatch — defends against record-substitution). Direct `File(...).readBytes()` / `writeBytes()` is forbidden for source files. +- Don't preserve the legacy `chat_store/` (plaintext) or `chat_documents/sources/` (plaintext) directories on a v2 build. `ChatRepository.init` and `SourceFileVault.init` both `deleteRecursively()` them on first launch — this is the migration path that closes the historical leakage. Re-introducing those dirs (e.g. for a "compatibility shim") puts plaintext back on disk. +- Don't unwrap the DEK in `:server` or `:inference`. `:server` runs in its own process and never opens HXS — token, model path, and config are pushed via AIDL `start(configJson)`. `:inference` is similar. Only `:app` (main process) holds the DEK, and only the main process derives signer-bound user-keys. Cross-process key handoff would defeat the binding. +- Don't link `:native-server` against BoringSSL / OpenSSL / zlib. Header-only httplib + getrandom(2). +- Don't add a new HTTP route without auth pre-routing. Only `/`, `/index.html`, `/webui`, `/health` are in `auth::is_public_path`. Never make `/v1/models` or `/v1/chat/completions` public. +- `RemoteServerService` lives in `:server` (its own process). Don't fold it back into `:app`. `:server` MUST NOT open HXS — token / model path / config / asset HTML are passed in via AIDL `start(configJson)`. Token rotation pushes from `:app` via `rotateToken(newToken)`. +- Don't remove the `startService(Intent(this, RemoteServerService::class.java))` call inside `handleStart` (just before `startForeground`). The service is otherwise bind-only and gets destroyed when `:app` dies / swipes from recents. The self-start transitions it to the "started" lifecycle so a foreground notification + `stopWithTask="false"` keeps it alive across client death. +- Don't let any UI escape the server lockdown. `LaunchedEffect(serverRunning, currentRoute)` with `popUpTo(0) { inclusive = true }`; `BackHandler(enabled = running) {}` in ServerScreen; drawer gated by `showDrawer = currentRoute == HomeScreen.route && !serverRunning`. +- Don't persist the server token outside the encrypted HXS `app_prefs` vault. +- `/v1/models` returns the full enabled-engine catalog (every installed model across `gguf` / `vlm` / `embedding` / `tts` / `stt` / `image_gen` / `image_upscaler`) with per-entry `type` + `owned_by`. Each model's JSON entry must carry `id`, `path`, `type`, `config_json`; for `vlm` rows also `mmproj_path`. Don't shrink this back to "currently loaded only" — clients pick the model per request via the OpenAI `model` field, and the registry lazy-loads on first use. Don't silently expose models with `pathType == CONTENT_URI`; they're filtered in `ServerController.buildEnginesCatalog` because the server has no SAF trampoline. +- Don't bypass `ServerEngineRegistry` from any new route. Per-kind locks (`Mutex` for the suspend-based loaders, plain `synchronized` for the sherpa-onnx ones) serialise concurrent loads — a second request for the same engine waits for the first load to complete instead of racing it. New endpoints add a new typed `Kind` to `server_models::Kind`, a new wrapper class, a new registry method, a new `InferenceBridge.start` upcall, and a new `nativeFeedReply*` consumption path. Don't shortcut by reaching into `GGMLEngine` / `StableDiffusionManager` directly from the route handler — every engine touch must go through the registry so loading discipline is preserved. +- Don't route `/v1/chat/completions` to the VLM engine unless the request has at least one `image_url` content part. `oai::extract_images_from_messages` is the single decision point — it sets `request.has_images = true` only when a part of type `image_url` is detected in any message. Don't move that detection upstream into the rate-limit / auth layer; pre-routing must stay pure. +- Don't accept network URLs in `image_url` parts on the server. Privacy / offline-only scope: only `data:image/...;base64,...` is parsed; `http(s)://...` returns 400 `invalid_image`. Adding outbound fetch from `:server` would mean adding curl-impersonate to the native server, which doubles its native deps and breaks the "no BoringSSL/OpenSSL/zlib in `:native-server`" rule. +- Don't pass big binary payloads (TTS audio, generated images, multipart image/mask inputs, multipart WAV) as `byte[]` over JNI. The contract is: write the bytes to `/server-staging/tn__` via `server_staging`, hand the **path** across JNI as a Java string. The C++ route reads/writes the file directly. Cleanup happens after the response is sent (`staging::unlink_safe`) and at every server stop (`staging::purge_all`). JNI byte[] copies for 4 MB images are measurable overhead and have caused OOMs in adjacent products. +- Don't change `reply_session.wait(timeout_ms)` to take 0 or -1 unconditionally. The defaults exist for a reason: embeddings 120s, TTS 180s, STT 180s, image gen 600s, image upscale 300s. On a Snapdragon 8 Gen 1 a 20-step SDXL run can flirt with 5 min; the 600s ceiling is the cliff before we say "something's wedged". +- Don't drop the in-process `ServerImageEngine.ensureRuntime()` check that gates on `/ai_sd_runtime/qnnlibs.tar.xz` existing. The user must have triggered the SD runtime download via the in-app Image Task screen at least once before the server's image endpoints work. Image gen routes return a clean 500 with "image engine unavailable or runtime not installed" if not. +- Don't share `StableDiffusionManager` state between `:app` and `:server`. Each process has its own `getInstance(context)` (class loaders are per-process). They cooperate only via the shared on-disk `/ai_sd_runtime/` directory; whichever process initialises first extracts qnnlibs, the other sees the existing files and skips re-extraction. Don't add IPC between the two image stacks. +- Don't fold the `:server`-side `ServerTtsEngine` / `ServerSttEngine` back into using `InferenceClient` AIDL. `:server` MUST own its sherpa-onnx instances directly — the AIDL hop would mean (a) crossing into `:inference` which has its own active TTS/STT for in-app voice, (b) yanking that state under the user's feet when the chat-side mic is in flight. The two stacks are intentionally independent. +- Don't read voice / embedding "active" preferences from inside `:server`. The server doesn't open HXS. `ServerTtsEngine.ttsFor(modelId)` falls back to `catalog.firstOf(TTS)` when the requested id isn't found — which is "whatever's listed first in `ModelRepository.models`" (install order). If the request omits `model`, the server picks the first installed engine of that kind. Clients that want a specific voice MUST send `model` explicitly. +- Don't broaden the OpenAI streaming contract to non-chat routes. Embeddings, TTS, STT, image gen are all single-response. Adding `stream:true` support to them would require either a new SSE schema (no OpenAI precedent for images), or partial-bytes-over-chunked-transfer (sherpa-onnx and SD AAR are batch-only; no per-step callback exposes individual samples). Stay aligned with OpenAI's published shapes. +- Don't trip the `server_->set_payload_max_length` setting back to 1 MB. The 64 MB cap is sized to fit base64-encoded VLM image_urls (4 MB raw ≈ 5.4 MB b64) plus multipart audio uploads (whisper-friendly 30s WAV at 16kHz 16-bit ≈ 960 KB) plus 4× upscale inputs. Same applies to the read/write timeouts (60s / 120s) — TTS synthesis of a single sentence on a Tensor G3 takes ~3s, but a 200-character paragraph runs longer. +- Don't add audio transcoding to `/v1/audio/transcriptions`. WAV-only is the documented contract. Bringing in ffmpeg or symphonia would balloon the native footprint. If clients want MP3/AAC support, they decode client-side before upload (the bundled web UI's STT panel already only accepts `.wav`). +- Don't bind the server only to the Wi-Fi IP by default. `ALL_INTERFACES` (0.0.0.0) is the default so the loopback URL is reachable from the device's own browser regardless of Wi-Fi state. Display two URLs: loopback (always works) + LAN (when Wi-Fi is up). +- Don't display `serverPort` from raw HXS without validation. Getter validates [1024..65535]; setter clamps. Effective port (post-bind) is written back from `nativeBoundPort()`. +- Don't drop the `serverController.isBusy` gating on chat-side load/unload/send. The server owns the loaded model; uncontrolled chat-side reload would yank state mid-request. +- Don't add a `modelType: String` field to `ModelInfo`. `ProviderType` is canonical. `HuggingFaceModel.modelType: String` is the pre-install hint mapped at insert time. +- Don't add a streaming-synthesize AIDL method. The AAR's `OfflineTts.generate` is synchronous. Streaming TTS = client-side text chunking. +- Don't record STT at anything other than 16 kHz mono `ENCODING_PCM_FLOAT` from `MediaRecorder.AudioSource.VOICE_RECOGNITION`. +- Don't skip the mid-chunk cancellation check in `TtsPlayer`'s write loop. +- Don't pass `dataDir` / `espeak-ng-data` as `content://`. sherpa-onnx wants filesystem paths. +- Don't resurrect a BYOM / SAF directory import path for voice. Store-only. +- Don't make the Mic button conditional on `voiceSttAvailable` — always rendered; the click handler routes to Store if no STT model is installed. +- Don't skip `voiceManager.unloadStt()` in `HomeViewModel.stopRecordingAndTranscribe`'s `finally`. +- Don't auto-request `FOREGROUND_SERVICE_MICROPHONE`. +- Don't cram more than three quick-links into a single drawer `SpaceEvenly` row. The drawer layout is two separate rows: a "chat tools" row under the New Chat button (Store / Docs / Server) and an "info" row at the bottom (Guide / Dev Notes / Credits). Settings sits as a gear `ActionButton` in the drawer header next to the title — it is not in either row because it is global, not chat-related. Keep three-and-three; pushing four into either row reintroduces the touch-target squashing the original 6-in-a-row had. +- Don't move the Credits screen out of fullscreen. Route is `NavScreens.Credits("credits")`. `AppScaffold.isFullscreen` includes it alongside Intro and Password so the AppTopBar / AppBottomBar are hidden — the screen owns the full viewport and draws its own theme-coloured background. Audio is `R.raw.credits` (mp3 in `app/src/main/res/raw/`); `MediaPlayer.create` is acquired via `remember`, started in `DisposableEffect`, released on dispose. `setOnCompletionListener { onExit() }` exits when the audio ends. Scroll is a `verticalScroll(scrollState, enabled = false)` Column animated by `scrollState.animateScrollTo(maxValue, tween(durationMillis = mediaPlayer.duration, easing = LinearEasing))` keyed on `scrollState.maxValue` and `durationMs` so the first emission with non-zero `maxValue` triggers the crawl. User scroll is disabled to keep the timing deterministic; tap or back exits. Colours pull from `MaterialTheme.colorScheme` (`surface` background, `primary` title, `onSurface` lines, `onSurfaceVariant` section labels) — adapts to dark/light theme. +- Don't let `VoiceModelManager` construct `AppPreferences` eagerly. `dagger.Lazy`. +- Don't add `*.md` spec / plan / research / TODO docs at the repo root. Project memory lives here. Implementation roadmaps belong in conversation context. +- Don't auto-scroll on every streaming token via `LazyListState.scrollToItem(index)`. That fights the user's drag and locks manual scroll mid-generation. The pattern is: track `stickToBottom` from `snapshotFlow { isScrollInProgress }` (re-evaluate when scroll settles via `!canScrollForward`), gate the auto-scroll `LaunchedEffect` on `stickToBottom && !isScrollInProgress`, and use `scrollToItem(last, scrollOffset = Int.MAX_VALUE)` so the bottom of the growing streaming bubble stays in view. +- Don't drop the `:native-server` consumer-rules.pro keeps for `NativeServer`, `NativeServer$*`, `InferenceBridge`, `BindMode`. The native HTTP server's JVM bridge invokes `InferenceBridge.startGeneration / cancelGeneration / onRequestEvent` via JNI on a `NewGlobalRef`'d jobject; renaming or stripping breaks dlsym at runtime. +- Don't drop the `com.dark.networking.**` keep + `dontwarn` block from `app/proguard-rules.pro` or the `WebNative` / `WebResponse` / `WebBytesResponse` / `WebSearchResult` keeps from `networking/consumer-rules.pro`. `WebNative` is a Kotlin `object` with `@JvmStatic external fun nativeFetch / nativeFetchBytes / nativeSearch / nativeHasBackend / nativeBackendName / nativeSetCaBundle / nativeSetProfile`; the JNI binding is `Java_com_dark_networking_WebNative_nativeFetch` etc., so the class FQCN must survive R8. Without these keeps, every HF Explorer / web-search / model-catalog HTTP call dies on release with `UnsatisfiedLinkError` (build is green; runtime crash). Verify post-R8: `grep com.dark.networking.WebNative app/build/outputs/mapping/release/mapping.txt` should show `WebNative -> com.dark.networking.WebNative` (identity mapping). +- Don't keep `com.dark.ai_sd.**` in `app/proguard-rules.pro`. The AAR was removed in commit `9d79a3b` — the rule is dead weight. +- Don't drop the `com.dark.gguf_lib.**` / `com.dark.ai_sherpa.**` keep + `dontwarn` block. The prebuilt AARs are already minified and rely on specific class+method names for JNI lookup. +- Don't strip `lockedUntilMs` and `wiped` from `PasswordScreen`. Both flow through `PasswordViewModel` from `VerifyResult.LockedOut(retryAtMs)` and `VerifyResult.Wiped`. The screen branches `wiped → WipedScreen → "Restart" → finishAffinity + Process.killProcess(myPid)` (post-`hardWipe`, `PolicyEngine.markTampered()` is latched and the process is unrecoverable in-place). `lockedUntilMs > now → LockedOutScreen` with a 500 ms-tick countdown that clears itself once the timestamp passes. +- Don't add a "set panic PIN" path that doesn't go through `SecurityManager.setPanicPin`. It gates on `securityMode == APP_PASSWORD` (NOT `session.isAllowed(AUTH_DISABLE)` — that was a non-deterministic timing trap; see Panic PIN section) and writes the second Argon2id hash into the same encrypted `AuthState` blob. UI lives in `Settings → Privacy` and only renders when `isLockEnabled`. `SettingsViewModel._panicPinSet` mirrors `security.hasPanicPin` and must reset to false on `setPassword`, `disableLock`, and `Wiped`. Same gate change applies to `clearPanicPin` and `disableLock`. +- Don't fan out `RemoteCallbackList` broadcasts from `InferenceService` without holding `sdBroadcastLock`. `RemoteCallbackList.beginBroadcast()` is not nestable — calling it from one thread while another is between `beginBroadcast()` and `finishBroadcast()` throws `IllegalStateException: beginBroadcast() called while already in a broadcast` and kills `:inference`. `startSdForwarding` launches five parallel collectors (backend / generation / isGenerating / upscale / runtimeSetup) on `Dispatchers.IO` — without serialisation they race on `sdClients.beginBroadcast()` and the service crash-loops immediately. The fix is `synchronized(sdBroadcastLock)` around the entire `fanoutSd` body; the same lock can serve `tnClients` if a similar pattern is ever added there. +- Don't bump `TAG_MSG_WEBSEARCH_RUN` away from `14` or `TAG_MSG_WEBSEARCH_STATE` away from `15`. Older messages without these tags decode with `webSearchRunId = null` and `webSearchState = ""`. New chat-message fields must use TAG ≥ 16. +- Don't reintroduce a runtime-only `webSearchEvents: Map` in HomeViewModel. The card's state must come from `WebSearchUiState.fromJson(message.webSearchState)` because (a) opening a different chat while a run is in flight should NOT bleed the running run's state into a completed chat's card; (b) after process restart, completed web-search cards must keep showing their Done state. `HomeViewModel.handleWebSearchEvent` is the single write point — looks up `(chatId, messageId)` via `webSearchMessages[runId]`, reads the message, applies the event, writes back. +- Don't lift the web-search lockdown. While `webSearchActive.value`, `HomeViewModel.sendMessage / loadModel / unloadModel` all early-return — the chat LLM is borrowed for both the GenerateQueries and Synthesize calls. +- Don't drop the web-search content swap in `InferenceCoordinator.buildMessagesJson`. Web-search cards store the user's query in `msg.content` (used by the card Header) and the synthesized answer in `msg.webSearchState`. The single point of LLM-history assembly MUST swap `content` for `WebSearchUiState.fromJson(webSearchState).answer.trim()` when `webSearchRunId != null`, and SKIP the message entirely when the answer is blank (in-flight / cancelled / failed). Without the swap the model sees `assistant: ""` instead of the actual research, and the next chat turn proceeds as if the search never happened. Without the skip, in-flight / failed cards inject an empty assistant turn that confuses the model. +- Don't replace `WebNative.search` with `HttpURLConnection` or any other client. The `:networking` curl-impersonate Chrome116 fingerprint is the single allowed pipe for DDG. The 3 queries × 5 results contract assumes that backend. +- Don't expand the web-search flow back into a multi-iteration pipeline. The user-facing contract is "3 queries, snippets, answer, done". Adding iterations / fetches / per-page extraction is what the old Research pipeline was; it was deleted on 2026-05-15 because it was minutes-slow and barely better than snippet-only synthesis. +- Don't switch `WebSearchCoordinator` from `tryEmit` to suspending `emit(...)` in the `catch (CancellationException)` / `catch (Throwable)` blocks. The catch fires on a cancelled Job, and `withContext(Dispatchers.Default)` throws CE immediately on a cancelled context — so the Cancelled event never reaches the SharedFlow and the card freezes mid-flight. `_events` has 64-slot buffer; tryEmit always succeeds. +- Don't change `WebSearchPrompts.QUERY_LINE_REGEX`. The synthesis prompt asks for the format `1. ` / `2. ` / `3. ` and the regex `^\s*(?:\d+[.)\-:]|[-*•])\s+(.+)$` parses that AND tolerates common LLM deviations (`-`/`*`/`•` bullets, `:` after the number). Tightening the regex breaks smaller models that don't follow numbered-list instructions perfectly; loosening it picks up the LLM's preamble lines. +- Don't fall back to `java.net.HttpURLConnection` for any HuggingFace API call — search, model info, tree, raw README, tags-by-type, trending, quicksearch, resolve. Every HF request goes through `:networking` (`WebNative.fetch`) so it inherits the curl-impersonate Chrome116 fingerprint + bundled `cacert.pem` + strict cert verify. The hub is `repo/HuggingFaceApi.kt` (Hilt singleton class, not an object); `repo/hf/HfClient.kt` builds typed endpoints on top. `ModelCatalog` and `RepositoryValidator` inject `HuggingFaceApi`. Same rule applies for any future HF or non-HF HTTP target — `:networking` is the only allowed pipe. +- Don't change `WebNative.fetch` back to `Result`. The contract is `Result` where `WebResponse(status: Int, body: String, error: String?)`. Result.failure is reserved for transport-layer issues (DNS, TLS handshake, native call collapse). HTTP non-2xx comes back as `Result.success(WebResponse(status=4xx, ...))` — callers can react to 429 (rate limited) vs 404 (not found) vs 401/403 (auth). Old behavior of returning `null` on non-2xx silently masked HF API bugs (e.g. invalid `expand=` params returning 400) for years. +- Don't log full URLs (with query string) to `ANDROID_LOG_WARN` from `net_jni.cpp`. Use the `host_of(url)` helper. Search queries are user PII (typed model names, sometimes sensitive). Status code + host is the maximum log surface. +- Don't add per-keystroke quicksearch / autocomplete to the HF Explorer search bar. Search fires on the Search button, the IME `Search` action, or a filter chip touch — never on every typed character. The HF API has a 500-call/5min unauthenticated rate limit per IP; per-keystroke autocomplete burns it on a single typing session. Slider drags fire only on `onValueChangeFinished` (one call per drag). Post-filter sliders (`minDownloads`, `minLikes`, `recentDays` in `HfPostFilters`) update `visibleResults()` locally without an API call. `HfClient.quickSearch` exists for future use but the UI must not wire it to typing. +- Don't write the HF tags catalog (`/api/models-tags-by-type` payload) anywhere outside the encrypted `app_prefs` vault. Keys are `hf_tags_catalog_v1` (JSON string) and `hf_tags_catalog_v1_at` (Long unix-millis), 24h TTL. The catalog feeds the dynamic filter chips; falling back to `HfFilterTaxonomy` constants is OK but only for the brief window before the catalog hydrates. Plaintext-on-disk is forbidden — use the encrypted prefs API only. +- Don't pass any device-identifying value to `WebNative.fetch`'s `headers` map. The map is intended for protocol headers (`Accept`, `Accept-Encoding`, future `Authorization`). Adding `X-Install-Id`, `User-Agent` overrides with TN-identifying suffixes, or anything that would let HF (or any future server) fingerprint a specific install is a privacy regression. +- Don't add the `expand=tags`, `expand=downloads`, etc. parameters back to `searchUrl`. Those query params are for `/api/models/{id}/tree/...`, NOT `/api/models?search=...`. HF returns 400 when they're present on the list endpoint. The minimal list response already includes `id`, `author`, `gated`, `tags`, `pipeline_tag`, `downloads`, `likes`, `lastModified`, `createdAt` — sufficient for `HfModelSummary` without `full=true`. +- Don't snake_case the `sort` URL param. HF API wants camelCase: `trendingScore`, `lastModified`, `createdAt`, `downloads`, `likes`. The legacy code emitted `trending_score` / `last_modified` / `created_at` and HF returned 400. Source of truth is `HfSort.apiKey` in `repo/HfFilters.kt`. +- Don't add speculative URL params to `HuggingFaceApi.searchUrl` without verifying they're documented for `/api/models`. `apps=`, `inference_provider=`, `inference=warm`, `filter=region:us` (with `region:` prefix), `filter=dataset:foo` (with `dataset:` prefix), `pipeline_tag=` — all of these were added historically and have been removed because they trigger HF 400. Only documented params are kept: `search`, `author`, `filter` (plain tag values, stackable), `gated`, `num_parameters`, `sort`, `limit`, `skip`. If you need a new filter, verify it works against a curl-built URL first; don't add it to the URL builder on the assumption that HF tolerates it. +- Don't reintroduce post-filter sliders (`minDownloads`, `minLikes`, `recentDays`) into `HfFilters` / `HfPostFilters` / the VM. They were UI clutter without unlocking new searches — `sort=downloads` already gets the user "popular" results in the right order, and "recent" is `sort=lastModified`. The user explicitly asked to drop them; bringing them back without consent is a regression. +- Don't change the SoC-bucket mapping in `data/SocBucket.kt` without first verifying `xororz/sd-qnn` still uses the same `_8gen1.zip` / `_8gen2.zip` / `_min.zip` filename suffixes. We pull our QNN model archives directly from LocalDream's HF repo, so a bucket rename or new chip class needs a re-validation against the actual `tree/main` listing. 8gen3 / 8 Elite intentionally route to the `8gen2` bucket because Qualcomm's HTP V73 contexts are forward-compatible — don't add a new "8gen3" bucket without uploading new archives first. +- Don't show NPU image-gen rows on non-Snapdragon devices. `imageModels()` is gated on `SocBucket.bucket(soc) != null`. Falling back through to NPU rows on Tensor / Dimensity / Exynos would download QNN contexts that can't load — surface only the `xororz/sd-mnn` CPU/MNN variants on those devices. +- Don't show SDXL rows on a SOC that's not in `SocBucket.SDXL_ELIGIBLE_SOCS`. The SDXL contexts ship only as `_8gen3.zip` and Qualcomm AI Hub hasn't compiled them for older NPUs. The rest of the pipeline still uses 768-dim CLIP under the hood, so even if you forced the download, generation would crash on the dimension mismatch — keep both gates (SDXL row visibility + 2048-dim future pipeline work) in lock-step. +- Don't bypass the path-traversal check in `unzipInto`. Each entry's canonical path must start with `target.canonicalPath + File.separator` (or equal `target.canonicalPath` itself for the top-level dir). Skip `..` entries pre-canonicalization too. The QNN ZIPs from `xororz/sd-qnn` have flat layouts today, but a malicious mirror could craft `../../files/key.bin` entries; the check is the only line of defense. +- Don't unwrap the SDK runtime onto an external dir. `/ai_sd_runtime/` is the correct location — internal storage, app-private, survives backups (`allowBackup=false` is set elsewhere). The QNN `.so`s extracted there are device-specific and shouldn't roam. +- Don't open a fresh `StableDiffusionManager` per request. It's a process singleton (`getInstance(context)`), wrapped by Hilt's `ImageGenManager`. The init-mutex inside `ensureRuntime()` covers the qnnlibs.tar.xz extraction. Calling `initialize()` twice is a no-op but rebuilding the manager would tear down the persistent native sessions used across generations. +- Don't ship the release AAR yet. `ai_sd-release.aar` ran R8 on the SDK side and renamed `StableDiffusionManager.Companion.getInstance` past Kotlin's compile-time visibility. Keep `ai_sd-debug.aar` copied as `libs/ai_sd-release.aar` until `:ai_sd`'s `consumer-rules.pro` adds `-keep class com.dark.ai_sd.StableDiffusionManager$Companion { *; }` and the AAR is rebuilt. +- Don't remove the standalone QNN upscaler implementation. The original AAR's `nativeLoadUpscaler` for QNN was a stub that only stashed the model path — `nativeUpscaleImage` would then fail with "Upscaler model not provided" because the QnnModel was never built. Filled in 2026-05-08 by porting LocalDream's per-request load pattern: `sd_pipeline::loadStandaloneQnnUpscaler(modelPath)` in `model_loader.cpp` calls `createQnnModel(path, "upscaler")` + `initializeQnnApp("Upscaler", upscalerApp)` and assigns to the global `upscalerApp`, mirroring `main.cpp:3203` in LocalDream. Prerequisite: `sd_pipeline::ensureQnnSystemReady(systemLibPath, backendPath)` must be called first to populate `g_qnnSystemFuncs` + `g_backendPathCmd` — `ai_sd_jni.cpp::nativeInitRuntime` does this best-effort using `/libQnn{System,Htp}.so`. The Kotlin caller (`ImageGenManager.loadUpscaler`) just calls `sdk.loadUpscaler(path, useMnn=path.endsWith(".mnn"), useOpenCL=...)` and the AAR's JNI dispatches to the right load path. Don't restore the .mnn-only IllegalStateException guard — the QNN path works now. +- Don't remove the adaptive upscale pipeline. `ImageUpscalePipeline` wraps the fixed native 4× primitive with scale presets (2× / 4× / 8× / custom), Auto / Fast / Safe tiled processing, memory estimation, tiled region decode, overlap trim, and file-backed PNG output for large results. Native upscaler input tiles must stay at or below 1024 max-edge; final output has no hard resolution cap and should route through tiled/file-backed output when one-shot is risky. +- Don't declare `commons-compress` and `xz` as anything weaker than `implementation` in `app/build.gradle.kts`. They are required by the AAR's runtime extraction path; `implementation(files(...))` AAR consumption does NOT pull transitive POM deps, so without explicit declarations the app crashes with `NoClassDefFoundError: org.tukaani.xz.XZInputStream` on first init. +- Don't switch image-gen tasks to a separate ViewModel per task. `ImageTaskViewModel` is the single VM for all four modes (Generate, Img2Img, Inpaint, Upscale) — sharing prompt / model selection / preview state across modes is intentional so the user can tweak a prompt then quickly switch from Generate to Edit without re-typing. +- Don't reuse the chat model picker for image-gen models. They're separate `ProviderType` rows (`IMAGE_GEN`, `IMAGE_UPSCALER`) on `ModelInfo` and live in `/sd_models/` / `/sd_upscalers/`, never in the GGUF chat model dir. The store routes them through `finalizeImageGenDownload` / `finalizeImageUpscalerDownload` and they should not appear in `chatModels` filters anywhere. +- Don't drop the `context.packageName` self-exclusion in `AccessibilityGuard.scan`. Our own `IslandAccessibilityService` registers under our own package; without the self-skip every user who enables the smart-dodge accessibility service would trip our own one-time `RootWarningDialog` ("Suspicious accessibility services attached: com.dark.tool_neuron"). The exclusion is targeted — only OUR pkg is allowed; every other non-allowlisted service still surfaces in the dialog. +- Don't duplicate the island pill / dodge geometry constants between `IslandOverlayRoot.kt` and `IslandAccessibilityService.kt`. Both read from `IslandGeometry` (PILL_W_DP, PILL_H_DP, OUTER_PADDING_DP, DODGE_MARGIN_DP, MAX_DODGE_DP). If the pill grows or shrinks, the service's pillRect computation diverges from where the overlay actually draws → dodge will compute against the wrong rect and either over-dodge (visible jitter) or under-dodge (pill stays on top of buttons). +- Don't call `HxdManager.enqueue(...)` from any new code path without immediately following it with `downloadCoordinator.registerLabel(hxdId, displayName, type)`. The Downloads screen's history is built from those labels — skipping the call means the history row falls back to `url.substringAfterLast('/')`, which is unreadable for the user (full hex repo IDs, etc.). The two enforced sites today are `ModelStoreViewModel.downloadModel` and `ImageGenManager.downloadRuntime`; new sites must follow the same pattern. +- Don't inject `DownloadCoordinator` or `DownloadHistoryRepository` from any class that runs in `:server` or `:inference`. Both are `:app`-process-only by construction (HXS access lives in main process; cross-process injection would attempt to unwrap the DEK from a process that doesn't have the Keystore alias scoped to it and crash). If a service-process feature needs download state, route through AIDL. +- Don't bump `DownloadHistoryRepository.MAX_ENTRIES` past 50 without a UI plan for the list growing. 50 entries × ~256 bytes is ~13 KB at rest — trivial — but the screen's LazyColumn isn't paginated and the section header has no count limit. Larger caps need at least a "show more" affordance or per-day grouping. +- Don't write into the `tn.download_history.*` info-string namespace from anywhere other than `DownloadHistoryRepository`. The v2 suffix is also load-bearing — bumping to v3 invalidates every existing user's history vault (the open-or-rebuild helper will wipe it). If the schema needs a breaking change, write a migration that reads via the v2 key and re-inserts under v3, then bump. +- Don't compute the AccessibilityService's pill rect against the current `nudge` value. Use the calibrated `IslandPositionStore.position.value` only. Dodge math must answer "where should the pill move to be clear", not "where would the pill stay if it's already nudged" — feeding nudge back into the calc creates an oscillation loop (the pill dodges off the button → button no longer overlaps → nudge resets to 0 → button overlaps again → dodge → …). +- Don't clear the `IslandPositionStore.nudge` on `TYPE_WINDOW_CONTENT_CHANGED`. Only `TYPE_WINDOW_STATE_CHANGED` (app/activity switch) resets the nudge before re-scanning. Content changes (RecyclerView scrolls, dialog toggles, text edits) fire dozens per second; clearing on every one would make the pill flash back to its calibrated home position constantly. The scan that follows each content event publishes a fresh nudge if needed; idempotent if the new nudge equals the old one. +- Don't lift the 150 ms coalesce on the accessibility-event handler. `TYPE_WINDOW_CONTENT_CHANGED` can fire many times per second in any UI with animations or live updates (chat message streaming, video player UI, progress bars). Walking the entire node tree per event will pin a CPU core; debouncing to one scan per 150 ms keeps the smart dodge under ~1 % CPU on a mid-tier device. +- Don't hard-code `node.recycle()` calls in `IslandAccessibilityService.collectClickableRects`. On API ≥ 33 `recycle()` is a no-op (or worse, can throw `IllegalStateException` if the node was already pooled); minSdk 29 means most install bases are post-33 by now. GC is the right cleanup path. If you ever need to back-port to API < 30, the recycle should be guarded by `Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU`. +- Don't widen `IslandAccessibilityService`'s event filter beyond `typeWindowStateChanged|typeWindowContentChanged`. The XML config gates the system from delivering us anything else; `typeViewClicked`, `typeViewFocused`, etc. would fire on every user interaction and add no value (we only care about WHERE clickable things are, not WHEN they're touched). Privacy: narrowest event set = least screen-content exposure. +- Don't bind the IslandAccessibilityService to anything other than the system's `BIND_ACCESSIBILITY_SERVICE` permission gate. The manifest entry is `android:exported="true"` (required by OS binder) with `permission="BIND_ACCESSIBILITY_SERVICE"` (only the system has it). Removing `exported="true"` breaks the bind; removing the permission opens the service to arbitrary IPC. +- Don't move the island pill via Compose `Modifier.offset` for placement (user offset or smart-dodge nudge). The pill's position must come from `WindowManager.LayoutParams.x/y` via `windowManager.updateViewLayout`. Two specific failures when you offset via Compose: (1) the WindowManager window stays at the original WRAP_CONTENT rect, so touches keep landing on the original screen location — a back/menu button under the original pill spot remains untappable even after the pill visually moves; (2) the pill Surface renders past the wrapped window bounds and gets clipped (visually disappears or half-cuts) once the offset exceeds the wrapped content height. The morph animation (pill → card) still grows the WRAP_CONTENT window correctly because that's a size change, not a position change. +- Don't drop the `flagRetrieveInteractiveWindows` from `xml/island_accessibility_service.xml`. The accessibility service needs `getWindows()` access to find our own overlay's actual on-screen rect via `AccessibilityWindowInfo.getBoundsInScreen()`. Without this flag, `windows` returns null and the service falls back to the manual rect computation (calibrated position + status-bar inset + padding), which is fragile across OEMs that handle status-bar / display-cutout differently. +- Don't make slider drags animate the WindowManager position. `IslandPositionStore.position.drop(1).collect { animY.snapTo(...) }` — `snapTo` not `animateTo`. The slider is a calibration tool; animating each tick lags the drag and feels sluggish. Only the smart-dodge `dodgeY` flow uses `animY.animateTo(target, dodgeSpring)`. Both observers feed the same Animatable, so the snapshotFlow → updateViewLayout collector handles whichever wrote last. +- Don't merge `IslandContinuity` (in `IslandShapes.kt`) with the global `TnContinuity` (in `Shapes.kt`). They are intentionally separate: the island wants a slightly more "puffed out" iOS-squircle profile (`bezierCurvatureScale = 1.2, arcCurvatureScale = 1.2, extendedFraction = 0.6`) than the rest of the app's surfaces. Sharing them means tuning one changes the other — and the island's aesthetic should be free to drift with iOS / OneUI Dynamic-Island design trends without dragging buttons / cards / chips along. +- Don't put `dp` literals for outer padding, card padding, action-button background, or column spacing in `IslandOverlayRoot`. Use `LocalDimens.current.spacingSm` / `.spacingLg` / `.actionIconSize` / `.iconMd` / `.iconLg` and `LocalTnShapes.current.full` for the action button surface. The `IslandGeometry` static constants are reserved for pixel-sized identity values (PILL_W/H, CARD_W/H/CORNER, PRESS_SCALE, SWIPE_THRESHOLD) that genuinely don't belong in the global theme system, and for service-side values (OUTER_PADDING_DP, DODGE_MARGIN_DP, MAX_DODGE_DP) the AccessibilityService needs outside Compose. +- Don't fragment the morph back into multiple `transition.animateFloat` calls — one per value with its own spec. The morph is ONE `progress: Float` animated value; width / height / cornerRadius / pillAlpha / cardAlpha are all `lerp(...)`-derived from progress in the same composition pass. Reasons: (a) any two independent `animateFloat` calls with different visibility thresholds can land on their target on different frames, producing a "settle stutter" where the surface jitters at the end; (b) when the morph value derivations all read from one State, Compose snapshots them atomically — no torn frame where width is at the target but corner is still mid-flight. The lerp-from-progress pattern also forces all springs to share a single dynamic, which is the only way to guarantee they reach the target together. +- Don't use `motionScheme.defaultSpatialSpec` for the morph progress. The custom `spring(dampingRatio = 0.85f, stiffness = StiffnessMediumLow)` was picked specifically over the M3 Expressive default because the default's overshoot, when applied simultaneously to corner radius and size, reads as a jittery "ripple" instead of a smooth squircle morph. Press scale, mode-swap slide, and cross-fades still use `motionScheme.*Spec`; only the main progress driver overrides. If you swap the override back, the morph will feel choppy on mid-tier devices regardless of how fast the frame timing is. +- Don't drop the `cornerRadius.roundToInt()` key on the `remember { islandShape(...) }` block. Animating shape allocation per-frame (60 allocations per second during the morph) churns the kyant `ContinuousRoundedRectangle` path computation and causes visible jitter on devices without aggressive Skia caching. 1dp granularity is visually imperceptible at this scale and reduces allocations by ~4x. +- Don't drop the pill-mode badge or restrict swipe to expanded state. The mode glyph must render in BOTH states (Sparkles for ASSISTANT, Sliders for CONTROL) so the user knows what mode the next expansion will open into. Swipe must work in BOTH states for the same reason — the user should be able to pre-select a mode while the pill is collapsed, then tap to expand directly into that mode. Gating swipe on `expanded` makes the pill feel like a black-box icon with no obvious affordance. +- Don't put the cross-fade alpha calc on its own `animateFloat`. `pillAlpha = (1f - progress * 2f).coerceIn(0f, 1f)` and `cardAlpha = ((progress - 0.5f) * 2f).coerceIn(0f, 1f)` are derived from the SAME progress as size/corner. The handoff at progress=0.5 (both alphas at 0) is exact, frame-perfect, and never visible. Adding a separate alpha animation re-introduces the desync problem. +- Don't drop the `pressed` scale feedback. The `onPress` lambda in `detectTapGestures` flips `pressed = true`, awaits release via `tryAwaitRelease()` in a `try { ... } finally { pressed = false }` block, and `pressScale` animates between 1.0 and `IslandGeometry.PRESS_SCALE = 0.92` via `graphicsLayer`. Without the `finally` block, a cancelled gesture (e.g. swipe initiated mid-press) leaves the surface visually pressed forever. +- Don't put tap and drag detectors in the same `pointerInput` block — keep them in separate modifier calls. Compose's per-modifier gesture isolation lets `detectTapGestures` and `detectHorizontalDragGestures` coexist on the same surface: tap claims on no-slop release, drag claims on slop-exceeded movement. Combined into one block they'd race for the same pointer stream and one would always lose. The drag block must early-return when `!expanded` so pill-mode taps aren't swallowed by stillborn drag-detector setup. +- Don't drop the haptic feedback calls (`view.performHapticFeedback(HapticFeedbackConstants.*)`) on any island gesture: `CONFIRM` on tap and action-button press, `LONG_PRESS` on long-press, `GESTURE_END` on mode-swap completion. Haptic is the only feedback that confirms the island registered the gesture — the visual scale is too subtle on its own. The constants are HapticFeedbackConstants, not the older deprecated `VirtualKey`/`LongPress` IDs. +- Don't make `IslandMode` state live in the service or `IslandPositionStore`. It's purely UI state — local `var mode by remember { mutableStateOf(IslandMode.ASSISTANT) }` inside `IslandSurface`. The mode resets to ASSISTANT each time the overlay is re-attached because the user almost always wants the default mode on first interaction. If a persistent "preferred mode" preference is ever needed, store it in HXS / SharedPreferences and seed the `mutableStateOf` from it — don't bring the service in to mediate. +- Don't reintroduce X-axis movement (slider, dodge, or otherwise). The pill is permanently centered horizontally via `gravity = TOP|CENTER_HORIZONTAL + x=0`. The dodge primitive is Y-only: if obstacles overlap the center-top pill rect, push down; otherwise no movement. Adding X back would require a separate horizontal slot decision (left? right? cheaper distance?) — exactly the heuristic we dropped because it's not deterministic across OEMs / app layouts. Center-top is reliably free space on almost every app (apps avoid centering buttons there because of the camera cutout / status bar); on the rare apps that DO have a center-top button, push-down handles it. +- Don't use `kotlinx.coroutines.Dispatchers.Main` for the IslandOverlayService scope. Compose `Animatable.animateTo()` requires a `MonotonicFrameClock` in the coroutine context, and plain `Dispatchers.Main` (kotlinx) doesn't carry one — `androidx.compose.ui.platform.AndroidUiDispatcher.Main` does (it's the Choreographer-backed dispatcher Compose itself uses). Without this the service crashes with `IllegalStateException: A MonotonicFrameClock is not available in this CoroutineContext` the first time it tries to animate the placement. Same rule applies to any future Service that wants to drive `Animatable.animateTo` from a service-owned CoroutineScope. +- Don't compute the pill rect for the accessibility service via the `windows` API. Use the manual computation in `IslandAccessibilityService.computeNaturalPillRect` — `screenWidth`, `IslandGeometry.PILL_W_DP`, `statusBarTopInsetPx()`, and `IslandPositionStore.position.value.offsetYDp` give the natural rect deterministically. Using `windows` returns the CURRENT animated position which oscillates: dodge → pill moves down → next scan finds no overlap → dodge=0 → pill snaps back → overlap → dodge → … . Manual computation answers "would the pill overlap at its natural position", which is the right question. +- Don't drop the `setDodgeY(0f)` clear on `TYPE_WINDOW_STATE_CHANGED` in `IslandAccessibilityService.onAccessibilityEvent`. Window-state-changed is the app-switch boundary; clearing the dodge first means the pill snaps back to centered position immediately when the user switches apps, then the next scan establishes the correct dodge for the new app. Without the clear, a dodge from app A leaks into app B for the ~150 ms coalesce window — visually the pill stays pushed down when you switch to an app that doesn't need it. +- Don't drop the launcher skip in `IslandAccessibilityService.scanAndPublish`. Any foreground package whose name contains `"launcher"` (case-insensitive substring) → publish `dodgeY = 0f` and return without scanning. Launchers are entirely clickable surfaces (app icons / widgets / search bars / quick toggles) — virtually every pixel under the pill is a clickable node, so the dodge math would push the pill down to the cap (`MAX_DODGE_DP = 96`) and leave it there permanently on the home screen. The pill belongs at center-top on the launcher because the user can just move it themselves if they need to tap something specific. Substring match misses launchers without "launcher" in the package (e.g. `com.miui.home`, `com.sec.android.app.launcher` actually matches, `com.huawei.android.launcher` matches); if a specific OEM launcher needs to be added later, expand the check rather than swapping to a `PackageManager.resolveActivity(CATEGORY_HOME)` query — the substring is cheap and runs on every coalesced scan (don't make a PackageManager round-trip hot-path code). + +--- + +## Housekeeping + +Whenever you change anything on the list below, update **this file** as part of the same change: + +- Security architecture or threat model +- Any auth flow or API surface (SecurityManager, SessionHolder, PolicyEngine, AuthNative, BootIntegrity) +- Any sealed state layout (AuthState, NativeIntegrity manifest, license blob) +- New feature IDs or reshuffling of the pro-feature range +- New persistent keys in HXS (`app_prefs` or `app_bootstrap`) +- New integrity checks, obfuscation scheme, crypto primitives +- New DI bindings touching the security graph +- Changes to release-build hardening (ProGuard, signing, manifest flags) +- Anything in "Things still deferred" moving in or out of scope +- Any new "Things NOT to regress" item discovered along the way + +If the AGENTS.md update isn't part of your diff, the change isn't finished. diff --git a/CLAUDE.md b/CLAUDE.md index 037c614d..f9279734 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -649,17 +649,47 @@ The PlusMenu's old "Documents" button is gone — attachments live entirely in t ## Web search -Replaces the prior Research pipeline (2026-05-15). Single-shot LLM-driven web search. User flips the Web Search toggle on the bottom action bar (or types `/search `); next chat send becomes a web-search run. +**Adaptive iterative researcher (re-pivoted 2026-06-22).** Was a single-pass "3 queries → snippets → answer" run; rebuilt — at the user's explicit direction ("need things functioning right", waiving the prior no-iteration / no-page-fetch rule) — into a controlled multi-round research loop with depth modes, a multi-engine search backend, per-round query refinement + evidence compaction, and page-fetching in the deep modes. This deliberately supersedes the 2026-05-15 deletion of the old Research pipeline; the key difference from that pipeline is that the loop is **mechanically bounded** (round/query/time caps + low-new-source-gain), so it can't run away on a weak on-device model. -Flow (`viewmodel/WebSearchCoordinator.kt`): +User flips the Web Search toggle (or types a slash command); the next send becomes a web-search run. -1. **Plan** — coordinator emits `WebSearchEvent.Plan(userQuery)` so the card renders immediately. -2. **GenerateQueries** — one LLM call (`WebSearchPrompts.generateQueries`) asking for exactly 3 numbered queries. Regex-parsed via `QUERY_LINE_REGEX = ^\s*(?:\d+[.)\-:]|[-*•])\s+(.+)$`. Failures fall through to `WebSearchEvent.Failed`. -3. **Search** — for each of the 3 queries, `WebSearcher.search(query, maxResults=5, idx)` via `WebNative.search` (DDG HTML). Per-query results are deduped against a session-wide `seenUrls` set so cross-query overlap doesn't double-feed the synthesizer. Total cap: 3 queries × 5 results = 15 unique snippets. -4. **Synthesize** — one LLM call (`WebSearchPrompts.synthesize`) with the user query + numbered `[i]` snippet list. Output is markdown with inline `[1]/[2]/[3]` citations and a trailing Sources section. -5. **Done** — emits `WebSearchEvent.Done(answer, sources)`; card renders the markdown answer + collapsible tappable source list (chip → `LocalUriHandler.openUri`). +### Depth modes (`repo/web_search/WebSearchMode.kt`) -No URL fetching. No document extraction. No iteration loop. The user-visible difference vs. old Research: seconds instead of minutes, single inline result instead of a "research document" archive screen. +| Mode | Trigger | Rounds | Total queries | Page fetch | +|------|---------|-------:|--------------:|------------| +| Quick | auto: short factual ask (≤5 words, no deep signal) | 1 | 3 | no | +| Normal | auto: default | 3 | 10 | no | +| Deep | auto: compare/vs/latest/best/research/in-depth/… or `/deep`,`/research` | 10 | 40 | top 6/round | +| Exhaustive | `/exhaustive`, or text "exhaustive"/"deep research" | 20 | 80 | top 8/round | + +`WebSearchMode.resolve(rawText, default)` → `(mode, cleanedQuery)`. **Priority: slash command > saved UI/Settings mode (`default`) > Auto-classify.** `classify` is a **pure heuristic** (no LLM call) on `?`-count / deep-signal keywords / word-count; the "exhaustive"/"deep research" keyword bump lives in `classifyAuto`, which only fires when `default == null` (Auto). `HomeViewModel.parseWebSearchInput` recognizes `/search`,`/deep`,`/research`,`/exhaustive` and the toggle, and passes `WebSearchMode.fromPref(_webSearchMode.value)` as the default. + +### Depth control UI (visible + persisted) + +Default depth is a persisted preference, **stored as a stable lowercase key, never a display label** (`auto`/`quick`/`normal`/`deep`/`exhaustive`; `AppPreferences.webSearchMode`, default `auto`). `WebSearchMode` owns the key↔mode↔label mapping: `key`, `SELECTABLE_KEYS`, `fromPref` (key→mode, null=Auto), `sanitizePref` (coerce unknown→`auto`), `labelForKey`. + +- **Chat** (`ToolsPickerWindow`): a "Search depth" `ActionToggleGroup` over `SELECTABLE_KEYS` sits under the Web-search card — **always visible but `enabled = webSearchEnabled`** (dimmed until web search is on). Selecting calls `HomeViewModel.setWebSearchMode`. +- **Settings** (`SettingsViewModel.webSearchSection`, section id `web_search`, route `NavScreens.SettingsWebSearch`): a "Default search depth" `SettingsItem.Choice` with the same keys + speed-aware descriptions. +- Both write the **same** `appPrefs.webSearchMode` — one source of truth. The card mode badge shows the **resolved concrete** mode (Auto resolves to Quick/Normal/Deep before `start`), never "Auto". + +### The loop (`viewmodel/WebSearchCoordinator.kt`) + +1. **Plan** — emit `WebSearchEvent.Plan(userQuery, mode)`. One LLM call (`WebSearchPrompts.initialQueries`) for the round-0 queries (count = `mode.initialQueries`); URL-in-query and Play-Store asks bypass the LLM with built queries; parse failure falls back to the raw user query. +2. **Round loop** (`while round < maxRounds && nextQueries.isNotEmpty() && totalQueries < maxQueries && !timeUp`): + - `RoundStart(round, maxRounds, focus)`; queries capped to remaining budget; `QueriesGenerated` emits the **cumulative** query list (global query index across rounds). + - per query: `delay(SEARCH_THROTTLE_MS=1200)`, `WebSearcher.search(q, mode.resultsPerQuery, idx)`, dedupe vs `seenUrls` (normalized), `SearchHits`. Accumulate into the evidence pool. + - Deep/Exhaustive: `Status("Reading pages…")`, fetch top-N ranked not-yet-fetched URLs via `PageFetcher` (concurrent, `HtmlText.extract` → ≤2500-char excerpts). + - Quick (or `followUpQueries==0`): break after one round, no digest. + - **Digest** (one LLM call, `WebSearchPrompts.roundDigest`): updated running summary + still-missing + up-to-K follow-up queries + coverage(0-100). Tolerant `SUMMARY:/MISSING:/QUERIES:/COVERAGE:` section parse; follow-ups parsed only from the `QUERIES:` block (avoids capturing summary prose), deduped against all prior queries. + - **Stop** if: coverage ≥ 85 · new-unique-sources < 2 for 2 consecutive rounds · no follow-up queries · budget/time hit. +3. **Synthesize** — `SynthesizeStart`; `WebSearchPrompts.synthesize(userQuery, summary, top-6 sources, excerpts)` (grounded, may cite `[n]`). Link/download asks short-circuit via `directAnswerIfPossible`. +4. **Done** — `Done(answer, sources)` (sources = ranked pool, ≤12); card renders markdown + collapsible source list. + +**Control flow is mechanical-first**: the LLM coverage score can only *early-stop*, never force-continue. Quick = plan + synth (2 LLM calls); Deep/Exhaustive add one digest call per round. + +### Multi-engine backend (`:networking`) + +`WebNative.search` → native `net::ddg::search` is now a **multi-engine chain: DuckDuckGo-html → Bing → Mojeek → DuckDuckGo-lite** (despite the legacy `net::ddg` name). Accumulates unique results, **stops early** once `max_results` is reached: DDG-ok = one request (same cost as before), DDG-blocked (202/429) = falls through to Bing/Mojeek instead of returning nothing. JNI/`nativeSearch` signatures unchanged. Parsers: `html::extract_ddg_results` / `extract_bing_results` (unwraps `bing.com/ck/a?…&u=a1` via `url::unwrap_bing_redirect`) / `extract_mojeek_results` (direct URLs). GET engines use a chrome116-consistent header set. ### Persistence @@ -673,26 +703,28 @@ The card message stores the user's original query in `msg.content` (read by the ### Lockdown -Same pattern as the old research lockdown — `webSearchActive: StateFlow` is derived from `webSearchCoordinator.activeRuns.isNotEmpty()`. `sendMessage`, `loadModel`, and `unloadModel` all early-return while a run is active because the chat LLM is borrowed for both the GenerateQueries and Synthesize calls. +Same pattern as the old research lockdown — `webSearchActive: StateFlow` is derived from `webSearchCoordinator.activeRuns.isNotEmpty()`. `sendMessage`, `loadModel`, and `unloadModel` all early-return while a run is active because the chat LLM is borrowed for every plan / digest / synthesis call in the loop. ### Card UI `ui/screens/web_search/WebSearchCard.kt` is a single Surface with: -- Header (Globe icon, "Web search", user query) -- Queries strip (3 rows with per-query progress indicators — Circle / spinner / Check + hit count) -- AnimatedContent for current phase (Plan / Queries / Search / Synthesize / Done / Cancelled / Failed) -- Stop button while in flight +- Header (Globe icon, "Web search", a **mode badge** chip when `state.mode` is set, user query) +- Queries strip (grows across rounds; per-query progress indicators — Circle / spinner / Check + hit count) +- AnimatedContent for current phase; the Search row shows `Round n/N · ` (status carries "Reading pages…" / "Round n reviewed", and coverage via `WebSearchEvent.Status.coverage`) +- Stop button while in flight (cancels the loop mid-round) - For Done: markdown answer + collapsible `N sources` accordion with `[i]` chips opening URLs externally ### File map -- `model/WebSearchEvent.kt` — sealed event class + `WebSearchHit` data class. -- `model/WebSearchUiState.kt` — phase machine + JSON serde. -- `repo/web_search/WebSearcher.kt` — thin wrapper over `WebNative.search`. -- `repo/web_search/WebSearchPrompts.kt` — prompt templates + `QUERY_LINE_REGEX`. -- `viewmodel/WebSearchCoordinator.kt` — single coordinator (no repository, stateless across runs). +- `model/WebSearchEvent.kt` — sealed events (+ `Plan.mode`, `RoundStart`, `Status(message, coverage)`) + `WebSearchHit`. +- `model/WebSearchUiState.kt` — phase machine + JSON serde (+ `mode`/`round`/`maxRounds`/`status`/`coverage`; `fromJson` zero-fills, TAG 15 unchanged). +- `repo/web_search/WebSearchMode.kt` — depth enum + per-mode budgets + `resolve`/`classify` heuristic. +- `repo/web_search/WebSearcher.kt` — thin wrapper over `WebNative.search` (multi-engine). +- `repo/web_search/PageFetcher.kt` + `HtmlText.kt` — concurrent page fetch + readable-text extraction (Deep/Exhaustive only). +- `repo/web_search/WebSearchPrompts.kt` — `initialQueries` / `roundDigest` / `synthesize` + `QUERY_LINE_REGEX`. +- `viewmodel/WebSearchCoordinator.kt` — the iterative controller (injects `WebSearcher` + `PageFetcher`). - `ui/screens/web_search/WebSearchCard.kt` — the only UI. -- Modified: `ChatMessage` (+ webSearchRunId, webSearchState), `ChatRepository` (TAG 14/15 renamed), `HomeViewModel` (toggle, slash parse, coordinator wiring, event mirror), `HomeScreen{Body,BottomBar}` + `ToolsPickerWindow` (Web search toggle), `ChatMessageList` (WebSearchCard render). +- Modified: `ChatMessage` (webSearchRunId, webSearchState), `HomeViewModel` (slash parse → `WebSearchMode`, coordinator wiring, event mirror), `HomeScreen{Body,BottomBar}` + `ToolsPickerWindow` (toggle), `ChatMessageList` (WebSearchCard render). --- @@ -1055,12 +1087,16 @@ Hub + 7 detail screens, all **single-Scaffold** (accept `innerPadding: PaddingVa - Don't fan out `RemoteCallbackList` broadcasts from `InferenceService` without holding `sdBroadcastLock`. `RemoteCallbackList.beginBroadcast()` is not nestable — calling it from one thread while another is between `beginBroadcast()` and `finishBroadcast()` throws `IllegalStateException: beginBroadcast() called while already in a broadcast` and kills `:inference`. `startSdForwarding` launches five parallel collectors (backend / generation / isGenerating / upscale / runtimeSetup) on `Dispatchers.IO` — without serialisation they race on `sdClients.beginBroadcast()` and the service crash-loops immediately. The fix is `synchronized(sdBroadcastLock)` around the entire `fanoutSd` body; the same lock can serve `tnClients` if a similar pattern is ever added there. - Don't bump `TAG_MSG_WEBSEARCH_RUN` away from `14` or `TAG_MSG_WEBSEARCH_STATE` away from `15`. Older messages without these tags decode with `webSearchRunId = null` and `webSearchState = ""`. New chat-message fields must use TAG ≥ 16. - Don't reintroduce a runtime-only `webSearchEvents: Map` in HomeViewModel. The card's state must come from `WebSearchUiState.fromJson(message.webSearchState)` because (a) opening a different chat while a run is in flight should NOT bleed the running run's state into a completed chat's card; (b) after process restart, completed web-search cards must keep showing their Done state. `HomeViewModel.handleWebSearchEvent` is the single write point — looks up `(chatId, messageId)` via `webSearchMessages[runId]`, reads the message, applies the event, writes back. -- Don't lift the web-search lockdown. While `webSearchActive.value`, `HomeViewModel.sendMessage / loadModel / unloadModel` all early-return — the chat LLM is borrowed for both the GenerateQueries and Synthesize calls. +- Don't lift the web-search lockdown. While `webSearchActive.value`, `HomeViewModel.sendMessage / loadModel / unloadModel` all early-return — the chat LLM is borrowed for every plan / per-round digest / synthesis call in the loop. - Don't drop the web-search content swap in `InferenceCoordinator.buildMessagesJson`. Web-search cards store the user's query in `msg.content` (used by the card Header) and the synthesized answer in `msg.webSearchState`. The single point of LLM-history assembly MUST swap `content` for `WebSearchUiState.fromJson(webSearchState).answer.trim()` when `webSearchRunId != null`, and SKIP the message entirely when the answer is blank (in-flight / cancelled / failed). Without the swap the model sees `assistant: ""` instead of the actual research, and the next chat turn proceeds as if the search never happened. Without the skip, in-flight / failed cards inject an empty assistant turn that confuses the model. -- Don't replace `WebNative.search` with `HttpURLConnection` or any other client. The `:networking` curl-impersonate Chrome116 fingerprint is the single allowed pipe for DDG. The 3 queries × 5 results contract assumes that backend. -- Don't expand the web-search flow back into a multi-iteration pipeline. The user-facing contract is "3 queries, snippets, answer, done". Adding iterations / fetches / per-page extraction is what the old Research pipeline was; it was deleted on 2026-05-15 because it was minutes-slow and barely better than snippet-only synthesis. +- Don't replace `WebNative.search` with `HttpURLConnection` or any other client. The `:networking` curl-impersonate Chrome116 fingerprint is the single allowed pipe — for `WebNative.search` (DDG→Bing→Mojeek→DDG-lite) AND `WebNative.fetch` (Deep/Exhaustive page fetch). Same applies to any new engine: it goes inside `net::ddg::search`'s chain, not a new HTTP client. +- Don't make the web-search loop unbounded. It is iterative now (re-pivoted 2026-06-22, user-directed, superseding the 2026-05-15 single-pass deletion), but EVERY run must stay bounded by `WebSearchMode`'s `maxRounds` / `maxQueries` / `timeBudgetMs` plus the low-new-source-gain stop. The LLM coverage score may only *early-stop*, never extend — a weak on-device model must not be able to wedge the loop. Quick mode stays single-round (plan + synth, no digest). +- Don't gate page-fetching on anything but `WebSearchMode.fetchPages` (Deep/Exhaustive only). Quick/Normal are snippet-only and must stay fast. `PageFetcher` excerpts are capped (`HtmlText.extract`, ≤2500 chars) and the synthesis prompt re-caps per source — don't feed whole pages into the prompt (context blowup on small models). +- Don't break `WebSearchMode.resolve`/`classify` being a pure heuristic (no LLM call). Mode selection must be instant. Slash triggers are `/search` (auto-classify), `/deep`+`/research` (Deep), `/exhaustive` (Exhaustive); `HomeViewModel.SEARCH_SLASHES` must list all four. Resolution priority is fixed: **slash > saved mode (`default`) > Auto-classify**. +- Don't store web-search depth as a display label or split it across two prefs. The single source of truth is `AppPreferences.webSearchMode`, holding a **stable lowercase key** (`auto/quick/normal/deep/exhaustive`); the chat `ToolsPickerWindow` selector and Settings `web_search` section both read/write it. Convert key→label only at render via `WebSearchMode.labelForKey`; run any saved value through `WebSearchMode.sanitizePref` (unknown → `auto`) so old/hand-edited values never crash. The card badge shows the resolved concrete mode, not "Auto". +- Don't make the chat depth selector appear only when web search is on. It's always rendered with `enabled = webSearchEnabled` (dimmed when off) for discoverability. Adding a new Settings section needs all four wiring points: `NavScreens`, the `SETTINGS_LANDING_CARDS` card, the `TNavigation` composable (sectionId = `SettingsViewModel.SECTION_*`), and the `AppTopBar` `when` case. - Don't switch `WebSearchCoordinator` from `tryEmit` to suspending `emit(...)` in the `catch (CancellationException)` / `catch (Throwable)` blocks. The catch fires on a cancelled Job, and `withContext(Dispatchers.Default)` throws CE immediately on a cancelled context — so the Cancelled event never reaches the SharedFlow and the card freezes mid-flight. `_events` has 64-slot buffer; tryEmit always succeeds. -- Don't change `WebSearchPrompts.QUERY_LINE_REGEX`. The synthesis prompt asks for the format `1. ` / `2. ` / `3. ` and the regex `^\s*(?:\d+[.)\-:]|[-*•])\s+(.+)$` parses that AND tolerates common LLM deviations (`-`/`*`/`•` bullets, `:` after the number). Tightening the regex breaks smaller models that don't follow numbered-list instructions perfectly; loosening it picks up the LLM's preamble lines. +- Don't change `WebSearchPrompts.QUERY_LINE_REGEX`. The query prompts ask for numbered lines (`1. `) and the regex `^\s*(?:\d+[.)\-:]|[-*•])\s+(.+)$` parses that AND tolerates `-`/`*`/`•` bullets + `:` after the number. The `roundDigest` parser also depends on it (parsed only from the `QUERIES:` block so summary prose isn't captured as queries). Tightening breaks small models; loosening picks up preamble. - Don't fall back to `java.net.HttpURLConnection` for any HuggingFace API call — search, model info, tree, raw README, tags-by-type, trending, quicksearch, resolve. Every HF request goes through `:networking` (`WebNative.fetch`) so it inherits the curl-impersonate Chrome116 fingerprint + bundled `cacert.pem` + strict cert verify. The hub is `repo/HuggingFaceApi.kt` (Hilt singleton class, not an object); `repo/hf/HfClient.kt` builds typed endpoints on top. `ModelCatalog` and `RepositoryValidator` inject `HuggingFaceApi`. Same rule applies for any future HF or non-HF HTTP target — `:networking` is the only allowed pipe. - Don't change `WebNative.fetch` back to `Result`. The contract is `Result` where `WebResponse(status: Int, body: String, error: String?)`. Result.failure is reserved for transport-layer issues (DNS, TLS handshake, native call collapse). HTTP non-2xx comes back as `Result.success(WebResponse(status=4xx, ...))` — callers can react to 429 (rate limited) vs 404 (not found) vs 401/403 (auth). Old behavior of returning `null` on non-2xx silently masked HF API bugs (e.g. invalid `expand=` params returning 400) for years. - Don't log full URLs (with query string) to `ANDROID_LOG_WARN` from `net_jni.cpp`. Use the `host_of(url)` helper. Search queries are user PII (typed model names, sometimes sensitive). Status code + host is the maximum log surface. diff --git a/app/build.gradle.kts b/app/build.gradle.kts index cba41259..5b4f7e91 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -126,6 +126,7 @@ dependencies { implementation(files("../libs/ai_sd-release.aar")) implementation(libs.kotlinx.coroutines.android) implementation(libs.kotlinx.serialization.json) + implementation(libs.onnxruntime.android) // DI implementation(libs.hilt.android) diff --git a/app/src/main/assets/server_webui.css b/app/src/main/assets/server_webui.css new file mode 100644 index 00000000..288fbfa3 --- /dev/null +++ b/app/src/main/assets/server_webui.css @@ -0,0 +1,419 @@ +:root{ + color-scheme:dark light; + --bg:#1A1A1D;--side:#141416;--raise:#232328;--raise-2:#2A2A30; + --hover:rgba(255,255,255,.055);--hover-2:rgba(255,255,255,.09); + --line:#2D2D34;--line-2:#3A3A43; + --tx:#ECECEE;--tx-dim:#B6B6BD;--tx-mute:#85858E; + --user:#2A2A31; + --accent:#7C8CFF;--accent-2:#9AA6FF;--accent-ink:#0C1024; + --accent-soft:rgba(124,140,255,.14);--accent-edge:rgba(124,140,255,.38); + --danger:#FF6B72;--good:#46C99A;--code:#0F0F13; + --r-s:8px;--r-m:12px;--r-l:18px;--r-xl:26px; + --sans:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Inter,"Helvetica Neue",system-ui,sans-serif; + --mono:ui-monospace,"SF Mono","JetBrains Mono",Menlo,Consolas,"Roboto Mono",monospace; + --side-w:268px; + --shadow:0 24px 60px -22px rgba(0,0,0,.75); +} +@media (prefers-color-scheme:light){:root:not([data-theme]){ + --bg:#FFFFFF;--side:#F7F7F9;--raise:#FFFFFF;--raise-2:#F2F2F5; + --hover:rgba(0,0,0,.045);--hover-2:rgba(0,0,0,.075); + --line:#E7E7EC;--line-2:#D8D8E0; + --tx:#1A1A1D;--tx-dim:#46464E;--tx-mute:#76767F; + --user:#F0F1F6; + --accent:#4F62E8;--accent-2:#3F52DC;--accent-ink:#FFFFFF; + --accent-soft:rgba(79,98,232,.10);--accent-edge:rgba(79,98,232,.32); + --danger:#D63A52;--good:#1F9D6B;--code:#F5F5F8; + --shadow:0 24px 60px -24px rgba(20,20,40,.28); +}} +html[data-theme=light]{ + --bg:#FFFFFF;--side:#F7F7F9;--raise:#FFFFFF;--raise-2:#F2F2F5; + --hover:rgba(0,0,0,.045);--hover-2:rgba(0,0,0,.075); + --line:#E7E7EC;--line-2:#D8D8E0; + --tx:#1A1A1D;--tx-dim:#46464E;--tx-mute:#76767F; + --user:#F0F1F6; + --accent:#4F62E8;--accent-2:#3F52DC;--accent-ink:#FFFFFF; + --accent-soft:rgba(79,98,232,.10);--accent-edge:rgba(79,98,232,.32); + --danger:#D63A52;--good:#1F9D6B;--code:#F5F5F8; + --shadow:0 24px 60px -24px rgba(20,20,40,.28); +} +html[data-theme=dark]{ + --bg:#1A1A1D;--side:#141416;--raise:#232328;--raise-2:#2A2A30; + --hover:rgba(255,255,255,.055);--hover-2:rgba(255,255,255,.09); + --line:#2D2D34;--line-2:#3A3A43; + --tx:#ECECEE;--tx-dim:#B6B6BD;--tx-mute:#85858E; + --user:#2A2A31;--code:#0F0F13; + --accent:#7C8CFF;--accent-2:#9AA6FF;--accent-ink:#0C1024; + --accent-soft:rgba(124,140,255,.14);--accent-edge:rgba(124,140,255,.38); + --danger:#FF6B72;--good:#46C99A; + --shadow:0 24px 60px -22px rgba(0,0,0,.75); +} + +*{box-sizing:border-box} +::selection{background:var(--accent-soft);color:var(--tx)} +html,body{height:100%;margin:0} +body{background:var(--bg);color:var(--tx);font:15px/1.6 var(--sans);-webkit-font-smoothing:antialiased;overflow:hidden} +button,input,textarea,select{font:inherit;color:inherit} +button{border:0;background:transparent;color:inherit;cursor:pointer} +a{color:var(--accent);text-decoration:none} +a:hover{text-decoration:underline} +svg{display:block} +:focus-visible{outline:2px solid var(--accent);outline-offset:2px} +::-webkit-scrollbar{width:10px;height:10px} +::-webkit-scrollbar-thumb{background:var(--line-2);border-radius:99px;border:3px solid transparent;background-clip:content-box} +::-webkit-scrollbar-thumb:hover{background:var(--tx-mute);background-clip:content-box} +*{scrollbar-width:thin;scrollbar-color:var(--line-2) transparent} + +.app{display:grid;grid-template-columns:var(--side-w) minmax(0,1fr);height:100dvh;transition:grid-template-columns .26s cubic-bezier(.4,0,.2,1)} +.app.tucked{grid-template-columns:0 minmax(0,1fr)} + +/* ============ sidebar ============ */ +.side{display:flex;flex-direction:column;min-width:0;background:var(--side);border-right:1px solid var(--line);overflow:hidden} +.app.tucked .side{border-right-color:transparent} +.side-top{display:flex;align-items:center;gap:8px;padding:13px 12px 10px} +.brand{display:flex;align-items:center;gap:9px;flex:1;min-width:0;padding:4px 6px} +.brand .glyph{width:26px;height:26px;flex:none;color:var(--accent)} +.brand .glyph svg{width:26px;height:26px} +.brand .name{font-weight:660;font-size:15px;letter-spacing:-.01em;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.iconbtn{width:34px;height:34px;flex:none;border-radius:var(--r-s);display:grid;place-items:center;color:var(--tx-mute);transition:background .14s,color .14s,transform .08s} +.iconbtn:hover{background:var(--hover);color:var(--tx)} +.iconbtn:active{transform:scale(.92)} +.iconbtn svg{width:19px;height:19px} + +.new-btn{margin:2px 12px 12px;height:42px;border-radius:var(--r-m);border:1px solid var(--line-2);background:var(--raise);color:var(--tx);font-size:14px;font-weight:560;display:flex;align-items:center;gap:10px;padding:0 14px;transition:background .14s,border-color .14s,transform .08s} +.new-btn:hover{background:var(--hover-2);border-color:var(--accent-edge)} +.new-btn:active{transform:translateY(1px)} +.new-btn svg{width:17px;height:17px;color:var(--accent)} + +.find{margin:0 12px 8px;display:flex;align-items:center;gap:9px;background:var(--raise);border:1px solid var(--line);border-radius:var(--r-s);padding:0 11px;height:38px} +.find svg{flex:none;color:var(--tx-mute);width:15px;height:15px} +.find input{border:0;background:transparent;outline:0;width:100%;font-size:14px} +.find input::placeholder{color:var(--tx-mute)} + +.threads{flex:1;overflow:auto;padding:4px 8px 10px;display:flex;flex-direction:column} +.daygroup{font-size:11px;font-weight:640;letter-spacing:.04em;text-transform:uppercase;color:var(--tx-mute);padding:14px 8px 6px} +.row{display:flex;align-items:center;gap:8px;border-radius:var(--r-s);padding:0 6px 0 10px;height:38px;color:var(--tx-dim);text-align:left;width:100%;position:relative;transition:background .12s,color .12s} +.row:hover{background:var(--hover)} +.row.on{background:var(--hover-2);color:var(--tx)} +.row .rtitle{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:14px;background:none;border:0;text-align:left;padding:9px 0;color:inherit} +.row .rkill{flex:none;width:26px;height:26px;border-radius:6px;display:grid;place-items:center;color:var(--tx-mute);opacity:0;transition:opacity .12s,color .12s,background .12s} +.row:hover .rkill,.row.on .rkill{opacity:.8} +.row .rkill:hover{opacity:1;color:var(--danger);background:var(--hover-2)} +.row .rkill svg{width:15px;height:15px} +.empty-threads{padding:24px 14px;text-align:center;color:var(--tx-mute);font-size:13px;line-height:1.5} + +.side-foot{border-top:1px solid var(--line);padding:8px} +.conn{display:flex;align-items:center;gap:10px;width:100%;padding:9px 11px;border-radius:var(--r-s);transition:background .12s;text-align:left} +.conn:hover{background:var(--hover)} +.dot{width:9px;height:9px;border-radius:50%;flex:none;background:var(--tx-mute);position:relative} +.dot.ok{background:var(--good);color:var(--good)}.dot.bad{background:var(--danger)}.dot.wait{background:var(--accent);color:var(--accent)} +.dot.ok::after,.dot.wait::after{content:"";position:absolute;inset:0;border-radius:50%;box-shadow:0 0 0 0 currentColor;animation:halo 2.2s ease-out infinite;opacity:.5} +@keyframes halo{0%{box-shadow:0 0 0 0 currentColor;opacity:.5}70%,100%{box-shadow:0 0 0 7px transparent;opacity:0}} +.conn .ctext{flex:1;min-width:0;display:flex;flex-direction:column;line-height:1.25} +.conn .clabel{font-size:13px;color:var(--tx);white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.conn .csub{font-size:11px;color:var(--tx-mute);white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.conn .cog{color:var(--tx-mute);flex:none} +.conn .cog svg{width:17px;height:17px} + +/* ============ main ============ */ +.main{display:flex;flex-direction:column;min-width:0;height:100dvh;position:relative} +.bar{display:flex;align-items:center;gap:8px;height:54px;padding:0 12px;flex:none} +.bar .spacer{flex:1} +.show-side{display:none} +.app.tucked .show-side{display:grid} + +.mpick{position:relative;min-width:0} +.mtrigger{display:flex;align-items:center;gap:7px;height:38px;padding:0 10px;border-radius:var(--r-s);transition:background .12s;max-width:min(52vw,420px)} +.mtrigger:hover{background:var(--hover)} +.mtrigger .mname{font-size:15px;font-weight:560;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.mtrigger .mcar{color:var(--tx-mute);flex:none;transition:transform .18s} +.mtrigger[aria-expanded=true] .mcar{transform:rotate(180deg)} +.mbadge{font-size:10px;font-weight:680;letter-spacing:.06em;text-transform:uppercase;color:var(--accent);background:var(--accent-soft);border-radius:5px;padding:2px 6px;flex:none} +.menu{position:absolute;top:46px;left:0;width:min(420px,calc(100vw - 24px));background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-m);box-shadow:var(--shadow);padding:8px;z-index:60;opacity:0;transform:translateY(-8px) scale(.98);transform-origin:top left;pointer-events:none;transition:.16s cubic-bezier(.2,.8,.2,1)} +.menu.open{opacity:1;transform:none;pointer-events:auto} +.menu .find{margin:0 0 7px;background:var(--raise-2)} +.mlist{max-height:340px;overflow:auto;display:flex;flex-direction:column;gap:2px} +.mopt{display:flex;align-items:center;gap:10px;justify-content:space-between;padding:10px 11px;border-radius:var(--r-s);text-align:left;width:100%;transition:background .12s} +.mopt:hover{background:var(--hover-2)} +.mopt.on{background:var(--accent-soft)} +.mopt .ml{display:flex;align-items:center;gap:10px;min-width:0} +.mopt .mtitle{font-size:14px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.mopt.on .mtitle{color:var(--accent);font-weight:560} +.mopt .mk{font-size:10px;font-weight:660;letter-spacing:.05em;text-transform:uppercase;color:var(--tx-mute);border:1px solid var(--line-2);border-radius:99px;padding:2px 8px;flex:none} +.mopt.on .mk{border-color:var(--accent-edge);color:var(--accent)} +.mopt .check{color:var(--accent);opacity:0;flex:none} +.mopt.on .check{opacity:1} +.menu-empty{padding:20px;text-align:center;color:var(--tx-mute);font-size:13px;line-height:1.5} + +/* ============ thread ============ */ +.stream{flex:1;overflow-y:auto;overflow-x:hidden;scroll-behavior:smooth} +.wrap{max-width:768px;margin:0 auto;padding:18px 24px 10px;width:100%} + +/* empty / welcome */ +.welcome{height:100%;min-height:60vh;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;padding:24px 24px 40px;max-width:720px;margin:0 auto} +.welcome .wmark{width:52px;height:52px;color:var(--accent);margin-bottom:20px} +.welcome .wmark svg{width:52px;height:52px} +.welcome h1{margin:0;font-size:30px;font-weight:680;letter-spacing:-.02em;line-height:1.2} +.welcome p{margin:12px 0 0;color:var(--tx-mute);font-size:15.5px;max-width:460px} +.chips{margin-top:30px;display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px;width:100%;max-width:560px} +.chip{text-align:left;padding:14px 16px;border:1px solid var(--line);background:var(--raise);border-radius:var(--r-m);transition:border-color .15s,background .14s,transform .1s} +.chip:hover{border-color:var(--accent-edge);background:var(--hover-2);transform:translateY(-2px)} +.chip .ct{font-size:14px;font-weight:580;color:var(--tx)} +.chip .cd{margin-top:3px;font-size:13px;color:var(--tx-mute);line-height:1.4} + +/* messages */ +.msg{padding:14px 0;animation:rise .24s ease both} +@keyframes rise{from{opacity:0;transform:translateY(6px)}to{opacity:1;transform:none}} +.msg.user{display:flex;flex-direction:column;align-items:flex-end} +.bubble{max-width:88%;background:var(--user);border:1px solid var(--line);border-radius:var(--r-l);border-bottom-right-radius:7px;padding:11px 16px;font-size:15.5px;line-height:1.6;white-space:pre-wrap;overflow-wrap:anywhere} +.atts{display:flex;flex-wrap:wrap;gap:8px;justify-content:flex-end;margin-bottom:8px;max-width:88%} +.msg.ai .atts{justify-content:flex-start} +.att{display:flex;align-items:center;gap:8px;font-size:12.5px;color:var(--tx-dim);background:var(--raise);border:1px solid var(--line);border-radius:10px;padding:6px 11px;max-width:240px} +.att .afile{white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.att img{width:34px;height:34px;border-radius:7px;object-fit:cover;margin:-2px -3px} +.att svg{width:15px;height:15px;flex:none;color:var(--tx-mute)} + +.ai-head{display:flex;align-items:center;gap:9px;margin-bottom:8px} +.ai-head .av{width:26px;height:26px;border-radius:8px;flex:none;display:grid;place-items:center;background:var(--accent-soft);color:var(--accent)} +.ai-head .av svg{width:16px;height:16px} +.ai-head .who{font-size:13px;font-weight:620;color:var(--tx-dim)} + +/* reasoning / thinking */ +.think{margin:0 0 12px;border:1px solid var(--line);border-radius:var(--r-m);background:var(--raise);overflow:hidden} +.think summary{list-style:none;display:flex;align-items:center;gap:9px;padding:10px 13px;cursor:pointer;font-size:13px;color:var(--tx-dim);user-select:none} +.think summary::-webkit-details-marker{display:none} +.think summary .tcar{margin-left:auto;color:var(--tx-mute);transition:transform .18s} +.think[open] summary .tcar{transform:rotate(90deg)} +.think summary .tspark{color:var(--accent);display:flex} +.think summary .tspark svg{width:15px;height:15px} +.think.live summary{color:var(--accent)} +.think.live summary .tlabel{background:linear-gradient(90deg,var(--tx-mute),var(--tx),var(--tx-mute));background-size:200% 100%;-webkit-background-clip:text;background-clip:text;-webkit-text-fill-color:transparent;animation:shine 1.6s linear infinite} +@keyframes shine{to{background-position:-200% 0}} +.think .tbody{padding:2px 14px 13px;font-size:13.5px;line-height:1.65;color:var(--tx-mute);border-top:1px solid var(--line);white-space:pre-wrap;overflow-wrap:anywhere} + +/* prose */ +.prose{overflow-wrap:anywhere;line-height:1.72;font-size:15.5px;color:var(--tx)} +.prose>:first-child{margin-top:0}.prose>:last-child{margin-bottom:0} +.prose p{margin:0 0 14px} +.prose strong{font-weight:660} +.prose em{font-style:italic} +.prose h1,.prose h2,.prose h3,.prose h4{margin:24px 0 12px;line-height:1.32;font-weight:680} +.prose h1{font-size:22px}.prose h2{font-size:18px}.prose h3{font-size:16px}.prose h4{font-size:15px} +.prose ul,.prose ol{padding-left:24px;margin:12px 0} +.prose li{margin:6px 0} +.prose li::marker{color:var(--tx-mute)} +.prose blockquote{margin:14px 0;padding:6px 0 6px 16px;border-left:3px solid var(--accent-edge);color:var(--tx-dim)} +.prose :not(pre)>code{font-family:var(--mono);font-size:.86em;background:var(--raise-2);border:1px solid var(--line);padding:1.5px 6px;border-radius:6px} +.prose hr{border:0;border-top:1px solid var(--line);margin:20px 0} +.prose table{border-collapse:collapse;display:block;overflow:auto;margin:14px 0;font-size:14px} +.prose th,.prose td{border:1px solid var(--line);padding:8px 13px;text-align:left} +.prose th{background:var(--raise);font-weight:640} +.prose a{text-decoration:underline;text-underline-offset:2px} +.cursor::after{content:"";display:inline-block;width:8px;height:16px;background:var(--accent);border-radius:2px;margin-left:2px;vertical-align:-2px;animation:blink 1.05s step-end infinite} +@keyframes blink{50%{opacity:0}} + +.fence{margin:14px 0;border:1px solid var(--line);border-radius:var(--r-m);overflow:hidden;background:var(--code)} +.fence-top{display:flex;align-items:center;justify-content:space-between;padding:7px 8px 7px 14px;border-bottom:1px solid var(--line);font-family:var(--mono);font-size:11.5px;color:var(--tx-mute)} +.fence-top button{display:inline-flex;align-items:center;gap:6px;font-size:11.5px;color:var(--tx-mute);padding:5px 9px;border-radius:7px;transition:color .12s,background .12s} +.fence-top button:hover{color:var(--tx);background:var(--hover-2)} +.fence-top button svg{width:14px;height:14px} +.fence pre{margin:0;padding:14px 16px;overflow:auto;font-family:var(--mono);font-size:13px;line-height:1.6;color:var(--tx)} +.fence pre code{background:none;border:0;padding:0;font-size:inherit} + +.interrupted{display:inline-flex;align-items:center;gap:7px;margin-top:2px;font-size:12.5px;color:var(--tx-mute)} +.interrupted svg{width:14px;height:14px;color:var(--danger)} + +/* per-message actions (always visible, dim) */ +.acts{display:flex;align-items:center;gap:2px;margin-top:8px} +.msg.user .acts{justify-content:flex-end} +.act{display:inline-flex;align-items:center;gap:6px;font-size:12.5px;color:var(--tx-mute);padding:6px 9px;border-radius:7px;transition:color .12s,background .12s} +.act:hover{color:var(--tx);background:var(--hover)} +.act.live{color:var(--accent)} +.act svg{width:15px;height:15px} +.act .alabel{font-size:12px} + +/* inline edit */ +.editor{width:100%} +.editor textarea{width:100%;min-height:84px;resize:vertical;background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-m);padding:12px 14px;outline:0;font-size:15px;line-height:1.55;font-family:var(--sans)} +.editor textarea:focus{border-color:var(--accent);box-shadow:0 0 0 3px var(--accent-soft)} +.editor-row{display:flex;gap:8px;justify-content:flex-end;margin-top:9px} +.btn-sm{height:34px;padding:0 15px;border-radius:8px;font-size:13px;font-weight:560;transition:filter .14s,background .12s,border-color .12s} +.btn-sm.fill{background:var(--accent);color:var(--accent-ink)} +.btn-sm.fill:hover{filter:brightness(1.07)} +.btn-sm.ghost{border:1px solid var(--line-2);color:var(--tx-dim)} +.btn-sm.ghost:hover{border-color:var(--tx-mute);color:var(--tx)} + +/* ============ composer ============ */ +.dock{flex:none;padding:6px 16px 14px;background:linear-gradient(to top,var(--bg) 62%,transparent)} +.composer{max-width:768px;margin:0 auto;background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-xl);padding:8px;transition:border-color .16s,box-shadow .16s} +.composer.focus{border-color:var(--accent-edge);box-shadow:0 0 0 3px var(--accent-soft)} +.pending{display:flex;flex-wrap:wrap;gap:8px;padding:6px 6px 8px} +.pending:empty{display:none} +.pchip{display:flex;align-items:center;gap:9px;background:var(--raise-2);border:1px solid var(--line);border-radius:10px;padding:7px 9px 7px 10px;font-size:12.5px;max-width:220px} +.pchip img{width:36px;height:36px;border-radius:7px;object-fit:cover;margin:-3px -3px -3px -4px} +.pchip svg.fic{width:16px;height:16px;color:var(--tx-mute);flex:none} +.pchip .pname{white-space:nowrap;overflow:hidden;text-overflow:ellipsis;color:var(--tx-dim)} +.pchip .px{width:20px;height:20px;border-radius:50%;flex:none;display:grid;place-items:center;color:var(--tx-mute);background:var(--hover)} +.pchip .px:hover{color:var(--danger);background:var(--hover-2)} +.pchip .px svg{width:12px;height:12px} + +.crow{display:flex;align-items:flex-end;gap:6px} +.composer textarea{flex:1;min-height:28px;max-height:46dvh;resize:none;border:0;background:transparent;outline:0;padding:9px 6px;font-size:16px;line-height:1.5;font-family:var(--sans)} +.composer textarea::placeholder{color:var(--tx-mute)} +.c-btn{width:38px;height:38px;flex:none;border-radius:50%;display:grid;place-items:center;color:var(--tx-mute);transition:background .12s,color .12s,transform .08s} +.c-btn:hover{background:var(--hover-2);color:var(--tx)} +.c-btn:active{transform:scale(.92)} +.c-btn svg{width:19px;height:19px} +.c-btn.voice{color:var(--accent)} +.c-btn.voice:hover{background:var(--accent-soft);color:var(--accent)} +.act.live svg{animation:pulse 1.1s ease-in-out infinite} +.go{width:38px;height:38px;flex:none;border-radius:50%;background:var(--accent);color:var(--accent-ink);display:grid;place-items:center;transition:filter .14s,transform .08s,background .14s,opacity .14s} +.go:hover{filter:brightness(1.07)} +.go:active{transform:scale(.92)} +.go:disabled{background:var(--raise-2);color:var(--tx-mute);cursor:default} +.go svg{width:19px;height:19px} +.go.stop{background:var(--tx);color:var(--bg)} +.hint{max-width:768px;margin:8px auto 0;text-align:center;font-size:11.5px;color:var(--tx-mute)} +.hint b{color:var(--tx-dim);font-weight:600} + +/* recording bar */ +.recbar{display:none;align-items:center;gap:12px;padding:6px 6px 6px 14px} +.composer.recording .crow{display:none} +.composer.recording .recbar{display:flex} +.composer.recording .pending{display:none} +.recdot{width:11px;height:11px;border-radius:50%;background:var(--danger);flex:none;animation:pulse 1.1s ease-in-out infinite} +@keyframes pulse{0%,100%{opacity:1;transform:scale(1)}50%{opacity:.45;transform:scale(.82)}} +.rectime{font-family:var(--mono);font-size:14px;color:var(--tx);min-width:46px} +.waves{flex:1;display:flex;align-items:center;gap:3px;height:24px;overflow:hidden} +.waves i{width:3px;border-radius:3px;background:var(--accent);height:20%} +.rec-x,.rec-ok{height:36px;padding:0 14px;border-radius:99px;font-size:13px;font-weight:560;display:inline-flex;align-items:center;gap:7px} +.rec-x{color:var(--tx-dim);background:var(--hover)} +.rec-x:hover{background:var(--hover-2);color:var(--tx)} +.rec-ok{background:var(--accent);color:var(--accent-ink)} +.rec-ok svg,.rec-x svg{width:15px;height:15px} + +/* ============ modal ============ */ +.veil{position:fixed;inset:0;background:rgba(8,8,14,.6);backdrop-filter:blur(5px);display:none;place-items:center;padding:18px;z-index:90} +.veil.show{display:grid;animation:fade .14s ease} +@keyframes fade{from{opacity:0}to{opacity:1}} +.sheet{width:min(520px,100%);max-height:90dvh;overflow:auto;background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-l);box-shadow:var(--shadow);animation:pop .18s cubic-bezier(.2,.8,.2,1)} +@keyframes pop{from{opacity:0;transform:translateY(10px) scale(.98)}to{opacity:1;transform:none}} +.sheet-top{display:flex;align-items:center;justify-content:space-between;padding:18px 20px 0} +.sheet-top h2{margin:0;font-size:17px;font-weight:660} +.sheet-body{padding:16px 20px 22px;display:flex;flex-direction:column;gap:22px} +.grp{display:flex;flex-direction:column;gap:13px} +.grp-h{font-size:12px;font-weight:680;letter-spacing:.05em;text-transform:uppercase;color:var(--tx-mute);display:flex;align-items:center;gap:10px} +.grp-h::after{content:"";flex:1;height:1px;background:var(--line)} +.f{display:flex;flex-direction:column;gap:6px} +.f>span{font-size:13px;font-weight:560;color:var(--tx-dim)} +.f input,.f select,.f textarea{width:100%;background:var(--bg);border:1px solid var(--line);border-radius:var(--r-s);padding:11px 13px;outline:0;font-size:14px;transition:border-color .12s,box-shadow .12s} +.f input:focus,.f select:focus,.f textarea:focus{border-color:var(--accent);box-shadow:0 0 0 3px var(--accent-soft)} +.f textarea{min-height:78px;resize:vertical;line-height:1.5} +.two{display:grid;grid-template-columns:1fr 1fr;gap:12px} +.sw{display:flex;align-items:center;justify-content:space-between;gap:12px;font-size:14px;color:var(--tx-dim)} +.sw .swl{display:flex;flex-direction:column;gap:2px} +.sw .swl small{color:var(--tx-mute);font-size:12px} +.switch{position:relative;width:42px;height:24px;flex:none} +.switch input{opacity:0;width:0;height:0;position:absolute} +.track{position:absolute;inset:0;border-radius:99px;background:var(--line-2);transition:background .16s;cursor:pointer} +.track::before{content:"";position:absolute;top:3px;left:3px;width:18px;height:18px;border-radius:50%;background:#fff;transition:transform .16s} +.switch input:checked+.track{background:var(--accent)} +.switch input:checked+.track::before{transform:translateX(18px)} +.note{font-size:13px;border:1px solid var(--line);background:var(--bg);border-radius:var(--r-s);padding:11px 13px;color:var(--tx-mute);line-height:1.5;display:flex;gap:10px;align-items:flex-start} +.note .nd{width:7px;height:7px;border-radius:50%;background:var(--tx-mute);margin-top:7px;flex:none} +.note.ok{border-color:var(--accent-edge);color:var(--tx-dim)}.note.ok .nd{background:var(--good)} +.note.bad{border-color:var(--danger);color:var(--danger)}.note.bad .nd{background:var(--danger)} +.acts2{display:flex;gap:10px;flex-wrap:wrap} +.btn{flex:1;min-width:130px;height:44px;border-radius:var(--r-s);font-size:14px;font-weight:580;display:flex;align-items:center;justify-content:center;gap:8px;transition:filter .14s,border-color .14s,background .12s,transform .08s} +.btn:active{transform:translateY(1px)} +.btn svg{width:17px;height:17px} +.btn.fill{background:var(--accent);color:var(--accent-ink)} +.btn.fill:hover{filter:brightness(1.07)} +.btn.line{border:1px solid var(--line-2);color:var(--tx)} +.btn.line:hover{border-color:var(--accent-edge)} +.btn.warn{border:1px solid var(--line-2);color:var(--danger)} +.btn.warn:hover{border-color:var(--danger);background:rgba(255,107,114,.08)} + +.toast{position:fixed;left:50%;bottom:26px;transform:translate(-50%,16px);background:var(--tx);color:var(--bg);border-radius:99px;padding:11px 20px;font-size:13px;font-weight:560;box-shadow:var(--shadow);opacity:0;pointer-events:none;transition:.22s;z-index:99} +.toast.show{opacity:1;transform:translate(-50%,0)} +.drop{position:fixed;inset:14px;border:2px dashed var(--accent);border-radius:var(--r-l);background:var(--accent-soft);display:none;place-items:center;font-size:16px;font-weight:600;color:var(--accent);z-index:88} +.drop.show{display:grid} +.scrim{position:fixed;inset:0;background:rgba(0,0,0,.5);display:none;z-index:30} +.scrim.show{display:block} + +@media (max-width:880px){ + .app{grid-template-columns:0 minmax(0,1fr)} + .side{position:fixed;top:0;left:0;width:min(88vw,320px);height:100dvh;z-index:50;transform:translateX(-100%);transition:transform .24s cubic-bezier(.2,.8,.2,1);border-right:1px solid var(--line);padding-bottom:env(safe-area-inset-bottom)} + .side.open{transform:none} + .show-side{display:grid!important} + .side-top{padding-top:max(13px,env(safe-area-inset-top))} + .iconbtn,.row .rkill,.c-btn,.go{width:44px;height:44px} + .new-btn,.find,.row,.conn,.mtrigger,.mopt,.btn,.rec-x,.rec-ok{min-height:44px} + .row{height:auto;padding:4px 4px 4px 12px} + .row .rkill{opacity:.75} + .row .rtitle{padding:10px 0} + .main{height:100dvh} + .bar{height:auto;min-height:56px;padding:8px max(10px,env(safe-area-inset-right)) 6px max(10px,env(safe-area-inset-left));align-items:center;flex-wrap:wrap} + .mpick{order:3;width:100%;flex-basis:100%} + .mtrigger{width:100%;max-width:none;justify-content:space-between;border:1px solid var(--line);background:var(--raise)} + .menu{position:fixed;top:calc(58px + env(safe-area-inset-top));left:max(10px,env(safe-area-inset-left));right:max(10px,env(safe-area-inset-right));width:auto;max-height:min(68dvh,520px);overflow:auto;transform:translateY(-6px)} + .menu.open{transform:none} + .mlist{max-height:none} + .chips{grid-template-columns:1fr} + .welcome{min-height:52vh;padding:20px 16px 28px} + .welcome h1{font-size:25px} + .wrap{padding:12px max(12px,env(safe-area-inset-right)) 8px max(12px,env(safe-area-inset-left))} + .bubble,.atts{max-width:100%} + .acts{gap:4px;flex-wrap:wrap} + .act{min-height:36px;padding:7px 9px} + .act .alabel{display:none} + .dock{padding:6px max(10px,env(safe-area-inset-right)) calc(10px + env(safe-area-inset-bottom)) max(10px,env(safe-area-inset-left));background:linear-gradient(to top,var(--bg) 76%,transparent)} + .composer{border-radius:20px;padding:7px} + .crow{align-items:flex-end} + .composer textarea{min-height:44px;max-height:34dvh;padding:10px 4px;font-size:16px} + .pending{padding:5px 5px 7px} + .pchip{max-width:100%;width:100%} + .recbar{gap:8px;flex-wrap:wrap;padding:6px} + .waves{min-width:120px;flex-basis:100%;order:2} + .rec-x,.rec-ok{flex:1;justify-content:center} + .veil{align-items:end;padding:max(8px,env(safe-area-inset-top)) max(8px,env(safe-area-inset-right)) max(8px,env(safe-area-inset-bottom)) max(8px,env(safe-area-inset-left))} + .sheet{width:100%;max-height:calc(100dvh - env(safe-area-inset-top) - env(safe-area-inset-bottom) - 16px);border-radius:20px} + .sheet-top{padding:16px 16px 0} + .sheet-body{padding:14px 16px 18px;gap:18px} + .two{grid-template-columns:1fr} + .acts2{display:grid;grid-template-columns:1fr} + .btn{width:100%;min-width:0} + .hint{display:none} +} +@media (hover:none){ + .row .rkill{opacity:.75} + .chip:hover{transform:none} + .btn-sm.fill:hover,.btn.fill:hover,.go:hover{filter:none} +} +@media (max-height:520px) and (orientation:landscape){ + .welcome{min-height:auto;justify-content:flex-start;padding-top:10px} + .welcome .wmark,.welcome p,.chips{display:none} + .bar{min-height:46px;padding-top:4px;padding-bottom:4px} + .dock{padding-top:4px} + .composer textarea{max-height:24dvh} + .sheet{max-height:calc(100dvh - 12px)} +} +@media (min-width:881px) and (max-width:1180px){ + :root{--side-w:260px} + .wrap,.composer,.hint{max-width:min(760px,calc(100vw - var(--side-w) - 40px))} + .mtrigger{max-width:min(44vw,360px)} + .acts{flex-wrap:wrap} +} +@media (min-width:1181px){ + .wrap,.composer,.hint{max-width:820px} + .bubble,.atts{max-width:min(82%,760px)} +} +@media (min-width:1680px){ + :root{--side-w:320px} + .wrap,.composer,.hint{max-width:980px} + .welcome{max-width:900px} + .chips{max-width:720px;grid-template-columns:repeat(4,minmax(0,1fr))} +} +@media (prefers-reduced-motion:reduce){ + *{animation-duration:.001s!important;transition-duration:.06s!important;scroll-behavior:auto} + .cursor::after{animation:none;opacity:1} +} diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index d136e06d..4f6f6f91 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -5,364 +5,8 @@ ToolNeuron -
diff --git a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt index 679fbcd5..fe1fd3fe 100644 --- a/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt +++ b/app/src/main/java/com/dark/tool_neuron/data/AppPreferences.kt @@ -234,6 +234,10 @@ class AppPreferences @Inject constructor( get() = getString(KEY_PLUGIN_ONNX_EP, DEFAULT_PLUGIN_ONNX_EP) set(value) = putString(KEY_PLUGIN_ONNX_EP, value) + var webSearchMode: String + get() = getString(KEY_WEB_SEARCH_MODE, DEFAULT_WEB_SEARCH_MODE) + set(value) = putString(KEY_WEB_SEARCH_MODE, value) + fun readAuthState(): AuthState { val sealed = getBytes(KEY_AUTH_STATE) ?: return AuthState.DEFAULT val plaintext = try { @@ -323,6 +327,9 @@ class AppPreferences @Inject constructor( const val KEY_IDLE_UNLOAD_MINUTES = "idle_unload_minutes" const val DEFAULT_IDLE_UNLOAD_MINUTES = 20 + const val KEY_WEB_SEARCH_MODE = "web_search_mode" + const val DEFAULT_WEB_SEARCH_MODE = "auto" + const val KEY_PLUGIN_ONNX_EP = "plugin_onnx_ep" const val PLUGIN_ONNX_EP_CPU = "cpu" const val PLUGIN_ONNX_EP_NNAPI = "nnapi" diff --git a/app/src/main/java/com/dark/tool_neuron/model/ModelTaxonomy.kt b/app/src/main/java/com/dark/tool_neuron/model/ModelTaxonomy.kt index acd74262..b4c90234 100644 --- a/app/src/main/java/com/dark/tool_neuron/model/ModelTaxonomy.kt +++ b/app/src/main/java/com/dark/tool_neuron/model/ModelTaxonomy.kt @@ -7,13 +7,17 @@ enum class ModelFamily(val displayName: String, val priority: Int) { LFM("LFM", 10), QWEN("Qwen", 20), GEMMA("Google Gemma", 30), + SMOLVLM("SmolVLM", 35), + MINICPM("MiniCPM", 36), + LLAVA("LLaVA", 37), SMOLLM("SmolLM", 40), PHI("Phi", 50), DEEPSEEK("DeepSeek", 60), MISTRAL("Mistral", 70), EMBEDDING("Embedding / RAG", 80), - VISION("Vision", 90), + VISION("Other vision", 90), IMAGE_GEN("Image generation", 100), + UPSCALER("Upscalers", 110), OTHER("Other models", 200), } @@ -21,7 +25,6 @@ enum class ModelTask(val displayName: String) { CHAT("Chat"), INSTRUCT("Instruct"), THINKING("Thinking"), - TOOL_SEARCH("Tool/Search"), VISION_CHAT("Vision chat"), EMBEDDING("Embedding"), TTS("Text-to-speech"), @@ -39,10 +42,11 @@ object ModelTaxonomy { .lowercase() return when { - model.modelType == "image_gen" -> ModelFamily.IMAGE_GEN - model.modelType == "image_upscaler" || model.modelType == "tts" || model.modelType == "stt" -> ModelFamily.OTHER + model.isVlm || "vlm" in haystack || "vision" in haystack -> visionFamily(haystack) + model.modelType in IMAGE_MODEL_TYPES -> ModelFamily.IMAGE_GEN + model.modelType == "image_upscaler" -> ModelFamily.UPSCALER + model.modelType == "tts" || model.modelType == "stt" -> ModelFamily.OTHER model.modelType == "embedding" -> ModelFamily.EMBEDDING - model.isVlm || "vlm" in haystack || "vision" in haystack -> ModelFamily.VISION "lfm" in haystack -> ModelFamily.LFM "deepseek" in haystack -> ModelFamily.DEEPSEEK "qwen" in haystack -> ModelFamily.QWEN @@ -50,7 +54,6 @@ object ModelTaxonomy { "smollm" in haystack || "smallm" in haystack -> ModelFamily.SMOLLM "phi" in haystack -> ModelFamily.PHI "mistral" in haystack -> ModelFamily.MISTRAL - model.modelType == "tool_search" -> ModelFamily.OTHER else -> ModelFamily.OTHER } } @@ -58,10 +61,11 @@ object ModelTaxonomy { fun family(model: ModelInfo): ModelFamily { val haystack = listOf(model.id, model.name, model.providerType.name).joinToString(" ").lowercase() return when { - model.providerType == ProviderType.IMAGE_GEN -> ModelFamily.IMAGE_GEN - model.providerType in setOf(ProviderType.TTS, ProviderType.STT, ProviderType.IMAGE_UPSCALER) -> ModelFamily.OTHER + model.providerType == ProviderType.VISION_CHAT || "vl" in haystack || "vision" in haystack -> visionFamily(haystack) + model.providerType in IMAGE_PROVIDER_TYPES -> ModelFamily.IMAGE_GEN + model.providerType == ProviderType.IMAGE_UPSCALER -> ModelFamily.UPSCALER + model.providerType in setOf(ProviderType.TTS, ProviderType.STT) -> ModelFamily.OTHER model.providerType == ProviderType.EMBEDDING -> ModelFamily.EMBEDDING - model.providerType == ProviderType.VISION_CHAT || "vl" in haystack || "vision" in haystack -> ModelFamily.VISION "lfm" in haystack -> ModelFamily.LFM "deepseek" in haystack -> ModelFamily.DEEPSEEK "qwen" in haystack -> ModelFamily.QWEN @@ -84,10 +88,8 @@ object ModelTaxonomy { model.modelType == "tts" -> ModelTask.TTS model.modelType == "stt" -> ModelTask.STT model.modelType == "embedding" -> ModelTask.EMBEDDING - model.modelType == "tool_search" -> ModelTask.TOOL_SEARCH model.isVlm -> ModelTask.VISION_CHAT "thinking" in haystack || "reasoning" in haystack || "deepseek-r1" in haystack -> ModelTask.THINKING - "tool" in haystack || "function" in haystack -> ModelTask.TOOL_SEARCH "instruct" in haystack || "it" in haystack -> ModelTask.INSTRUCT model.modelType == "gguf" -> ModelTask.CHAT else -> ModelTask.OTHER @@ -97,7 +99,7 @@ object ModelTaxonomy { fun task(model: ModelInfo): ModelTask = when (model.providerType) { ProviderType.GGUF -> ModelTask.CHAT ProviderType.VISION_CHAT -> ModelTask.VISION_CHAT - ProviderType.TOOL_SEARCH -> ModelTask.TOOL_SEARCH + ProviderType.TOOL_SEARCH -> ModelTask.CHAT ProviderType.TTS -> ModelTask.TTS ProviderType.STT -> ModelTask.STT ProviderType.EMBEDDING -> ModelTask.EMBEDDING @@ -106,4 +108,22 @@ object ModelTaxonomy { } fun groupKey(model: HuggingFaceModel): String = family(model).name + + private fun visionFamily(haystack: String): ModelFamily = when { + "qwen" in haystack -> ModelFamily.QWEN + "lfm" in haystack || "liquid" in haystack -> ModelFamily.LFM + "gemma" in haystack -> ModelFamily.GEMMA + "smolvlm" in haystack || "smol-vlm" in haystack -> ModelFamily.SMOLVLM + "minicpm" in haystack || "mini-cpm" in haystack -> ModelFamily.MINICPM + "llava" in haystack || "llava" in haystack -> ModelFamily.LLAVA + else -> ModelFamily.VISION + } + + private val IMAGE_MODEL_TYPES = setOf( + "image_gen", + ) + + private val IMAGE_PROVIDER_TYPES = setOf( + ProviderType.IMAGE_GEN, + ) } diff --git a/app/src/main/java/com/dark/tool_neuron/model/NavScreens.kt b/app/src/main/java/com/dark/tool_neuron/model/NavScreens.kt index 377a436c..323fff81 100644 --- a/app/src/main/java/com/dark/tool_neuron/model/NavScreens.kt +++ b/app/src/main/java/com/dark/tool_neuron/model/NavScreens.kt @@ -31,6 +31,7 @@ sealed class NavScreens(val route: String) { object ModelManager : NavScreens("model_manager") object Settings : NavScreens("settings") object SettingsChatRag : NavScreens("settings_chat_rag") + object SettingsWebSearch : NavScreens("settings_web_search") object SettingsVoice : NavScreens("settings_voice") object SettingsTheming : NavScreens("settings_theming") object SettingsPrivacy : NavScreens("settings_privacy") diff --git a/app/src/main/java/com/dark/tool_neuron/model/VlmFileGroup.kt b/app/src/main/java/com/dark/tool_neuron/model/VlmFileGroup.kt new file mode 100644 index 00000000..79003777 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/model/VlmFileGroup.kt @@ -0,0 +1,120 @@ +package com.dark.tool_neuron.model + +import com.dark.tool_neuron.repo.hf.HfSibling +import com.dark.tool_neuron.util.extractQuantization + +data class VlmFileGroup( + val key: String, + val displayName: String, + val baseModels: List = emptyList(), + val projectors: List = emptyList(), + val baseFiles: List = emptyList(), + val projectorFiles: List = emptyList(), +) { + val totalSizeBytes: Long = + baseModels.sumOf { it.sizeBytes } + + projectors.sumOf { it.sizeBytes } + + baseFiles.sumOf { it.sizeBytes } + + projectorFiles.sumOf { it.sizeBytes } + + val quantizations: List = + (baseModels.mapNotNull { it.quantization.takeIf(String::isNotBlank) } + + baseFiles.mapNotNull { extractQuantization(it.path) }) + .distinct() + .sortedWith(quantComparator) + + val preferredModel: HuggingFaceModel? + get() = baseModels.sortedWith( + compareBy { quantRank(it.quantization) } + .thenBy { it.sizeBytes.takeIf { bytes -> bytes > 0 } ?: Long.MAX_VALUE } + .thenBy { it.name.lowercase() }, + ).firstOrNull() +} + +object VlmFileGroups { + fun fromModels(models: List): List = + models.groupBy { normalizedKey(it.fileName.ifBlank { it.name }) } + .map { (key, items) -> + val bases = items.filterNot { it.isMmproj }.sortedWith(modelSorter) + val projectors = items.filter { it.isMmproj }.sortedWith(modelSorter) + VlmFileGroup( + key = key, + displayName = displayNameFor(key, bases.firstOrNull()?.name ?: projectors.firstOrNull()?.name.orEmpty()), + baseModels = bases, + projectors = projectors, + ) + } + .sortedWith(compareBy { it.displayName.lowercase() }.thenBy { it.totalSizeBytes }) + + fun fromFiles(files: List): List = + files.groupBy { normalizedKey(it.path) } + .map { (key, siblings) -> + val bases = siblings.filterNot { it.path.contains("mmproj", ignoreCase = true) } + .sortedWith(fileSorter) + val projectors = siblings.filter { it.path.contains("mmproj", ignoreCase = true) } + .sortedWith(fileSorter) + VlmFileGroup( + key = key, + displayName = displayNameFor(key, bases.firstOrNull()?.path ?: projectors.firstOrNull()?.path.orEmpty()), + baseFiles = bases, + projectorFiles = projectors, + ) + } + .sortedWith(compareBy { it.displayName.lowercase() }.thenBy { it.totalSizeBytes }) + + private fun normalizedKey(raw: String): String { + val leaf = raw.substringAfterLast('/') + .removeSuffix(".gguf") + .removeSuffix(".GGUF") + .lowercase() + return leaf + .replace(Regex("mmproj|projector|vision-projector|multi-modal-projector"), "") + .replace(Regex("q[2-8](_k)?(_[msl])?|iq[1-4]_[a-z0-9_]+|f16|bf16|fp16|q4f16|int8|uint8|bnb4"), "") + .replace(Regex("[._\\-]+"), " ") + .replace(Regex("\\s+"), " ") + .trim() + .ifBlank { leaf } + } + + private fun displayNameFor(key: String, fallback: String): String { + val cleaned = key.split(' ') + .filter { it.isNotBlank() } + .joinToString(" ") { part -> + when { + part.equals("vl", ignoreCase = true) -> "VL" + part.equals("vlm", ignoreCase = true) -> "VLM" + part.equals("llm", ignoreCase = true) -> "LLM" + part.length <= 2 -> part.uppercase() + else -> part.replaceFirstChar { c -> c.titlecase() } + } + } + return cleaned.ifBlank { fallback.substringAfterLast('/').removeSuffix(".gguf") } + } + + private val modelSorter = compareBy { quantRank(it.quantization) } + .thenBy { it.sizeBytes.takeIf { bytes -> bytes > 0 } ?: Long.MAX_VALUE } + .thenBy { it.fileName.lowercase() } + + private val fileSorter = compareBy { quantRank(extractQuantization(it.path).orEmpty()) } + .thenBy { it.sizeBytes.takeIf { bytes -> bytes > 0 } ?: Long.MAX_VALUE } + .thenBy { it.path.lowercase() } +} + +private val quantComparator = Comparator { a, b -> + val rank = quantRank(a).compareTo(quantRank(b)) + if (rank != 0) rank else a.compareTo(b, ignoreCase = true) +} + +private fun quantRank(raw: String): Int = when (raw.uppercase()) { + "Q4_K_M" -> 0 + "Q4_K_S" -> 1 + "Q4_0" -> 2 + "Q5_K_M" -> 3 + "Q5_K_S" -> 4 + "Q6_K" -> 5 + "Q8_0" -> 6 + "F16", "FP16", "BF16" -> 7 + "MMPROJ" -> 100 + "" -> 50 + else -> 20 +} diff --git a/app/src/main/java/com/dark/tool_neuron/model/WebSearchEvent.kt b/app/src/main/java/com/dark/tool_neuron/model/WebSearchEvent.kt index 190e26a2..055a6595 100644 --- a/app/src/main/java/com/dark/tool_neuron/model/WebSearchEvent.kt +++ b/app/src/main/java/com/dark/tool_neuron/model/WebSearchEvent.kt @@ -6,6 +6,20 @@ sealed class WebSearchEvent { data class Plan( override val runId: String, val userQuery: String, + val mode: String = "", + ) : WebSearchEvent() + + data class RoundStart( + override val runId: String, + val round: Int, + val maxRounds: Int, + val focus: String = "", + ) : WebSearchEvent() + + data class Status( + override val runId: String, + val message: String, + val coverage: Int = -1, ) : WebSearchEvent() data class QueriesGenerated( diff --git a/app/src/main/java/com/dark/tool_neuron/model/WebSearchUiState.kt b/app/src/main/java/com/dark/tool_neuron/model/WebSearchUiState.kt index 0feeb6d8..6d7be0f9 100644 --- a/app/src/main/java/com/dark/tool_neuron/model/WebSearchUiState.kt +++ b/app/src/main/java/com/dark/tool_neuron/model/WebSearchUiState.kt @@ -6,6 +6,11 @@ import org.json.JSONObject data class WebSearchUiState( val phase: String = PHASE_PLAN, val userQuery: String = "", + val mode: String = "", + val round: Int = 0, + val maxRounds: Int = 0, + val status: String = "", + val coverage: Int = -1, val queries: List = emptyList(), val perQueryHits: Map> = emptyMap(), val currentQueryIndex: Int = -1, @@ -34,7 +39,21 @@ data class WebSearchUiState( } private fun applyEventInternal(event: WebSearchEvent): WebSearchUiState = when (event) { - is WebSearchEvent.Plan -> copy(phase = PHASE_PLAN, userQuery = event.userQuery) + is WebSearchEvent.Plan -> copy( + phase = PHASE_PLAN, + userQuery = event.userQuery, + mode = event.mode.ifBlank { mode }, + ) + is WebSearchEvent.RoundStart -> copy( + phase = PHASE_SEARCH, + round = event.round, + maxRounds = event.maxRounds, + status = event.focus, + ) + is WebSearchEvent.Status -> copy( + status = event.message, + coverage = if (event.coverage in 0..100) event.coverage else coverage, + ) is WebSearchEvent.QueriesGenerated -> copy( phase = PHASE_QUERIES, queries = event.queries, @@ -62,6 +81,11 @@ data class WebSearchUiState( val obj = JSONObject() obj.put("phase", phase) if (userQuery.isNotEmpty()) obj.put("uq", userQuery) + if (mode.isNotEmpty()) obj.put("mode", mode) + if (round > 0) obj.put("rnd", round) + if (maxRounds > 0) obj.put("mr", maxRounds) + if (status.isNotEmpty()) obj.put("st", status) + if (coverage in 0..100) obj.put("cov", coverage) if (queries.isNotEmpty()) { val arr = JSONArray() queries.forEach { arr.put(it) } @@ -132,6 +156,11 @@ data class WebSearchUiState( WebSearchUiState( phase = o.optString("phase", PHASE_PLAN).ifBlank { PHASE_PLAN }, userQuery = o.optString("uq"), + mode = o.optString("mode"), + round = o.optInt("rnd", 0), + maxRounds = o.optInt("mr", 0), + status = o.optString("st"), + coverage = o.optInt("cov", -1), queries = queries, perQueryHits = perQueryHits, currentQueryIndex = o.optInt("cqi", -1), diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ImageGenManager.kt b/app/src/main/java/com/dark/tool_neuron/repo/ImageGenManager.kt index b859a412..8c86df94 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/ImageGenManager.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/ImageGenManager.kt @@ -18,6 +18,7 @@ import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import java.io.File @@ -208,6 +209,16 @@ class ImageGenManager @Inject constructor( InferenceClient.sdUpscale(bitmap) } + suspend fun upscaleOnce(bitmap: Bitmap): Bitmap { + InferenceClient.sdResetUpscaleState() + InferenceClient.sdUpscale(bitmap) + return when (val state = upscaleState.first { it is UpscaleState.Complete || it is UpscaleState.Error }) { + is UpscaleState.Complete -> state.bitmap + is UpscaleState.Error -> throw IllegalStateException(state.message) + else -> throw IllegalStateException("Upscale stopped") + } + } + fun stop() { InferenceClient.sdStop() loadedModelId = "" diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ImageUpscalePipeline.kt b/app/src/main/java/com/dark/tool_neuron/repo/ImageUpscalePipeline.kt new file mode 100644 index 00000000..7f266101 --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/repo/ImageUpscalePipeline.kt @@ -0,0 +1,438 @@ +package com.dark.tool_neuron.repo + +import android.app.ActivityManager +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.BitmapRegionDecoder +import android.graphics.Rect +import android.net.Uri +import androidx.core.graphics.scale +import java.io.BufferedOutputStream +import java.io.File +import java.io.FileInputStream +import java.io.OutputStream +import java.io.RandomAccessFile +import java.util.UUID +import java.util.zip.CRC32 +import java.util.zip.Deflater +import java.util.zip.DeflaterOutputStream +import kotlin.math.ceil +import kotlin.math.ln +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +enum class UpscaleScalePreset { X2, X4, X8, CUSTOM } +enum class UpscaleProcessingMode { AUTO, FAST, SAFE_TILED } + +data class UpscalePipelineProgress( + val label: String, + val progress: Float, + val currentTile: Int = 0, + val totalTiles: Int = 0, + val currentPass: Int = 0, + val totalPasses: Int = 0, +) + +data class UpscalePipelineResult( + val preview: Bitmap, + val fullWidth: Int, + val fullHeight: Int, + val outputFile: File?, + val scale: Float, + val tiled: Boolean, + val elapsedMs: Long, +) + +class FastUpscaleBlockedException(message: String) : IllegalStateException(message) + +class ImageUpscalePipeline( + private val context: Context, + private val imageGen: ImageGenManager, +) { + suspend fun run( + imagePath: String, + scale: Float, + mode: UpscaleProcessingMode, + outputDir: File, + onProgress: (UpscalePipelineProgress) -> Unit, + ): UpscalePipelineResult { + val started = System.currentTimeMillis() + outputDir.mkdirs() + val bounds = readBounds(imagePath) + val targetWidth = (bounds.width * scale).roundToInt().coerceAtLeast(1) + val targetHeight = (bounds.height * scale).roundToInt().coerceAtLeast(1) + val oneShotRisky = isOneShotRisky(bounds.width, bounds.height, targetWidth, targetHeight, scale) + val tiled = when (mode) { + UpscaleProcessingMode.SAFE_TILED -> true + UpscaleProcessingMode.AUTO -> oneShotRisky + UpscaleProcessingMode.FAST -> { + if (oneShotRisky) { + throw FastUpscaleBlockedException("Fast mode is risky for this image. Switch to Safe tiled to process it reliably.") + } + false + } + } + onProgress(UpscalePipelineProgress(if (tiled) "Preparing tiles" else "Preparing", 0.02f)) + val result = if (tiled) runTiled(imagePath, bounds, scale, targetWidth, targetHeight, outputDir, onProgress) + else runOneShot(imagePath, scale, targetWidth, targetHeight, onProgress) + return result.copy(elapsedMs = (System.currentTimeMillis() - started).coerceAtLeast(1L)) + } + + private suspend fun runOneShot( + imagePath: String, + scale: Float, + targetWidth: Int, + targetHeight: Int, + onProgress: (UpscalePipelineProgress) -> Unit, + ): UpscalePipelineResult { + val source = decodeBitmap(imagePath) ?: error("Couldn't read input image") + val out = upscaleExact(source, scale, onProgress) + val exact = if (out.width == targetWidth && out.height == targetHeight) out + else out.scale(targetWidth, targetHeight) + onProgress(UpscalePipelineProgress("Writing image", 0.98f)) + return UpscalePipelineResult( + preview = exact, + fullWidth = exact.width, + fullHeight = exact.height, + outputFile = null, + scale = scale, + tiled = false, + elapsedMs = 0L, + ) + } + + private suspend fun runTiled( + imagePath: String, + bounds: ImageBounds, + scale: Float, + targetWidth: Int, + targetHeight: Int, + outputDir: File, + onProgress: (UpscalePipelineProgress) -> Unit, + ): UpscalePipelineResult { + val rawFile = File(outputDir, "upscale_${UUID.randomUUID()}.rgba") + val pngFile = File(outputDir, "upscale_${System.currentTimeMillis()}.png") + val tiles = buildTiles(bounds.width, bounds.height) + RawRgbaCanvas(rawFile, targetWidth, targetHeight).use { canvas -> + tiles.forEachIndexed { index, tile -> + onProgress( + UpscalePipelineProgress( + label = "Tile ${index + 1}/${tiles.size}", + progress = (index.toFloat() / tiles.size.toFloat()).coerceIn(0.02f, 0.92f), + currentTile = index + 1, + totalTiles = tiles.size, + ), + ) + val region = decodeRegion(imagePath, tile.readRect) ?: error("Couldn't decode tile") + val upscaled = upscaleExact(region, scale) { pass -> + onProgress( + pass.copy( + label = "Tile ${index + 1}/${tiles.size}", + progress = ((index.toFloat() + pass.progress) / tiles.size.toFloat()).coerceIn(0.02f, 0.92f), + currentTile = index + 1, + totalTiles = tiles.size, + ), + ) + } + val crop = tile.cropInReadRect(scale, upscaled.width, upscaled.height) + val exact = Bitmap.createBitmap(upscaled, crop.left, crop.top, crop.width(), crop.height()) + canvas.writeBitmap( + bitmap = exact, + dstX = (tile.contentRect.left * scale).roundToInt(), + dstY = (tile.contentRect.top * scale).roundToInt(), + ) + if (exact !== upscaled) exact.recycle() + upscaled.recycle() + region.recycle() + } + onProgress(UpscalePipelineProgress("Writing image", 0.96f)) + StreamingPngWriter.write(canvas.file, pngFile, targetWidth, targetHeight) + val preview = canvas.preview(maxEdge = 1600) + return UpscalePipelineResult( + preview = preview, + fullWidth = targetWidth, + fullHeight = targetHeight, + outputFile = pngFile, + scale = scale, + tiled = true, + elapsedMs = 0L, + ) + } + } + + private suspend fun upscaleExact( + source: Bitmap, + scale: Float, + onProgress: (UpscalePipelineProgress) -> Unit = {}, + ): Bitmap { + val passes = max(1, ceil(ln(scale.toDouble()) / ln(4.0)).toInt()) + var current = source + repeat(passes) { index -> + onProgress( + UpscalePipelineProgress( + label = "Pass ${index + 1}/$passes", + progress = index.toFloat() / passes.toFloat(), + currentPass = index + 1, + totalPasses = passes, + ), + ) + val next = imageGen.upscaleOnce(current) + if (current !== source) current.recycle() + current = next + } + val targetW = (source.width * scale).roundToInt().coerceAtLeast(1) + val targetH = (source.height * scale).roundToInt().coerceAtLeast(1) + return if (current.width == targetW && current.height == targetH) current else { + val exact = current.scale(targetW, targetH) + current.recycle() + exact + } + } + + private fun isOneShotRisky( + sourceWidth: Int, + sourceHeight: Int, + targetWidth: Int, + targetHeight: Int, + scale: Float, + ): Boolean { + if (max(sourceWidth, sourceHeight) > SAFE_NATIVE_INPUT_EDGE) return true + val nativePasses = max(1, ceil(ln(scale.toDouble()) / ln(4.0)).toInt()) + val largestMultiplier = if (nativePasses <= 1) 4f else 16f + val sourceBytes = sourceWidth.toLong() * sourceHeight.toLong() * 4L + val intermediateBytes = (sourceWidth * largestMultiplier).toLong() * + (sourceHeight * largestMultiplier).toLong() * 4L + val targetBytes = targetWidth.toLong() * targetHeight.toLong() * 4L + val riskyBytes = sourceBytes + intermediateBytes + targetBytes + val runtime = Runtime.getRuntime() + val available = (runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())).coerceAtLeast(0L) + val memoryClassBytes = ((context.getSystemService(Context.ACTIVITY_SERVICE) as? ActivityManager) + ?.largeMemoryClass ?: 256).toLong() * 1024L * 1024L + val budget = min(available, memoryClassBytes / 2L).coerceAtLeast(96L * 1024L * 1024L) + return riskyBytes > budget + } + + private fun buildTiles(width: Int, height: Int): List { + val tiles = mutableListOf() + var y = 0 + while (y < height) { + val contentBottom = min(y + TILE_CONTENT_EDGE, height) + var x = 0 + while (x < width) { + val contentRight = min(x + TILE_CONTENT_EDGE, width) + val content = Rect(x, y, contentRight, contentBottom) + val read = Rect( + max(0, content.left - TILE_OVERLAP), + max(0, content.top - TILE_OVERLAP), + min(width, content.right + TILE_OVERLAP), + min(height, content.bottom + TILE_OVERLAP), + ) + tiles += UpscaleTile(read, content) + x += TILE_CONTENT_EDGE + } + y += TILE_CONTENT_EDGE + } + return tiles + } + + private fun readBounds(path: String): ImageBounds { + val options = BitmapFactory.Options().apply { inJustDecodeBounds = true } + openInput(path).use { BitmapFactory.decodeStream(it, null, options) } + check(options.outWidth > 0 && options.outHeight > 0) { "Couldn't read input image bounds" } + return ImageBounds(options.outWidth, options.outHeight) + } + + private fun decodeBitmap(path: String): Bitmap? = + openInput(path).use { BitmapFactory.decodeStream(it) } + + @Suppress("DEPRECATION") + private fun decodeRegion(path: String, rect: Rect): Bitmap? = + openInput(path).use { input -> + val decoder = BitmapRegionDecoder.newInstance(input, false) ?: return null + try { + decoder.decodeRegion(rect, BitmapFactory.Options()) + } finally { + decoder.recycle() + } + } + + private fun openInput(path: String) = + if (path.startsWith("content://") || path.startsWith("file://")) { + context.contentResolver.openInputStream(Uri.parse(path)) + ?: error("Couldn't open input image") + } else { + FileInputStream(path) + } + + private data class ImageBounds(val width: Int, val height: Int) + + private data class UpscaleTile( + val readRect: Rect, + val contentRect: Rect, + ) { + fun cropInReadRect(scale: Float, upscaledWidth: Int, upscaledHeight: Int): Rect { + val left = ((contentRect.left - readRect.left) * scale).roundToInt().coerceIn(0, upscaledWidth - 1) + val top = ((contentRect.top - readRect.top) * scale).roundToInt().coerceIn(0, upscaledHeight - 1) + val right = (left + (contentRect.width() * scale).roundToInt()).coerceIn(left + 1, upscaledWidth) + val bottom = (top + (contentRect.height() * scale).roundToInt()).coerceIn(top + 1, upscaledHeight) + return Rect(left, top, right, bottom) + } + } + + private class RawRgbaCanvas( + val file: File, + private val width: Int, + private val height: Int, + ) : AutoCloseable { + private val raf = RandomAccessFile(file, "rw") + private val rgba = ByteArray(width * 4) + + init { + raf.setLength(width.toLong() * height.toLong() * 4L) + } + + fun writeBitmap(bitmap: Bitmap, dstX: Int, dstY: Int) { + val safeWidth = min(bitmap.width, width - dstX).coerceAtLeast(0) + val safeHeight = min(bitmap.height, height - dstY).coerceAtLeast(0) + if (safeWidth == 0 || safeHeight == 0) return + val pixels = IntArray(safeWidth) + val bytes = ByteArray(safeWidth * 4) + repeat(safeHeight) { y -> + bitmap.getPixels(pixels, 0, safeWidth, 0, y, safeWidth, 1) + pixelsToRgba(pixels, bytes, safeWidth) + raf.seek(((dstY + y).toLong() * width.toLong() + dstX.toLong()) * 4L) + raf.write(bytes) + } + } + + fun preview(maxEdge: Int): Bitmap { + val scale = min(1f, maxEdge.toFloat() / max(width, height).toFloat()) + val previewW = (width * scale).roundToInt().coerceAtLeast(1) + val previewH = (height * scale).roundToInt().coerceAtLeast(1) + val preview = Bitmap.createBitmap(previewW, previewH, Bitmap.Config.ARGB_8888) + val pixel = IntArray(1) + repeat(previewH) { py -> + val sy = ((py + 0.5f) / scale).toInt().coerceIn(0, height - 1) + repeat(previewW) { px -> + val sx = ((px + 0.5f) / scale).toInt().coerceIn(0, width - 1) + readPixel(sx, sy, pixel) + preview.setPixel(px, py, pixel[0]) + } + } + return preview + } + + private fun readPixel(x: Int, y: Int, out: IntArray) { + raf.seek((y.toLong() * width.toLong() + x.toLong()) * 4L) + raf.readFully(rgba, 0, 4) + val r = rgba[0].toInt() and 0xff + val g = rgba[1].toInt() and 0xff + val b = rgba[2].toInt() and 0xff + val a = rgba[3].toInt() and 0xff + out[0] = (a shl 24) or (r shl 16) or (g shl 8) or b + } + + override fun close() { + raf.close() + file.delete() + } + + private fun pixelsToRgba(pixels: IntArray, bytes: ByteArray, count: Int) { + repeat(count) { i -> + val argb = pixels[i] + val o = i * 4 + bytes[o] = ((argb shr 16) and 0xff).toByte() + bytes[o + 1] = ((argb shr 8) and 0xff).toByte() + bytes[o + 2] = (argb and 0xff).toByte() + bytes[o + 3] = ((argb ushr 24) and 0xff).toByte() + } + } + } + + private object StreamingPngWriter { + fun write(rawFile: File, pngFile: File, width: Int, height: Int) { + val zdat = File(pngFile.parentFile, "${pngFile.name}.zdat") + val row = ByteArray(width * 4) + FileInputStream(rawFile).use { raw -> + DeflaterOutputStream( + BufferedOutputStream(zdat.outputStream()), + Deflater(Deflater.DEFAULT_COMPRESSION), + ).use { deflated -> + repeat(height) { + deflated.write(0) + raw.readFully(row) + deflated.write(row) + } + } + } + BufferedOutputStream(pngFile.outputStream()).use { out -> + out.write(PNG_SIGNATURE) + writeChunk(out, "IHDR", ihdr(width, height)) + FileInputStream(zdat).use { input -> + val data = input.readBytes() + writeChunk(out, "IDAT", data) + } + writeChunk(out, "IEND", ByteArray(0)) + } + zdat.delete() + } + + private fun ihdr(width: Int, height: Int): ByteArray { + val data = ByteArray(13) + writeInt(data, 0, width) + writeInt(data, 4, height) + data[8] = 8 + data[9] = 6 + data[10] = 0 + data[11] = 0 + data[12] = 0 + return data + } + + private fun writeChunk(out: OutputStream, type: String, data: ByteArray) { + val typeBytes = type.encodeToByteArray() + val crc = CRC32() + writeInt(out, data.size) + out.write(typeBytes) + out.write(data) + crc.update(typeBytes) + crc.update(data) + writeInt(out, crc.value.toInt()) + } + + private fun writeInt(out: OutputStream, value: Int) { + out.write((value ushr 24) and 0xff) + out.write((value ushr 16) and 0xff) + out.write((value ushr 8) and 0xff) + out.write(value and 0xff) + } + + private fun writeInt(data: ByteArray, offset: Int, value: Int) { + data[offset] = ((value ushr 24) and 0xff).toByte() + data[offset + 1] = ((value ushr 16) and 0xff).toByte() + data[offset + 2] = ((value ushr 8) and 0xff).toByte() + data[offset + 3] = (value and 0xff).toByte() + } + + private val PNG_SIGNATURE = byteArrayOf( + 137.toByte(), 80, 78, 71, 13, 10, 26, 10, + ) + } + + companion object { + const val SAFE_NATIVE_INPUT_EDGE = 1024 + private const val TILE_OVERLAP = 32 + private const val TILE_CONTENT_EDGE = SAFE_NATIVE_INPUT_EDGE - (TILE_OVERLAP * 2) + } +} + +private fun FileInputStream.readFully(buffer: ByteArray) { + var offset = 0 + while (offset < buffer.size) { + val read = read(buffer, offset, buffer.size - offset) + if (read < 0) error("Unexpected EOF") + offset += read + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ModelCatalog.kt b/app/src/main/java/com/dark/tool_neuron/repo/ModelCatalog.kt index 7474d1ac..311c0fcc 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/ModelCatalog.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/ModelCatalog.kt @@ -1,12 +1,12 @@ package com.dark.tool_neuron.repo import android.content.Context +import com.dark.download_manager.formatBytes import com.dark.tool_neuron.data.SocBucket import com.dark.tool_neuron.model.HFRepository import com.dark.tool_neuron.model.HuggingFaceModel import com.dark.tool_neuron.util.VlmPaths import com.dark.tool_neuron.util.extractQuantization -import com.dark.download_manager.formatBytes import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -276,7 +276,12 @@ class ModelCatalog @Inject constructor( mmprojSizeBytes = mmprojFile?.second ?: 0L, )) } - models.sortedWith(compareByDescending { it.isMmproj }.thenBy { it.sizeBytes }) + models.sortedWith( + compareBy { it.name.lowercase() } + .thenBy { it.isMmproj } + .thenBy { it.quantization } + .thenBy { it.fileName.lowercase() } + ) } private fun inferModelType(repo: HFRepository, meta: JSONObject): String { @@ -450,40 +455,17 @@ class ModelCatalog @Inject constructor( modelType = "embedding", ), HuggingFaceModel( - id = "smollm2-360m-tool-search-q8_0", - name = "SmolLM2 360M Tool/Search (Q8)", - fileName = "smollm2-360m-instruct-q8_0.gguf", - fileUri = "${HuggingFaceApi.BASE}/HuggingFaceTB/SmolLM2-360M-Instruct-GGUF/resolve/main/smollm2-360m-instruct-q8_0.gguf", - approximateSize = "~386 MB", - sizeBytes = 386_000_000L, - repoId = "tool-search-built-in", - quantization = "Q8_0", - tags = listOf("Tool/Search", "Small", "Instruct", "SmolLM"), - modelType = "tool_search", - ), - HuggingFaceModel( - id = "qwen3-0.6b-tool-search-q8_0", - name = "Qwen3 0.6B Tool/Search (Q8)", - fileName = "Qwen3-0.6B-Q8_0.gguf", - fileUri = "${HuggingFaceApi.BASE}/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf", - approximateSize = "~650 MB", - sizeBytes = 650_000_000L, - repoId = "tool-search-built-in", - quantization = "Q8_0", - tags = listOf("Tool/Search", "Qwen", "Small"), - modelType = "tool_search", - ), - HuggingFaceModel( - id = "lfm25-350m-tool-search-q4_k_m", - name = "LFM 2.5 350M Tool/Search (Q4)", - fileName = "LFM2.5-350M-Q4_K_M.gguf", - fileUri = "${HuggingFaceApi.BASE}/LiquidAI/LFM2.5-350M-GGUF/resolve/main/LFM2.5-350M-Q4_K_M.gguf", - approximateSize = "~229 MB", - sizeBytes = 229_312_224L, - repoId = "tool-search-built-in", - quantization = "Q4_K_M", - tags = listOf("Tool/Search", "LFM", "Small"), - modelType = "tool_search", + id = "mnn-upscaler-nmkd-upgiflite-v2-x2", + name = "NMKD UpgifLiteV2 x2 (MNN)", + fileName = "2x_NMKD-UpgifLiteV2_210k.mnn", + fileUri = "${HuggingFaceApi.BASE}/tumuyan2/realsr-models/resolve/main/models-MNNSR/2x_NMKD-UpgifLiteV2_210k.mnn", + approximateSize = "~20 MB", + sizeBytes = 20_170_128L, + repoId = "upscaler-built-in", + tags = listOf("Upscaler", "MNN", "2x", "Use: General", "Style: Natural", "runtime=MNN", "scale=2x"), + modelType = "image_upscaler", + isUpscaler = true, + featureLabel = "Upscale 2x", ), HuggingFaceModel( id = "mnn-upscaler-nmkd-upgiflite-v2-x4", @@ -493,7 +475,7 @@ class ModelCatalog @Inject constructor( approximateSize = "~20 MB", sizeBytes = 20_207_664L, repoId = "upscaler-built-in", - tags = listOf("Upscaler", "MNN", "4x", "Small"), + tags = listOf("Upscaler", "MNN", "4x", "Use: General", "Style: Balanced", "runtime=MNN", "scale=4x"), modelType = "image_upscaler", isUpscaler = true, featureLabel = "Upscale 4x", @@ -506,7 +488,7 @@ class ModelCatalog @Inject constructor( approximateSize = "~18 MB", sizeBytes = 17_934_208L, repoId = "upscaler-built-in", - tags = listOf("Upscaler", "MNN", "4x", "Anime"), + tags = listOf("Upscaler", "MNN", "4x", "Use: Anime", "Style: Line art", "runtime=MNN", "scale=4x"), modelType = "image_upscaler", isUpscaler = true, featureLabel = "Upscale 4x", @@ -519,7 +501,59 @@ class ModelCatalog @Inject constructor( approximateSize = "~34 MB", sizeBytes = 33_566_440L, repoId = "upscaler-built-in", - tags = listOf("Upscaler", "MNN", "4x", "Photo"), + tags = listOf("Upscaler", "MNN", "4x", "Use: Photo", "Style: Natural detail", "runtime=MNN", "scale=4x"), + modelType = "image_upscaler", + isUpscaler = true, + featureLabel = "Upscale 4x", + ), + HuggingFaceModel( + id = "mnn-upscaler-realesrgan-sourcebook-x4", + name = "RealESRGAN SourceBook x4 (MNN)", + fileName = "RealESRGAN-SourceBook-latest.mnn", + fileUri = "${HuggingFaceApi.BASE}/tumuyan2/realsr-models/resolve/main/models-MNNSR/RealESRGAN-SourceBook-latest.mnn", + approximateSize = "~34 MB", + sizeBytes = 33_648_656L, + repoId = "upscaler-built-in", + tags = listOf("Upscaler", "MNN", "4x", "Use: Portrait", "Style: Soft realistic", "runtime=MNN", "scale=4x"), + modelType = "image_upscaler", + isUpscaler = true, + featureLabel = "Upscale 4x", + ), + HuggingFaceModel( + id = "mnn-upscaler-wtp-colords-x4", + name = "ESRGAN WTP ColorDS x4 (MNN)", + fileName = "ESRGAN-WTP-ColorDS-x4.mnn", + fileUri = "${HuggingFaceApi.BASE}/tumuyan2/realsr-models/resolve/main/models-MNNSR/ESRGAN-WTP-ColorDS-x4.mnn", + approximateSize = "~34 MB", + sizeBytes = 33_637_744L, + repoId = "upscaler-built-in", + tags = listOf("Upscaler", "MNN", "4x", "Use: Sharp cleanup", "Style: Crisp", "runtime=MNN", "scale=4x"), + modelType = "image_upscaler", + isUpscaler = true, + featureLabel = "Upscale 4x", + ), + HuggingFaceModel( + id = "mnn-upscaler-moesr-illustration-fix1-x4", + name = "MoeSR Illustration fix1 x4 (MNN)", + fileName = "MoeSR-ESRGAN-jp_Illustration-fix1-d-x4.mnn", + fileUri = "${HuggingFaceApi.BASE}/tumuyan2/realsr-models/resolve/main/models-MNNSR/MoeSR-ESRGAN-jp_Illustration-fix1-d-x4.mnn", + approximateSize = "~34 MB", + sizeBytes = 33_637_600L, + repoId = "upscaler-built-in", + tags = listOf("Upscaler", "MNN", "4x", "Use: Anime", "Style: Illustration", "runtime=MNN", "scale=4x"), + modelType = "image_upscaler", + isUpscaler = true, + featureLabel = "Upscale 4x", + ), + HuggingFaceModel( + id = "mnn-upscaler-moesr-illustration-fix2-x4", + name = "MoeSR Illustration fix2 x4 (MNN)", + fileName = "MoeSR-ESRGAN-jp_Illustration-fix2-x4.mnn", + fileUri = "${HuggingFaceApi.BASE}/tumuyan2/realsr-models/resolve/main/models-MNNSR/MoeSR-ESRGAN-jp_Illustration-fix2-x4.mnn", + approximateSize = "~34 MB", + sizeBytes = 33_637_600L, + repoId = "upscaler-built-in", + tags = listOf("Upscaler", "MNN", "4x", "Use: Other", "Style: Illustration", "runtime=MNN", "scale=4x"), modelType = "image_upscaler", isUpscaler = true, featureLabel = "Upscale 4x", diff --git a/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt b/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt index 91ddae72..d797c725 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/ModelRepository.kt @@ -40,6 +40,26 @@ class ModelRepository @Inject constructor( } fun refresh() { + val records = storage.getAll(COL_MODELS) + val removed = records.filter { it.getString(TAG_PROVIDER_TYPE) in REMOVED_IMAGE_OPERATION_TYPES } + val toolSearch = records.filter { it.getString(TAG_PROVIDER_TYPE) == LEGACY_TOOL_SEARCH_PROVIDER } + if (removed.isNotEmpty()) { + removed.forEach { record -> + val modelId = record.getString(TAG_ID) + storage.delete(COL_MODELS, record.id) + storage.queryString(COL_CONFIG, TAG_CONFIG_MODEL_ID, modelId) + .forEach { storage.delete(COL_CONFIG, it.id) } + } + storage.flushAll() + File(context.filesDir, "image_onnx").deleteRecursively() + } + if (toolSearch.isNotEmpty()) { + toolSearch.forEach { record -> + record.putString(TAG_PROVIDER_TYPE, ProviderType.GGUF.name) + storage.update(COL_MODELS, record) + } + storage.flush(COL_MODELS) + } val all = storage.getAll(COL_MODELS).map { it.toModelInfo() } val seen = HashSet(all.size) _models.value = all.filter { seen.add(it.id) } @@ -183,11 +203,15 @@ class ModelRepository @Inject constructor( return File(parent, leaf) } - - companion object { private const val COL_MODELS = "models" private const val COL_CONFIG = "model_config" + private val REMOVED_IMAGE_OPERATION_TYPES = setOf( + "IMAGE_BACKGROUND_REMOVAL", + "IMAGE_SEGMENTATION", + "IMAGE_INPAINTER", + ) + private const val LEGACY_TOOL_SEARCH_PROVIDER = "TOOL_SEARCH" private const val TAG_ID = 1 private const val TAG_NAME = 2 diff --git a/app/src/main/java/com/dark/tool_neuron/repo/StorageInspector.kt b/app/src/main/java/com/dark/tool_neuron/repo/StorageInspector.kt index 55ba76c5..18715b13 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/StorageInspector.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/StorageInspector.kt @@ -2,9 +2,11 @@ package com.dark.tool_neuron.repo import android.content.Context import com.dark.tool_neuron.model.enums.ProviderType +import com.dark.tool_neuron.util.VlmPaths import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.json.JSONObject import java.io.File import javax.inject.Inject import javax.inject.Singleton @@ -30,6 +32,26 @@ data class StorageSnapshot( val categories: Map, ) +enum class StorageMaintenanceMode { QUICK_CLEAN, DETAILED_CHECK, DEEP_MODEL_TEST } + +enum class StorageIssueSeverity { INFO, WARNING, ERROR } + +data class StorageMaintenanceIssue( + val title: String, + val detail: String, + val severity: StorageIssueSeverity, + val fixed: Boolean = false, +) + +data class StorageMaintenanceReport( + val mode: StorageMaintenanceMode, + val checkedCount: Int, + val issueCount: Int, + val fixedCount: Int, + val skippedCount: Int, + val issues: List, +) + @Singleton class StorageInspector @Inject constructor( @ApplicationContext private val context: Context, @@ -116,6 +138,91 @@ class StorageInspector @Inject constructor( } } + suspend fun maintenance(mode: StorageMaintenanceMode): StorageMaintenanceReport = withContext(Dispatchers.IO) { + val issues = mutableListOf() + var checked = 0 + var fixed = 0 + var skipped = 0 + + fun add(issue: StorageMaintenanceIssue) { + issues += issue + if (issue.fixed) fixed++ + } + + if (mode == StorageMaintenanceMode.QUICK_CLEAN) { + val roots = listOf(context.filesDir, context.cacheDir) + roots.forEach { root -> + root.walkTopDown() + .filter { it.isFile && (it.name.endsWith(".hxd_tmp") || it.name.startsWith("_archive_")) } + .forEach { file -> + checked++ + val bytes = file.length() + val deleted = file.delete() + add(StorageMaintenanceIssue( + title = if (deleted) "Removed leftover file" else "Couldn't remove leftover file", + detail = "${file.name} · ${bytes} bytes", + severity = if (deleted) StorageIssueSeverity.INFO else StorageIssueSeverity.WARNING, + fixed = deleted, + )) + } + } + context.cacheDir.deleteRecursively() + context.cacheDir.mkdirs() + add(StorageMaintenanceIssue("Cache refreshed", "Temporary cache directory recreated", StorageIssueSeverity.INFO, fixed = true)) + return@withContext StorageMaintenanceReport(mode, checked, issues.size, fixed, skipped, issues.take(80)) + } + + modelRepo.models.value.forEach { model -> + checked++ + val path = model.path + val file = File(path) + when { + path.isBlank() -> add(StorageMaintenanceIssue("Broken model record", model.name, StorageIssueSeverity.ERROR)) + !path.startsWith("content://") && !file.exists() -> + add(StorageMaintenanceIssue("Missing model file", "${model.name} · $path", StorageIssueSeverity.ERROR)) + !path.startsWith("content://") && file.isFile && file.length() <= 0L -> + add(StorageMaintenanceIssue("Empty model file", "${model.name} · $path", StorageIssueSeverity.ERROR)) + !path.startsWith("content://") && file.isDirectory && file.walkTopDown().none { it.isFile && it.length() > 0L } -> + add(StorageMaintenanceIssue("Empty model folder", "${model.name} · $path", StorageIssueSeverity.ERROR)) + } + if (model.providerType == ProviderType.VISION_CHAT && !path.startsWith("content://")) { + val projector = VlmPaths.colocatedMmproj(file) + if (projector == null || !projector.exists() || projector.length() <= 0L) { + add(StorageMaintenanceIssue("Vision projector missing", model.name, StorageIssueSeverity.ERROR)) + } + } + if (mode == StorageMaintenanceMode.DEEP_MODEL_TEST) { + skipped += deepCheck(model.providerType, file, path, model.id, model.name, ::add) + } + } + + val referenced = modelRepo.models.value.asSequence() + .mapNotNull { it.path.takeIf { p -> p.isNotBlank() && !p.startsWith("content://") } } + .map { File(it).canonicalFile } + .toSet() + val modelRoots = listOf( + File(context.filesDir, "models"), + File(context.filesDir, "sd_models"), + File(context.filesDir, "sd_upscalers"), + File(context.filesDir, "voice"), + ) + modelRoots.filter { it.exists() }.forEach { root -> + root.walkTopDown() + .filter { it.isFile && it.length() > 0L } + .forEach { file -> + val canonical = file.canonicalFile + val owned = referenced.any { ref -> + canonical == ref || canonical.path.startsWith(ref.path + File.separator) || ref.path.startsWith(canonical.parentFile?.path.orEmpty()) + } + if (!owned && mode == StorageMaintenanceMode.DETAILED_CHECK) { + add(StorageMaintenanceIssue("Possible orphan file", file.path, StorageIssueSeverity.WARNING)) + } + } + } + + StorageMaintenanceReport(mode, checked, issues.size, fixed, skipped, issues.take(120)) + } + private fun clearChatModels() { val models = modelRepo.models.value.filter { isChatModel(it.path, it.providerType) } models.forEach { model -> @@ -192,4 +299,53 @@ class StorageInspector @Inject constructor( } return total } + + private fun deepCheck( + type: ProviderType, + file: File, + path: String, + modelId: String, + name: String, + add: (StorageMaintenanceIssue) -> Unit, + ): Int { + if (path.startsWith("content://")) { + add(StorageMaintenanceIssue("Skipped content URI", name, StorageIssueSeverity.INFO)) + return 1 + } + if (!file.exists()) return 0 + when (type) { + ProviderType.GGUF, ProviderType.VISION_CHAT, ProviderType.TOOL_SEARCH, ProviderType.EMBEDDING -> { + val gguf = if (file.isFile) file else file.walkTopDown().firstOrNull { it.isFile && it.extension.equals("gguf", true) } + if (gguf == null) add(StorageMaintenanceIssue("GGUF file missing", name, StorageIssueSeverity.ERROR)) + else if (!hasMagic(gguf, "GGUF")) add(StorageMaintenanceIssue("GGUF header check failed", gguf.name, StorageIssueSeverity.ERROR)) + } + ProviderType.IMAGE_UPSCALER -> { + if (!file.extension.equals("mnn", true) && !file.extension.equals("bin", true)) { + add(StorageMaintenanceIssue("Unexpected upscaler file type", file.name, StorageIssueSeverity.WARNING)) + } + } + ProviderType.IMAGE_GEN -> { + val configLike = file.walkTopDown().any { it.isFile && it.name in setOf("tokenizer.json", "unet.bin", "unet.mnn", "clip.mnn", "vae_decoder.mnn") } + if (!configLike) add(StorageMaintenanceIssue("Image model layout looks incomplete", name, StorageIssueSeverity.ERROR)) + } + ProviderType.TTS, ProviderType.STT -> { + val hasConfig = file.walkTopDown().any { it.isFile && it.extension.equals("json", true) } + if (!hasConfig) add(StorageMaintenanceIssue("Voice config not found", name, StorageIssueSeverity.WARNING)) + } + } + modelRepo.getConfig(modelId)?.let { config -> + runCatching { JSONObject(config.loadingParamsJson.ifBlank { "{}" }) } + .onFailure { add(StorageMaintenanceIssue("Loading config JSON invalid", name, StorageIssueSeverity.ERROR)) } + runCatching { JSONObject(config.inferenceParamsJson.ifBlank { "{}" }) } + .onFailure { add(StorageMaintenanceIssue("Inference config JSON invalid", name, StorageIssueSeverity.ERROR)) } + } + return 0 + } + + private fun hasMagic(file: File, magic: String): Boolean = runCatching { + file.inputStream().use { input -> + val bytes = ByteArray(magic.length) + input.read(bytes) == bytes.size && bytes.decodeToString() == magic + } + }.getOrDefault(false) } diff --git a/app/src/main/java/com/dark/tool_neuron/repo/web_search/HtmlText.kt b/app/src/main/java/com/dark/tool_neuron/repo/web_search/HtmlText.kt new file mode 100644 index 00000000..380a2e1c --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/repo/web_search/HtmlText.kt @@ -0,0 +1,63 @@ +package com.dark.tool_neuron.repo.web_search + +object HtmlText { + + private val DOT = setOf(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL) + + private val SCRIPT = Regex("]*>.*?", DOT) + private val STYLE = Regex("]*>.*?", DOT) + private val NOSCRIPT = Regex("]*>.*?", DOT) + private val HEAD = Regex("]*>.*?", DOT) + private val COMMENT = Regex("", setOf(RegexOption.DOT_MATCHES_ALL)) + private val BLOCK_END = Regex("(?i)") + private val BR = Regex("(?i)") + private val TAG = Regex("<[^>]+>") + private val MULTI_NL = Regex("\\n{2,}") + private val INLINE_WS = Regex("[ \\t]+") + private val NUM_ENTITY = Regex("&#(x?[0-9a-fA-F]+);") + + private val NAMED = mapOf( + "&" to "&", "<" to "<", ">" to ">", """ to "\"", + "'" to "'", "'" to "'", " " to " ", "—" to "—", + "–" to "–", "…" to "…", "’" to "'", "‘" to "'", + "“" to "\"", "”" to "\"", "™" to "™", "®" to "®", + ) + + fun extract(html: String, maxChars: Int): String { + if (html.isBlank()) return "" + var s = html + s = SCRIPT.replace(s, " ") + s = STYLE.replace(s, " ") + s = NOSCRIPT.replace(s, " ") + s = HEAD.replace(s, " ") + s = COMMENT.replace(s, " ") + s = BR.replace(s, "\n") + s = BLOCK_END.replace(s, "\n") + s = TAG.replace(s, " ") + s = decodeEntities(s) + s = INLINE_WS.replace(s, " ") + s = s.lines().joinToString("\n") { it.trim() } + s = MULTI_NL.replace(s, "\n").trim() + if (s.length > maxChars) s = s.substring(0, maxChars).substringBeforeLast(' ').trim() + return s + } + + private fun decodeEntities(input: String): String { + var s = input + NAMED.forEach { (k, v) -> if (s.contains(k, ignoreCase = true)) s = s.replace(k, v, ignoreCase = true) } + s = NUM_ENTITY.replace(s) { m -> + val raw = m.groupValues[1] + val code = if (raw.startsWith("x") || raw.startsWith("X")) { + raw.drop(1).toIntOrNull(16) + } else { + raw.toIntOrNull() + } + if (code != null && code in 32..0x10FFFF) { + runCatching { String(Character.toChars(code)) }.getOrDefault(m.value) + } else { + m.value + } + } + return s + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/web_search/PageFetcher.kt b/app/src/main/java/com/dark/tool_neuron/repo/web_search/PageFetcher.kt new file mode 100644 index 00000000..f94a565a --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/repo/web_search/PageFetcher.kt @@ -0,0 +1,42 @@ +package com.dark.tool_neuron.repo.web_search + +import android.content.Context +import com.dark.networking.WebNative +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import javax.inject.Inject +import javax.inject.Singleton + +data class FetchedPage( + val url: String, + val text: String, +) + +@Singleton +class PageFetcher @Inject constructor( + @ApplicationContext private val context: Context, +) { + suspend fun fetch(urls: List, maxCharsPerPage: Int = PAGE_CHAR_CAP): List { + if (urls.isEmpty()) return emptyList() + WebNative.ensureReady(context) + return coroutineScope { + urls.distinct().map { url -> + async(Dispatchers.IO) { + val resp = WebNative.fetch(url = url, timeoutMs = PAGE_TIMEOUT_MS).getOrNull() + if (resp == null || resp.status !in 200..399 || resp.body.isBlank()) return@async null + val text = HtmlText.extract(resp.body, maxCharsPerPage) + if (text.length < MIN_USEFUL_CHARS) null else FetchedPage(url, text) + } + }.awaitAll().filterNotNull() + } + } + + private companion object { + const val PAGE_TIMEOUT_MS = 12_000 + const val PAGE_CHAR_CAP = 2500 + const val MIN_USEFUL_CHARS = 120 + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchMode.kt b/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchMode.kt new file mode 100644 index 00000000..e4face3e --- /dev/null +++ b/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchMode.kt @@ -0,0 +1,91 @@ +package com.dark.tool_neuron.repo.web_search + +enum class WebSearchMode( + val label: String, + val maxRounds: Int, + val maxQueries: Int, + val initialQueries: Int, + val followUpQueries: Int, + val resultsPerQuery: Int, + val fetchPages: Boolean, + val pagesPerRound: Int, + val timeBudgetMs: Long, +) { + QUICK("Quick", maxRounds = 1, maxQueries = 3, initialQueries = 3, followUpQueries = 0, resultsPerQuery = 6, fetchPages = false, pagesPerRound = 0, timeBudgetMs = 90_000), + NORMAL("Normal", maxRounds = 3, maxQueries = 10, initialQueries = 4, followUpQueries = 3, resultsPerQuery = 8, fetchPages = false, pagesPerRound = 0, timeBudgetMs = 240_000), + DEEP("Deep", maxRounds = 10, maxQueries = 40, initialQueries = 5, followUpQueries = 4, resultsPerQuery = 8, fetchPages = true, pagesPerRound = 6, timeBudgetMs = 300_000), + EXHAUSTIVE("Exhaustive", maxRounds = 20, maxQueries = 80, initialQueries = 6, followUpQueries = 5, resultsPerQuery = 8, fetchPages = true, pagesPerRound = 8, timeBudgetMs = 600_000); + + val key: String get() = name.lowercase() + + companion object { + const val AUTO_KEY = "auto" + + val SELECTABLE_KEYS = listOf(AUTO_KEY, "quick", "normal", "deep", "exhaustive") + + fun fromPref(value: String?): WebSearchMode? = when (value?.trim()?.lowercase()) { + "quick" -> QUICK + "normal" -> NORMAL + "deep" -> DEEP + "exhaustive" -> EXHAUSTIVE + else -> null + } + + fun sanitizePref(value: String?): String { + val v = value?.trim()?.lowercase() + return if (v != null && v in SELECTABLE_KEYS) v else AUTO_KEY + } + + fun labelForKey(key: String): String = + if (key.trim().equals(AUTO_KEY, ignoreCase = true)) "Auto" else (fromPref(key)?.label ?: "Auto") + + fun resolve(rawText: String, default: WebSearchMode? = null): Resolved { + val text = rawText.trim() + val lower = text.lowercase() + + forcedPrefix(lower, "/exhaustive")?.let { return Resolved(EXHAUSTIVE, strip(text, it)) } + forcedPrefix(lower, "/research")?.let { return Resolved(DEEP, strip(text, it)) } + forcedPrefix(lower, "/deep")?.let { return Resolved(DEEP, strip(text, it)) } + forcedPrefix(lower, "/search")?.let { + val q = strip(text, it) + return Resolved(default ?: classifyAuto(q), q) + } + + return Resolved(default ?: classifyAuto(text), text) + } + + private fun classifyAuto(query: String): WebSearchMode { + val lower = query.lowercase() + if (lower.contains("exhaustive") || lower.contains("deep research")) return EXHAUSTIVE + return classify(query) + } + + fun classify(query: String): WebSearchMode { + val q = query.trim() + if (q.isEmpty()) return NORMAL + val lower = q.lowercase() + + if (lower.count { it == '?' } >= 2) return DEEP + if (DEEP_SIGNALS.any { lower.contains(it) }) return DEEP + + val words = q.split(Regex("\\s+")).filter { it.isNotBlank() }.size + return if (words <= 5) QUICK else NORMAL + } + + private fun forcedPrefix(lower: String, prefix: String): String? = + if (lower == prefix || lower.startsWith("$prefix ") || lower.startsWith("$prefix\n")) prefix else null + + private fun strip(text: String, prefix: String): String = + text.substring(prefix.length).trim() + + private val DEEP_SIGNALS = listOf( + "compare", "comparison", "versus", " vs ", " vs. ", "difference between", + "pros and cons", "advantages and disadvantages", "best ", "top ", + "latest", "newest", "up to date", "up-to-date", "alternatives", + "benchmark", "in depth", "in-depth", "comprehensive", "detailed", + "review", "research", "everything about", "deep dive", + ) + } + + data class Resolved(val mode: WebSearchMode, val query: String) +} diff --git a/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchPrompts.kt b/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchPrompts.kt index f119d352..e38568c0 100644 --- a/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchPrompts.kt +++ b/app/src/main/java/com/dark/tool_neuron/repo/web_search/WebSearchPrompts.kt @@ -3,51 +3,92 @@ package com.dark.tool_neuron.repo.web_search import com.dark.tool_neuron.model.WebSearchHit object WebSearchPrompts { - - fun generateQueries(userQuery: String): String = buildString { + fun initialQueries(userQuery: String, count: Int): String = buildString { + val n = count.coerceIn(1, 8) append("You are a search-query generator. The user wants information about the topic below.") - append("\n\nWrite exactly THREE concise web search queries that, together, will surface the most useful pages to answer the user.") - append(" Each query should target a different angle (e.g. overview, recent news, specific examples).") - append(" Each query should be 3-8 words. Do not repeat the same query in different words.") - append("\n\nReturn ONLY the queries as a numbered list, one per line, in this exact format:") - append("\n1. ") - append("\n2. ") - append("\n3. ") + append("\n\nWrite exactly $n concise web search queries that together surface the most useful pages to answer the user.") + append(" Each query should target a different angle (overview, specifics, recent/latest, comparisons, official source).") + append(" Each query should be 3-8 words. Keep the user's key terms. Do not repeat a query in different words.") + append("\n\nReturn ONLY the queries as a numbered list, one per line:") + for (i in 1..n) append("\n$i. ") append("\n\nNo preamble, no explanation, no extra text.") append("\n\nUser topic: ") append(userQuery.trim()) } - fun synthesize(userQuery: String, hits: List): String = buildString { - // Small chat models (LFM2-350M, Qwen 0.5B, etc.) struggle when asked - // to write markdown AND emit inline citations AND append a sources - // section all at once — they typically output only the sources block - // or a 1-line deflection. Strip the prompt down to a single task: - // write a short factual summary. The sources accordion in the UI - // already lists every URL, so the model doesn't need to repeat them. - append("Summarize the search results below to answer the user's question. ") - append("Write a direct answer first, then add brief context only if useful. ") - append("If the user asked for a link or download page, provide the best official URL from the results first. ") - append("Use only information from the snippets — do not invent details. ") - append("If the snippets don't cover the question, say so in one sentence.") + fun roundDigest( + userQuery: String, + priorSummary: String, + findings: List, + excerpts: Map, + followUpCount: Int, + ): String = buildString { + val k = followUpCount.coerceIn(1, 8) + append("You are researching a question using web search results.") + append("\n\nQuestion: ") + append(userQuery.trim()) + append("\n\nWhat we already know:\n") + append(priorSummary.ifBlank { "(nothing yet)" }) + append("\n\nNew findings from the latest search:\n") + findings.forEachIndexed { i, h -> + append("\n") + append(i + 1) + append(". ") + append(h.title.ifBlank { h.url }) + val body = excerpts[h.url]?.takeIf { it.isNotBlank() } ?: h.snippet + if (body.isNotBlank()) { + append(" — ") + append(body.trim().replace('\n', ' ').take(EVIDENCE_CHAR_CAP)) + } + append('\n') + } + append("\nUsing ONLY these findings (do not invent facts), reply in EXACTLY this format:") + append("\nSUMMARY:") + append("\n") + append("\nMISSING:") + append("\n") + append("\nQUERIES:") + append("\n") + append("\nCOVERAGE:") + append("\n") + } + + fun synthesize( + userQuery: String, + summary: String, + hits: List, + excerpts: Map, + ): String = buildString { + append("Answer the user's question using the research notes and sources below. ") + append("Write a direct, complete answer first, then add useful context. ") + append("If the user asked for a link or download page, give the best official URL first. ") + append("Use only the information provided — do not invent details. ") + append("You may cite sources inline as [1], [2] where helpful. ") + append("If the sources don't fully cover the question, answer what you can and say what's missing.") append("\n\nQuestion: ") append(userQuery.trim()) - append("\n\nSearch results:\n") + if (summary.isNotBlank()) { + append("\n\nResearch notes:\n") + append(summary.trim().take(SUMMARY_CHAR_CAP)) + } + append("\n\nSources:\n") hits.forEachIndexed { i, h -> append("\n") append(i + 1) append(". ") append(h.title.ifBlank { h.url }) - if (h.snippet.isNotBlank()) { + val body = excerpts[h.url]?.takeIf { it.isNotBlank() } ?: h.snippet + if (body.isNotBlank()) { append(" — ") - append(h.snippet.trim().replace('\n', ' ').take(SNIPPET_CHAR_CAP)) + append(body.trim().replace('\n', ' ').take(EVIDENCE_CHAR_CAP)) } append('\n') } - append("\nSummary:\n") + append("\nAnswer:\n") } - private const val SNIPPET_CHAR_CAP = 320 + private const val EVIDENCE_CHAR_CAP = 700 + private const val SUMMARY_CHAR_CAP = 1600 val QUERY_LINE_REGEX = Regex("^\\s*(?:\\d+[.)\\-:]|[-*•])\\s+(.+)$") } diff --git a/app/src/main/java/com/dark/tool_neuron/service/inference/InferenceClient.kt b/app/src/main/java/com/dark/tool_neuron/service/inference/InferenceClient.kt index ec30b3fc..8f7c97a1 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/inference/InferenceClient.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/inference/InferenceClient.kt @@ -949,6 +949,10 @@ object InferenceClient { } } + fun sdResetUpscaleState() { + _sdUpscaleState.value = UpscaleState.Idle + } + fun sdStop() { try { _service.value?.sdStop() } catch (_: Throwable) {} } diff --git a/app/src/main/java/com/dark/tool_neuron/service/server/RemoteServerService.kt b/app/src/main/java/com/dark/tool_neuron/service/server/RemoteServerService.kt index f2b32971..4af96ac0 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/server/RemoteServerService.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/server/RemoteServerService.kt @@ -123,6 +123,7 @@ class RemoteServerService : Service() { val bindModeS = cfg.optString("bindMode", BindMode.ALL_INTERFACES.name) val bindMode = runCatching { BindMode.valueOf(bindModeS) }.getOrDefault(BindMode.ALL_INTERFACES) val webUiHtml = cfg.optString("webUiHtml", "") + val webUiCss = cfg.optString("webUiCss", "") val docsHtml = cfg.optString("docsHtml", "") val engines = cfg.optJSONArray("engines") ?: JSONArray() @@ -185,6 +186,7 @@ class RemoteServerService : Service() { NativeServer.nativeSetToken(token) NativeServer.nativeSetModelsCatalog(catalog.toJsonArray().toString()) if (webUiHtml.isNotBlank()) NativeServer.nativeSetWebUiHtml(webUiHtml) + if (webUiCss.isNotBlank()) NativeServer.nativeSetWebUiCss(webUiCss) if (docsHtml.isNotBlank()) NativeServer.nativeSetDocsHtml(docsHtml) val ok = NativeServer.nativeStart(resolution.host, port) @@ -224,8 +226,10 @@ class RemoteServerService : Service() { try { NativeServer.nativeSetModelsCatalog(catalog.toJsonArray().toString()) } catch (_: Exception) {} val webUiHtml = cfg.optString("webUiHtml", "") + val webUiCss = cfg.optString("webUiCss", "") val docsHtml = cfg.optString("docsHtml", "") if (webUiHtml.isNotBlank()) try { NativeServer.nativeSetWebUiHtml(webUiHtml) } catch (_: Exception) {} + if (webUiCss.isNotBlank()) try { NativeServer.nativeSetWebUiCss(webUiCss) } catch (_: Exception) {} if (docsHtml.isNotBlank()) try { NativeServer.nativeSetDocsHtml(docsHtml) } catch (_: Exception) {} val displayEntry = catalog.primary() diff --git a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt index c9f80664..bd21bd7c 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt @@ -150,6 +150,7 @@ class ServerController @Inject constructor( put("port", prefs.serverPort) put("bindMode", prefs.serverBindMode) put("webUiHtml", loadAsset("server_webui.html")) + put("webUiCss", loadAsset("server_webui.css")) put("docsHtml", loadAsset("server_docs.html")) } diff --git a/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt b/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt index 68aa5b98..32494589 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/navigation/TNavigation.kt @@ -363,6 +363,15 @@ fun TNavigation( onNavigate = { route -> navController.navigate(route) }, ) } + composable(NavScreens.SettingsWebSearch.route) { + val viewModel: SettingsViewModel = hiltViewModel() + SettingsSectionScreen( + innerPadding = innerPadding, + sectionId = SettingsViewModel.SECTION_WEB_SEARCH, + viewModel = viewModel, + onNavigate = { route -> navController.navigate(route) }, + ) + } composable(NavScreens.SettingsVoice.route) { val viewModel: SettingsViewModel = hiltViewModel() SettingsSectionScreen( diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/downloads/DownloadsScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/downloads/DownloadsScreen.kt index adacb2f7..1ed26c3e 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/downloads/DownloadsScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/downloads/DownloadsScreen.kt @@ -76,7 +76,12 @@ fun DownloadsScreen( if (active.isNotEmpty()) { item { SectionHeader(label = "Active", count = active.size) } items(active, key = { it.hxdId }) { item -> - ActiveRow(item = item, onCancel = { viewModel.cancel(item.hxdId) }) + ActiveRow( + item = item, + onCancel = { viewModel.cancel(item.hxdId) }, + onRetry = { viewModel.retry(item.hxdId) }, + onClear = { viewModel.clear(item.hxdId) }, + ) } item { Spacer(Modifier.height(dimens.spacingSm)) } } @@ -131,7 +136,12 @@ private fun SectionHeader( } @Composable -private fun ActiveRow(item: ActiveDownloadItem, onCancel: () -> Unit) { +private fun ActiveRow( + item: ActiveDownloadItem, + onCancel: () -> Unit, + onRetry: () -> Unit, + onClear: () -> Unit, +) { val dimens = LocalDimens.current val shapes = LocalTnShapes.current val state = item.state @@ -158,11 +168,24 @@ private fun ActiveRow(item: ActiveDownloadItem, onCancel: () -> Unit) { maxLines = 1, overflow = TextOverflow.Ellipsis, ) - ActionButton( - onClickListener = onCancel, - icon = TnIcons.X, - contentDescription = "Cancel download", - ) + if (state.status == HxdStatus.FAILED) { + ActionButton( + onClickListener = onRetry, + icon = TnIcons.Refresh, + contentDescription = "Retry download", + ) + ActionButton( + onClickListener = onClear, + icon = TnIcons.Trash, + contentDescription = "Clear failed download", + ) + } else { + ActionButton( + onClickListener = onCancel, + icon = TnIcons.X, + contentDescription = "Cancel download", + ) + } } ActiveProgress(state = state) } @@ -201,6 +224,13 @@ private fun ActiveProgress(state: HxdState) { color = MaterialTheme.colorScheme.onSurfaceVariant, ) } + HxdStatus.FAILED -> { + Text( + text = state.error ?: "Download failed", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + } else -> { if (state.totalBytes <= 0L) { TnIndeterminateProgressBar(modifier = Modifier.fillMaxWidth()) diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/hf_explorer/HfRepoDetailScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/hf_explorer/HfRepoDetailScreen.kt index 5f9de28b..5caac413 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/hf_explorer/HfRepoDetailScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/hf_explorer/HfRepoDetailScreen.kt @@ -1,5 +1,6 @@ package com.dark.tool_neuron.ui.screens.hf_explorer +import androidx.compose.animation.AnimatedVisibility import androidx.compose.foundation.background import androidx.compose.foundation.clickable import androidx.compose.foundation.horizontalScroll @@ -26,6 +27,9 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontFamily @@ -41,6 +45,7 @@ import com.dark.tool_neuron.repo.hf.HfGated import com.dark.tool_neuron.repo.hf.HfGgufMeta import com.dark.tool_neuron.repo.hf.HfModelDetail import com.dark.tool_neuron.repo.hf.HfSibling +import com.dark.tool_neuron.model.VlmFileGroup import com.dark.tool_neuron.ui.components.ActionButton import com.dark.tool_neuron.ui.components.ActionTextButton import com.dark.tool_neuron.ui.components.CaptionText @@ -102,6 +107,7 @@ fun HfRepoDetailScreen( bucket = bucket, isAdded = existing.contains(s.detail.summary.id.lowercase()), visibleFiles = vm.visibleFiles(s.detail), + visibleFileGroups = vm.visibleFileGroups(s.detail), onAdd = { vm.addRepository(s.detail.summary.id, s.detail.summary.id.substringAfter("/")) }, onFilterChange = vm::setFileFilter, onBucketChange = vm::setFileSizeBucket, @@ -174,6 +180,7 @@ private fun DetailContent( bucket: HfFileSizeBucket, isAdded: Boolean, visibleFiles: List, + visibleFileGroups: List, onAdd: () -> Unit, onFilterChange: (HfFileFilter) -> Unit, onBucketChange: (HfFileSizeBucket) -> Unit, @@ -213,8 +220,21 @@ private fun DetailContent( }, ) } - items(visibleFiles, key = { it.path }) { file -> - FileRow(file = file) + if (visibleFileGroups.isNotEmpty()) { + items(visibleFileGroups, key = { it.key }) { group -> + VlmFileGroupRow(group = group) + } + val groupedPaths = visibleFileGroups.flatMap { group -> + group.baseFiles.map { it.path } + group.projectorFiles.map { it.path } + }.toSet() + val remaining = visibleFiles.filterNot { it.path in groupedPaths } + items(remaining, key = { it.path }) { file -> + FileRow(file = file) + } + } else { + items(visibleFiles, key = { it.path }) { file -> + FileRow(file = file) + } } if (visibleFiles.isEmpty()) { item { CaptionText("No files match the current filter.") } @@ -247,6 +267,67 @@ private fun DetailContent( } } +@Composable +private fun VlmFileGroupRow(group: VlmFileGroup) { + val dimens = LocalDimens.current + val tnShapes = LocalTnShapes.current + var expanded by remember(group.key) { mutableStateOf(false) } + val baseCount = group.baseFiles.size + val projectorCount = group.projectorFiles.size + Surface( + shape = tnShapes.cardSmall, + color = MaterialTheme.colorScheme.primary.copy(alpha = 0.07f), + modifier = Modifier.fillMaxWidth(), + onClick = { expanded = !expanded }, + ) { + Column( + modifier = Modifier.padding(dimens.spacingSm), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(dimens.spacingSm), + ) { + Icon( + imageVector = TnIcons.Eye, + contentDescription = null, + tint = MaterialTheme.colorScheme.primary, + modifier = Modifier.size(18.dp), + ) + Column(modifier = Modifier.weight(1f)) { + Text( + text = group.displayName, + style = MaterialTheme.typography.bodyMedium, + fontWeight = FontWeight.SemiBold, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) + CaptionText( + text = buildList { + add("$baseCount model ${if (baseCount == 1) "file" else "files"}") + if (projectorCount > 0) add("$projectorCount projector ${if (projectorCount == 1) "file" else "files"}") + if (group.quantizations.isNotEmpty()) add(group.quantizations.joinToString(", ")) + }.joinToString(" · "), + ) + } + CaptionText(text = formatBytes(group.totalSizeBytes)) + Icon( + imageVector = if (expanded) TnIcons.ChevronUp else TnIcons.ChevronDown, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.size(18.dp), + ) + } + AnimatedVisibility(visible = expanded) { + Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { + group.baseFiles.forEach { file -> FileRow(file) } + group.projectorFiles.forEach { file -> FileRow(file) } + } + } + } + } +} + @Composable private fun HeaderCard( detail: HfModelDetail, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/HomeScreenBottomBar.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/HomeScreenBottomBar.kt index 959d3306..b998f89f 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/HomeScreenBottomBar.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/HomeScreenBottomBar.kt @@ -87,6 +87,7 @@ fun HomeScreenBottomBar( val thinkingEnabled by viewModel.thinkingEnabled.collectAsStateWithLifecycle() val webSearchEnabled by viewModel.webSearchEnabled.collectAsStateWithLifecycle() + val webSearchMode by viewModel.webSearchMode.collectAsStateWithLifecycle() val supportsThinking by viewModel.supportsThinking.collectAsStateWithLifecycle() val isModelLoaded by InferenceClient.isModelLoaded.collectAsStateWithLifecycle() val isGenerating by viewModel.isGenerating.collectAsStateWithLifecycle() @@ -235,6 +236,7 @@ fun HomeScreenBottomBar( thinkingEnabled = thinkingEnabled, thinkingSupported = supportsThinking, webSearchEnabled = webSearchEnabled, + webSearchMode = webSearchMode, canAttachImage = isVlmLoaded, canAttachFiles = embeddingModelInstalled, canCompact = isModelLoaded @@ -245,6 +247,7 @@ fun HomeScreenBottomBar( ?.kind != com.dark.tool_neuron.model.MessageKind.CompactSummary, onToggleThinking = viewModel::toggleThinking, onToggleWebSearch = viewModel::toggleWebSearch, + onSetWebSearchMode = viewModel::setWebSearchMode, onAttachImage = { if (isVlmLoaded) { toolsWindowOpen = false diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/ToolsPickerWindow.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/ToolsPickerWindow.kt index d95cd5d1..b8371f04 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/ToolsPickerWindow.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/home_screen/ToolsPickerWindow.kt @@ -24,6 +24,8 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp +import com.dark.tool_neuron.repo.web_search.WebSearchMode +import com.dark.tool_neuron.ui.components.ActionToggleGroup import com.dark.tool_neuron.ui.components.SectionHeader import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.theme.LocalDimens @@ -35,11 +37,13 @@ fun ToolsPickerWindow( thinkingEnabled: Boolean, thinkingSupported: Boolean, webSearchEnabled: Boolean, + webSearchMode: String, canAttachImage: Boolean, canAttachFiles: Boolean, canCompact: Boolean, onToggleThinking: () -> Unit, onToggleWebSearch: () -> Unit, + onSetWebSearchMode: (String) -> Unit, onAttachImage: () -> Unit, onAttachFiles: () -> Unit, onCompactChat: () -> Unit, @@ -81,13 +85,30 @@ fun ToolsPickerWindow( Modifier.weight(1f), icon = TnIcons.Globe, label = "Web search", - description = "3 queries, snippets, answer", + description = "Multi-engine research", enabled = true, active = webSearchEnabled, onClick = onToggleWebSearch, ) } + Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingXxs)) { + Text( + text = "Search depth", + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(start = dimens.spacingXs), + ) + ActionToggleGroup( + items = WebSearchMode.SELECTABLE_KEYS, + selectedItem = WebSearchMode.sanitizePref(webSearchMode), + onItemSelected = onSetWebSearchMode, + itemLabel = { WebSearchMode.labelForKey(it) }, + enabled = webSearchEnabled, + modifier = Modifier.fillMaxWidth(), + ) + } + Row( modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(dimens.spacingSm), diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageTaskScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageTaskScreen.kt index 739defd4..93951f08 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageTaskScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageTaskScreen.kt @@ -9,22 +9,29 @@ import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.Image import androidx.compose.foundation.clickable +import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxWithConstraints import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.offset import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.LazyListScope import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.Icon import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Slider import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable @@ -37,9 +44,12 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.clipToBounds import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -55,6 +65,8 @@ import com.dark.tool_neuron.ui.components.TnTextField import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.theme.LocalDimens import com.dark.tool_neuron.ui.theme.LocalTnShapes +import com.dark.tool_neuron.repo.UpscaleProcessingMode +import com.dark.tool_neuron.repo.UpscaleScalePreset import com.dark.tool_neuron.viewmodel.ImageTaskMode import com.dark.tool_neuron.viewmodel.ImageTaskUi import com.dark.tool_neuron.viewmodel.ImageTaskViewModel @@ -63,6 +75,8 @@ import com.dark.tool_neuron.viewmodel.ModelLoadPhase import com.dark.tool_neuron.viewmodel.RuntimePhase import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileInputStream @Composable fun ImageTaskScreen( @@ -85,14 +99,18 @@ fun ImageTaskScreen( ActivityResultContracts.OpenDocument() ) { uri -> uri?.let(viewModel::setInputImage) } var pendingSaveElsewhereBitmap by remember { mutableStateOf(null) } + var pendingSaveElsewhereFile by remember { mutableStateOf(null) } var handledSaveElsewhereToken by remember { mutableStateOf("") } val saveElsewhere = rememberLauncherForActivityResult( ActivityResultContracts.CreateDocument("image/png"), ) { uri -> val bitmap = pendingSaveElsewhereBitmap + val file = pendingSaveElsewhereFile pendingSaveElsewhereBitmap = null + pendingSaveElsewhereFile = null if (uri != null && bitmap != null) { - val result = writeBitmapToUri(context, bitmap, uri) + val result = file?.let { copyFileToUri(context, it, uri) } + ?: writeBitmapToUri(context, bitmap, uri) Toast.makeText( context, if (result.isSuccess) "Saved image" else "Save failed: ${result.exceptionOrNull()?.message}", @@ -125,6 +143,7 @@ fun ImageTaskScreen( image != null ) { handledSaveElsewhereToken = ui.outputToken + pendingSaveElsewhereFile = ui.upscaleOutput?.outputPath?.let(::File)?.takeIf { it.exists() } pendingSaveElsewhereBitmap = image val prefix = when (ui.mode) { ImageTaskMode.GENERATE -> "tn_generate" @@ -261,6 +280,7 @@ private fun LazyListScope.upscaleBody( ) } item("u-output-policy") { OutputPolicyCard(ui = ui, viewModel = viewModel) } + item("u-scale") { UpscaleSettingsCard(ui = ui, viewModel = viewModel) } item("u-run") { RunCard(ui = ui, viewModel = viewModel) } item("u-output") { UpscaleOutputCard(ui = ui) } } @@ -294,9 +314,9 @@ private fun OutputPolicyCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { modifier = Modifier.fillMaxWidth(), ) CaptionText( - text = when (ui.outputAction) { - ImageOutputAction.KEEP -> "Keep the result in this screen. You can still Save Photos or Save as from the result card." - ImageOutputAction.REPLACE_INPUT -> "Use the new image as the next input and clear the mask." + text = when (ui.outputAction) { + ImageOutputAction.KEEP -> "Keep the result in this screen. You can still Save Photos or Save as from the result card." + ImageOutputAction.REPLACE_INPUT -> "Use the new image as the next input and clear the mask." ImageOutputAction.SAVE_PHOTOS -> "Auto-save to Android Photos: Pictures/ToolNeuron." ImageOutputAction.SAVE_ELSEWHERE -> "Open Android's save dialog when the new image is ready." }, @@ -308,6 +328,88 @@ private fun OutputPolicyCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { } } +@Composable +private fun UpscaleSettingsCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { + val dimens = LocalDimens.current + val targetText = ui.inputImagePath?.let { path -> + val context = LocalContext.current + val source by produceState?>(initialValue = null, path) { + value = withContext(Dispatchers.IO) { + decodeBitmap(context, path)?.let { bitmap -> + val size = bitmap.width to bitmap.height + bitmap.recycle() + size + } + } + } + source?.let { (w, h) -> + "${(w * ui.upscaleScale).toInt()}×${(h * ui.upscaleScale).toInt()}" + } + } ?: "Pick an image first" + StandardCard( + title = "Upscale", + icon = TnIcons.Zap, + description = "Choose output scale and processing mode.", + ) { + Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingSm)) { + CaptionText(text = "Scale: ${"%.2f".format(ui.upscaleScale).trimEnd('0').trimEnd('.')}x · $targetText") + ActionToggleGroup( + items = listOf( + UpscaleScalePreset.X2, + UpscaleScalePreset.X4, + UpscaleScalePreset.X8, + UpscaleScalePreset.CUSTOM, + ), + selectedItem = ui.upscaleScalePreset, + onItemSelected = viewModel::setUpscaleScalePreset, + itemLabel = { + when (it) { + UpscaleScalePreset.X2 -> "2x" + UpscaleScalePreset.X4 -> "4x" + UpscaleScalePreset.X8 -> "8x" + UpscaleScalePreset.CUSTOM -> "Custom" + } + }, + modifier = Modifier.fillMaxWidth(), + ) + if (ui.upscaleScalePreset == UpscaleScalePreset.CUSTOM) { + Slider( + value = ui.customUpscaleScale, + onValueChange = viewModel::setCustomUpscaleScale, + valueRange = 1.25f..8f, + steps = 26, + modifier = Modifier.fillMaxWidth(), + ) + } + CaptionText(text = "Processing") + ActionToggleGroup( + items = listOf( + UpscaleProcessingMode.AUTO, + UpscaleProcessingMode.FAST, + UpscaleProcessingMode.SAFE_TILED, + ), + selectedItem = ui.upscaleProcessingMode, + onItemSelected = viewModel::setUpscaleProcessingMode, + itemLabel = { + when (it) { + UpscaleProcessingMode.AUTO -> "Auto" + UpscaleProcessingMode.FAST -> "Fast" + UpscaleProcessingMode.SAFE_TILED -> "Safe tiled" + } + }, + modifier = Modifier.fillMaxWidth(), + ) + CaptionText( + text = when (ui.upscaleProcessingMode) { + UpscaleProcessingMode.AUTO -> "Automatically tiles only when needed." + UpscaleProcessingMode.FAST -> "Uses one-shot processing when it is safe." + UpscaleProcessingMode.SAFE_TILED -> "Slower, safer processing for large images." + }, + ) + } + } +} + @Composable private fun ModelLoadCard( phase: ModelLoadPhase, @@ -422,7 +524,7 @@ private fun RuntimeCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { RuntimePhase.READY -> "Runtime — ready" } val description = when (ui.runtimePhase) { - RuntimePhase.NEEDS_DOWNLOAD -> "One-time ~30 MB download. Required to run any image model." + RuntimePhase.NEEDS_DOWNLOAD -> "One-time ~30 MB download. Required for SD generation, SD inpaint, and upscalers." RuntimePhase.DOWNLOADING -> { val total = ui.runtimeDownloadTotal val got = ui.runtimeDownloadBytes @@ -431,7 +533,7 @@ private fun RuntimeCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { } RuntimePhase.READY_TO_INITIALIZE -> "Initializing automatically…" RuntimePhase.INITIALIZING -> ui.statusText.ifBlank { "Extracting native libs…" } - RuntimePhase.READY -> "Image gen + upscale are ready." + RuntimePhase.READY -> "SD image gen + upscale are ready." } StandardCard( title = title, @@ -512,27 +614,88 @@ private fun ModeCard( icon = TnIcons.Sparkles, description = "Pick what to do with image models", ) { - ActionToggleGroup( - items = listOf( - ImageTaskMode.GENERATE, - ImageTaskMode.INPAINT, - ImageTaskMode.UPSCALE, - ), - selectedItem = ui.mode, - onItemSelected = onSelect, - itemLabel = { mode -> - when (mode) { - ImageTaskMode.GENERATE -> "Generate" - ImageTaskMode.INPAINT -> "Inpaint" - ImageTaskMode.UPSCALE -> "Upscale" + Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { + imageTaskRows.forEach { row -> + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + row.forEach { mode -> + ImageTaskTile( + mode = mode, + selected = ui.mode == mode, + onClick = { onSelect(mode) }, + modifier = Modifier.weight(1f), + ) + } } - }, - modifier = Modifier.fillMaxWidth(), - height = dimens.toggleGroupHeight + 8.dp, - ) + } + } } } +private val imageTaskRows = listOf( + listOf(ImageTaskMode.GENERATE, ImageTaskMode.INPAINT), + listOf(ImageTaskMode.UPSCALE), +) + +@Composable +private fun ImageTaskTile( + mode: ImageTaskMode, + selected: Boolean, + onClick: () -> Unit, + modifier: Modifier = Modifier, +) { + val dimens = LocalDimens.current + val shapes = LocalTnShapes.current + Surface( + modifier = modifier + .heightIn(min = 74.dp) + .clickable(onClick = onClick), + color = if (selected) MaterialTheme.colorScheme.primary.copy(alpha = 0.14f) + else MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.34f), + shape = shapes.cardSmall, + ) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(dimens.spacingSm), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(dimens.spacingSm), + ) { + Icon( + imageVector = mode.taskIcon(), + contentDescription = null, + tint = if (selected) MaterialTheme.colorScheme.primary + else MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.size(dimens.iconMd), + ) + Column(modifier = Modifier.weight(1f)) { + BodyLabel(text = mode.shortLabel()) + CaptionText(text = mode.shortDescription()) + } + } + } +} + +private fun ImageTaskMode.shortLabel(): String = when (this) { + ImageTaskMode.GENERATE -> "Create" + ImageTaskMode.INPAINT -> "Inpaint" + ImageTaskMode.UPSCALE -> "Upscale" +} + +private fun ImageTaskMode.shortDescription(): String = when (this) { + ImageTaskMode.GENERATE -> "Text to image" + ImageTaskMode.INPAINT -> "Prompt fill" + ImageTaskMode.UPSCALE -> "2x, 4x, 8x" +} + +private fun ImageTaskMode.taskIcon() = when (this) { + ImageTaskMode.GENERATE -> TnIcons.Sparkles + ImageTaskMode.INPAINT -> TnIcons.Edit + ImageTaskMode.UPSCALE -> TnIcons.Zap +} + @Composable private fun ModelPickCard( ui: ImageTaskUi, @@ -541,14 +704,22 @@ private fun ModelPickCard( onOpenStore: () -> Unit, ) { val dimens = LocalDimens.current - val needsUpscaler = ui.mode == ImageTaskMode.UPSCALE - val list: List = if (needsUpscaler) ui.installedUpscalers else ui.installedDiffusionModels - val activeId = if (needsUpscaler) ui.activeUpscalerId else ui.activeModelId + val list: List = when (ui.mode) { + ImageTaskMode.UPSCALE -> ui.installedUpscalers + else -> ui.installedDiffusionModels + } + val activeId = when (ui.mode) { + ImageTaskMode.UPSCALE -> ui.activeUpscalerId + else -> ui.activeModelId + } val pendingId = (ui.modelLoadPhase as? ModelLoadPhase.Loading) ?.let { activeId } ?: "" StandardCard( - title = if (needsUpscaler) "Upscaler" else "Image model", + title = when (ui.mode) { + ImageTaskMode.UPSCALE -> "Upscaler" + else -> "Image model" + }, icon = TnIcons.Photo, description = if (list.isEmpty()) "Install one from the Store first" @@ -569,7 +740,10 @@ private fun ModelPickCard( selected = model.id == activeId, loading = model.id == pendingId, onClick = { - if (needsUpscaler) onPickUpscaler(model.id) else onPickModel(model.id) + when (ui.mode) { + ImageTaskMode.UPSCALE -> onPickUpscaler(model.id) + else -> onPickModel(model.id) + } }, ) } @@ -632,6 +806,7 @@ private fun PromptCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { val dimens = LocalDimens.current val promptPlaceholder = when (ui.mode) { ImageTaskMode.INPAINT -> "Describe what to fill in the painted area…" + ImageTaskMode.UPSCALE -> "Optional note for this upscale…" else -> "Describe the image you want…" } StandardCard(title = "Prompt", icon = TnIcons.Edit) { @@ -853,6 +1028,14 @@ private fun RunCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { color = MaterialTheme.colorScheme.error, style = MaterialTheme.typography.labelMedium, ) + if (ui.mode == ImageTaskMode.UPSCALE && ui.upscaleProcessingMode == UpscaleProcessingMode.FAST) { + ActionTextButton( + onClickListener = viewModel::switchToSafeTiled, + icon = TnIcons.Zap, + text = "Use Safe tiled", + modifier = Modifier.fillMaxWidth(), + ) + } } if (ui.isBusy && ui.progress > 0f) { LinearProgressIndicator( @@ -870,7 +1053,7 @@ private fun RunCard(ui: ImageTaskUi, viewModel: ImageTaskViewModel) { text = when (ui.mode) { ImageTaskMode.GENERATE -> "Generate" ImageTaskMode.INPAINT -> "Inpaint" - ImageTaskMode.UPSCALE -> "Upscale 4×" + ImageTaskMode.UPSCALE -> "Upscale ${"%.2f".format(ui.upscaleScale).trimEnd('0').trimEnd('.')}x" }, enabled = !ui.isBusy, modifier = Modifier.weight(1f), @@ -974,14 +1157,113 @@ private fun formatDuration(ms: Long): String { @Composable private fun UpscaleOutputCard(ui: ImageTaskUi) { - val image = ui.upscaleResultImage ?: return - ImageViewerCard( - bitmap = image, - title = "Upscaled 4×", - saveNamePrefix = "tn_upscale", - showUpscale = false, - onUpscaleClick = null, - ) + val output = ui.upscaleOutput ?: return + val context = LocalContext.current + val original by produceState(initialValue = null, ui.inputImagePath) { + value = withContext(Dispatchers.IO) { + ui.inputImagePath?.let { decodeBitmap(context, it) } + } + } + val source = original + if (source == null) { + ImageViewerCard( + bitmap = output.preview, + title = "Upscaled ${"%.2f".format(output.scale).trimEnd('0').trimEnd('.')}x", + saveNamePrefix = "tn_upscale", + showUpscale = false, + onUpscaleClick = null, + sourceFilePath = output.outputPath, + displayWidth = output.fullWidth, + displayHeight = output.fullHeight, + ) + return + } + val dimens = LocalDimens.current + val shapes = LocalTnShapes.current + Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingMd)) { + StandardCard( + title = "Before / after", + icon = TnIcons.Eye, + description = if (output.tiled) "Safe tiled output · ${output.fullWidth}×${output.fullHeight}" + else "One-shot output · ${output.fullWidth}×${output.fullHeight}", + ) { + BeforeAfterSlider( + before = source, + after = output.preview, + modifier = Modifier + .fillMaxWidth() + .clip(shapes.cardSmall), + ) + } + ImageViewerCard( + bitmap = output.preview, + title = "Upscaled ${"%.2f".format(output.scale).trimEnd('0').trimEnd('.')}x", + saveNamePrefix = "tn_upscale", + showUpscale = false, + onUpscaleClick = null, + sourceFilePath = output.outputPath, + displayWidth = output.fullWidth, + displayHeight = output.fullHeight, + ) + } +} + +@Composable +private fun BeforeAfterSlider( + before: Bitmap, + after: Bitmap, + modifier: Modifier = Modifier, +) { + var fraction by remember { mutableStateOf(0.5f) } + val density = LocalDensity.current + BoxWithConstraints( + modifier = modifier + .aspectRatio(after.width.toFloat() / after.height.toFloat()) + .clipToBounds() + .pointerInput(Unit) { + detectDragGestures { change, dragAmount -> + val width = size.width.toFloat().coerceAtLeast(1f) + fraction = ((change.position.x + dragAmount.x) / width).coerceIn(0.05f, 0.95f) + } + }, + ) { + Image( + bitmap = before.asImageBitmap(), + contentDescription = null, + modifier = Modifier.fillMaxSize(), + contentScale = ContentScale.Crop, + ) + Box( + modifier = Modifier + .fillMaxHeight() + .width(maxWidth * fraction) + .clipToBounds(), + ) { + Image( + bitmap = after.asImageBitmap(), + contentDescription = null, + modifier = Modifier.fillMaxSize(), + contentScale = ContentScale.Crop, + ) + } + Surface( + modifier = Modifier + .offset(x = with(density) { (maxWidth.toPx() * fraction).toDp() } - 1.dp) + .width(2.dp) + .fillMaxHeight(), + color = MaterialTheme.colorScheme.primary, + ) {} + Row( + modifier = Modifier + .fillMaxWidth() + .align(Alignment.BottomCenter) + .padding(8.dp), + horizontalArrangement = Arrangement.SpaceBetween, + ) { + CaptionText(text = "Upscaled") + CaptionText(text = "Original") + } + } } private fun decodeBitmap(context: android.content.Context, path: String): Bitmap? = try { @@ -1003,3 +1285,10 @@ private fun writeBitmapToUri(context: android.content.Context, bitmap: Bitmap, u } } } + +private fun copyFileToUri(context: android.content.Context, source: File, uri: Uri): Result = runCatching { + context.contentResolver.openOutputStream(uri).use { out -> + requireNotNull(out) { "openOutputStream returned null" } + FileInputStream(source).use { input -> input.copyTo(out) } + } +} diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageViewerCard.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageViewerCard.kt index bd640b65..a8204f1a 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageViewerCard.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/image_task/ImageViewerCard.kt @@ -51,6 +51,8 @@ import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.theme.LocalDimens import com.dark.tool_neuron.ui.theme.LocalTnShapes import com.dark.tool_neuron.util.ImageExport +import java.io.File +import java.io.FileInputStream import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext @@ -65,6 +67,9 @@ fun ImageViewerCard( saveNamePrefix: String = "tool_neuron", showUpscale: Boolean = false, onUpscaleClick: (() -> Unit)? = null, + sourceFilePath: String? = null, + displayWidth: Int = bitmap.width, + displayHeight: Int = bitmap.height, ) { val dimens = LocalDimens.current val tnShapes = LocalTnShapes.current @@ -74,6 +79,7 @@ fun ImageViewerCard( var saving by remember { mutableStateOf(false) } var savingAs by remember { mutableStateOf(false) } var pendingSaveAsBitmap by remember { mutableStateOf(null) } + var pendingSaveAsFile by remember { mutableStateOf(null) } val saveAsLauncher = rememberLauncherForActivityResult( ActivityResultContracts.CreateDocument("image/png"), ) { uri -> @@ -85,10 +91,12 @@ fun ImageViewerCard( val pending = pendingSaveAsBitmap ?: bitmap scope.launch { val result = withContext(Dispatchers.IO) { - writeBitmapToUri(context, pending, outUri) + pendingSaveAsFile?.let { copyFileToUri(context, it, outUri) } + ?: writeBitmapToUri(context, pending, outUri) } savingAs = false pendingSaveAsBitmap = null + pendingSaveAsFile = null toast( context, if (result.isSuccess) "Saved image" @@ -124,7 +132,7 @@ fun ImageViewerCard( modifier = Modifier.weight(1f), ) Text( - text = "${bitmap.width}×${bitmap.height}", + text = "${displayWidth}×${displayHeight}", style = MaterialTheme.typography.labelSmall, color = MaterialTheme.colorScheme.onSurfaceVariant, ) @@ -147,11 +155,20 @@ fun ImageViewerCard( onClickListener = { saving = true scope.launch { - val result = ImageExport.saveBitmapToGallery( - context = context, - bitmap = bitmap, - displayName = "${saveNamePrefix}_${System.currentTimeMillis()}", - ) + val file = sourceFilePath?.let(::File)?.takeIf { it.exists() } + val result = if (file != null) { + ImageExport.savePngFileToGallery( + context = context, + source = file, + displayName = "${saveNamePrefix}_${System.currentTimeMillis()}", + ) + } else { + ImageExport.saveBitmapToGallery( + context = context, + bitmap = bitmap, + displayName = "${saveNamePrefix}_${System.currentTimeMillis()}", + ) + } saving = false toast( context, @@ -169,7 +186,8 @@ fun ImageViewerCard( ActionTextButton( onClickListener = { savingAs = true - pendingSaveAsBitmap = bitmap + pendingSaveAsFile = sourceFilePath?.let(::File)?.takeIf { it.exists() } + pendingSaveAsBitmap = if (pendingSaveAsFile == null) bitmap else null saveAsLauncher.launch("${saveNamePrefix}_${System.currentTimeMillis()}.png") }, icon = TnIcons.Download, @@ -189,7 +207,7 @@ fun ImageViewerCard( ActionTextButton( onClickListener = onUpscaleClick, icon = TnIcons.Sparkles, - text = "Upscale 4×", + text = "Upscale", modifier = Modifier.weight(1f), ) } @@ -214,6 +232,13 @@ private fun writeBitmapToUri(context: Context, bitmap: Bitmap, uri: Uri): Result } } +private fun copyFileToUri(context: Context, source: File, uri: Uri): Result = runCatching { + context.contentResolver.openOutputStream(uri).use { out -> + requireNotNull(out) { "openOutputStream returned null" } + FileInputStream(source).use { input -> input.copyTo(out) } + } +} + @Composable private fun ZoomableBitmap( bitmap: Bitmap, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt index 4ded1ab8..a18f79e6 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_config/ModelConfigScreen.kt @@ -201,7 +201,8 @@ fun ModelConfigScreen( copyJson(loadingJson, loading) loading.put("numThreads", sttThreads) } - ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER -> { + ProviderType.IMAGE_GEN, + ProviderType.IMAGE_UPSCALER -> { copyJson(loadingJson, loading) } } @@ -285,7 +286,8 @@ fun ModelConfigScreen( } } when (selectedProviderType) { - ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER -> { + ProviderType.IMAGE_GEN, + ProviderType.IMAGE_UPSCALER -> { item { Spacer(Modifier.height(dimens.spacingSm)) StandardCard(title = "Image model") { @@ -778,7 +780,6 @@ private val MIROSTAT_LABELS = listOf("Off", "v1", "v2") private val MODEL_TYPES = listOf( ProviderType.GGUF, ProviderType.VISION_CHAT, - ProviderType.TOOL_SEARCH, ProviderType.EMBEDDING, ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER, @@ -825,7 +826,7 @@ private fun ModelIdentityOption( private fun modelTypeLabel(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat" ProviderType.VISION_CHAT -> "Vision" - ProviderType.TOOL_SEARCH -> "Tool/Search" + ProviderType.TOOL_SEARCH -> "Chat" ProviderType.EMBEDDING -> "RAG" ProviderType.IMAGE_GEN -> "Image Gen" ProviderType.IMAGE_UPSCALER -> "Upscaler" @@ -836,7 +837,7 @@ private fun modelTypeLabel(type: ProviderType): String = when (type) { private fun modelTypeDescription(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat model: used by normal chat and OpenAI-compatible text completions." ProviderType.VISION_CHAT -> "Vision chat model: used by normal chat and image-attached chat when a projector is available." - ProviderType.TOOL_SEARCH -> "Tool/Search model: intended for search query planning, source filtering, and structured tool-call output." + ProviderType.TOOL_SEARCH -> "Chat model: used by normal chat and OpenAI-compatible text completions." ProviderType.EMBEDDING -> "Embedding model: used for document indexing and RAG retrieval." ProviderType.IMAGE_GEN -> "Image generation model: used by image create/inpaint tasks." ProviderType.IMAGE_UPSCALER -> "Image upscaler: used by image upscale tasks." diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt index 05a7421e..aa1ebf9f 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_manager/ModelManagerScreen.kt @@ -492,7 +492,6 @@ private data class Section( private val SECTIONS = listOf( Section(ProviderType.GGUF, "Chat models", "Used for the conversation in the chat screen.", "Chat"), Section(ProviderType.VISION_CHAT, "Vision chat models", "Used for chat with image attachments.", "Vision"), - Section(ProviderType.TOOL_SEARCH, "Tool/Search models", "Used for query planning and structured tool calls.", "Tools"), Section(ProviderType.EMBEDDING, "Embedding models", "Used to index documents you attach to chats (RAG).", "RAG"), Section(ProviderType.IMAGE_GEN, "Image generation", "Used by the local image workspace.", "Image"), Section(ProviderType.IMAGE_UPSCALER, "Image upscalers", "Used by image upscale mode.", "Upscale"), diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt index cf7688ac..732651c2 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/BrowseModelsTab.kt @@ -25,7 +25,9 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontWeight @@ -58,6 +60,7 @@ internal fun BrowseModelsTab( viewModel: ModelStoreViewModel, onDownload: (HuggingFaceModel) -> Unit, onCancelDownload: (String) -> Unit, + onClearDownload: (String) -> Unit, onRetry: () -> Unit ) { val dimens = LocalDimens.current @@ -105,7 +108,7 @@ internal fun BrowseModelsTab( if (repoKey == null) { RepoCardListView(viewModel, isLoading, downloadStates, installStates) } else { - RepoDetailView(repoKey, viewModel, isLoading, downloadStates, installStates, extractingIds, extractingFile, installedModelIds, onDownload, onCancelDownload) + RepoDetailView(repoKey, viewModel, isLoading, downloadStates, installStates, extractingIds, extractingFile, installedModelIds, onDownload, onCancelDownload, onClearDownload) } } } @@ -209,7 +212,8 @@ internal fun RepoDetailView( extractingFile: Map, installedModelIds: Set, onDownload: (HuggingFaceModel) -> Unit, - onCancelDownload: (String) -> Unit + onCancelDownload: (String) -> Unit, + onClearDownload: (String) -> Unit, ) { val dimens = LocalDimens.current val filteredModels by viewModel.filteredModels.collectAsStateWithLifecycle() @@ -263,7 +267,8 @@ internal fun RepoDetailView( isExtracting = model.id in extractingIds, extractingEntryName = extractingFile[model.id], onDownload = { onDownload(model) }, - onCancel = { onCancelDownload(model.id) } + onCancel = { onCancelDownload(model.id) }, + onClear = { onClearDownload(model.id) }, ) } } diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt index d151e64f..1e039dc8 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelCard.kt @@ -51,6 +51,7 @@ fun CatalogModelCard( extractingEntryName: String? = null, onDownload: () -> Unit, onCancel: () -> Unit, + onClear: () -> Unit = {}, ) { val dimens = LocalDimens.current val shapes = LocalTnShapes.current @@ -136,11 +137,18 @@ fun CatalogModelCard( ) } isFailed -> { - ActionButton( - onClickListener = onDownload, - icon = TnIcons.Refresh, - contentDescription = "Retry" - ) + Row(horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { + ActionButton( + onClickListener = onDownload, + icon = TnIcons.Refresh, + contentDescription = "Retry" + ) + ActionButton( + onClickListener = onClear, + icon = TnIcons.Trash, + contentDescription = "Clear failed download" + ) + } } else -> { ActionButton( @@ -188,7 +196,11 @@ fun CatalogModelCard( ) } - model.tags.take(2).forEach { tag -> + val displayTags = buildList { + if (model.isVlm) add("projector auto") + addAll(model.tags) + }.distinct().take(3) + displayTags.forEach { tag -> Text( text = tag, style = MaterialTheme.typography.labelSmall, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelFilters.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelFilters.kt index bdeae200..673f406f 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelFilters.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelFilters.kt @@ -83,6 +83,16 @@ private val SORT_OPTIONS = listOf( FilterOption("Recent", SortOption.RECENTLY_ADDED), ) +private val UPSCALER_USE_CASE_OPTIONS: List> = listOf( + FilterOption("Manual", null), + FilterOption("General", "General"), + FilterOption("Photo", "Photo"), + FilterOption("Portrait", "Portrait"), + FilterOption("Anime", "Anime"), + FilterOption("Sharp cleanup", "Sharp cleanup"), + FilterOption("Other", "Other"), +) + private val PARAMETER_BUCKETS = listOf("0.5B", "1B", "3B", "6.7B", "8B", "32B", "70B") private val QUANT_BUCKETS = listOf("Q4_0", "Q5_0", "Q8_0", "Q4_K_M", "Q5_K_M", "Q6_K") @@ -121,6 +131,7 @@ fun ModelFiltersSection(viewModel: ModelStoreViewModel) { val selectedQuantizations by viewModel.selectedQuantizations.collectAsStateWithLifecycle() val selectedSizeCategory by viewModel.selectedSizeCategory.collectAsStateWithLifecycle() val selectedTags by viewModel.selectedTags.collectAsStateWithLifecycle() + val selectedUpscalerUseCase by viewModel.selectedUpscalerUseCase.collectAsStateWithLifecycle() val showNsfw by viewModel.showNsfw.collectAsStateWithLifecycle() val showOver2GbModels by viewModel.showOver2GbModels.collectAsStateWithLifecycle() val executionTarget by viewModel.executionTarget.collectAsStateWithLifecycle() @@ -138,6 +149,7 @@ fun ModelFiltersSection(viewModel: ModelStoreViewModel) { !showNsfw, showOver2GbModels, executionTarget != null, + selectedUpscalerUseCase != null, sortBy != SortOption.NAME, ).count { it } @@ -193,6 +205,32 @@ fun ModelFiltersSection(viewModel: ModelStoreViewModel) { } } + if (selectedModelType == "image_upscaler") { + Column( + modifier = Modifier.padding(horizontal = dimens.spacingLg), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + Text( + text = "Upscale use case", + style = MaterialTheme.typography.labelMedium, + fontWeight = FontWeight.SemiBold, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + FlowRow( + horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + UPSCALER_USE_CASE_OPTIONS.forEach { option -> + FilterChip( + selected = selectedUpscalerUseCase == option.value, + onClick = { viewModel.setUpscalerUseCase(option.value) }, + label = { Text(option.label) }, + ) + } + } + } + } + // Advanced toggle Row( modifier = Modifier diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt index 1fd21d50..8662c03a 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelImportTypePicker.kt @@ -108,7 +108,7 @@ internal fun ModelTypePickerDialog( internal fun providerTypeLabel(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Chat (GGUF)" ProviderType.VISION_CHAT -> "Vision chat" - ProviderType.TOOL_SEARCH -> "Tool/Search" + ProviderType.TOOL_SEARCH -> "Chat" ProviderType.EMBEDDING -> "Embedding (RAG)" ProviderType.IMAGE_GEN -> "Image generation" ProviderType.IMAGE_UPSCALER -> "Image upscaler" @@ -119,7 +119,7 @@ internal fun providerTypeLabel(type: ProviderType): String = when (type) { private fun providerTypeBlurb(type: ProviderType): String = when (type) { ProviderType.GGUF -> "Conversation models used by chat and the remote server." ProviderType.VISION_CHAT -> "Chat models that can accept image input when a projector is available." - ProviderType.TOOL_SEARCH -> "Small instruction models used for query planning and tool-call JSON." + ProviderType.TOOL_SEARCH -> "Chat model." ProviderType.EMBEDDING -> "Vector models for document search and RAG." ProviderType.IMAGE_GEN -> "Diffusion models used by the image workspace." ProviderType.IMAGE_UPSCALER -> "Upscaling models shown in image upscale mode." @@ -130,7 +130,6 @@ private fun providerTypeBlurb(type: ProviderType): String = when (type) { private val PROVIDER_OPTIONS = listOf( ProviderType.GGUF, ProviderType.VISION_CHAT, - ProviderType.TOOL_SEARCH, ProviderType.EMBEDDING, ProviderType.IMAGE_GEN, ProviderType.IMAGE_UPSCALER, diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt index 14d07fc7..7b388952 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/model_store/ModelStoreScreen.kt @@ -206,6 +206,7 @@ fun ModelStoreScreen( viewModel = viewModel, onDownload = { viewModel.downloadModel(it) }, onCancelDownload = { viewModel.cancelDownload(it) }, + onClearDownload = { viewModel.clearDownloadState(it) }, onRetry = { viewModel.loadModels() } ) StoreTab.INSTALLED -> InstalledModelsTab( diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt index a1de7cfd..dc758656 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt @@ -11,6 +11,7 @@ import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.itemsIndexed import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue @@ -46,12 +47,21 @@ fun SettingsScreen( ), verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), ) { - itemsIndexed(SETTINGS_LANDING_CARDS, key = { _, card -> card.route }) { index, card -> - AnimatedSettingsCard( - visible = visible, - stagger = index * 40, - ) { - SettingsLandingCard(card = card, onClick = { onNavigate(card.route) }) + SETTING_GROUPS.forEachIndexed { groupIndex, group -> + item(key = "group-${group.title}") { + Text( + text = group.title, + style = MaterialTheme.typography.labelLarge, + color = MaterialTheme.colorScheme.primary, + ) + } + itemsIndexed(group.cards, key = { _, card -> "${group.title}-${card.route}" }) { index, card -> + AnimatedSettingsCard( + visible = visible, + stagger = (groupIndex * 80) + (index * 35), + ) { + SettingsLandingCard(card = card, onClick = { onNavigate(card.route) }) + } } } } @@ -107,71 +117,133 @@ private data class LandingCard( val icon: ImageVector, ) -private val SETTINGS_LANDING_CARDS = listOf( - LandingCard( - route = NavScreens.SettingsChatRag.route, - title = "Chat & RAG", - description = "Embeddings, rerank, retrieval debugging", - icon = TnIcons.MessageCircle, - ), - LandingCard( - route = NavScreens.SettingsVoice.route, - title = "Voice", - description = "Default text-to-speech and speech-to-text", - icon = TnIcons.Volume, - ), - LandingCard( - route = NavScreens.SettingsTheming.route, - title = "Theming", - description = "Mode, palette, and live preview", - icon = TnIcons.Sparkles, +private data class SettingsGroup( + val title: String, + val cards: List, +) + +private val SETTING_GROUPS = listOf( + SettingsGroup( + title = "Models", + cards = listOf( + LandingCard( + route = NavScreens.SettingsChatRag.route, + title = "Chat & RAG", + description = "Embeddings, rerank, retrieval debugging", + icon = TnIcons.MessageCircle, + ), + LandingCard( + route = NavScreens.SettingsVision.route, + title = "Vision", + description = "Image quality for VLM models", + icon = TnIcons.Photo, + ), + LandingCard( + route = NavScreens.SettingsModel.route, + title = "Model performance", + description = "Performance and per-model configuration", + icon = TnIcons.Sliders, + ), + ), ), - LandingCard( - route = NavScreens.SettingsVision.route, - title = "Vision", - description = "Image quality for VLM (multimodal) models", - icon = TnIcons.Photo, + SettingsGroup( + title = "Storage", + cards = listOf( + LandingCard( + route = NavScreens.Storage.route, + title = "Storage & maintenance", + description = "Usage, cleanup, and model health checks", + icon = TnIcons.HardDrive, + ), + ), ), - LandingCard( - route = NavScreens.SettingsServerRoles.route, - title = "Server model roles", - description = "Manual identities for remote API models", - icon = TnIcons.Server, + SettingsGroup( + title = "Downloads", + cards = listOf( + LandingCard( + route = NavScreens.Downloads.route, + title = "Download history", + description = "Active downloads, retries, and failures", + icon = TnIcons.Download, + ), + ), ), - LandingCard( - route = NavScreens.SettingsModel.route, - title = "Model", - description = "Performance and per-model configuration", - icon = TnIcons.Sliders, + SettingsGroup( + title = "Remote Server", + cards = listOf( + LandingCard( + route = NavScreens.SettingsServerRoles.route, + title = "Server model roles", + description = "Remote API model identities", + icon = TnIcons.Server, + ), + ), ), - LandingCard( - route = NavScreens.SettingsPlugins.route, - title = "Plugins", - description = "ONNX execution provider, installed plugins", - icon = TnIcons.Puzzle, + SettingsGroup( + title = "Web Search", + cards = listOf( + LandingCard( + route = NavScreens.SettingsWebSearch.route, + title = "Web search", + description = "Default search depth for web queries", + icon = TnIcons.Globe, + ), + ), ), - LandingCard( - route = NavScreens.SettingsPrivacy.route, - title = "Privacy", - description = "App lock, panic PIN, vault", - icon = TnIcons.Shield, + SettingsGroup( + title = "Privacy & Security", + cards = listOf( + LandingCard( + route = NavScreens.SettingsPrivacy.route, + title = "Privacy", + description = "App lock, panic PIN, vault", + icon = TnIcons.Shield, + ), + ), ), - LandingCard( - route = NavScreens.SettingsDiagnostics.route, - title = "Diagnostics", - description = "Recent errors, crashes, exportable bundle", - icon = TnIcons.AlertTriangle, + SettingsGroup( + title = "Appearance", + cards = listOf( + LandingCard( + route = NavScreens.SettingsTheming.route, + title = "Theming", + description = "Mode, palette, and live preview", + icon = TnIcons.Sparkles, + ), + ), ), - LandingCard( - route = NavScreens.Storage.route, - title = "Storage", - description = "Disk usage by category", - icon = TnIcons.HardDrive, + SettingsGroup( + title = "Advanced", + cards = listOf( + LandingCard( + route = NavScreens.SettingsVoice.route, + title = "Voice", + description = "Default text-to-speech and speech-to-text", + icon = TnIcons.Volume, + ), + LandingCard( + route = NavScreens.SettingsPlugins.route, + title = "Plugins", + description = "ONNX execution provider, installed plugins", + icon = TnIcons.Puzzle, + ), + LandingCard( + route = NavScreens.SettingsDiagnostics.route, + title = "Diagnostics", + description = "Recent errors, crashes, exportable bundle", + icon = TnIcons.AlertTriangle, + ), + ), ), - LandingCard( - route = NavScreens.SettingsAbout.route, + SettingsGroup( title = "About", - description = "Version, license, terms, credits", - icon = TnIcons.Info, + cards = listOf( + LandingCard( + route = NavScreens.SettingsAbout.route, + title = "About", + description = "Version, license, terms, credits", + icon = TnIcons.Info, + ), + ), ), ) diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/storage/StorageScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/storage/StorageScreen.kt index 45c82ce7..69d7c8e4 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/storage/StorageScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/storage/StorageScreen.kt @@ -17,6 +17,7 @@ import androidx.compose.foundation.shape.CircleShape import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button import androidx.compose.material3.ButtonDefaults +import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface @@ -38,6 +39,9 @@ import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.dark.download_manager.formatBytes import com.dark.tool_neuron.repo.StorageCategoryId import com.dark.tool_neuron.repo.StorageCategorySnapshot +import com.dark.tool_neuron.repo.StorageIssueSeverity +import com.dark.tool_neuron.repo.StorageMaintenanceMode +import com.dark.tool_neuron.repo.StorageMaintenanceReport import com.dark.tool_neuron.repo.StorageSnapshot import com.dark.tool_neuron.ui.components.ActionTextButton import com.dark.tool_neuron.ui.components.SectionHeader @@ -81,6 +85,17 @@ fun StorageScreen( HeroCard(snapshot = state.snapshot, isLoading = state.isLoading) } + item(key = "maintenance") { + MaintenanceCard( + isRunning = state.isMaintaining, + report = state.maintenanceReport, + onQuickClean = viewModel::runQuickClean, + onDetailedCheck = viewModel::runDetailedCheck, + onDeepTest = viewModel::runDeepModelTest, + onDismissReport = viewModel::clearMaintenanceReport, + ) + } + item(key = "warning") { DangerStrip() } item(key = "section-header") { @@ -184,6 +199,124 @@ private fun HeroCard(snapshot: StorageSnapshot?, isLoading: Boolean) { } } +@Composable +private fun MaintenanceCard( + isRunning: Boolean, + report: StorageMaintenanceReport?, + onQuickClean: () -> Unit, + onDetailedCheck: () -> Unit, + onDeepTest: () -> Unit, + onDismissReport: () -> Unit, +) { + val dimens = LocalDimens.current + val shapes = LocalTnShapes.current + Surface( + modifier = Modifier.fillMaxWidth(), + shape = shapes.card, + color = MaterialTheme.colorScheme.surfaceVariant.copy(alpha = 0.28f), + ) { + Column( + modifier = Modifier.padding(dimens.cardPadding), + verticalArrangement = Arrangement.spacedBy(dimens.spacingSm), + ) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(dimens.spacingSm), + verticalAlignment = Alignment.CenterVertically, + ) { + Icon(TnIcons.Search, null, Modifier.size(dimens.iconMd), MaterialTheme.colorScheme.primary) + Column(modifier = Modifier.weight(1f)) { + Text("Maintenance", style = MaterialTheme.typography.titleSmall, fontWeight = FontWeight.SemiBold) + Text( + "Clean leftovers, check model files, and verify installed records.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + if (isRunning) CircularProgressIndicator(Modifier.size(20.dp), strokeWidth = 2.dp) + } + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + Button(onClick = onQuickClean, enabled = !isRunning, modifier = Modifier.weight(1f)) { + Text("Quick clean", maxLines = 1, overflow = TextOverflow.Ellipsis) + } + Button(onClick = onDetailedCheck, enabled = !isRunning, modifier = Modifier.weight(1f)) { + Text("Detailed check", maxLines = 1, overflow = TextOverflow.Ellipsis) + } + } + TextButton(onClick = onDeepTest, enabled = !isRunning) { + Text("Deep model test") + } + if (report != null) { + MaintenanceReportCard(report = report, onDismiss = onDismissReport) + } + } + } +} + +@Composable +private fun MaintenanceReportCard(report: StorageMaintenanceReport, onDismiss: () -> Unit) { + val dimens = LocalDimens.current + val title = when (report.mode) { + StorageMaintenanceMode.QUICK_CLEAN -> "Quick clean" + StorageMaintenanceMode.DETAILED_CHECK -> "Detailed check" + StorageMaintenanceMode.DEEP_MODEL_TEST -> "Deep model test" + } + Surface( + modifier = Modifier.fillMaxWidth(), + shape = LocalTnShapes.current.cardSmall, + color = MaterialTheme.colorScheme.surface.copy(alpha = 0.7f), + ) { + Column( + modifier = Modifier.padding(dimens.spacingMd), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + Text( + "$title result", + style = MaterialTheme.typography.labelLarge, + fontWeight = FontWeight.SemiBold, + modifier = Modifier.weight(1f), + ) + TextButton(onClick = onDismiss) { Text("Dismiss") } + } + Text( + "Checked ${report.checkedCount} · Issues ${report.issueCount} · Fixed ${report.fixedCount} · Skipped ${report.skippedCount}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + report.issues.take(6).forEach { issue -> + val color = when (issue.severity) { + StorageIssueSeverity.INFO -> MaterialTheme.colorScheme.primary + StorageIssueSeverity.WARNING -> MaterialTheme.colorScheme.tertiary + StorageIssueSeverity.ERROR -> MaterialTheme.colorScheme.error + } + Row(horizontalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { + Box( + modifier = Modifier + .padding(top = 6.dp) + .size(7.dp) + .clip(CircleShape) + .background(color), + ) + Column(modifier = Modifier.weight(1f)) { + Text(issue.title, style = MaterialTheme.typography.bodySmall, fontWeight = FontWeight.SemiBold) + Text( + issue.detail, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + } + } + } + } +} + @Composable private fun SegmentedBar(snapshot: StorageSnapshot, palette: Map) { val tnShapes = LocalTnShapes.current diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/system_ui/AppTopBar.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/system_ui/AppTopBar.kt index c645ff5c..1fd7f737 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/system_ui/AppTopBar.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/system_ui/AppTopBar.kt @@ -64,6 +64,11 @@ fun AppTopBar( title = "Chat & RAG", subtitle = "Defaults for indexing and retrieval", ) + NavScreens.SettingsWebSearch.route -> SettingsTopBar( + onBack = onBack, + title = "Web search", + subtitle = "Default search depth", + ) NavScreens.SettingsVoice.route -> SettingsTopBar( onBack = onBack, title = "Voice", diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/web_search/WebSearchCard.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/web_search/WebSearchCard.kt index f7b19c58..8d3536fb 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/web_search/WebSearchCard.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/web_search/WebSearchCard.kt @@ -72,7 +72,7 @@ fun WebSearchCard( .padding(vertical = dimens.spacingXs), ) { Column(modifier = Modifier.padding(dimens.spacingMd)) { - Header(userQuery = message.content) + Header(userQuery = message.content, mode = state.mode) Spacer(Modifier.size(dimens.spacingSm)) QueriesStrip(state) @@ -84,7 +84,10 @@ fun WebSearchCard( ) { phase -> when (phase) { WebSearchUiState.PHASE_PLAN -> StatusRow(TnIcons.Sparkles, "Planning search…") - WebSearchUiState.PHASE_QUERIES -> StatusRow(TnIcons.Search, "Generated 3 queries") + WebSearchUiState.PHASE_QUERIES -> StatusRow( + TnIcons.Search, + "Generated ${state.queries.size} ${if (state.queries.size == 1) "query" else "queries"}", + ) WebSearchUiState.PHASE_SEARCH -> SearchProgressRow(state) WebSearchUiState.PHASE_SYNTHESIZE -> StatusRow(TnIcons.Sparkles, "Writing answer from sources…") WebSearchUiState.PHASE_STOPPING -> StatusRow(TnIcons.PlayerStop, "Stopping…") @@ -156,7 +159,7 @@ fun WebSearchCard( } @Composable -private fun Header(userQuery: String) { +private fun Header(userQuery: String, mode: String) { val dimens = LocalDimens.current Row(verticalAlignment = Alignment.CenterVertically) { Icon( @@ -167,12 +170,18 @@ private fun Header(userQuery: String) { ) Spacer(Modifier.size(dimens.spacingXs)) Column(modifier = Modifier.fillMaxWidth()) { - Text( - text = "Web search", - style = MaterialTheme.typography.labelLarge, - color = MaterialTheme.colorScheme.primary, - fontWeight = FontWeight.SemiBold, - ) + Row(verticalAlignment = Alignment.CenterVertically) { + Text( + text = "Web search", + style = MaterialTheme.typography.labelLarge, + color = MaterialTheme.colorScheme.primary, + fontWeight = FontWeight.SemiBold, + ) + if (mode.isNotBlank()) { + Spacer(Modifier.size(dimens.spacingXs)) + ModeBadge(mode) + } + } if (userQuery.isNotBlank()) { Text( text = userQuery, @@ -268,12 +277,36 @@ private fun StatusRow(icon: androidx.compose.ui.graphics.vector.ImageVector, lab } } +@Composable +private fun ModeBadge(mode: String) { + val dimens = LocalDimens.current + Surface( + shape = LocalTnShapes.current.full, + color = MaterialTheme.colorScheme.primary.copy(alpha = 0.16f), + ) { + Text( + text = mode, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.primary, + fontWeight = FontWeight.Medium, + modifier = Modifier.padding(horizontal = dimens.spacingSm, vertical = 1.dp), + ) + } +} + @Composable private fun SearchProgressRow(state: WebSearchUiState) { - val idx = state.currentQueryIndex.coerceAtLeast(0) - val current = state.queries.getOrNull(idx).orEmpty() - val total = state.queries.size - val label = if (current.isBlank()) "Searching…" else "Searching ${idx + 1}/$total: $current" + val label = when { + state.status.isNotBlank() && state.maxRounds > 1 -> + "Round ${state.round.coerceAtLeast(1)}/${state.maxRounds} · ${state.status}" + state.status.isNotBlank() -> state.status + else -> { + val idx = state.currentQueryIndex.coerceAtLeast(0) + val current = state.queries.getOrNull(idx).orEmpty() + val total = state.queries.size + if (current.isBlank()) "Searching…" else "Searching ${idx + 1}/$total: $current" + } + } StatusRow(TnIcons.Search, label) } diff --git a/app/src/main/java/com/dark/tool_neuron/util/ImageExport.kt b/app/src/main/java/com/dark/tool_neuron/util/ImageExport.kt index 8733b627..15d16c58 100644 --- a/app/src/main/java/com/dark/tool_neuron/util/ImageExport.kt +++ b/app/src/main/java/com/dark/tool_neuron/util/ImageExport.kt @@ -8,6 +8,7 @@ import android.os.Build import android.os.Environment import android.provider.MediaStore import java.io.File +import java.io.FileInputStream object ImageExport { @@ -48,4 +49,37 @@ object ImageExport { } uri } + + suspend fun savePngFileToGallery( + context: Context, + source: File, + displayName: String = "tool_neuron_${System.currentTimeMillis()}", + ): Result = runCatching { + val resolver = context.contentResolver + val collection = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) + } else { + MediaStore.Images.Media.EXTERNAL_CONTENT_URI + } + val values = ContentValues().apply { + put(MediaStore.Images.Media.DISPLAY_NAME, "$displayName.png") + put(MediaStore.Images.Media.MIME_TYPE, "image/png") + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + put(MediaStore.Images.Media.RELATIVE_PATH, "${Environment.DIRECTORY_PICTURES}/ToolNeuron") + put(MediaStore.Images.Media.IS_PENDING, 1) + } + } + val uri = resolver.insert(collection, values) + ?: throw IllegalStateException("MediaStore.insert returned null") + resolver.openOutputStream(uri).use { out -> + requireNotNull(out) { "openOutputStream returned null" } + FileInputStream(source).use { input -> input.copyTo(out) } + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + values.clear() + values.put(MediaStore.Images.Media.IS_PENDING, 0) + resolver.update(uri, values, null, null) + } + uri + } } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/DownloadsViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/DownloadsViewModel.kt index 0b42a681..b448f080 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/DownloadsViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/DownloadsViewModel.kt @@ -1,5 +1,6 @@ package com.dark.tool_neuron.viewmodel +import android.content.Context import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.dark.download_manager.HxdManager @@ -10,6 +11,7 @@ import com.dark.tool_neuron.repo.DownloadCoordinator import com.dark.tool_neuron.repo.DownloadHistoryRepository import com.dark.tool_neuron.repo.DownloadLabel import dagger.hilt.android.lifecycle.HiltViewModel +import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.combine @@ -26,6 +28,7 @@ data class ActiveDownloadItem( @HiltViewModel class DownloadsViewModel @Inject constructor( + @ApplicationContext private val context: Context, private val coordinator: DownloadCoordinator, private val historyRepo: DownloadHistoryRepository, ) : ViewModel() { @@ -39,7 +42,8 @@ class DownloadsViewModel @Inject constructor( it.status == HxdStatus.QUEUED || it.status == HxdStatus.CONNECTING || it.status == HxdStatus.DOWNLOADING || - it.status == HxdStatus.PAUSED + it.status == HxdStatus.PAUSED || + it.status == HxdStatus.FAILED } .map { state -> val label = labels[state.id] ?: DownloadLabel.fromUrl(state.url) @@ -62,7 +66,16 @@ class DownloadsViewModel @Inject constructor( HxdManager.cancel(hxdId) } + fun retry(hxdId: Int) { + HxdManager.resume(context, hxdId) + } + + fun clear(hxdId: Int) { + HxdManager.clear(hxdId) + } + fun clearHistory() { historyRepo.clearAll() } + } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/HfExplorerViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/HfExplorerViewModel.kt index 07e775c5..16ac5356 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/HfExplorerViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/HfExplorerViewModel.kt @@ -4,6 +4,8 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.dark.tool_neuron.data.AppPreferences import com.dark.tool_neuron.model.HFRepository +import com.dark.tool_neuron.model.VlmFileGroup +import com.dark.tool_neuron.model.VlmFileGroups import com.dark.tool_neuron.repo.GatedFilter import com.dark.tool_neuron.repo.HfApiError import com.dark.tool_neuron.repo.HfFilters @@ -250,9 +252,15 @@ class HfExplorerViewModel @Inject constructor( } val cap = _fileSizeBucket.value.maxBytes if (cap != Long.MAX_VALUE) list = list.filter { it.sizeBytes in 1..cap } - return list.sortedBy { it.sizeBytes } + return list.sortedWith(compareBy { it.path.substringAfterLast('/').lowercase() }.thenBy { it.sizeBytes }) } + fun visibleFileGroups(detail: HfModelDetail): List = + VlmFileGroups.fromFiles(visibleFiles(detail).filter { + it.path.endsWith(".gguf", ignoreCase = true) || + it.path.contains("mmproj", ignoreCase = true) + }) + private fun loadTagsCatalog() { viewModelScope.launch { client.tagsCatalog().onSuccess { _tagsCatalog.value = it } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt index 2db7dd20..69729e1b 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/HomeViewModel.kt @@ -20,6 +20,7 @@ import com.dark.gguf_lib.ImageQuality import com.dark.tool_neuron.data.AppPreferences import com.dark.tool_neuron.data.AppVisibility import com.dark.tool_neuron.data.UserNotificationManager +import com.dark.tool_neuron.repo.web_search.WebSearchMode import com.dark.tool_neuron.repo.ChatRepository import com.dark.tool_neuron.repo.DownloadCoordinator import com.dark.tool_neuron.repo.ModelRepository @@ -194,6 +195,9 @@ class HomeViewModel @Inject constructor( private val _webSearchEnabled = MutableStateFlow(false) val webSearchEnabled = _webSearchEnabled.asStateFlow() + private val _webSearchMode = MutableStateFlow(WebSearchMode.sanitizePref(appPrefs.webSearchMode)) + val webSearchMode = _webSearchMode.asStateFlow() + val webSearchActive: StateFlow = webSearchCoordinator.activeRuns .map { it.isNotEmpty() } .stateIn(viewModelScope, SharingStarted.Eagerly, false) @@ -400,6 +404,12 @@ class HomeViewModel @Inject constructor( if (next) _thinkingEnabled.value = false } + fun setWebSearchMode(key: String) { + val k = WebSearchMode.sanitizePref(key) + appPrefs.webSearchMode = k + _webSearchMode.value = k + } + fun toggleThinking() { val next = !_thinkingEnabled.value _thinkingEnabled.value = next @@ -715,9 +725,9 @@ class HomeViewModel @Inject constructor( return } - val webSearchQuery = parseWebSearchInput(trimmed) - if (webSearchQuery != null) { - startWebSearch(active, webSearchQuery) + val webSearch = parseWebSearchInput(trimmed) + if (webSearch != null) { + startWebSearch(active, webSearch.mode, webSearch.query) return } @@ -753,12 +763,15 @@ class HomeViewModel @Inject constructor( } } - private fun parseWebSearchInput(text: String): String? { - if (text.startsWith("/search", ignoreCase = true)) { - val q = text.removePrefix("/search").removePrefix("/SEARCH").trim() - return q.ifBlank { null } + private fun parseWebSearchInput(text: String): WebSearchMode.Resolved? { + val lower = text.trimStart().lowercase() + val isSlash = SEARCH_SLASHES.any { lower == it || lower.startsWith("$it ") || lower.startsWith("$it\n") } + val default = WebSearchMode.fromPref(_webSearchMode.value) + if (isSlash) { + val resolved = WebSearchMode.resolve(text, default) + return if (resolved.query.isBlank()) null else resolved } - if (_webSearchEnabled.value) return text + if (_webSearchEnabled.value) return WebSearchMode.resolve(text, default) return null } @@ -784,7 +797,7 @@ class HomeViewModel @Inject constructor( return "$priorTopic $normalized" } - private fun startWebSearch(active: ModelInfo, userQuery: String) { + private fun startWebSearch(active: ModelInfo, mode: WebSearchMode, userQuery: String) { if (webSearchActive.value) return val chatId = ensureChat(active) val previousMessages = chatRepo.getMessages(chatId) @@ -812,13 +825,18 @@ class HomeViewModel @Inject constructor( kind = MessageKind.Text, modelName = active.name, webSearchRunId = placeholderRunId, - webSearchState = WebSearchUiState(userQuery = searchQuery).toJson(), + webSearchState = WebSearchUiState( + userQuery = searchQuery, + mode = mode.label, + maxRounds = mode.maxRounds, + ).toJson(), ) chatRepo.addMessage(cardMessage) val runId = webSearchCoordinator.start( scope = viewModelScope, userQuery = searchQuery, + mode = mode, ) webSearchMessages[runId] = chatId to cardMessageId @@ -843,12 +861,15 @@ class HomeViewModel @Inject constructor( // as a plain chat send). if (lastAssistant.webSearchRunId != null) { val query = lastAssistant.content + val priorMode = WebSearchUiState.fromJson(lastAssistant.webSearchState).mode + val mode = WebSearchMode.entries.firstOrNull { it.label == priorMode } + ?: WebSearchMode.classify(query) chatRepo.deleteMessage(lastAssistant.id) val msgs = chatRepo.getMessages(chatId) val lastUser = msgs.lastOrNull { it.role == ROLE_USER } if (lastUser != null) chatRepo.deleteMessage(lastUser.id) _messages.value = chatRepo.getMessages(chatId) - startWebSearch(active, query) + startWebSearch(active, mode, query) return } chatRepo.deleteMessage(lastAssistant.id) @@ -1053,6 +1074,8 @@ class HomeViewModel @Inject constructor( // mid-range device. We never let the bar exceed 97% before Done so // a fast summary doesn't visibly "finish then re-snap" at completion. private const val GEN_PHASE_TARGET_MS = 12_000L + + private val SEARCH_SLASHES = listOf("/search", "/deep", "/research", "/exhaustive") } private fun runGeneration(chatId: String, isFirstTurn: Boolean, userText: String) { diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/ImageTaskViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/ImageTaskViewModel.kt index 3804a33d..48cadceb 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/ImageTaskViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/ImageTaskViewModel.kt @@ -6,7 +6,6 @@ import android.graphics.Bitmap import android.graphics.BitmapFactory import android.net.Uri import android.util.Log -import androidx.core.graphics.scale import java.io.File import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope @@ -20,7 +19,11 @@ import com.dark.download_manager.HxdStatus import com.dark.tool_neuron.model.ModelInfo import com.dark.tool_neuron.model.enums.ProviderType import com.dark.tool_neuron.repo.ImageGenManager +import com.dark.tool_neuron.repo.FastUpscaleBlockedException +import com.dark.tool_neuron.repo.ImageUpscalePipeline import com.dark.tool_neuron.repo.ModelRepository +import com.dark.tool_neuron.repo.UpscaleProcessingMode +import com.dark.tool_neuron.repo.UpscaleScalePreset import com.dark.tool_neuron.util.ImageExport import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.Dispatchers @@ -38,6 +41,15 @@ import javax.inject.Inject enum class ImageTaskMode { GENERATE, INPAINT, UPSCALE } enum class ImageOutputAction { KEEP, REPLACE_INPUT, SAVE_PHOTOS, SAVE_ELSEWHERE } +data class UpscaleOutput( + val preview: Bitmap, + val fullWidth: Int, + val fullHeight: Int, + val outputPath: String?, + val scale: Float, + val tiled: Boolean, +) + private data class ImageTaskInputs( val mode: ImageTaskMode = ImageTaskMode.GENERATE, val prompt: String = "", @@ -54,6 +66,15 @@ private data class ImageTaskInputs( val maskImagePath: String? = null, val activeModelId: String = "", val activeUpscalerId: String = "", + val upscaleScalePreset: UpscaleScalePreset = UpscaleScalePreset.X4, + val customUpscaleScale: Float = 4f, + val upscaleProcessingMode: UpscaleProcessingMode = UpscaleProcessingMode.AUTO, + val upscaleProgressLabel: String = "", + val upscaleProgress: Float = 0f, + val upscaleTileCurrent: Int = 0, + val upscaleTileTotal: Int = 0, + val upscalePassCurrent: Int = 0, + val upscalePassTotal: Int = 0, val localStatus: String = "", val localError: String? = null, val pendingLoadModelId: String = "", @@ -68,6 +89,7 @@ private data class ImageTaskInputs( val outputToken: String = "", val generationStartedAtMs: Long = 0L, val upscaleStartedAtMs: Long = 0L, + val upscaleBusy: Boolean = false, val lastUpscaleEstimateMs: Long = 45_000L, val processedOutputToken: String = "", /** @@ -128,6 +150,16 @@ data class ImageTaskUi( val installedUpscalers: List = emptyList(), val activeModelId: String = "", val activeUpscalerId: String = "", + val upscaleScalePreset: UpscaleScalePreset = UpscaleScalePreset.X4, + val customUpscaleScale: Float = 4f, + val upscaleScale: Float = 4f, + val upscaleProcessingMode: UpscaleProcessingMode = UpscaleProcessingMode.AUTO, + val upscaleProgressLabel: String = "", + val upscaleTileCurrent: Int = 0, + val upscaleTileTotal: Int = 0, + val upscalePassCurrent: Int = 0, + val upscalePassTotal: Int = 0, + val upscaleOutput: UpscaleOutput? = null, val isBusy: Boolean = false, val runtimeReady: Boolean = false, val runtimePhase: RuntimePhase = RuntimePhase.NEEDS_DOWNLOAD, @@ -164,6 +196,7 @@ class ImageTaskViewModel @Inject constructor( // image and a blank slot. Holds onto the most recent non-null preview; // resets when generation starts / completes / errors. private val cachedPreview = MutableStateFlow(null) + private val upscaleOutput = MutableStateFlow(null) private val clock = MutableStateFlow(System.currentTimeMillis()) private val context: Context get() = getApplication() @@ -233,18 +266,6 @@ class ImageTaskViewModel @Inject constructor( } } - viewModelScope.launch { - imageGen.upscaleState.collect { state -> - if (state is UpscaleState.Complete) { - handleCompletedOutput( - token = "up_${state.width}_${state.height}_${state.timeMs}_${System.nanoTime()}", - bitmap = state.bitmap, - prefix = "tn_upscale", - finalUpscaleTimeMs = state.timeMs.toLong().coerceAtLeast(1L), - ) - } - } - } } val ui: StateFlow = combine( @@ -256,6 +277,7 @@ class ImageTaskViewModel @Inject constructor( imageGen.backendState, imageGen.runtimeDownload, cachedPreview, + upscaleOutput, clock, ) { values -> @Suppress("UNCHECKED_CAST") @@ -268,7 +290,8 @@ class ImageTaskViewModel @Inject constructor( val backend = values[5] as DiffusionBackendState val runtimeDl = values[6] as HxdState? val sticky = values[7] as Bitmap? - val now = values[8] as Long + val upscaleOut = values[8] as UpscaleOutput? + val now = values[9] as Long val archivePresent = imageGen.isRuntimeArchivePresent() val runtimeReady = runtime is RuntimeSetupState.Complete @@ -302,7 +325,7 @@ class ImageTaskViewModel @Inject constructor( val name = models.firstOrNull { it.id == i.loadedUpscalerId }?.name ?: i.loadedUpscalerId ModelLoadPhase.Loaded(name) } - i.mode != ImageTaskMode.UPSCALE && i.loadedDiffusionId.isNotBlank() -> { + i.mode in setOf(ImageTaskMode.GENERATE, ImageTaskMode.INPAINT) && i.loadedDiffusionId.isNotBlank() -> { val name = models.firstOrNull { it.id == i.loadedDiffusionId }?.name ?: i.loadedDiffusionId ModelLoadPhase.Loaded(name) } @@ -311,7 +334,6 @@ class ImageTaskViewModel @Inject constructor( val sdkProgress = (gen as? DiffusionGenerationState.Progress)?.progress ?: 0f val diffusionResult = (gen as? DiffusionGenerationState.Complete)?.bitmap - val upscaleResult = (upscale as? UpscaleState.Complete)?.bitmap val metrics = progressMetrics(i, gen, upscale, now) val sdkError = (gen as? DiffusionGenerationState.Error)?.message @@ -339,8 +361,18 @@ class ImageTaskViewModel @Inject constructor( installedUpscalers = models.filter { it.providerType == ProviderType.IMAGE_UPSCALER }, activeModelId = i.activeModelId, activeUpscalerId = i.activeUpscalerId, + upscaleScalePreset = i.upscaleScalePreset, + customUpscaleScale = i.customUpscaleScale, + upscaleScale = i.upscaleScale(), + upscaleProcessingMode = i.upscaleProcessingMode, + upscaleProgressLabel = i.upscaleProgressLabel, + upscaleTileCurrent = i.upscaleTileCurrent, + upscaleTileTotal = i.upscaleTileTotal, + upscalePassCurrent = i.upscalePassCurrent, + upscalePassTotal = i.upscalePassTotal, diffusionResultImage = diffusionResult, - upscaleResultImage = upscaleResult, + upscaleResultImage = upscaleOut?.preview, + upscaleOutput = upscaleOut, previewImage = sticky, progress = metrics.progress.takeIf { it > 0f } ?: sdkProgress, metrics = metrics, @@ -353,7 +385,8 @@ class ImageTaskViewModel @Inject constructor( || runtime is RuntimeSetupState.Extracting || runtime is RuntimeSetupState.InitializingRuntime || backend is DiffusionBackendState.Starting - || i.pendingLoadModelId.isNotBlank(), + || i.pendingLoadModelId.isNotBlank() + || i.upscaleBusy, runtimePhase = runtimePhase, runtimeDownloadProgress = dlProgress, runtimeDownloadBytes = runtimeDl?.downloadedBytes ?: 0L, @@ -399,8 +432,17 @@ class ImageTaskViewModel @Inject constructor( inputs.update { it.copy(denoiseStrength = value.coerceIn(0f, 1f)) } fun setOutputAction(action: ImageOutputAction) = inputs.update { it.copy(outputAction = action, outputStatus = "") } + fun setUpscaleScalePreset(value: UpscaleScalePreset) = + inputs.update { it.copy(upscaleScalePreset = value, localError = null) } + fun setCustomUpscaleScale(value: Float) = + inputs.update { it.copy(customUpscaleScale = value.coerceIn(1.25f, 8f), localError = null) } + fun setUpscaleProcessingMode(value: UpscaleProcessingMode) = + inputs.update { it.copy(upscaleProcessingMode = value, localError = null) } + fun switchToSafeTiled() = + inputs.update { it.copy(upscaleProcessingMode = UpscaleProcessingMode.SAFE_TILED, localError = null) } fun setInputImage(uri: Uri) { + upscaleOutput.value = null inputs.update { it.copy( inputImagePath = uri.toString(), @@ -433,32 +475,24 @@ class ImageTaskViewModel @Inject constructor( maskImagePath = null, localError = if (installedUpscalers == 0) "Install an upscaler from the Store first" - else "Pick an upscaler — then tap Upscale 4×", + else "Pick an upscaler, then choose a scale.", ) } } return } - val maxEdge = maxOf(bitmap.width, bitmap.height) - val targetEdge = maxEdge.coerceAtMost(1024) - val safe = if (maxEdge == targetEdge) bitmap else { - val scale = targetEdge.toFloat() / maxEdge.toFloat() - bitmap.scale( - (bitmap.width * scale).toInt(), - (bitmap.height * scale).toInt(), - ) - } viewModelScope.launch(Dispatchers.IO) { val path = saveBitmapToCache(bitmap) + upscaleOutput.value = null inputs.update { it.copy( mode = ImageTaskMode.UPSCALE, inputImagePath = path, maskImagePath = null, localError = null, + localStatus = "Ready to upscale", ) } - imageGen.upscale(safe) } } @@ -757,32 +791,81 @@ class ImageTaskViewModel @Inject constructor( return } viewModelScope.launch(Dispatchers.IO) { - val bitmap = decodeBitmap(path) ?: run { - inputs.update { it.copy(localError = "Couldn't read input image") } - return@launch - } - // Cap input at 1024 max-edge: 4× output = 4096² ≈ 64 MB which fits - // the default JVM heap. The SDK can technically take 2048² but that - // produces an 8192² ≈ 256 MB bitmap which OOMs the heap on result - // delivery (bitmap allocation in DiffusionManager.createBitmapFromRgb). - val maxEdge = maxOf(bitmap.width, bitmap.height) - val targetEdge = maxEdge.coerceAtMost(1024) - val safe = if (maxEdge == targetEdge) bitmap else { - val scale = targetEdge.toFloat() / maxEdge.toFloat() - bitmap.scale( - (bitmap.width * scale).toInt(), - (bitmap.height * scale).toInt(), - ) - } + val pipeline = ImageUpscalePipeline(context, imageGen) inputs.update { it.copy( localError = null, outputStatus = "", upscaleStartedAtMs = System.currentTimeMillis(), + upscaleBusy = true, + upscaleProgressLabel = "Preparing", + upscaleProgress = 0.02f, + upscaleTileCurrent = 0, + upscaleTileTotal = 0, + upscalePassCurrent = 0, + upscalePassTotal = 0, processedOutputToken = "", ) } - imageGen.upscale(safe) + runCatching { + pipeline.run( + imagePath = path, + scale = s.upscaleScale(), + mode = s.upscaleProcessingMode, + outputDir = File(context.cacheDir, "upscale_outputs"), + ) { progress -> + inputs.update { current -> + current.copy( + localStatus = progress.label, + upscaleProgressLabel = progress.label, + upscaleProgress = progress.progress, + upscaleTileCurrent = progress.currentTile, + upscaleTileTotal = progress.totalTiles, + upscalePassCurrent = progress.currentPass, + upscalePassTotal = progress.totalPasses, + ) + } + } + }.onSuccess { result -> + val output = UpscaleOutput( + preview = result.preview, + fullWidth = result.fullWidth, + fullHeight = result.fullHeight, + outputPath = result.outputFile?.absolutePath, + scale = result.scale, + tiled = result.tiled, + ) + upscaleOutput.value = output + handleCompletedOutput( + token = "up_${result.fullWidth}_${result.fullHeight}_${System.nanoTime()}", + bitmap = result.preview, + prefix = "tn_upscale", + finalUpscaleTimeMs = result.elapsedMs, + outputFile = result.outputFile, + ) + inputs.update { current -> + current.copy( + upscaleBusy = false, + localStatus = "Upscale done", + localError = null, + upscaleProgressLabel = "", + upscaleProgress = 1f, + ) + } + }.onFailure { error -> + inputs.update { state -> + state.copy( + upscaleBusy = false, + upscaleProgressLabel = "", + upscaleProgress = 0f, + localStatus = "", + localError = when (error) { + is FastUpscaleBlockedException -> error.message + else -> error.message ?: "Upscale failed" + }, + ) + } + } } } @@ -791,6 +874,7 @@ class ImageTaskViewModel @Inject constructor( bitmap: Bitmap, prefix: String, finalUpscaleTimeMs: Long? = null, + outputFile: File? = null, ) { val state = inputs.value if (state.processedOutputToken == token) return @@ -807,7 +891,7 @@ class ImageTaskViewModel @Inject constructor( } } ImageOutputAction.REPLACE_INPUT -> { - val path = withContext(Dispatchers.IO) { saveBitmapToCache(bitmap) } + val path = outputFile?.absolutePath ?: withContext(Dispatchers.IO) { saveBitmapToCache(bitmap) } inputs.update { it.copy( inputImagePath = path, @@ -818,11 +902,15 @@ class ImageTaskViewModel @Inject constructor( } } ImageOutputAction.SAVE_PHOTOS -> { - val result = ImageExport.saveBitmapToGallery( - context = context, - bitmap = bitmap, - displayName = "${prefix}_${System.currentTimeMillis()}", - ) + val result = if (outputFile != null) { + ImageExport.savePngFileToGallery(context, outputFile, "${prefix}_${System.currentTimeMillis()}") + } else { + ImageExport.saveBitmapToGallery( + context = context, + bitmap = bitmap, + displayName = "${prefix}_${System.currentTimeMillis()}", + ) + } inputs.update { it.copy( processedOutputToken = token, @@ -866,10 +954,11 @@ class ImageTaskViewModel @Inject constructor( runtime is RuntimeSetupState.CopyingSafetyChecker -> "Setting up runtime: safety checker" runtime is RuntimeSetupState.InitializingRuntime -> "Initializing native runtime…" backend is DiffusionBackendState.Starting -> "Loading model…" + i.upscaleBusy -> i.upscaleProgressLabel.ifBlank { i.localStatus } gen is DiffusionGenerationState.Progress -> "Step ${gen.currentStep}/${gen.totalSteps} (${"%.0f".format(gen.progress * 100)}%)" gen is DiffusionGenerationState.Complete -> "Done" - upscale is UpscaleState.Processing -> "Upscaling…" + upscale is UpscaleState.Processing -> i.upscaleProgressLabel.ifBlank { "Upscaling…" } upscale is UpscaleState.Complete -> "Upscale done" else -> i.localStatus } @@ -903,19 +992,21 @@ class ImageTaskViewModel @Inject constructor( detail = "Real step progress from native engine", ) } - upscale is UpscaleState.Processing -> { + i.upscaleBusy -> { val started = i.upscaleStartedAtMs.takeIf { it > 0L } ?: now val elapsed = (now - started).coerceAtLeast(0L) val estimate = i.lastUpscaleEstimateMs.coerceAtLeast(8_000L) + val progress = i.upscaleProgress.takeIf { it > 0f } + ?: (elapsed.toFloat() / estimate.toFloat()).coerceIn(0.03f, 0.97f) ImageTaskMetrics( active = true, - progress = (elapsed.toFloat() / estimate.toFloat()).coerceIn(0.03f, 0.97f), + progress = progress.coerceIn(0.03f, 0.98f), elapsedMs = elapsed, etaMs = (estimate - elapsed).coerceAtLeast(0L), - width = i.width * 4, - height = i.height * 4, - label = "Upscaling", - detail = "Estimated from this device's last upscale time", + currentStep = i.upscaleTileCurrent.takeIf { it > 0 } ?: i.upscalePassCurrent.takeIf { it > 0 }, + totalSteps = i.upscaleTileTotal.takeIf { it > 0 } ?: i.upscalePassTotal.takeIf { it > 0 }, + label = i.upscaleProgressLabel.ifBlank { "Upscaling" }, + detail = "Adaptive upscale pipeline", ) } upscale is UpscaleState.Complete -> ImageTaskMetrics( @@ -950,3 +1041,10 @@ class ImageTaskViewModel @Inject constructor( private inline fun MutableStateFlow.update(transform: (T) -> T) { value = transform(value) } + +private fun ImageTaskInputs.upscaleScale(): Float = when (upscaleScalePreset) { + UpscaleScalePreset.X2 -> 2f + UpscaleScalePreset.X4 -> 4f + UpscaleScalePreset.X8 -> 8f + UpscaleScalePreset.CUSTOM -> customUpscaleScale +} diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt index 511874f4..cb0ed7e8 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/ModelStoreViewModel.kt @@ -179,6 +179,9 @@ class ModelStoreViewModel @Inject constructor( private val _selectedTags = MutableStateFlow>(emptySet()) val selectedTags: StateFlow> = _selectedTags.asStateFlow() + private val _selectedUpscalerUseCase = MutableStateFlow(null) + val selectedUpscalerUseCase: StateFlow = _selectedUpscalerUseCase.asStateFlow() + private val _showNsfw = MutableStateFlow(true) val showNsfw: StateFlow = _showNsfw.asStateFlow() @@ -405,6 +408,14 @@ class ModelStoreViewModel @Inject constructor( } } + if (_selectedModelType.value == "image_upscaler") { + _selectedUpscalerUseCase.value?.let { useCase -> + filtered = filtered.filter { model -> + model.tags.any { it.equals("Use: $useCase", ignoreCase = true) } + } + } + } + if (!_showNsfw.value) { filtered = filtered.filter { "NSFW" !in it.tags } } @@ -442,7 +453,11 @@ class ModelStoreViewModel @Inject constructor( } fun filterModels(query: String) { _searchQuery.value = query; applyAllFilters() } - fun filterByModelType(type: String?) { _selectedModelType.value = type; applyAllFilters() } + fun filterByModelType(type: String?) { + _selectedModelType.value = type + if (type != "image_upscaler") _selectedUpscalerUseCase.value = null + applyAllFilters() + } fun filterByCategory(cat: ModelCategory?) { _selectedCategory.value = cat; applyAllFilters() } fun toggleParameterFilter(p: String) { _selectedParameters.value = if (p in _selectedParameters.value) _selectedParameters.value - p else _selectedParameters.value + p @@ -458,6 +473,7 @@ class ModelStoreViewModel @Inject constructor( _selectedTags.value = if (tag in _selectedTags.value) _selectedTags.value - tag else _selectedTags.value + tag applyAllFilters() } + fun setUpscalerUseCase(useCase: String?) { _selectedUpscalerUseCase.value = useCase; applyAllFilters() } fun setShowNsfw(show: Boolean) { _showNsfw.value = show; applyAllFilters() } fun setShowOver2GbModels(show: Boolean) { _showOver2GbModels.value = show; applyAllFilters() } fun setExecutionTarget(t: String?) { _executionTarget.value = t; applyAllFilters() } @@ -465,7 +481,7 @@ class ModelStoreViewModel @Inject constructor( _selectedModelType.value = null; _selectedCategory.value = null _selectedParameters.value = emptySet(); _selectedQuantizations.value = emptySet() _selectedSizeCategory.value = null; _selectedTags.value = emptySet() - _showNsfw.value = true; _executionTarget.value = null + _showNsfw.value = true; _executionTarget.value = null; _selectedUpscalerUseCase.value = null _showOver2GbModels.value = false _sortBy.value = SortOption.NAME; _searchQuery.value = "" applyAllFilters() @@ -482,7 +498,7 @@ class ModelStoreViewModel @Inject constructor( fun getGroupedRepos(): Map { val grouped = mutableMapOf() - _filteredModels.value.groupBy { ModelTaxonomy.groupKey(it) }.forEach { (key, models) -> + _filteredModels.value.filterNot { it.isMmproj }.groupBy { ModelTaxonomy.groupKey(it) }.forEach { (key, models) -> val family = ModelTaxonomy.family(models.first()) val taskSummary = models .map { ModelTaxonomy.task(it).displayName } @@ -502,7 +518,7 @@ class ModelStoreViewModel @Inject constructor( } fun getModelsForRepo(repoKey: String): List = - _filteredModels.value.filter { ModelTaxonomy.groupKey(it) == repoKey } + _filteredModels.value.filter { !it.isMmproj && ModelTaxonomy.groupKey(it) == repoKey } .sortedWith(compareBy { ModelTaxonomy.task(it).ordinal } .thenBy { sizeBytesOf(it).takeIf { bytes -> bytes > 0 } ?: Long.MAX_VALUE } .thenBy { it.name.lowercase() }) @@ -512,7 +528,6 @@ class ModelStoreViewModel @Inject constructor( .groupBy { ModelTaxonomy.task(it).displayName } .map { (task, items) -> task to items } - fun downloadByQuickStartId(modelId: String) { viewModelScope.launch { val pool = _filteredModels.value.takeIf { it.isNotEmpty() } @@ -661,6 +676,8 @@ class ModelStoreViewModel @Inject constructor( _downloadStates.value = _downloadStates.value - model.id } else if (model.modelType == "tts" || model.modelType == "stt") { finalizeVoiceDownload(model, destFile) + } else if (model.modelType == "image_gen") { + finalizeImageGenDownload(model, destFile) } else if (model.modelType == "image_gen") { finalizeImageGenDownload(model, destFile) } else if (model.modelType == "image_upscaler") { @@ -1044,13 +1061,7 @@ class ModelStoreViewModel @Inject constructor( return } setInstallState(model, destFile, ModelInstallPhase.VERIFYING) - val provider = when (model.modelType) { - "tts" -> ProviderType.TTS - "stt" -> ProviderType.STT - "embedding" -> ProviderType.EMBEDDING - "tool_search" -> ProviderType.TOOL_SEARCH - else -> ProviderType.GGUF - } + val provider = providerForModelType(model.modelType) modelRepo.insert(ModelInfo( id = model.id, name = model.name, path = destFile.absolutePath, pathType = PathType.FILE, @@ -1101,12 +1112,26 @@ class ModelStoreViewModel @Inject constructor( else -> "gguf" } + private fun providerForModelType(modelType: String): ProviderType = when (modelType) { + "tts" -> ProviderType.TTS + "stt" -> ProviderType.STT + "embedding" -> ProviderType.EMBEDDING + else -> ProviderType.GGUF + } + fun cancelDownload(modelId: String) { _downloadIds.value[modelId]?.let { HxdManager.cancel(it) } _downloadIds.value = _downloadIds.value - modelId _downloadStates.value = _downloadStates.value - modelId } + fun clearDownloadState(modelId: String) { + _downloadIds.value[modelId]?.let { HxdManager.clear(it) } + _downloadIds.value = _downloadIds.value - modelId + _downloadStates.value = _downloadStates.value - modelId + _installStates.value = _installStates.value - modelId + } + fun deleteModel(model: ModelInfo) { viewModelScope.launch(Dispatchers.IO) { diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt index c734e9cb..cca6b4fc 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/SettingsViewModel.kt @@ -14,6 +14,7 @@ import com.dark.tool_neuron.model.NavScreens import com.dark.tool_neuron.model.enums.ProviderType import com.dark.tool_neuron.repo.ModelRepository import com.dark.tool_neuron.repo.RagManager +import com.dark.tool_neuron.repo.web_search.WebSearchMode import com.dark.tool_neuron.service.server.ServerModelRole import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.screens.crash_report.CrashReportActivity @@ -74,6 +75,7 @@ class SettingsViewModel @Inject constructor( private val _pluginOnnxEp = MutableStateFlow(prefs.pluginOnnxEp) private val _serverModelRolesJson = MutableStateFlow(prefs.serverModelRolesJson) private val _serverRoleDefaultsJson = MutableStateFlow(prefs.serverRoleDefaultsJson) + private val _webSearchMode = MutableStateFlow(WebSearchMode.sanitizePref(prefs.webSearchMode)) private val _installedPluginCount = pluginExecutor.registry.installed private val appVersion: String = resolveVersion() @@ -98,6 +100,7 @@ class SettingsViewModel @Inject constructor( _serverModelRolesJson, _serverRoleDefaultsJson, _installedPluginCount, + _webSearchMode, ) { values -> @Suppress("UNCHECKED_CAST") val models = values[0] as List @@ -122,6 +125,7 @@ class SettingsViewModel @Inject constructor( @Suppress("UNCHECKED_CAST") val installedPlugins = values[19] as List val pluginCount = installedPlugins.size + val webSearchMode = values[20] as String SettingsState( sections = buildSections( @@ -143,6 +147,7 @@ class SettingsViewModel @Inject constructor( serverModelRolesJson = serverModelRolesJson, serverRoleDefaultsJson = serverRoleDefaultsJson, pluginCount = pluginCount, + webSearchMode = webSearchMode, ), dialog = dialog, snackbarMessage = snackbar, @@ -188,8 +193,10 @@ class SettingsViewModel @Inject constructor( serverModelRolesJson: String, serverRoleDefaultsJson: String, pluginCount: Int, + webSearchMode: String, ): List = listOf( chatAndRagSection(models, defaultEmbedding, ragSmartRerank, ragMultiQuery, ragDeepResearch), + webSearchSection(webSearchMode), voiceSection(models, activeTts, activeStt, voiceTtsBackend, voiceSttBackend), visionSection(vlmImageQuality), serverRolesSection(models, serverModelRolesJson, serverRoleDefaultsJson, activeTts, activeStt), @@ -432,7 +439,7 @@ class SettingsViewModel @Inject constructor( role == wanted } - private fun defaultServerRole(model: ModelInfo): ServerModelRole = when (model.providerType) { + private fun defaultServerRole(model: ModelInfo): ServerModelRole? = when (model.providerType) { ProviderType.GGUF -> { inferRoleFromName(model)?.let { return it } val modelsRoot = modelRepo.getModelsDir() @@ -609,6 +616,47 @@ class SettingsViewModel @Inject constructor( ) } + private fun webSearchSection(webSearchMode: String): SettingsSection { + val current = WebSearchMode.sanitizePref(webSearchMode) + val options = WebSearchMode.SELECTABLE_KEYS.map { key -> + SettingsChoiceOption( + key = key, + label = WebSearchMode.labelForKey(key), + description = webSearchModeDescription(key), + ) + } + return SettingsSection( + id = SECTION_WEB_SEARCH, + title = "Web search", + description = "How deep web searches go by default.", + icon = TnIcons.Globe, + items = listOf( + SettingsItem.Choice( + id = ID_WEB_SEARCH_DEPTH, + title = "Default search depth", + subtitle = "Auto picks per question. Slash commands (/deep, /exhaustive) override per message.", + icon = TnIcons.Globe, + selectedKey = current, + options = options, + onSelect = { key -> + val k = WebSearchMode.sanitizePref(key) + prefs.webSearchMode = k + _webSearchMode.value = k + _dialog.value = null + }, + ), + ), + ) + } + + private fun webSearchModeDescription(key: String): String = when (key) { + "quick" -> "Fastest, simple lookups" + "normal" -> "Balanced search" + "deep" -> "More rounds, may read pages (slower)" + "exhaustive" -> "Maximum research, slowest" + else -> "Pick depth from the question" + } + private fun voiceSection( models: List, activeTts: String, @@ -954,6 +1002,7 @@ class SettingsViewModel @Inject constructor( companion object { const val SECTION_CHAT_RAG = "chat_rag" + const val SECTION_WEB_SEARCH = "web_search" const val SECTION_VOICE = "voice" const val SECTION_VISION = "vision" const val SECTION_SERVER_ROLES = "server_roles" @@ -974,6 +1023,7 @@ class SettingsViewModel @Inject constructor( private const val ID_RAG_MULTI_QUERY = "rag_multi_query" private const val ID_RAG_DEEP_RESEARCH = "rag_deep_research" private const val ID_VLM_IMAGE_QUALITY = "vlm_image_quality" + private const val ID_WEB_SEARCH_DEPTH = "web_search_depth" private const val ID_SERVER_DEFAULT_PREFIX = "server_default_" private const val ID_SERVER_ROLE_PREFIX = "server_role_" private const val ID_THREAD_MODE = "thread_mode" diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/StorageViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/StorageViewModel.kt index ccadb954..d64c4cd8 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/StorageViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/StorageViewModel.kt @@ -4,6 +4,8 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.dark.tool_neuron.repo.StorageCategoryId import com.dark.tool_neuron.repo.StorageInspector +import com.dark.tool_neuron.repo.StorageMaintenanceMode +import com.dark.tool_neuron.repo.StorageMaintenanceReport import com.dark.tool_neuron.repo.StorageSnapshot import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.flow.MutableStateFlow @@ -15,7 +17,9 @@ import javax.inject.Inject data class StorageUiState( val snapshot: StorageSnapshot? = null, val isLoading: Boolean = true, + val isMaintaining: Boolean = false, val pendingClear: StorageCategoryId? = null, + val maintenanceReport: StorageMaintenanceReport? = null, val message: String? = null, ) @@ -66,6 +70,31 @@ class StorageViewModel @Inject constructor( _state.value = _state.value.copy(message = null) } + fun runQuickClean() = runMaintenance(StorageMaintenanceMode.QUICK_CLEAN) + + fun runDetailedCheck() = runMaintenance(StorageMaintenanceMode.DETAILED_CHECK) + + fun runDeepModelTest() = runMaintenance(StorageMaintenanceMode.DEEP_MODEL_TEST) + + fun clearMaintenanceReport() { + _state.value = _state.value.copy(maintenanceReport = null) + } + + private fun runMaintenance(mode: StorageMaintenanceMode) { + viewModelScope.launch { + _state.value = _state.value.copy(isMaintaining = true, maintenanceReport = null) + val report = inspector.maintenance(mode) + val snap = inspector.snapshot() + _state.value = _state.value.copy( + snapshot = snap, + isLoading = false, + isMaintaining = false, + maintenanceReport = report, + message = maintenanceMessage(report), + ) + } + } + private fun clearedMessage(id: StorageCategoryId): String = when (id) { StorageCategoryId.CHAT_MODELS -> "Chat models cleared" StorageCategoryId.VLM -> "Vision models cleared" @@ -75,4 +104,10 @@ class StorageViewModel @Inject constructor( StorageCategoryId.CACHE -> "Cache cleared" StorageCategoryId.SYSTEM -> "" } + + private fun maintenanceMessage(report: StorageMaintenanceReport): String = when (report.mode) { + StorageMaintenanceMode.QUICK_CLEAN -> "Cleaned ${report.fixedCount} item${if (report.fixedCount == 1) "" else "s"}" + StorageMaintenanceMode.DETAILED_CHECK -> "Check found ${report.issueCount} issue${if (report.issueCount == 1) "" else "s"}" + StorageMaintenanceMode.DEEP_MODEL_TEST -> "Deep test found ${report.issueCount} issue${if (report.issueCount == 1) "" else "s"}" + } } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/WebSearchCoordinator.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/WebSearchCoordinator.kt index 678852e8..2007aa87 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/WebSearchCoordinator.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/WebSearchCoordinator.kt @@ -2,6 +2,8 @@ package com.dark.tool_neuron.viewmodel import com.dark.tool_neuron.model.WebSearchEvent import com.dark.tool_neuron.model.WebSearchHit +import com.dark.tool_neuron.repo.web_search.PageFetcher +import com.dark.tool_neuron.repo.web_search.WebSearchMode import com.dark.tool_neuron.repo.web_search.WebSearchPrompts import com.dark.tool_neuron.repo.web_search.WebSearcher import com.dark.tool_neuron.service.inference.InferenceClient @@ -29,6 +31,7 @@ import kotlin.coroutines.cancellation.CancellationException @Singleton class WebSearchCoordinator @Inject constructor( private val webSearcher: WebSearcher, + private val pageFetcher: PageFetcher, ) { private val _events = MutableSharedFlow(extraBufferCapacity = 64) val events: SharedFlow = _events.asSharedFlow() @@ -41,11 +44,12 @@ class WebSearchCoordinator @Inject constructor( fun start( scope: CoroutineScope, userQuery: String, + mode: WebSearchMode, ): String { val runId = "ws_" + UUID.randomUUID().toString() markActive(runId, true) val job = scope.launch(Dispatchers.IO) { - run(runId = runId, userQuery = userQuery) + run(runId = runId, userQuery = userQuery, mode = mode) } jobs[runId] = job job.invokeOnCompletion { @@ -63,54 +67,123 @@ class WebSearchCoordinator @Inject constructor( jobs.keys.toList().forEach { cancel(it, reason) } } - private suspend fun run(runId: String, userQuery: String) { + private suspend fun run(runId: String, userQuery: String, mode: WebSearchMode) { try { - emit(WebSearchEvent.Plan(runId, userQuery)) + emit(WebSearchEvent.Plan(runId, userQuery, mode.label)) if (!InferenceClient.isModelLoaded.value) { emit(WebSearchEvent.Failed(runId, "Load a chat model first")) return } + val startTime = System.currentTimeMillis() val requestedUrl = extractUrl(userQuery) - val queries = if (requestedUrl != null) { - buildUrlFocusedQueries(userQuery, requestedUrl) - } else if (wantsPlayStore(userQuery)) { - buildPlayStoreQueries(userQuery) - } else { - generateQueries(userQuery).take(MAX_QUERIES) + + val allQueries = mutableListOf() + val seenUrls = mutableSetOf() + val evidence = mutableListOf() + val excerpts = mutableMapOf() + var summary = "" + var coverage = -1 + var lowGainStreak = 0 + var globalIdx = 0 + var totalQueries = 0 + + var nextQueries: List = when { + requestedUrl != null -> buildUrlFocusedQueries(userQuery, requestedUrl, mode.initialQueries) + wantsPlayStore(userQuery) -> buildPlayStoreQueries(userQuery, mode.initialQueries) + else -> generateInitialQueries(userQuery, mode.initialQueries) } - if (queries.isEmpty()) { + if (nextQueries.isEmpty()) { emit(WebSearchEvent.Failed(runId, "Couldn't generate search queries")) return } - emit(WebSearchEvent.QueriesGenerated(runId, queries)) - val allHits = mutableListOf() - val seenUrls = mutableSetOf() - queries.forEachIndexed { idx, q -> - // Throttle between back-to-back queries. DDG flags 3 sequential - // POSTs from the same IP in <1s as bot traffic and answers with - // an HTTP 202 anti-bot challenge. - if (idx > 0) delay(SEARCH_THROTTLE_MS) - emit(WebSearchEvent.SearchStart(runId, idx, q)) - val result = webSearcher.search(q, RESULTS_PER_QUERY, idx) - val hits = result.getOrDefault(emptyList()) - val deduped = hits.filter { it.url.isNotBlank() && seenUrls.add(it.url) } - emit(WebSearchEvent.SearchHits(runId, idx, deduped)) - allHits.addAll(deduped) + var round = 0 + while (round < mode.maxRounds && nextQueries.isNotEmpty() && totalQueries < mode.maxQueries) { + if (System.currentTimeMillis() - startTime > mode.timeBudgetMs) break + + emit(WebSearchEvent.RoundStart( + runId, round + 1, mode.maxRounds, + if (round == 0) "Searching" else "Refining search", + )) + + val budgetLeft = mode.maxQueries - totalQueries + val roundQueries = nextQueries.take(budgetLeft) + allQueries.addAll(roundQueries) + emit(WebSearchEvent.QueriesGenerated(runId, allQueries.toList())) + + val roundHits = mutableListOf() + for (q in roundQueries) { + if (totalQueries >= mode.maxQueries) break + if (totalQueries > 0) delay(SEARCH_THROTTLE_MS) + val idx = globalIdx + emit(WebSearchEvent.SearchStart(runId, idx, q)) + val result = webSearcher.search(q, mode.resultsPerQuery, idx) + val hits = result.getOrDefault(emptyList()) + val deduped = hits.filter { it.url.isNotBlank() && seenUrls.add(normalizeUrl(it.url)) } + emit(WebSearchEvent.SearchHits(runId, idx, deduped)) + roundHits.addAll(deduped) + evidence.addAll(deduped) + globalIdx++ + totalQueries++ + } + + val newGain = roundHits.size + + if (mode.fetchPages && roundHits.isNotEmpty()) { + emit(WebSearchEvent.Status(runId, "Reading pages…")) + val ranked = filterAndRankHits(userQuery, requestedUrl, evidence, RANK_POOL) + val toFetch = ranked + .filter { it.url.isNotBlank() && !excerpts.containsKey(it.url) } + .take(mode.pagesPerRound) + .map { it.url } + runCatching { pageFetcher.fetch(toFetch) } + .getOrDefault(emptyList()) + .forEach { excerpts[it.url] = it.text } + } + + round++ + + if (mode == WebSearchMode.QUICK || mode.followUpQueries == 0) break + + emit(WebSearchEvent.Status(runId, "Reviewing results…")) + val findings = filterAndRankHits(userQuery, requestedUrl, roundHits, DIGEST_FINDINGS) + val digest = if (findings.isEmpty()) null + else runDigest(userQuery, summary, findings, excerpts, mode.followUpQueries) + + if (digest != null) { + if (digest.summary.isNotBlank()) summary = digest.summary + if (digest.coverage in 0..100) coverage = digest.coverage + val seenLower = allQueries.map { it.lowercase() }.toSet() + nextQueries = digest.queries.filter { it.lowercase() !in seenLower } + emit(WebSearchEvent.Status(runId, "Round ${round} reviewed", coverage)) + } else { + nextQueries = emptyList() + } + + if (newGain < LOW_GAIN_THRESHOLD) lowGainStreak++ else lowGainStreak = 0 + + if (coverage >= COVERAGE_STOP) break + if (lowGainStreak >= 2) break + if (nextQueries.isEmpty()) break } - val filteredHits = filterAndRankHits(userQuery, requestedUrl, allHits) - if (filteredHits.isEmpty()) { + val finalPool = filterAndRankHits(userQuery, requestedUrl, evidence, FINAL_SOURCES) + if (finalPool.isEmpty()) { emit(WebSearchEvent.Failed(runId, "No search results found")) return } emit(WebSearchEvent.SynthesizeStart(runId)) - val answer = directAnswerIfPossible(userQuery, filteredHits) - ?: runInference(WebSearchPrompts.synthesize(userQuery, filteredHits), SYNTHESIZE_MAX_TOKENS) - emit(WebSearchEvent.Done(runId, answer, filteredHits)) + val synthSources = finalPool.take(SYNTH_SOURCES) + val synthTokens = if (mode.fetchPages) SYNTH_TOKENS_DEEP else SYNTH_TOKENS_SHALLOW + val answer = directAnswerIfPossible(userQuery, finalPool) + ?: runInference( + WebSearchPrompts.synthesize(userQuery, summary, synthSources, excerpts), + synthTokens, + ) + emit(WebSearchEvent.Done(runId, answer, finalPool)) } catch (ce: CancellationException) { _events.tryEmit(WebSearchEvent.Cancelled(runId, ce.message ?: "Cancelled")) throw ce @@ -119,43 +192,79 @@ class WebSearchCoordinator @Inject constructor( } } - private suspend fun generateQueries(userQuery: String): List { - val raw = runInference(WebSearchPrompts.generateQueries(userQuery), QUERY_GEN_MAX_TOKENS) - val parsed = parseQueries(raw) - // Small chat models (LFM2-350M, Qwen 0.5B) often ignore the "exactly 3 - // numbered" format and emit one terse query or unstructured text. The - // regex won't match those, leaving us with zero queries and no way to - // search. Fall back to the user's original query as a single search so - // the flow always produces something. + private suspend fun generateInitialQueries(userQuery: String, count: Int): List { + val raw = runInference(WebSearchPrompts.initialQueries(userQuery, count), QUERY_GEN_MAX_TOKENS) + val parsed = parseQueries(raw, count, allowLooseLines = true) if (parsed.isEmpty()) return listOf(userQuery) return parsed } - private fun parseQueries(raw: String): List { + private data class Digest( + val summary: String, + val queries: List, + val coverage: Int, + ) + + private suspend fun runDigest( + userQuery: String, + priorSummary: String, + findings: List, + excerpts: Map, + followUpCount: Int, + ): Digest { + val raw = runInference( + WebSearchPrompts.roundDigest(userQuery, priorSummary, findings, excerpts, followUpCount), + DIGEST_MAX_TOKENS, + ) + val summary = sectionBetween(raw, "SUMMARY:", listOf("MISSING:", "QUERIES:", "COVERAGE:")) + .ifBlank { priorSummary } + val queriesBlock = sectionBetween(raw, "QUERIES:", listOf("COVERAGE:")) + val queries = if (queriesBlock.isBlank() || queriesBlock.trim().equals("none", ignoreCase = true)) { + emptyList() + } else { + parseQueries(queriesBlock, followUpCount, allowLooseLines = false) + .filter { !it.equals("none", ignoreCase = true) } + } + val coverage = Regex("COVERAGE:?\\s*(\\d{1,3})", RegexOption.IGNORE_CASE) + .find(raw)?.groupValues?.get(1)?.toIntOrNull()?.coerceIn(0, 100) ?: -1 + return Digest(summary.trim(), queries, coverage) + } + + private fun sectionBetween(text: String, startHeader: String, endHeaders: List): String { + val startIdx = text.indexOf(startHeader, ignoreCase = true) + if (startIdx < 0) return "" + val from = startIdx + startHeader.length + var end = text.length + for (h in endHeaders) { + val e = text.indexOf(h, from, ignoreCase = true) + if (e in 0 until end) end = e + } + return text.substring(from, end).trim() + } + + private fun parseQueries(raw: String, max: Int, allowLooseLines: Boolean): List { if (raw.isBlank()) return emptyList() val out = mutableListOf() val seen = mutableSetOf() for (line in raw.lines()) { val trimmed = line.trim() if (trimmed.isEmpty()) continue - // First try the strict numbered/bullet format val m = WebSearchPrompts.QUERY_LINE_REGEX.matchEntire(trimmed) val candidate = if (m != null) { m.groupValues[1].trim().trim('"', '\'') - } else { - // Fall back to "any line that looks like a search phrase": - // 3+ words, no trailing colon (skips section headers like - // "Queries:"), no markdown emphasis chars dominating. + } else if (allowLooseLines) { val cleaned = trimmed.trim('"', '\'', '*', '_', '`') if (cleaned.endsWith(':')) continue if (cleaned.split(Regex("\\s+")).size < 2) continue if (cleaned.length > 120) continue cleaned + } else { + continue } if (candidate.length < 3) continue val key = candidate.lowercase() if (seen.add(key)) out.add(candidate) - if (out.size >= MAX_QUERIES) break + if (out.size >= max) break } return out } @@ -163,17 +272,17 @@ class WebSearchCoordinator @Inject constructor( private fun extractUrl(text: String): String? = Regex("""https?://[^\s<>"']+""").find(text)?.value?.trimEnd('.', ',', ')', ']') - private fun buildUrlFocusedQueries(userQuery: String, url: String): List { + private fun buildUrlFocusedQueries(userQuery: String, url: String, max: Int): List { val domain = domainOf(url).orEmpty() val cleaned = userQuery.replace(url, "").trim().ifBlank { "summary" } return listOfNotNull( url, domain.takeIf { it.isNotBlank() }?.let { "site:$it $cleaned" }, domain.takeIf { it.isNotBlank() }?.let { "$it ${pathWords(url)} $cleaned" }, - ).distinct().take(MAX_QUERIES) + ).distinct().take(max) } - private fun buildPlayStoreQueries(userQuery: String): List { + private fun buildPlayStoreQueries(userQuery: String, max: Int): List { val appName = userQuery .replace(Regex("""(?i)\b(give|me|the|direct|exact|download|link|url|for|to|on|google|play|store|app|please)\b"""), " ") .replace(Regex("""\s+"""), " ") @@ -183,13 +292,14 @@ class WebSearchCoordinator @Inject constructor( "$appName site:play.google.com/store/apps/details", "$appName Google Play Store", "$appName official Android app", - ).distinct().take(MAX_QUERIES) + ).distinct().take(max) } private fun filterAndRankHits( userQuery: String, requestedUrl: String?, hits: List, + limit: Int, ): List { val requestedDomain = requestedUrl?.let(::domainOf) val wantsPlayStore = wantsPlayStore(userQuery) @@ -221,7 +331,7 @@ class WebSearchCoordinator @Inject constructor( .sortedWith(compareByDescending> { it.second }.thenBy { it.first.sourceQueryIndex }) .map { it.first } .distinctBy { normalizeUrl(it.url) } - .take(MAX_FILTERED_SOURCES) + .take(limit) } private fun wantsPlayStore(userQuery: String): Boolean = @@ -298,11 +408,16 @@ class WebSearchCoordinator @Inject constructor( } companion object { - private const val MAX_QUERIES = 3 - private const val RESULTS_PER_QUERY = 5 - private const val QUERY_GEN_MAX_TOKENS = 200 - private const val SYNTHESIZE_MAX_TOKENS = 1024 - private const val SEARCH_THROTTLE_MS = 1800L - private const val MAX_FILTERED_SOURCES = 8 + private const val SEARCH_THROTTLE_MS = 1200L + private const val QUERY_GEN_MAX_TOKENS = 256 + private const val DIGEST_MAX_TOKENS = 384 + private const val SYNTH_TOKENS_SHALLOW = 1024 + private const val SYNTH_TOKENS_DEEP = 1536 + private const val DIGEST_FINDINGS = 5 + private const val FINAL_SOURCES = 12 + private const val SYNTH_SOURCES = 6 + private const val RANK_POOL = 24 + private const val LOW_GAIN_THRESHOLD = 2 + private const val COVERAGE_STOP = 85 } } diff --git a/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt b/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt index b17b15fb..c4140b66 100644 --- a/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt +++ b/download_manager/src/main/java/com/dark/download_manager/HxdManager.kt @@ -13,7 +13,9 @@ import java.util.concurrent.ConcurrentHashMap data class HxdOptions( val expectedSha256: String? = null, val userAgent: String = "", - val headers: Map = emptyMap() + val headers: Map = emptyMap(), + val maxAttempts: Int = 3, + val stallTimeoutMs: Long = 45_000L, ) enum class HxdStatus { QUEUED, CONNECTING, DOWNLOADING, PAUSED, COMPLETED, FAILED, CANCELLED } @@ -26,7 +28,11 @@ data class HxdState( val totalBytes: Long, val speedBps: Long, val status: HxdStatus, - val error: String? = null + val error: String? = null, + val attempt: Int = 1, + val maxAttempts: Int = 3, + val lastByteAtMs: Long = System.currentTimeMillis(), + val nextRetryAtMs: Long = 0L, ) { val progress: Float get() = if (totalBytes > 0) downloadedBytes.toFloat() / totalBytes.toFloat() else -1f @@ -69,7 +75,8 @@ object HxdManager { taskRegistry[id] = task pendingQueue.add(task) _tasks.value = _tasks.value + (id to HxdState( - id, url, destPath, 0L, -1L, 0L, HxdStatus.QUEUED + id, url, destPath, 0L, -1L, 0L, HxdStatus.QUEUED, + maxAttempts = options.maxAttempts.coerceAtLeast(1), )) } @@ -115,6 +122,14 @@ object HxdManager { synchronized(lock) { taskRegistry.remove(id) } } + fun clear(id: Int) { + val task = synchronized(lock) { taskRegistry.remove(id) } + HxdNative.nativeCancel(id) + task?.let { File(it.destPath + ".hxd_tmp").delete() } + _tasks.value = _tasks.value - id + lastProgressUpdate.remove(id) + } + fun cancelAll() { val ids = synchronized(lock) { taskRegistry.keys.toList() } ids.forEach { cancel(it) } @@ -126,6 +141,7 @@ object HxdManager { val prog = HxdNative.nativeGetProgress(id) val current = _tasks.value[id] ?: return val status = HxdStatus.entries.getOrNull(prog[3].toInt()) ?: current.status + val lastByteAt = if (prog[0] > current.downloadedBytes) System.currentTimeMillis() else current.lastByteAtMs if (status == HxdStatus.DOWNLOADING) { val now = System.currentTimeMillis() @@ -140,7 +156,8 @@ object HxdManager { downloadedBytes = prog[0], totalBytes = prog[1], speedBps = prog[2], - status = status + status = status, + lastByteAtMs = lastByteAt, )) } @@ -149,6 +166,17 @@ object HxdManager { _tasks.value = _tasks.value + (id to current.copy(status = status, error = error)) } + internal fun updateAttempt(id: Int, attempt: Int, nextRetryAtMs: Long = 0L, error: String? = null) { + val current = _tasks.value[id] ?: return + _tasks.value = _tasks.value + (id to current.copy( + attempt = attempt, + nextRetryAtMs = nextRetryAtMs, + error = error, + )) + } + + internal fun isRegistered(id: Int): Boolean = synchronized(lock) { taskRegistry.containsKey(id) } + internal fun saveQueue(queueFile: File) { val snap = synchronized(lock) { taskRegistry.values.toList() } if (snap.isEmpty()) { queueFile.delete(); return } diff --git a/download_manager/src/main/java/com/dark/download_manager/HxdService.kt b/download_manager/src/main/java/com/dark/download_manager/HxdService.kt index c5036fe9..c657d223 100644 --- a/download_manager/src/main/java/com/dark/download_manager/HxdService.kt +++ b/download_manager/src/main/java/com/dark/download_manager/HxdService.kt @@ -92,13 +92,24 @@ class HxdService : Service() { } private suspend fun executeDownload(task: HxdTask) = withContext(Dispatchers.IO) { + val maxAttempts = task.options.maxAttempts.coerceAtLeast(1) + var attempt = 1 + while (attempt <= maxAttempts && HxdManager.isRegistered(task.id)) { + HxdManager.updateAttempt(task.id, attempt) + val retry = executeAttempt(task, attempt, maxAttempts) + if (!retry) return@withContext + attempt++ + } + } + + private suspend fun executeAttempt(task: HxdTask, attempt: Int, maxAttempts: Int): Boolean = withContext(Dispatchers.IO) { val id = task.id - HxdManager.updateStatus(id, HxdStatus.CONNECTING) + HxdManager.updateStatus(id, HxdStatus.CONNECTING, retryMessage(attempt, maxAttempts)) val resumeOffset = HxdNative.nativePrepare(id, task.destPath) if (resumeOffset < 0) { HxdManager.updateStatus(id, HxdStatus.FAILED, "Cannot create destination file") - return@withContext + return@withContext false } val digest: MessageDigest? = task.options.expectedSha256 @@ -111,8 +122,7 @@ class HxdService : Service() { if (code != 200 && code != 206) { HxdNative.nativeFail(id) - HxdManager.updateStatus(id, HxdStatus.FAILED, "HTTP $code") - return@withContext + return@withContext retryOrFail(id, attempt, maxAttempts, "HTTP $code", isTransientHttp(code)) } val contentLen = conn.contentLengthLong @@ -150,7 +160,7 @@ class HxdService : Service() { } else { HxdManager.updateStatus(id, HxdStatus.PAUSED) } - return@withContext + return@withContext false } if (digest != null) { @@ -158,7 +168,7 @@ class HxdService : Service() { if (!actual.equals(task.options.expectedSha256, ignoreCase = true)) { HxdNative.nativeFail(id) HxdManager.updateStatus(id, HxdStatus.FAILED, "Checksum mismatch") - return@withContext + return@withContext false } } @@ -167,15 +177,47 @@ class HxdService : Service() { } else { HxdManager.updateStatus(id, HxdStatus.FAILED, "Failed to finalize file") } + false } catch (e: Exception) { HxdNative.nativeFail(id) - HxdManager.updateStatus(id, HxdStatus.FAILED, e.message ?: "Network error") + retryOrFail(id, attempt, maxAttempts, e.message ?: "Network error", transient = true) } finally { conn?.disconnect() } } + private suspend fun retryOrFail( + id: Int, + attempt: Int, + maxAttempts: Int, + reason: String, + transient: Boolean, + ): Boolean { + if (!transient || attempt >= maxAttempts) { + HxdManager.updateStatus(id, HxdStatus.FAILED, reason) + return false + } + val delayMs = retryDelayMs(attempt) + val next = System.currentTimeMillis() + delayMs + HxdManager.updateAttempt(id, attempt, next, "$reason · retrying") + HxdManager.updateStatus(id, HxdStatus.QUEUED, "$reason · retrying") + delay(delayMs) + return HxdManager.isRegistered(id) + } + + private fun retryMessage(attempt: Int, maxAttempts: Int): String? = + if (attempt > 1) "Retry $attempt/$maxAttempts" else null + + private fun retryDelayMs(attempt: Int): Long = when (attempt) { + 1 -> 2_000L + 2 -> 6_000L + else -> 15_000L + } + + private fun isTransientHttp(code: Int): Boolean = + code == 408 || code == 425 || code == 429 || code in 500..599 + private fun openConnection(task: HxdTask, resumeOffset: Long): HttpURLConnection { return (URL(task.url).openConnection() as HttpURLConnection).apply { connectTimeout = 30_000 diff --git a/native-server/src/main/cpp/native_server.cpp b/native-server/src/main/cpp/native_server.cpp index 6b1522ea..6d1a5f15 100644 --- a/native-server/src/main/cpp/native_server.cpp +++ b/native-server/src/main/cpp/native_server.cpp @@ -234,10 +234,19 @@ Java_com_dark_native_1server_NativeServer_nativeSetWebUiHtml( else tn::server::webui::set_html(h); } +JNIEXPORT void JNICALL +Java_com_dark_native_1server_NativeServer_nativeSetWebUiCss( + JNIEnv* env, jobject, jstring css) { + std::string c = jstringToStd(env, css); + if (c.empty()) tn::server::webui::clear_css(); + else tn::server::webui::set_css(c); +} + JNIEXPORT void JNICALL Java_com_dark_native_1server_NativeServer_nativeClearWebUi( JNIEnv*, jobject) { tn::server::webui::clear_html(); + tn::server::webui::clear_css(); } JNIEXPORT void JNICALL diff --git a/native-server/src/main/cpp/server_auth.cpp b/native-server/src/main/cpp/server_auth.cpp index 666302f0..faa419dd 100644 --- a/native-server/src/main/cpp/server_auth.cpp +++ b/native-server/src/main/cpp/server_auth.cpp @@ -52,6 +52,7 @@ namespace tn::server::auth { path == "/" || path == "/index.html" || path == "/webui" || + path == "/webui.css" || path == "/docs" || path == "/docs/" || path == "/docs/index.html"; diff --git a/native-server/src/main/cpp/server_core.cpp b/native-server/src/main/cpp/server_core.cpp index d1356afb..962ed891 100644 --- a/native-server/src/main/cpp/server_core.cpp +++ b/native-server/src/main/cpp/server_core.cpp @@ -466,6 +466,14 @@ namespace tn::server { server_->Get("/", serve_webui); server_->Get("/index.html", serve_webui); server_->Get("/webui", serve_webui); + server_->Get("/webui.css", [](const httplib::Request&, httplib::Response& res) { + if (!webui::has_css()) { + res.status = 503; + res.set_content("Web UI CSS not configured", "text/plain"); + return; + } + res.set_content(webui::get_css(), "text/css; charset=utf-8"); + }); auto serve_docs = [](const httplib::Request&, httplib::Response& res) { if (!docs::has_html()) { diff --git a/native-server/src/main/cpp/server_webui.cpp b/native-server/src/main/cpp/server_webui.cpp index 8ed04e98..e6c6c69e 100644 --- a/native-server/src/main/cpp/server_webui.cpp +++ b/native-server/src/main/cpp/server_webui.cpp @@ -7,6 +7,7 @@ namespace tn::server::webui { namespace { std::mutex g_mu; std::string g_html; + std::string g_css; } void set_html(const std::string& html) { @@ -14,19 +15,39 @@ namespace tn::server::webui { g_html = html; } + void set_css(const std::string& css) { + std::lock_guard lock(g_mu); + g_css = css; + } + void clear_html() { std::lock_guard lock(g_mu); g_html.clear(); } + void clear_css() { + std::lock_guard lock(g_mu); + g_css.clear(); + } + std::string get_html() { std::lock_guard lock(g_mu); return g_html; } + std::string get_css() { + std::lock_guard lock(g_mu); + return g_css; + } + bool has_html() { std::lock_guard lock(g_mu); return !g_html.empty(); } + bool has_css() { + std::lock_guard lock(g_mu); + return !g_css.empty(); + } + } diff --git a/native-server/src/main/cpp/server_webui.h b/native-server/src/main/cpp/server_webui.h index 3fed9ace..16308d3e 100644 --- a/native-server/src/main/cpp/server_webui.h +++ b/native-server/src/main/cpp/server_webui.h @@ -4,7 +4,11 @@ namespace tn::server::webui { void set_html(const std::string& html); + void set_css(const std::string& css); void clear_html(); + void clear_css(); std::string get_html(); + std::string get_css(); bool has_html(); + bool has_css(); } diff --git a/native-server/src/main/java/com/dark/native_server/NativeServer.kt b/native-server/src/main/java/com/dark/native_server/NativeServer.kt index ecfcfb45..82e8cc45 100644 --- a/native-server/src/main/java/com/dark/native_server/NativeServer.kt +++ b/native-server/src/main/java/com/dark/native_server/NativeServer.kt @@ -52,6 +52,7 @@ object NativeServer { external fun nativeResetRateLimit() external fun nativeSetWebUiHtml(html: String) + external fun nativeSetWebUiCss(css: String) external fun nativeClearWebUi() external fun nativeSetDocsHtml(html: String) diff --git a/networking/src/main/cpp/ddg_client.cpp b/networking/src/main/cpp/ddg_client.cpp index 9b536176..75f87228 100644 --- a/networking/src/main/cpp/ddg_client.cpp +++ b/networking/src/main/cpp/ddg_client.cpp @@ -5,11 +5,14 @@ #include "url_util.h" #include +#include namespace net::ddg { namespace { +using Parser = std::vector (*)(const std::string&, int); + constexpr const char* kHtmlHost = "https://html.duckduckgo.com/html/"; constexpr const char* kLiteHost = "https://lite.duckduckgo.com/lite/"; @@ -20,12 +23,6 @@ HttpRequest build_post(const std::string& host, const std::string& query, req.method = "POST"; std::string kl = locale.empty() ? "wt-wt" : locale; req.body = "q=" + url::encode(query) + "&kl=" + url::encode(kl) + "&kd=-1&b="; - // Header set must agree with the curl-impersonate "chrome116" TLS - // fingerprint we set globally. Missing Sec-CH-UA + Sec-CH-UA-Mobile + - // Sec-CH-UA-Platform on a Chrome 116 TLS handshake is itself a tell — - // DDG returns the anomaly page when these disagree. Sec-Fetch-Site is - // "same-origin" because we're POSTing back to the same site we - // referred from (the DDG search form). req.headers = { {"User-Agent", ua}, {"Content-Type", "application/x-www-form-urlencoded"}, @@ -62,10 +59,32 @@ std::vector to_results(std::vector&& entries) { return out; } -SearchOutcome try_host(const std::string& host, const std::string& query, const std::string& ua, - int max_results, const std::string& locale) { +HttpRequest build_get(const std::string& url, const std::string& ua) { + HttpRequest req; + req.url = url; + req.method = "GET"; + req.headers = { + {"User-Agent", ua}, + {"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"}, + {"Accept-Language", "en-US,en;q=0.9"}, + {"Accept-Encoding", "gzip, deflate, br"}, + {"Sec-CH-UA", "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"116\", \"Google Chrome\";v=\"116\""}, + {"Sec-CH-UA-Mobile", "?0"}, + {"Sec-CH-UA-Platform", "\"Windows\""}, + {"Sec-Fetch-Dest", "document"}, + {"Sec-Fetch-Mode", "navigate"}, + {"Sec-Fetch-Site", "none"}, + {"Sec-Fetch-User", "?1"}, + {"Upgrade-Insecure-Requests", "1"}, + }; + req.timeout_ms = 15000; + req.follow_redirects = true; + return req; +} + +SearchOutcome run_engine(const HttpRequest& req, Parser parse, int max_results) { SearchOutcome out; - auto resp = http_execute(build_post(host, query, ua, locale)); + auto resp = http_execute(req); if (!resp.has_value()) { out.error = {"http backend unavailable"}; return out; @@ -82,40 +101,73 @@ SearchOutcome try_host(const std::string& host, const std::string& query, const out.error = {"http status " + std::to_string(resp->status)}; return out; } - auto entries = html::extract_ddg_results(resp->body, max_results); - out.results = to_results(std::move(entries)); + out.results = to_results(parse(resp->body, max_results)); out.ok = true; return out; } +std::string bing_url(const std::string& query, int count) { + return "https://www.bing.com/search?q=" + url::encode(query) + + "&count=" + std::to_string(count) + "&setlang=en-US"; +} + +std::string mojeek_url(const std::string& query) { + return "https://www.mojeek.com/search?q=" + url::encode(query); +} + } SearchOutcome search(const std::string& query, const std::string& user_agent, int max_results, const std::string& locale) { + SearchOutcome merged; if (query.empty()) { - SearchOutcome out; - out.error = {"empty query"}; - return out; + merged.error = {"empty query"}; + return merged; } - int capped = std::clamp(max_results, 1, 30); - - auto primary = try_host(kHtmlHost, query, user_agent, capped, locale); - if (primary.ok) return primary; + const int capped = std::clamp(max_results, 1, 30); + std::unordered_set seen; + std::string last_error; + + auto enough = [&]() { return static_cast(merged.results.size()) >= capped; }; + auto absorb = [&](SearchOutcome eo) { + if (!eo.ok) { + if (!eo.error.message.empty()) last_error = eo.error.message; + return; + } + for (auto& r : eo.results) { + if (r.url.empty()) continue; + if (!seen.insert(r.url).second) continue; + merged.results.push_back(std::move(r)); + } + }; - // If DDG answered with the 202 anti-bot challenge, it set a tracking - // cookie on the response. The CURLSH-shared cookie jar in http_curl.cpp - // captured it, so an immediate retry against the same host carries the - // cookie and almost always clears the challenge. - if (primary.error.message.find("status 202") != std::string::npos) { - auto retry = try_host(kHtmlHost, query, user_agent, capped, locale); - if (retry.ok) return retry; + absorb(run_engine(build_post(kHtmlHost, query, user_agent, locale), + html::extract_ddg_results, capped)); + if (merged.results.empty() && last_error.find("status 202") != std::string::npos) { + absorb(run_engine(build_post(kHtmlHost, query, user_agent, locale), + html::extract_ddg_results, capped)); } - auto fallback = try_host(kLiteHost, query, user_agent, capped, locale); - if (fallback.ok) return fallback; + if (!enough()) { + absorb(run_engine(build_get(bing_url(query, capped), user_agent), + html::extract_bing_results, capped)); + } + if (!enough()) { + absorb(run_engine(build_get(mojeek_url(query), user_agent), + html::extract_mojeek_results, capped)); + } + if (merged.results.empty()) { + absorb(run_engine(build_post(kLiteHost, query, user_agent, locale), + html::extract_ddg_results, capped)); + } - return primary.error.message.empty() ? fallback : primary; + if (static_cast(merged.results.size()) > capped) merged.results.resize(capped); + merged.ok = !merged.results.empty(); + if (!merged.ok) { + merged.error = {last_error.empty() ? "no results" : last_error}; + } + return merged; } } diff --git a/networking/src/main/cpp/html_extract.cpp b/networking/src/main/cpp/html_extract.cpp index bf932815..e268dc20 100644 --- a/networking/src/main/cpp/html_extract.cpp +++ b/networking/src/main/cpp/html_extract.cpp @@ -1,5 +1,7 @@ #include "html_extract.h" +#include "url_util.h" + #include #include @@ -62,15 +64,6 @@ std::vector extract_ddg_results(const std::string& html, int max_results) std::vector out; if (max_results <= 0) return out; - // DDG silently changed their HTML structure: the old class="result" - // / result__a / result__snippet selectors no longer match (confirmed - // 2026-05-15 — zero matches in fresh fetches even though the page - // contains real results). What DOES remain stable across redesigns - // is their click-tracking redirect URL format `/l/?uddg=...` on - // every result title anchor. We anchor parsing on that pattern and - // grab the surrounding context for title + snippet. - // - // Pass 1 (current DDG layout): try the new structure first. static const std::regex kRedirectAnchor( R"rx(]*href="((?:https?:)?//(?:duckduckgo\.com)?/l/\?[^"]*uddg=[^"]+)"[^>]*>([\s\S]*?))rx", std::regex::icase | std::regex::optimize @@ -83,7 +76,6 @@ std::vector extract_ddg_results(const std::string& html, int max_results) Entry e; e.href = (*it)[1].str(); e.title = strip_tags((*it)[2].str()); - // Trim trailing/leading whitespace introduced by inline tags. while (!e.title.empty() && (e.title.front() == ' ' || e.title.front() == '\n' || e.title.front() == '\t' || e.title.front() == '\r')) { e.title.erase(0, 1); @@ -94,17 +86,11 @@ std::vector extract_ddg_results(const std::string& html, int max_results) } if (e.href.empty() || e.title.empty()) continue; - // Snippet: find the next chunk of plain text after this anchor, - // bounded by the next anchor opening or a heavy structural tag. - // Heuristic: take the substring between the anchor's and the - // next "position() + it->length(); if (after < html.size()) { std::size_t cap_end = std::min(after + 1024, html.size()); std::string window = html.substr(after, cap_end - after); - // Strip tags inside the window and trim. std::string snippet = strip_tags(window); - // Collapse whitespace. std::string compact; compact.reserve(snippet.size()); bool prev_space = false; @@ -119,9 +105,6 @@ std::vector extract_ddg_results(const std::string& html, int max_results) } } while (!compact.empty() && compact.front() == ' ') compact.erase(0, 1); - // Cut at the first occurrence of common UI affordances or - // when we hit the title text again (DDG repeats title near - // the URL display). const char* kCutMarkers[] = {"More results", "Next page", "Prev page", nullptr}; for (int i = 0; kCutMarkers[i]; ++i) { auto pos = compact.find(kCutMarkers[i]); @@ -137,9 +120,6 @@ std::vector extract_ddg_results(const std::string& html, int max_results) if (!out.empty()) return out; - // Pass 2 (legacy layout): the old result block / result__a / result__snippet - // structure. Kept as a fallback in case DDG rolls back, or so the parser - // works on older mirrored HTML in tests. static const std::regex kResultBlock( R"rx(]*class="[^"]*\bresult\b[^"]*"[^>]*>([\s\S]*?)(?=]*class="[^"]*\bresult\b|
\s*
\s*]*class="nav-link))rx", std::regex::icase | std::regex::optimize @@ -179,4 +159,97 @@ std::vector extract_ddg_results(const std::string& html, int max_results) return out; } +namespace { + +std::string trim_ws(std::string s) { + while (!s.empty() && (s.front() == ' ' || s.front() == '\n' || + s.front() == '\t' || s.front() == '\r')) { + s.erase(0, 1); + } + while (!s.empty() && (s.back() == ' ' || s.back() == '\n' || + s.back() == '\t' || s.back() == '\r')) { + s.pop_back(); + } + return s; +} + +std::string collapse_ws(const std::string& in, std::size_t cap) { + std::string out; + out.reserve(in.size()); + bool prev_space = false; + for (char c : in) { + if (c == '\n' || c == '\t' || c == '\r') c = ' '; + if (c == ' ') { + if (!prev_space) out.push_back(c); + prev_space = true; + } else { + out.push_back(c); + prev_space = false; + } + } + out = trim_ws(std::move(out)); + if (out.size() > cap) out.resize(cap); + return out; +} + +std::vector extract_h2_results(const std::string& html, int max_results, + const std::regex& snippet_re, bool bing_unwrap) { + std::vector out; + if (max_results <= 0) return out; + + static const std::regex kTitle( + R"rx(]*>\s*]*href="([^"]+)"[^>]*>([\s\S]*?))rx", + std::regex::icase | std::regex::optimize + ); + + auto begin = std::sregex_iterator(html.begin(), html.end(), kTitle); + auto end = std::sregex_iterator(); + + for (auto it = begin; it != end && static_cast(out.size()) < max_results; ++it) { + Entry e; + e.href = (*it)[1].str(); + if (bing_unwrap) e.href = url::unwrap_bing_redirect(e.href); + e.title = trim_ws(strip_tags((*it)[2].str())); + if (e.href.empty() || e.title.empty()) continue; + if (e.href.rfind("http", 0) != 0 && e.href.rfind("//", 0) != 0) continue; + if (e.href.rfind("//", 0) == 0) e.href = "https:" + e.href; + + std::size_t after = it->position() + it->length(); + if (after < html.size()) { + std::size_t cap_end = std::min(after + 2000, html.size()); + std::string window = html.substr(after, cap_end - after); + std::smatch m; + if (std::regex_search(window, m, snippet_re)) { + e.snippet = collapse_ws(strip_tags(m[1].str()), 500); + } + } + out.push_back(std::move(e)); + } + return out; +} + +} + +std::vector extract_bing_results(const std::string& html, int max_results) { + static const std::regex kSnippet( + R"rx(]*>([\s\S]*?)

)rx", + std::regex::icase | std::regex::optimize + ); + return extract_h2_results(html, max_results, kSnippet, true); +} + +std::vector extract_mojeek_results(const std::string& html, int max_results) { + static const std::regex kSnippet( + R"rx(]*class="[^"]*\bs\b[^"]*"[^>]*>([\s\S]*?)

)rx", + std::regex::icase | std::regex::optimize + ); + auto out = extract_h2_results(html, max_results, kSnippet, false); + if (!out.empty()) return out; + static const std::regex kAnyP( + R"rx(]*>([\s\S]*?)

)rx", + std::regex::icase | std::regex::optimize + ); + return extract_h2_results(html, max_results, kAnyP, false); +} + } diff --git a/networking/src/main/cpp/html_extract.h b/networking/src/main/cpp/html_extract.h index 33288065..bb149557 100644 --- a/networking/src/main/cpp/html_extract.h +++ b/networking/src/main/cpp/html_extract.h @@ -13,6 +13,10 @@ struct Entry { std::vector extract_ddg_results(const std::string& html, int max_results); +std::vector extract_bing_results(const std::string& html, int max_results); + +std::vector extract_mojeek_results(const std::string& html, int max_results); + std::string strip_tags(const std::string& html); std::string decode_entities(const std::string& in); diff --git a/networking/src/main/cpp/url_util.cpp b/networking/src/main/cpp/url_util.cpp index 072f4353..7a9c2fb6 100644 --- a/networking/src/main/cpp/url_util.cpp +++ b/networking/src/main/cpp/url_util.cpp @@ -66,4 +66,51 @@ std::string unwrap_ddg_redirect(const std::string& href) { return decode(encoded); } +namespace { + +int b64_value(char c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+' || c == '-') return 62; + if (c == '/' || c == '_') return 63; + return -1; +} + +std::string base64url_decode(const std::string& in) { + std::string out; + out.reserve(in.size() * 3 / 4 + 1); + int buf = 0; + int bits = 0; + for (char c : in) { + if (c == '=' || c == '\0') break; + int v = b64_value(c); + if (v < 0) continue; + buf = (buf << 6) | v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out.push_back(static_cast((buf >> bits) & 0xFF)); + } + } + return out; +} + +} + +std::string unwrap_bing_redirect(const std::string& href) { + if (href.find("/ck/a") == std::string::npos) return href; + const std::string marker = "u=a1"; + auto pos = href.find(marker); + if (pos == std::string::npos) return href; + pos += marker.size(); + auto end = href.find('&', pos); + std::string payload = (end == std::string::npos) + ? href.substr(pos) + : href.substr(pos, end - pos); + std::string decoded = base64url_decode(payload); + if (decoded.rfind("http", 0) == 0) return decoded; + return href; +} + } diff --git a/networking/src/main/cpp/url_util.h b/networking/src/main/cpp/url_util.h index 4f195b94..b158d2c4 100644 --- a/networking/src/main/cpp/url_util.h +++ b/networking/src/main/cpp/url_util.h @@ -10,4 +10,6 @@ std::string decode(const std::string& in); std::string unwrap_ddg_redirect(const std::string& href); +std::string unwrap_bing_redirect(const std::string& href); + } From c199f801a32046e36944a7f276aaa92e21b0fc94 Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Tue, 23 Jun 2026 17:01:17 +0600 Subject: [PATCH 10/13] fix: stabilize settings section rendering --- .../ui/screens/settings/SettingsScreen.kt | 71 ++++++------------- 1 file changed, 21 insertions(+), 50 deletions(-) diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt index dc758656..5c8bdd4a 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/settings/SettingsScreen.kt @@ -1,31 +1,24 @@ package com.dark.tool_neuron.ui.screens.settings -import androidx.compose.animation.AnimatedVisibility -import androidx.compose.animation.fadeIn -import androidx.compose.animation.slideInVertically import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.lazy.LazyColumn -import androidx.compose.foundation.lazy.itemsIndexed +import androidx.compose.foundation.lazy.items import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.vector.ImageVector import com.dark.tool_neuron.model.NavScreens import com.dark.tool_neuron.ui.components.StandardCard import com.dark.tool_neuron.ui.icons.TnIcons import com.dark.tool_neuron.ui.theme.LocalDimens -import com.dark.tool_neuron.ui.theme.Motion -import kotlinx.coroutines.delay @Composable fun SettingsScreen( @@ -34,9 +27,6 @@ fun SettingsScreen( ) { val dimens = LocalDimens.current - var visible by remember { mutableStateOf(false) } - LaunchedEffect(Unit) { delay(40); visible = true } - LazyColumn( modifier = Modifier.fillMaxSize(), contentPadding = PaddingValues( @@ -45,49 +35,30 @@ fun SettingsScreen( top = innerPadding.calculateTopPadding() + dimens.spacingSm, bottom = innerPadding.calculateBottomPadding() + dimens.spacingSm, ), - verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), + verticalArrangement = Arrangement.spacedBy(dimens.spacingMd), ) { - SETTING_GROUPS.forEachIndexed { groupIndex, group -> - item(key = "group-${group.title}") { - Text( - text = group.title, - style = MaterialTheme.typography.labelLarge, - color = MaterialTheme.colorScheme.primary, - ) - } - itemsIndexed(group.cards, key = { _, card -> "${group.title}-${card.route}" }) { index, card -> - AnimatedSettingsCard( - visible = visible, - stagger = (groupIndex * 80) + (index * 35), - ) { - SettingsLandingCard(card = card, onClick = { onNavigate(card.route) }) - } - } + items(SETTING_GROUPS, key = { it.title }) { group -> + SettingsGroupBlock(group = group, onNavigate = onNavigate) } } } @Composable -private fun AnimatedSettingsCard( - visible: Boolean, - stagger: Int, - content: @Composable () -> Unit, -) { - var ready by remember { mutableStateOf(false) } - LaunchedEffect(visible) { - if (visible) { - if (stagger > 0) delay(stagger.toLong()) - ready = true - } else { - ready = false - } - } - AnimatedVisibility( - visible = ready, - enter = fadeIn(Motion.entrance()) + - slideInVertically(Motion.entrance()) { it / 8 }, +private fun SettingsGroupBlock(group: SettingsGroup, onNavigate: (String) -> Unit) { + val dimens = LocalDimens.current + Column( + modifier = Modifier.fillMaxWidth(), + verticalArrangement = Arrangement.spacedBy(dimens.spacingXs), ) { - content() + Text( + text = group.title, + style = MaterialTheme.typography.labelLarge, + color = MaterialTheme.colorScheme.primary, + modifier = Modifier.padding(start = dimens.spacingXxs), + ) + group.cards.forEach { card -> + SettingsLandingCard(card = card, onClick = { onNavigate(card.route) }) + } } } From ee515d6abf677bcfec2adb345b0ad3e517f3098b Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Tue, 23 Jun 2026 17:06:15 +0600 Subject: [PATCH 11/13] fix: make remote web ui fit mobile --- app/src/main/assets/server_webui.css | 65 +++++++-- app/src/main/assets/server_webui.html | 191 +++++++++++++++++++++++++- 2 files changed, 238 insertions(+), 18 deletions(-) diff --git a/app/src/main/assets/server_webui.css b/app/src/main/assets/server_webui.css index 288fbfa3..1b32e644 100644 --- a/app/src/main/assets/server_webui.css +++ b/app/src/main/assets/server_webui.css @@ -50,7 +50,7 @@ html[data-theme=dark]{ *{box-sizing:border-box} ::selection{background:var(--accent-soft);color:var(--tx)} -html,body{height:100%;margin:0} +html,body{height:100%;margin:0;width:100%;max-width:100%;overflow:hidden} body{background:var(--bg);color:var(--tx);font:15px/1.6 var(--sans);-webkit-font-smoothing:antialiased;overflow:hidden} button,input,textarea,select{font:inherit;color:inherit} button{border:0;background:transparent;color:inherit;cursor:pointer} @@ -63,7 +63,7 @@ svg{display:block} ::-webkit-scrollbar-thumb:hover{background:var(--tx-mute);background-clip:content-box} *{scrollbar-width:thin;scrollbar-color:var(--line-2) transparent} -.app{display:grid;grid-template-columns:var(--side-w) minmax(0,1fr);height:100dvh;transition:grid-template-columns .26s cubic-bezier(.4,0,.2,1)} +.app{display:grid;grid-template-columns:var(--side-w) minmax(0,1fr);width:100%;max-width:100vw;height:100dvh;overflow:hidden;transition:grid-template-columns .26s cubic-bezier(.4,0,.2,1)} .app.tucked{grid-template-columns:0 minmax(0,1fr)} /* ============ sidebar ============ */ @@ -115,7 +115,7 @@ svg{display:block} .conn .cog svg{width:17px;height:17px} /* ============ main ============ */ -.main{display:flex;flex-direction:column;min-width:0;height:100dvh;position:relative} +.main{display:flex;flex-direction:column;min-width:0;width:100%;max-width:100%;height:100dvh;overflow:hidden;position:relative} .bar{display:flex;align-items:center;gap:8px;height:54px;padding:0 12px;flex:none} .bar .spacer{flex:1} .show-side{display:none} @@ -145,8 +145,8 @@ svg{display:block} .menu-empty{padding:20px;text-align:center;color:var(--tx-mute);font-size:13px;line-height:1.5} /* ============ thread ============ */ -.stream{flex:1;overflow-y:auto;overflow-x:hidden;scroll-behavior:smooth} -.wrap{max-width:768px;margin:0 auto;padding:18px 24px 10px;width:100%} +.stream{flex:1;min-width:0;overflow-y:auto;overflow-x:hidden;scroll-behavior:smooth} +.wrap{max-width:768px;margin:0 auto;padding:18px 24px 10px;width:100%;min-width:0} /* empty / welcome */ .welcome{height:100%;min-height:60vh;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;padding:24px 24px 40px;max-width:720px;margin:0 auto} @@ -243,8 +243,8 @@ svg{display:block} .btn-sm.ghost:hover{border-color:var(--tx-mute);color:var(--tx)} /* ============ composer ============ */ -.dock{flex:none;padding:6px 16px 14px;background:linear-gradient(to top,var(--bg) 62%,transparent)} -.composer{max-width:768px;margin:0 auto;background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-xl);padding:8px;transition:border-color .16s,box-shadow .16s} +.dock{flex:none;min-width:0;padding:6px 16px 14px;background:linear-gradient(to top,var(--bg) 62%,transparent)} +.composer{max-width:768px;width:100%;min-width:0;margin:0 auto;background:var(--raise);border:1px solid var(--line-2);border-radius:var(--r-xl);padding:8px;transition:border-color .16s,box-shadow .16s} .composer.focus{border-color:var(--accent-edge);box-shadow:0 0 0 3px var(--accent-soft)} .pending{display:flex;flex-wrap:wrap;gap:8px;padding:6px 6px 8px} .pending:empty{display:none} @@ -256,7 +256,7 @@ svg{display:block} .pchip .px:hover{color:var(--danger);background:var(--hover-2)} .pchip .px svg{width:12px;height:12px} -.crow{display:flex;align-items:flex-end;gap:6px} +.crow{display:flex;align-items:flex-end;gap:6px;min-width:0} .composer textarea{flex:1;min-height:28px;max-height:46dvh;resize:none;border:0;background:transparent;outline:0;padding:9px 6px;font-size:16px;line-height:1.5;font-family:var(--sans)} .composer textarea::placeholder{color:var(--tx-mute)} .c-btn{width:38px;height:38px;flex:none;border-radius:50%;display:grid;place-items:center;color:var(--tx-mute);transition:background .12s,color .12s,transform .08s} @@ -340,9 +340,43 @@ svg{display:block} .scrim{position:fixed;inset:0;background:rgba(0,0,0,.5);display:none;z-index:30} .scrim.show{display:block} -@media (max-width:880px){ - .app{grid-template-columns:0 minmax(0,1fr)} - .side{position:fixed;top:0;left:0;width:min(88vw,320px);height:100dvh;z-index:50;transform:translateX(-100%);transition:transform .24s cubic-bezier(.2,.8,.2,1);border-right:1px solid var(--line);padding-bottom:env(safe-area-inset-bottom)} +/* ============ workspace switcher ============ */ +.wsswitch{display:flex;gap:4px;margin:0 12px 10px;padding:3px;background:var(--raise);border:1px solid var(--line);border-radius:var(--r-m)} +.wsbtn{flex:1;min-width:0;height:34px;border-radius:var(--r-s);font-size:13px;font-weight:560;color:var(--tx-dim);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:background .12s,color .12s} +.wsbtn:hover{color:var(--tx);background:var(--hover)} +.wsbtn.on{background:var(--accent-soft);color:var(--accent)} + +/* ============ studio panels (embeddings / image) ============ */ +.studio{flex:1;min-width:0;overflow-y:auto;overflow-x:hidden} +.studio-wrap{max-width:768px;margin:0 auto;padding:20px 24px 40px;width:100%;min-width:0;display:flex;flex-direction:column;gap:16px} +.studio-head h2{margin:0;font-size:22px;font-weight:680;letter-spacing:-.01em} +.studio-head p{margin:5px 0 0;color:var(--tx-mute);font-size:13px} +.studio-head code{font-family:var(--mono);font-size:.86em;background:var(--raise-2);border:1px solid var(--line);padding:1px 5px;border-radius:5px} +.studio .f{display:flex;flex-direction:column;gap:6px} +.studio .f>span{font-size:13px;font-weight:560;color:var(--tx-dim)} +.studio .f input,.studio .f select,.studio .f textarea{width:100%;background:var(--bg);border:1px solid var(--line);border-radius:var(--r-s);padding:11px 13px;outline:0;font-size:16px;transition:border-color .12s,box-shadow .12s} +.studio .f textarea{resize:vertical;line-height:1.5;font-family:var(--sans)} +.studio .f input:focus,.studio .f select:focus,.studio .f textarea:focus{border-color:var(--accent);box-shadow:0 0 0 3px var(--accent-soft)} +.studio #imagePromptWrap,.studio #imageInputWrap,.studio #imageMaskWrap{display:flex;flex-direction:column;gap:16px} +.four{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px} +.seg-group{display:flex;gap:4px;padding:3px;background:var(--raise);border:1px solid var(--line);border-radius:var(--r-m)} +.seg{flex:1;min-width:0;height:38px;border-radius:var(--r-s);font-size:13px;font-weight:560;color:var(--tx-dim);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;transition:background .12s,color .12s} +.seg:hover{color:var(--tx);background:var(--hover)} +.seg.on{background:var(--accent-soft);color:var(--accent)} +.studio-status{font-size:13px;color:var(--tx-mute);min-height:18px} +.studio-status.ok{color:var(--good)} +.studio-status.err{color:var(--danger)} +.studio-out{margin:0;padding:14px 16px;background:var(--code);border:1px solid var(--line);border-radius:var(--r-m);font-family:var(--mono);font-size:12.5px;line-height:1.6;color:var(--tx);white-space:pre-wrap;overflow-wrap:anywhere;max-height:50dvh;overflow:auto} +.studio-out:empty{display:none} +.studio-image:empty{display:none} +.studio-image{border:1px solid var(--line);border-radius:var(--r-m);overflow:hidden;background:var(--raise)} +.studio-image img{display:block;width:100%;height:auto} + +@media (max-width:980px),(pointer:coarse) and (max-width:1180px){ + html,body{position:relative;width:100%;max-width:100vw;overflow:hidden} + .app{display:block;width:100vw;max-width:100vw;height:100dvh;overflow:hidden} + .app.tucked{display:block} + .side{position:fixed;top:0;left:0;width:min(88vw,360px);max-width:calc(100vw - 48px);height:100dvh;z-index:50;transform:translate3d(-105%,0,0);transition:transform .24s cubic-bezier(.2,.8,.2,1);border-right:1px solid var(--line);box-shadow:var(--shadow);padding-bottom:env(safe-area-inset-bottom)} .side.open{transform:none} .show-side{display:grid!important} .side-top{padding-top:max(13px,env(safe-area-inset-top))} @@ -351,7 +385,7 @@ svg{display:block} .row{height:auto;padding:4px 4px 4px 12px} .row .rkill{opacity:.75} .row .rtitle{padding:10px 0} - .main{height:100dvh} + .main{width:100vw;max-width:100vw;height:100dvh;min-width:0;overflow:hidden} .bar{height:auto;min-height:56px;padding:8px max(10px,env(safe-area-inset-right)) 6px max(10px,env(safe-area-inset-left));align-items:center;flex-wrap:wrap} .mpick{order:3;width:100%;flex-basis:100%} .mtrigger{width:100%;max-width:none;justify-content:space-between;border:1px solid var(--line);background:var(--raise)} @@ -361,13 +395,16 @@ svg{display:block} .chips{grid-template-columns:1fr} .welcome{min-height:52vh;padding:20px 16px 28px} .welcome h1{font-size:25px} - .wrap{padding:12px max(12px,env(safe-area-inset-right)) 8px max(12px,env(safe-area-inset-left))} + .wrap{max-width:none;padding:12px max(12px,env(safe-area-inset-right)) 8px max(12px,env(safe-area-inset-left))} + .studio-wrap{max-width:none;padding:16px max(14px,env(safe-area-inset-right)) 32px max(14px,env(safe-area-inset-left))} + .four{grid-template-columns:1fr 1fr} + .wsbtn,.seg{min-height:42px} .bubble,.atts{max-width:100%} .acts{gap:4px;flex-wrap:wrap} .act{min-height:36px;padding:7px 9px} .act .alabel{display:none} .dock{padding:6px max(10px,env(safe-area-inset-right)) calc(10px + env(safe-area-inset-bottom)) max(10px,env(safe-area-inset-left));background:linear-gradient(to top,var(--bg) 76%,transparent)} - .composer{border-radius:20px;padding:7px} + .composer{max-width:none;border-radius:20px;padding:7px} .crow{align-items:flex-end} .composer textarea{min-height:44px;max-height:34dvh;padding:10px 4px;font-size:16px} .pending{padding:5px 5px 7px} diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index 4f6f6f91..0111b8af 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -26,6 +26,11 @@ New chat +
+ + + +
@@ -115,6 +120,69 @@

What can I help with?

Enter to send · Shift+Enter for a new line · everything stays on your device
+ + + +
@@ -183,14 +251,14 @@

Settings

var $=function(id){return document.getElementById(id);}; var KEY="tn.webui.v4"; var state=load(); - var chatModels=[],ttsModels=[],sttModels=[]; + var chatModels=[],ttsModels=[],sttModels=[],embedModels=[],imageModels=[]; var active=null,abort=null,generating=false; var atts=[]; var pendingVoiceReply=false; - var mq=window.matchMedia("(max-width:880px)"); + var mq=window.matchMedia("(max-width:980px), (pointer:coarse) and (max-width:1180px)"); function defaults(){return{serverUrl:location.origin,token:"",model:"",systemPrompt:"",maxTokens:1024,temp:0.7,stream:true,theme:"", - sttModel:"",ttsModel:"",ttsVoice:0,ttsSpeed:1,tucked:false,chats:[]};} + sttModel:"",ttsModel:"",ttsVoice:0,ttsSpeed:1,tucked:false,chats:[],workspace:"chat",embedModel:"",imageModel:""};} function load(){try{return Object.assign(defaults(),JSON.parse(localStorage.getItem(KEY)||"{}"));}catch(e){return defaults();}} function save(){try{localStorage.setItem(KEY,JSON.stringify(state));}catch(e){}} function uid(){return "c"+Date.now().toString(36)+Math.random().toString(36).slice(2,7);} @@ -418,6 +486,95 @@

Settings

} function note(text,kind){$("note").className="note"+(kind?" "+kind:"");$("noteText").textContent=text;} + /* ---------- studio: embeddings + image ---------- */ + function fillModelSelect(sel,arr,cur,emptyLabel){ + if(!sel)return;sel.innerHTML=""; + if(!arr.length){var e=document.createElement("option");e.value="";e.textContent=emptyLabel;sel.appendChild(e);return;} + arr.forEach(function(m){var o=document.createElement("option");o.value=m.id;o.textContent=m.id+(m.type==="image_upscaler"?" · upscaler":"");if(m.id===cur)o.selected=true;sel.appendChild(o);}); + } + function fillStudioSelects(){ + fillModelSelect($("embedModel"),embedModels,state.embedModel,"No embedding models"); + fillModelSelect($("imageModel"),imageModels,state.imageModel,"No image models"); + } + function switchWorkspace(ws){ + state.workspace=ws;save(); + Array.prototype.forEach.call(document.querySelectorAll(".wsbtn"),function(b){ + var on=b.getAttribute("data-ws")===ws;b.classList.toggle("on",on);b.setAttribute("aria-selected",on?"true":"false"); + }); + var chatMain=$("stream"),dock=document.querySelector(".dock"); + var showChat=ws==="chat"; + chatMain.style.display=showChat?"":"none"; + if(dock)dock.style.display=showChat?"":"none"; + $("studioEmbed").hidden=ws!=="embed"; + $("studioImage").hidden=ws!=="image"; + $("mtrigger").style.display=showChat?"":"none"; + $("exportChat").style.display=showChat?"":"none"; + if(ws!=="chat")loadModels(); + closeRail(); + } + function currentImageMode(){var el=document.querySelector("#imageMode .seg.on");return el?el.getAttribute("data-mode"):"generate";} + function applyImageMode(){ + var mode=currentImageMode(); + $("imagePromptWrap").hidden=(mode==="upscale"); + $("imageInputWrap").hidden=(mode==="generate"); + $("imageMaskWrap").hidden=(mode!=="inpaint"); + $("imageRun").textContent=mode==="upscale"?"Upscale":(mode==="generate"?"Generate":(mode==="edit"?"Edit":"Inpaint")); + } + function runEmbeddings(){ + var model=$("embedModel").value; + var lines=($("embedInput").value||"").split("\n").map(function(s){return s.trim();}).filter(function(s){return s.length>0;}); + var status=$("embedStatus"),out=$("embedOutput"); + if(!model){status.className="studio-status err";status.textContent="Select an embedding model";return;} + if(!lines.length){status.className="studio-status err";status.textContent="Enter at least one line";return;} + status.className="studio-status";status.textContent="Running…";out.textContent=""; + fetch(base()+"/v1/embeddings",{method:"POST",headers:headers(),body:JSON.stringify({model:model,input:lines})}).then(function(r){ + return r.json().then(function(j){if(!r.ok)throw new Error((j.error&&(j.error.message||j.error))||("HTTP "+r.status));return j;}); + }).then(function(j){ + var data=j.data||[]; + var dim=(data[0]&&data[0].embedding)?data[0].embedding.length:0; + status.className="studio-status ok";status.textContent=data.length+" vector"+(data.length===1?"":"s")+" · dim="+dim; + out.textContent=data.map(function(d,i){ + var preview=(d.embedding||[]).slice(0,8).map(function(v){return v.toFixed(4);}).join(", "); + return "["+i+"] dim="+(d.embedding||[]).length+" → ["+preview+" …]"; + }).join("\n"); + }).catch(function(e){status.className="studio-status err";status.textContent=e.message||"Failed";}); + } + function runImage(){ + var mode=currentImageMode(),model=$("imageModel").value; + var status=$("imageStatus"),preview=$("imagePreview"); + var inputEl=$("imageInput"),maskEl=$("imageMask"); + if(!model){status.className="studio-status err";status.textContent="Select an image model";return;} + status.className="studio-status";status.textContent="Running…";preview.innerHTML=""; + var p; + try{ + if(mode==="generate"){ + var body={model:model,prompt:$("imagePrompt").value,negative_prompt:$("imageNeg").value, + steps:parseInt($("imageSteps").value,10)||20,cfg:parseFloat($("imageCfg").value)||7, + width:parseInt($("imageWidth").value,10)||512,height:parseInt($("imageHeight").value,10)||512}; + p=fetch(base()+"/v1/images/generations",{method:"POST",headers:headers(),body:JSON.stringify(body)}); + }else if(mode==="upscale"){ + if(!inputEl.files||!inputEl.files[0])throw new Error("Select an input image"); + var fd=new FormData();fd.append("model",model);fd.append("image",inputEl.files[0],inputEl.files[0].name); + p=fetch(base()+"/v1/images/upscale",{method:"POST",headers:authMultipart(),body:fd}); + }else{ + if(!inputEl.files||!inputEl.files[0])throw new Error("Select an input image"); + var fd2=new FormData();fd2.append("model",model);fd2.append("image",inputEl.files[0],inputEl.files[0].name); + if(mode==="inpaint"){if(!maskEl.files||!maskEl.files[0])throw new Error("Select a mask image");fd2.append("mask",maskEl.files[0],maskEl.files[0].name);} + fd2.append("prompt",$("imagePrompt").value);fd2.append("negative_prompt",$("imageNeg").value); + fd2.append("steps",$("imageSteps").value);fd2.append("cfg",$("imageCfg").value); + fd2.append("width",$("imageWidth").value);fd2.append("height",$("imageHeight").value); + p=fetch(base()+"/v1/images/edits",{method:"POST",headers:authMultipart(),body:fd2}); + } + }catch(e){status.className="studio-status err";status.textContent=e.message;return;} + p.then(function(r){return r.json().then(function(j){if(!r.ok)throw new Error((j.error&&(j.error.message||j.error))||("HTTP "+r.status));return j;});}).then(function(j){ + var b64=j.data&&j.data[0]&&j.data[0].b64_json; + if(!b64)throw new Error("No image in response"); + var img=document.createElement("img");img.src="data:image/png;base64,"+b64;preview.appendChild(img); + status.className="studio-status ok";status.textContent="Done"; + }).catch(function(e){status.className="studio-status err";status.textContent=e.message||"Failed";}); + } + function authMultipart(){var h={};if(state.token)h.Authorization="Bearer "+state.token;return h;} + function loadModels(){ setConn("wait","Connecting…",hostShort()); if(!state.token){setConn("bad","Not connected","Add your token in settings");note("Paste the token shown on the app's Server screen, then connect.","bad");renderModelMenu();return Promise.resolve();} @@ -430,7 +587,10 @@

Settings

chatModels=all.filter(function(m){return m.type==="gguf"||m.type==="vlm";}); ttsModels=all.filter(function(m){return m.type==="tts";}); sttModels=all.filter(function(m){return m.type==="stt";}); + embedModels=all.filter(function(m){return m.type==="embedding";}); + imageModels=all.filter(function(m){return m.type==="image_gen"||m.type==="image_upscaler";}); fillVoiceSelects(); + fillStudioSelects(); if(!chatModels.length){setConn("bad","No chat models","Assign one in the app");note("Linked, but no chat or vision models are exposed. Assign one in the app.","bad");renderModelMenu();render();return;} if(!state.model||!chatModels.some(function(m){return m.id===state.model;})){var d=chatModels.find(function(m){return m.default;});setModel(d?d.id:chatModels[0].id);} else setModel(state.model); @@ -679,7 +839,14 @@

Settings

if(mq.matches){if($("side").classList.contains("open"))closeRail();else openRail();} else{state.tucked=!state.tucked;$("app").classList.toggle("tucked",state.tucked);save();} } - function applyTuck(){if(!mq.matches)$("app").classList.toggle("tucked",!!state.tucked);else $("app").classList.remove("tucked");} + function applyTuck(){ + if(!mq.matches){ + $("side").classList.remove("open");$("scrim").classList.remove("show"); + $("app").classList.toggle("tucked",!!state.tucked); + }else{ + $("app").classList.remove("tucked"); + } + } /* ---------- settings ---------- */ function openSettings(){ @@ -745,6 +912,20 @@

Settings

c.onclick=function(){$("prompt").value=c.getAttribute("data-p");autosize();syncSend();$("prompt").focus();}; }); + Array.prototype.forEach.call(document.querySelectorAll(".wsbtn"),function(b){ + b.onclick=function(){switchWorkspace(b.getAttribute("data-ws"));}; + }); + Array.prototype.forEach.call(document.querySelectorAll("#imageMode .seg"),function(s){ + s.onclick=function(){ + Array.prototype.forEach.call(document.querySelectorAll("#imageMode .seg"),function(x){x.classList.remove("on");}); + s.classList.add("on");applyImageMode(); + }; + }); + $("embedRun").onclick=runEmbeddings; + $("imageRun").onclick=runImage; + $("embedModel").onchange=function(){state.embedModel=$("embedModel").value;save();}; + $("imageModel").onchange=function(){state.imageModel=$("imageModel").value;save();}; + var dragN=0; window.addEventListener("dragenter",function(e){if(e.dataTransfer&&Array.prototype.indexOf.call(e.dataTransfer.types||[],"Files")>=0){dragN++;$("drop").classList.add("show");}}); window.addEventListener("dragover",function(e){if($("drop").classList.contains("show"))e.preventDefault();}); @@ -774,6 +955,8 @@

Settings

/* ---------- boot ---------- */ applyTheme();applyTuck();wire();autosize();syncSend(); + applyImageMode(); + switchWorkspace(state.workspace||"chat"); $("mname").textContent=state.model||"Select a model"; if(state.chats.length)active=state.chats[0]; render(); From 1c0928c375b34afe893c52d2dfedcb0286d3337a Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Wed, 24 Jun 2026 10:11:11 +0600 Subject: [PATCH 12/13] fix: route server image chat through vlm --- AGENTS.md | 8 +- app/src/main/assets/server_webui.html | 102 +++++++++++++++--- .../service/server/ServerController.kt | 16 ++- .../ui/screens/server/ServerScreen.kt | 29 +++-- .../tool_neuron/viewmodel/ServerViewModel.kt | 27 ++++- native-server/src/main/cpp/server_core.cpp | 21 +++- native-server/src/main/cpp/server_models.cpp | 9 ++ native-server/src/main/cpp/server_models.h | 1 + 8 files changed, 187 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e71835fa..fe5eb98a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -427,7 +427,7 @@ native-server/src/main/cpp/ server_auth.{h,cpp} — bearer token store + constant-time compare + 401/403 server_crypto.{h,cpp} — getrandom(2) RNG, const_time_eq, base64url, base64 std, base64 decoder, secure_zero server_models.{h,cpp} — typed catalog: id + display_name + path + mmproj_path + config_json + Kind + created - + has_id_of_kind / first_of_kind / has_any_of_kind / build_list_response + + has_id_of_kind / first_of_kind / default_of_kind / count_of_kind / build_list_response server_audit.{h,cpp} — 128-entry ring buffer of request events server_rate_limit.{h,cpp} — per-client token bucket (cap=30, refill=1/s) + auth-fail ban (20 fails → 1 h) server_webui.{h,cpp} — set/clear/get/has HTML (mutex-protected std::string) @@ -450,18 +450,18 @@ Payload mechanics: - **Single-response** (embeddings / TTS / STT / image): `reply_session` — bridge calls `nativeFeedReplyText(replyId, body, mime)` for JSON/text or `nativeFeedReplyBinary(replyId, path, mime)` for staged binary; the route handler blocks on `session->wait(timeout)`. - **Big binary upload** (multipart image, mask, wav): cpp-httplib decodes multipart natively; the route writes each part to `/server-staging/tn__` via `server_staging::write_bytes`, hands the path to Java via JNI string (avoids byte[] JNI copies), and unlinks on response. - **Big binary download** (TTS wav, generated PNG): Kotlin writes the bytes to the staged path, hands the path back via `nativeFeedReplyBinary(path, mime)`, the C++ side reads + sends + unlinks. PNG responses are base64-encoded into JSON `b64_json` per OpenAI; WAV is sent as raw `audio/wav`. -- **VLM image_url parts**: only `data:image/...;base64,...` URLs are accepted. Network URLs return 400 (offline-only scope). Decoded bytes are staged to tmpfiles and the paths passed to `InferenceBridge.startGeneration(..., imagePaths=[...])`. Sanitised messages (image parts collapsed into text-only `content`) are forwarded to the engine alongside the path list — the Kotlin bridge reads each tmpfile and feeds the bytes to `GGMLEngine.generateVlmFlow(imageData = [...])`. +- **VLM image_url parts**: only `data:image/...;base64,...` URLs are accepted. Network URLs return 400 (offline-only scope). Decoded bytes are staged to tmpfiles and the paths passed to `InferenceBridge.startGeneration(..., imagePaths=[...])`. Sanitised messages (image parts collapsed into text-only `content`) are forwarded to the engine alongside the path list — the Kotlin bridge reads each tmpfile and feeds the bytes to `GGMLEngine.generateVlmFlow(imageData = [...])`. If an image-bearing request names a non-VLM model, native routing switches to the default VLM; if no default exists and exactly one VLM is enabled, it uses that single VLM. Multiple VLMs with no default return a clear model error. ### Web UI Bundled at `app/src/main/assets/server_webui.html`. Single Material-3 SPA with a sidebar tab strip that swaps the main panel between four workspaces: -- **Chat** — preserved from the prior build: localStorage history, markdown rendering, streaming with blinking cursor, settings dialog, connection indicator. Adds an attach-image button (📎) that converts the uploaded image to a `data:image/...;base64,...` URL and appends it as an OpenAI multi-part `image_url` content entry on the next send. Server auto-detects and routes to the VLM engine. +- **Chat** — preserved from the prior build: localStorage history, markdown rendering, streaming with blinking cursor, settings dialog, connection indicator. The attach-image button converts uploads to `data:image/...;base64,...` URLs. The Web UI sends image parts only for the current image turn, or for a later turn whose text explicitly refers back to a previous image; unrelated follow-up text stays on the selected chat model. If the selected model is not VLM, Web UI switches per request to the default VLM, the only VLM, or prompts the user when multiple VLMs exist and no default is set. - **Embeddings** — model select, multi-line input (one row per line), runs `/v1/embeddings`, shows vector count + first 8 dims of each row. - **Voice** — two cards. TTS: model + text + voice id + speed, plays the returned WAV inline. STT: model + WAV upload, shows transcribed text. - **Image** — segmented switch (Generate / Edit / Inpaint / Upscale). Prompt + negative + steps/CFG/width/height for diffusion modes. Input image file for Edit/Inpaint/Upscale. Mask file for Inpaint. Result is rendered inline from `b64_json`. -`refreshModelCache()` hits `/v1/models` once per tab activation and filters per-kind for the model dropdowns. JNI: `nativeSetWebUiHtml(html)` pushes the bundled file at server start; `nativeClearWebUi()` clears on stop. Same applies to `/docs` via `nativeSetDocsHtml` + `app/src/main/assets/server_docs.html`. The docs file documents every endpoint with copy-pasteable curl examples. +`refreshModelCache()` hits `/v1/models` once per tab activation and filters per-kind for the model dropdowns. Chat keeps `chatModels` and `vlmModels` separate even though both can answer `/v1/chat/completions`; `vlmModels` exists for per-turn image routing and the multi-VLM chooser. JNI: `nativeSetWebUiHtml(html)` pushes the bundled file at server start; `nativeClearWebUi()` clears on stop. Same applies to `/docs` via `nativeSetDocsHtml` + `app/src/main/assets/server_docs.html`. The docs file documents every endpoint with copy-pasteable curl examples. ### Start config (`configJson`) schema diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index 0111b8af..fc3b1363 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -251,13 +251,13 @@

Settings

var $=function(id){return document.getElementById(id);}; var KEY="tn.webui.v4"; var state=load(); - var chatModels=[],ttsModels=[],sttModels=[],embedModels=[],imageModels=[]; + var chatModels=[],vlmModels=[],ttsModels=[],sttModels=[],embedModels=[],imageModels=[]; var active=null,abort=null,generating=false; var atts=[]; var pendingVoiceReply=false; var mq=window.matchMedia("(max-width:980px), (pointer:coarse) and (max-width:1180px)"); - function defaults(){return{serverUrl:location.origin,token:"",model:"",systemPrompt:"",maxTokens:1024,temp:0.7,stream:true,theme:"", + function defaults(){return{serverUrl:location.origin,token:"",model:"",vlmModel:"",systemPrompt:"",maxTokens:1024,temp:0.7,stream:true,theme:"", sttModel:"",ttsModel:"",ttsVoice:0,ttsSpeed:1,tucked:false,chats:[],workspace:"chat",embedModel:"",imageModel:""};} function load(){try{return Object.assign(defaults(),JSON.parse(localStorage.getItem(KEY)||"{}"));}catch(e){return defaults();}} function save(){try{localStorage.setItem(KEY,JSON.stringify(state));}catch(e){}} @@ -584,6 +584,7 @@

Settings

return r.json(); }).then(function(j){ var all=j.data||[]; + vlmModels=all.filter(function(m){return m.type==="vlm";}); chatModels=all.filter(function(m){return m.type==="gguf"||m.type==="vlm";}); ttsModels=all.filter(function(m){return m.type==="tts";}); sttModels=all.filter(function(m){return m.type==="stt";}); @@ -592,30 +593,90 @@

Settings

fillVoiceSelects(); fillStudioSelects(); if(!chatModels.length){setConn("bad","No chat models","Assign one in the app");note("Linked, but no chat or vision models are exposed. Assign one in the app.","bad");renderModelMenu();render();return;} - if(!state.model||!chatModels.some(function(m){return m.id===state.model;})){var d=chatModels.find(function(m){return m.default;});setModel(d?d.id:chatModels[0].id);} + if(state.vlmModel&&!vlmModels.some(function(m){return m.id===state.vlmModel;})){state.vlmModel="";save();} + if(!state.model||!chatModels.some(function(m){return m.id===state.model;})){ + var d=chatModels.find(function(m){return m.default&&m.type==="gguf";})||chatModels.find(function(m){return m.type==="gguf";})||vlmModels.find(function(m){return m.default;})||vlmModels[0]; + setModel(d?d.id:chatModels[0].id); + } else setModel(state.model); setConn("ok","Connected",hostShort()); note("Connected to "+hostShort()+" · "+chatModels.length+" chat model"+(chatModels.length>1?"s":"")+" ready.","ok"); renderModelMenu();render(); }).catch(function(e){ - chatModels=[];ttsModels=[];sttModels=[];fillVoiceSelects();renderModelMenu(); + chatModels=[];vlmModels=[];ttsModels=[];sttModels=[];fillVoiceSelects();renderModelMenu(); setConn("bad","Can't connect",hostShort()); note(e&&e.soft?e.soft:"Can't reach "+hostShort()+". Check the address and that the server is running.","bad"); }); } /* ---------- compose / send ---------- */ - function buildMessages(){ + function isImageFile(f){return f&&f.kind==="image"&&f.dataUrl;} + function isVlmModel(id){return !!vlmModels.find(function(m){return m.id===id;});} + function lastUserMessage(){ + if(!active)return null; + for(var i=active.msgs.length-1;i>=0;i--){ + var m=active.msgs[i]; + if(m.role==="user"&&!m.pending)return m; + } + return null; + } + function imageRefText(text){ + return /\b(image|photo|picture|screenshot|diagram|chart|attached file|attachment|the attached|this visual|that visual|previous visual|last visual)\b/i.test(text||""); + } + function lastImageFiles(){ + if(!active)return[]; + for(var i=active.msgs.length-1;i>=0;i--){ + var m=active.msgs[i]; + if(m.role==="user"&&m.files){ + var imgs=m.files.filter(isImageFile); + if(imgs.length)return imgs; + } + } + return[]; + } + function chooseVlmModel(){ + if(!vlmModels.length){openSettings();note("Install or expose a VLM model before sending images.","bad");return"";} + if(isVlmModel(state.model))return state.model; + if(state.vlmModel&&isVlmModel(state.vlmModel))return state.vlmModel; + var d=vlmModels.find(function(m){return m.default;}); + if(d)return d.id; + if(vlmModels.length===1)return vlmModels[0].id; + var list=vlmModels.map(function(m,i){return (i+1)+". "+m.id;}).join("\n"); + var pick=prompt("Choose a vision model for this image request:\n\n"+list+"\n\nType a number or model id."); + if(!pick)return""; + pick=pick.trim(); + var byNum=/^\d+$/.test(pick)?vlmModels[Number(pick)-1]:null; + var byId=vlmModels.find(function(m){return m.id===pick;}); + var chosen=byNum||byId; + if(!chosen){toast("Vision model not changed");return"";} + state.vlmModel=chosen.id;save(); + return chosen.id; + } + function visionPlanFor(text,files){ + var imgs=(files||[]).filter(isImageFile); + if(imgs.length)return{images:imgs,reason:"attached"}; + if(imageRefText(text)){ + var prior=lastImageFiles(); + if(prior.length)return{images:prior,reason:"referenced"}; + } + return{images:[],reason:""}; + } + function buildMessages(visionImages){ var out=[]; + var lastUser=-1; + for(var j=active.msgs.length-1;j>=0;j--){if(active.msgs[j].role==="user"&&!active.msgs[j].pending){lastUser=j;break;}} if((state.systemPrompt||"").trim())out.push({role:"system",content:state.systemPrompt.trim()}); - active.msgs.forEach(function(m){ + active.msgs.forEach(function(m,i){ if(m.pending)return; if(m.role==="user"&&m.files&&m.files.length){ var parts=[];if(m.text)parts.push({type:"text",text:m.text}); m.files.forEach(function(f){ - if(f.kind==="image"&&f.dataUrl)parts.push({type:"image_url",image_url:{url:f.dataUrl}}); - else if(f.content)parts.push({type:"text",text:"Attached file \""+f.name+"\":\n"+f.content.slice(0,16000)}); + if(f.content)parts.push({type:"text",text:"Attached file \""+f.name+"\":\n"+f.content.slice(0,16000)}); + else if(isImageFile(f)&&i!==lastUser)parts.push({type:"text",text:"Previously attached image: "+(f.name||"image")}); }); + if(i===lastUser&&(visionImages||[]).length){ + visionImages.forEach(function(f){parts.push({type:"image_url",image_url:{url:f.dataUrl}});}); + } out.push({role:"user",content:parts.length?parts:(m.text||"")}); }else{ out.push({role:m.role,content:m.role==="assistant"?answerOf(m):(m.text||"")}); @@ -639,15 +700,32 @@

Settings

if(!state.token){openSettings();note("Paste your access token to start chatting.","bad");return;} if(!chatModels.length){openSettings();note("Connect to a server with a chat model first.","bad");return;} if(!state.model){openSettings();note("Pick a model — none is selected.","bad");return;} + var vision=visionPlanFor(text,atts); + var routeModel=state.model; + if(vision.images.length){ + routeModel=chooseVlmModel(); + if(!routeModel)return; + if(routeModel!==state.model)toast("Switching to vision model: "+routeModel); + } if(!active)newChat(); var speakBack=pendingVoiceReply;pendingVoiceReply=false; active.msgs.push({role:"user",text:text,files:atts.slice()}); titleFrom(active); $("prompt").value="";atts=[];renderPending();autosize(); - save();render();respond(speakBack); + save();render();respond(speakBack,{model:routeModel,images:vision.images,reason:vision.reason}); } - function respond(autoSpeak){ - var ph={role:"assistant",text:"",reasoning:"",pending:true,model:state.model,autoSpeak:!!autoSpeak}; + function respond(autoSpeak,route){ + if(!route){ + var lu=lastUserMessage(),vp=lu?visionPlanFor(lu.text,lu.files):{images:[],reason:""}; + var rm=state.model; + if(vp.images.length){ + rm=chooseVlmModel(); + if(!rm)return; + if(rm!==state.model)toast("Switching to vision model: "+rm); + } + route={model:rm,images:vp.images,reason:vp.reason}; + } + var ph={role:"assistant",text:"",reasoning:"",pending:true,model:route.model,autoSpeak:!!autoSpeak}; active.msgs.push(ph);save();render(); var el=$("list").firstChild.lastChild,prose=el._prose,thinkBody=el._think,thinkWrap=el._thinkWrap; setGenerating(true); @@ -667,7 +745,7 @@

Settings

if(stuck)scrollEnd(); } function bump(){if(!raf)raf=requestAnimationFrame(paint);} - var body={model:state.model,messages:buildMessages(),temperature:Number(state.temp)||0,max_tokens:Number(state.maxTokens)||1024,stream:!!state.stream}; + var body={model:route.model,messages:buildMessages(route.images||[]),temperature:Number(state.temp)||0,max_tokens:Number(state.maxTokens)||1024,stream:!!state.stream}; var url=base()+"/v1/chat/completions",p; if(state.stream){ p=fetch(url,{method:"POST",headers:headers(),body:JSON.stringify(body),signal:abort.signal}).then(function(r){ diff --git a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt index bd21bd7c..e336c92f 100644 --- a/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt +++ b/app/src/main/java/com/dark/tool_neuron/service/server/ServerController.kt @@ -174,12 +174,24 @@ class ServerController @Inject constructor( fun currentToken(): String = prefs.serverToken fun selectedModelId(): String = prefs.serverSelectedModelId + fun selectedVlmModelId(): String = + readServerRoleDefaults()[ServerEngineKind.VLM.token].orEmpty() + fun setSelectedModelId(modelId: String) { prefs.serverSelectedModelId = modelId } + fun setChatDefaultModelId(modelId: String) { prefs.serverSelectedModelId = modelId + setRoleDefaultModelId(ServerEngineKind.CHAT_GGUF, modelId) + } + + fun setVlmDefaultModelId(modelId: String) { + setRoleDefaultModelId(ServerEngineKind.VLM, modelId) + } + + private fun setRoleDefaultModelId(kind: ServerEngineKind, modelId: String) { val obj = runCatching { JSONObject(prefs.serverRoleDefaultsJson) }.getOrDefault(JSONObject()) - if (modelId.isBlank()) obj.remove(ServerEngineKind.CHAT_GGUF.token) - else obj.put(ServerEngineKind.CHAT_GGUF.token, modelId) + if (modelId.isBlank()) obj.remove(kind.token) + else obj.put(kind.token, modelId) prefs.serverRoleDefaultsJson = obj.toString() } diff --git a/app/src/main/java/com/dark/tool_neuron/ui/screens/server/ServerScreen.kt b/app/src/main/java/com/dark/tool_neuron/ui/screens/server/ServerScreen.kt index c1732caf..52d4fd84 100644 --- a/app/src/main/java/com/dark/tool_neuron/ui/screens/server/ServerScreen.kt +++ b/app/src/main/java/com/dark/tool_neuron/ui/screens/server/ServerScreen.kt @@ -76,7 +76,9 @@ fun ServerScreen( val tokenVisible by vm.tokenVisible.collectAsStateWithLifecycle() val anyEngineInstalled by vm.anyEngineInstalled.collectAsStateWithLifecycle() val chatModels by vm.chatModels.collectAsStateWithLifecycle() + val vlmModels by vm.vlmModels.collectAsStateWithLifecycle() val selectedChatModelId by vm.selectedChatModelId.collectAsStateWithLifecycle() + val selectedVlmModelId by vm.selectedVlmModelId.collectAsStateWithLifecycle() val dimens = LocalDimens.current val busy = state is ServerState.Running || @@ -99,13 +101,26 @@ fun ServerScreen( EndpointsCard(state as ServerState.Running) } - ServerChatModelCard( + ServerDefaultModelCard( + title = "Chat model", + description = "Used by the bundled Web UI and by API requests with a blank/default chat model.", + emptyMessage = "No GGUF chat model installed. Import or download a chat model first.", models = chatModels, selectedId = selectedChatModelId, disabled = busy, onPick = vm::setChatDefaultModel, ) + ServerDefaultModelCard( + title = "Vision model", + description = "Used when the Web UI or API receives image attachments. The selected VLM loads with its projector automatically.", + emptyMessage = "No VLM with a colocated projector is installed. Download a vision model first.", + models = vlmModels, + selectedId = selectedVlmModelId, + disabled = busy, + onPick = vm::setVlmDefaultModel, + ) + ConfigCard( port = port, bindMode = bindMode, @@ -277,7 +292,10 @@ private fun UrlRow(label: String, url: String, onCopy: () -> Unit) { } @Composable -private fun ServerChatModelCard( +private fun ServerDefaultModelCard( + title: String, + description: String, + emptyMessage: String, models: List, selectedId: String, disabled: Boolean, @@ -285,14 +303,14 @@ private fun ServerChatModelCard( ) { val dimens = LocalDimens.current StandardCard( - title = "Chat model", - description = "Used by the bundled Web UI and by API requests with a blank/default chat model.", + title = title, + description = description, icon = TnIcons.Cpu, ) { Column(verticalArrangement = Arrangement.spacedBy(dimens.spacingXs)) { if (models.isEmpty()) { CaptionText( - text = "No GGUF chat model installed. Import or download a chat model first.", + text = emptyMessage, color = MaterialTheme.colorScheme.error, ) } else { @@ -305,7 +323,6 @@ private fun ServerChatModelCard( ) } } - CaptionText("Other endpoint defaults, like embeddings, voice, image generation, and upscaling, live in Settings > Server model roles.") } } } diff --git a/app/src/main/java/com/dark/tool_neuron/viewmodel/ServerViewModel.kt b/app/src/main/java/com/dark/tool_neuron/viewmodel/ServerViewModel.kt index cf4c1066..c6296c94 100644 --- a/app/src/main/java/com/dark/tool_neuron/viewmodel/ServerViewModel.kt +++ b/app/src/main/java/com/dark/tool_neuron/viewmodel/ServerViewModel.kt @@ -12,6 +12,7 @@ import com.dark.tool_neuron.repo.ModelRepository import com.dark.tool_neuron.service.server.ServerController import com.dark.tool_neuron.service.server.ServerRequestEvent import com.dark.tool_neuron.service.server.ServerState +import com.dark.tool_neuron.util.VlmPaths import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted @@ -19,6 +20,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.map +import java.io.File import javax.inject.Inject @HiltViewModel @@ -46,13 +48,24 @@ class ServerViewModel @Inject constructor( val chatModels: StateFlow> = modelRepo.models .map { list -> - list.filter { it.pathType == PathType.FILE && it.providerType == ProviderType.GGUF } + list.filter { + it.pathType == PathType.FILE && + it.providerType == ProviderType.GGUF && + !isVlmCandidate(it) + } } .stateIn(viewModelScope, SharingStarted.Eagerly, emptyList()) + val vlmModels: StateFlow> = modelRepo.models + .map { list -> list.filter { it.pathType == PathType.FILE && isVlmCandidate(it) } } + .stateIn(viewModelScope, SharingStarted.Eagerly, emptyList()) + private val _selectedChatModelId = MutableStateFlow(controller.selectedModelId()) val selectedChatModelId: StateFlow = _selectedChatModelId.asStateFlow() + private val _selectedVlmModelId = MutableStateFlow(controller.selectedVlmModelId()) + val selectedVlmModelId: StateFlow = _selectedVlmModelId.asStateFlow() + fun start() { controller.setPort(_port.value) controller.setBindMode(_bindMode.value) @@ -81,6 +94,11 @@ class ServerViewModel @Inject constructor( controller.setChatDefaultModelId(modelId) } + fun setVlmDefaultModel(modelId: String) { + _selectedVlmModelId.value = modelId + controller.setVlmDefaultModelId(modelId) + } + fun revealToken(): Boolean { if (!session.isAllowed(PolicyEngine.Feature.AUTH_VERIFY)) return false _tokenVisible.value = true @@ -106,4 +124,11 @@ class ServerViewModel @Inject constructor( val head = raw.take(6) return "$head${"•".repeat(8)}" } + + private fun isVlmCandidate(model: ModelInfo): Boolean { + if (model.providerType == ProviderType.VISION_CHAT) return true + if (model.providerType != ProviderType.GGUF) return false + return VlmPaths.isInsideVlmFolder(model.path, modelRepo.getModelsDir()) && + VlmPaths.colocatedMmproj(File(model.path)) != null + } } diff --git a/native-server/src/main/cpp/server_core.cpp b/native-server/src/main/cpp/server_core.cpp index 962ed891..967209b5 100644 --- a/native-server/src/main/cpp/server_core.cpp +++ b/native-server/src/main/cpp/server_core.cpp @@ -100,6 +100,25 @@ namespace tn::server { return true; } + bool ensure_vlm_model_for_image_request(httplib::Response& res, std::string& model_id) { + if (!model_id.empty() && m::has_id_of_kind(model_id, m::Kind::Vlm)) return true; + + auto fallback = m::default_of_kind(m::Kind::Vlm); + if (fallback.id.empty() && m::count_of_kind(m::Kind::Vlm) == 1) { + fallback = m::first_of_kind(m::Kind::Vlm); + } + if (!fallback.id.empty()) { + model_id = fallback.id; + return true; + } + + std::string msg = m::has_any_of_kind(m::Kind::Vlm) + ? "image request needs a VLM model; choose one or set a default vision model" + : "no VLM model installed"; + respond_error(res, 404, "model_not_found", msg, "invalid_request_error"); + return false; + } + bool ensure_text_chat_model(httplib::Response& res, std::string& model_id) { if (model_id.empty()) model_id = resolve_model_id(model_id, m::Kind::ChatGguf); if (model_id.empty()) model_id = resolve_model_id(model_id, m::Kind::Vlm); @@ -504,7 +523,7 @@ namespace tn::server { std::string model_id = parsed.request.model; if (route_vlm) { - if (!ensure_model_for(res, model_id, m::Kind::Vlm, "VLM")) return; + if (!ensure_vlm_model_for_image_request(res, model_id)) return; } else { if (!ensure_text_chat_model(res, model_id)) return; } diff --git a/native-server/src/main/cpp/server_models.cpp b/native-server/src/main/cpp/server_models.cpp index 7d27bcc7..7444fb92 100644 --- a/native-server/src/main/cpp/server_models.cpp +++ b/native-server/src/main/cpp/server_models.cpp @@ -155,6 +155,15 @@ namespace tn::server::models { return {}; } + size_t count_of_kind(Kind k) { + std::lock_guard lock(g_mu); + size_t count = 0; + for (const auto& m : g_models) { + if (m.kind == k) ++count; + } + return count; + } + bool has_any_of_kind(Kind k) { std::lock_guard lock(g_mu); for (const auto& m : g_models) { diff --git a/native-server/src/main/cpp/server_models.h b/native-server/src/main/cpp/server_models.h index 25324f08..f0ba4f87 100644 --- a/native-server/src/main/cpp/server_models.h +++ b/native-server/src/main/cpp/server_models.h @@ -40,6 +40,7 @@ namespace tn::server::models { ModelRef get(const std::string& id); ModelRef first_of_kind(Kind k); ModelRef default_of_kind(Kind k); + size_t count_of_kind(Kind k); bool has_any_of_kind(Kind k); std::string build_list_response(); From 58d458275e0c24ef5653c8bdb16d20cacbef5708 Mon Sep 17 00:00:00 2001 From: sheikhti1205 Date: Wed, 24 Jun 2026 11:06:27 +0600 Subject: [PATCH 13/13] fix: improve text file attachment handling --- AGENTS.md | 4 +- app/src/main/assets/server_webui.html | 56 +++++++++++++++++-- .../com/dark/tool_neuron/repo/RagManager.kt | 25 +++++---- .../viewmodel/home_vm/InferenceCoordinator.kt | 9 +++ native-server/src/main/cpp/openai_schema.cpp | 42 ++++++++++++++ native-server/src/main/cpp/openai_schema.h | 2 + native-server/src/main/cpp/server_core.cpp | 1 + 7 files changed, 122 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fe5eb98a..eee00868 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -456,7 +456,7 @@ Payload mechanics: Bundled at `app/src/main/assets/server_webui.html`. Single Material-3 SPA with a sidebar tab strip that swaps the main panel between four workspaces: -- **Chat** — preserved from the prior build: localStorage history, markdown rendering, streaming with blinking cursor, settings dialog, connection indicator. The attach-image button converts uploads to `data:image/...;base64,...` URLs. The Web UI sends image parts only for the current image turn, or for a later turn whose text explicitly refers back to a previous image; unrelated follow-up text stays on the selected chat model. If the selected model is not VLM, Web UI switches per request to the default VLM, the only VLM, or prompts the user when multiple VLMs exist and no default is set. +- **Chat** — preserved from the prior build: localStorage history, markdown rendering, streaming with blinking cursor, settings dialog, connection indicator. The attach-image button converts uploads to `data:image/...;base64,...` URLs. The Web UI sends image parts only for the current image turn, or for a later turn whose text explicitly refers back to a previous image; unrelated follow-up text stays on the selected chat model. If the selected model is not VLM, Web UI switches per request to the default VLM, the only VLM, or prompts the user when multiple VLMs exist and no default is set. Non-image attachments are attempted as text by reading bytes and rejecting binary-looking payloads; do not reintroduce extension-only gating because `.ps1`, `.sql`, `.gradle`, unknown-extension notes, and similar files must work when they decode as readable text. - **Embeddings** — model select, multi-line input (one row per line), runs `/v1/embeddings`, shows vector count + first 8 dims of each row. - **Voice** — two cards. TTS: model + text + voice id + speed, plays the returned WAV inline. STT: model + WAV upload, shows transcribed text. - **Image** — segmented switch (Generate / Edit / Inpaint / Upscale). Prompt + negative + steps/CFG/width/height for diffusion modes. Input image file for Edit/Inpaint/Upscale. Mask file for Inpaint. Result is rendered inline from `b64_json`. @@ -588,6 +588,8 @@ Every attached document is stored content-addressed by SHA-256 of its bytes: `RagManager.hydrateChat(chatId)` re-ingests persisted records into the live RAG engine on chat-open (the engine itself is rebuilt fresh per process). It tracks `ingestedDocIds: MutableSet` to avoid duplicate ingests; the set clears on `engine.close()`. Hydration also re-populates the FTS5 BM25 index for text-format documents (idempotent — `keywordIndex.docCount(docId) > 0` check skips already-indexed). +Text-file handling is optimistic: first try the native parser, then readable-text fallback (UTF-8 / UTF-16 and binary-control checks). Unknown extensions are accepted when the bytes look like text, and binary-looking files are rejected. Keyword indexing uses the same readable-text fallback so scripts/configs without a friendly MIME type still participate in BM25 retrieval. + `RagManager.attachExisting(currentChatId, source)` is the prev-chat re-attach: builds the new compound docId, re-reads `.bin`, calls `engine.ingestBytes(...)`, persists the new record. Idempotent — if the chat already has the same `sourceId`, returns the existing record. `RagManager.removeDocument(docId)` removes the chunks from the engine + the FTS5 keyword rows + record from HXS, and deletes `.bin` only when no other record references that sourceId (`documentRepo.countWithSource(sourceId) == 0`). diff --git a/app/src/main/assets/server_webui.html b/app/src/main/assets/server_webui.html index fc3b1363..db69942e 100644 --- a/app/src/main/assets/server_webui.html +++ b/app/src/main/assets/server_webui.html @@ -97,7 +97,7 @@

What can I help with?

- +