diff --git a/lib/Listener/BeforeTemplateRenderedListener.php b/lib/Listener/BeforeTemplateRenderedListener.php index 2c4fd6f0..209f31d9 100644 --- a/lib/Listener/BeforeTemplateRenderedListener.php +++ b/lib/Listener/BeforeTemplateRenderedListener.php @@ -72,6 +72,8 @@ public function handle(Event $event): void { $this->initialStateService->provideInitialState('contextChatIndexingComplete', $indexingComplete); $this->initialStateService->provideInitialState('contextAgentToolSources', $this->assistantService->informationSources); $this->initialStateService->provideInitialState('audio_chat_available', $this->assistantService->isAudioChatAvailable()); + $multimodalChatAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes()); + $this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable); $autoplayAudioChat = $this->config->getUserValue($this->userId, Application::APP_ID, 'autoplay_audio_chat', '1') === '1'; $this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat); $agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes()); diff --git a/lib/Service/ChatService.php b/lib/Service/ChatService.php index ac39bd83..bcd81d50 100644 --- a/lib/Service/ChatService.php +++ b/lib/Service/ChatService.php @@ -433,13 +433,27 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $ $taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId()); } else { // for a text chat task, let's only use text in the history - $history = array_map(static function (Message $message) { + $historyMessages = array_map(static function (Message $message) { return json_encode([ 'role' => $message->getRole(), 'content' => $message->getContent(), ]); }, $history); - $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId); + if (count($lastAttachments) > 0 && $this->isMultimodalChatAvailable()) { + // for a multimodal chat task, let's use the attachments in the history as we don't know the structure in history + $attachmentsHistory = array_map(static function (Message $message) { + return $message->jsonSerialize()['attachments']; + }, $history); + // Just add all the attachments in the history as attachments for the latest message as we can't know the structure in history + $attachmentsHistory = array_merge($lastAttachments, ...$attachmentsHistory); + $attachmentsHistory = array_map(static function (array $attachment) { + return $attachment['file_id']; + }, $attachmentsHistory); + + $taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $attachmentsHistory); + } else { + $taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId); + } } } return $taskId; @@ -554,6 +568,13 @@ public function isContextAgentAudioAvailable(): bool { return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); } + public function isMultimodalChatAvailable(): bool { + if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) { + return false; + } + return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds()); + } + private function getAudioHistory(array $history): array { // history is a list of JSON strings @@ -670,6 +691,49 @@ private function scheduleLLMChatTask( return $task->getId() ?? 0; } + /** + * Schedule a Multimodal Chat task + * + * @throws BadRequestException + * @throws InternalException + */ + private function scheduleMultimodalChatTask( + ?string $userId, + string $content, + string $systemPrompt, + array $history, + int $sessionId, + array $attachmentsHistory, + ): int { + $customId = 'chatty-llm:' . $sessionId; + $this->checkIfSessionIsThinking($userId, $customId); + $input = [ + 'input' => $content, + 'system_prompt' => $systemPrompt, + 'history' => $history, + 'input_attachments' => $attachmentsHistory, + 'tools' => '[]', // Empty tools as there is not a non tools version + 'tool_message' => '', + ]; + /** @psalm-suppress UndefinedClass */ + $task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId); + /** @psalm-suppress UndefinedMethod */ + $task->setPreferStreaming(true); + try { + $this->taskProcessingManager->scheduleTask($task); + } catch (PreConditionNotMetException $e) { + throw new BadRequestException('pre_condition_not_met', previous: $e); + } catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) { + throw new BadRequestException('unauthorized', previous: $e); + } catch (ValidationException $e) { + throw new BadRequestException('validation_failed', previous: $e); + } catch (\OCP\TaskProcessing\Exception\Exception $e) { + $this->logger->error($e->getMessage(), ['exception' => $e]); + throw new InternalException(previous: $e); + } + return $task->getId() ?? 0; + } + /** * Schedule an agency chat task * diff --git a/src/components/ChattyLLM/ChattyLLMInputForm.vue b/src/components/ChattyLLM/ChattyLLMInputForm.vue index da2b310c..0d93d451 100644 --- a/src/components/ChattyLLM/ChattyLLMInputForm.vue +++ b/src/components/ChattyLLM/ChattyLLMInputForm.vue @@ -665,16 +665,20 @@ export default { return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation') }, - async handleSubmit(event) { + async handleSubmit(attachedFileIds) { if (this.chatContent.trim() === '') { console.debug('empty message') return } + const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId })) const role = Roles.HUMAN const content = this.chatContent.trim() const timestamp = +new Date() / 1000 | 0 console.debug('[Assistant] submit text', content) + if (attachedFileIds.length > 0) { + console.debug('[Assistant] submit attachments', attachments) + } if (this.active === null) { await this.newSession() @@ -686,10 +690,10 @@ export default { this.active.agencyAnswered = true } - this.messages.push({ role, content, timestamp, session_id: this.active.id }) + this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments }) this.chatContent = '' this.scrollToLastMessage() - await this.newMessage(role, content, timestamp, this.active.id) + await this.newMessage(role, content, timestamp, this.active.id, attachments) }, async handleSubmitAudio(fileId) { diff --git a/src/components/ChattyLLM/DocumentPreviews.vue b/src/components/ChattyLLM/DocumentPreviews.vue new file mode 100644 index 00000000..06e5192a --- /dev/null +++ b/src/components/ChattyLLM/DocumentPreviews.vue @@ -0,0 +1,82 @@ + + + + + + diff --git a/src/components/ChattyLLM/InputArea.vue b/src/components/ChattyLLM/InputArea.vue index a40c24c4..36ab2034 100644 --- a/src/components/ChattyLLM/InputArea.vue +++ b/src/components/ChattyLLM/InputArea.vue @@ -4,53 +4,97 @@ --> @@ -187,11 +294,16 @@ export default { .input-area { display: flex; - flex-direction: row; - justify-content: space-between; - align-items: end; + flex-direction: column; gap: 4px; + &__row { + display: flex; + flex-direction: row; + align-items: end; + gap: 4px; + } + :deep(&__thinking > div) { font-style: italic; animation: breathing 2s linear infinite normal; diff --git a/src/components/ChattyLLM/Message.vue b/src/components/ChattyLLM/Message.vue index fa05f801..fb46a410 100644 --- a/src/components/ChattyLLM/Message.vue +++ b/src/components/ChattyLLM/Message.vue @@ -103,6 +103,12 @@ :file-id="a.file_id" :task-id="message.role === 'human' ? undefined : (a.ocp_task_id ?? message.ocp_task_id)" :is-output="message.role === 'assistant'" /> + @@ -126,6 +132,7 @@ import { showSuccess } from '@nextcloud/dialogs' import { generateOcsUrl } from '@nextcloud/router' import axios from '@nextcloud/axios' import { SHAPE_TYPE_NAMES } from '../../constants.js' +import FileDisplay from '../fields/FileDisplay.vue' const PLAIN_URL_PATTERN = /(?:\s|^|\()((?:https?:\/\/)(?:[-A-Z0-9+_.]+(?::[0-9]+)?(?:\/[-A-Z0-9+&@#%?=~_|!:,.;()]*)*))(?:\s|$|\))/ig const MARKDOWN_LINK_PATTERN = /\[[-A-Z0-9+&@#%?=~_|!:,.;()]+\]\(((?:https?:\/\/)(?:[-A-Z0-9+_.]+(?::[0-9]+)?(?:\/[-A-Z0-9+&@#%?=~_|!:,.;]*)*))\)/ig @@ -135,6 +142,7 @@ export default { components: { AudioDisplay, + FileDisplay, AssistantIcon, NcAvatar, @@ -213,6 +221,9 @@ export default { audioAttachments() { return this.message.attachments?.filter(a => a.type === SHAPE_TYPE_NAMES.Audio) ?? [] }, + fileAttachments() { + return this.message.attachments?.filter(a => a.type === SHAPE_TYPE_NAMES.File) ?? [] + }, }, watch: { diff --git a/src/constants.js b/src/constants.js index 75034955..d71a33b7 100644 --- a/src/constants.js +++ b/src/constants.js @@ -4,6 +4,7 @@ */ export const MAX_TEXT_INPUT_LENGTH = 64_000 +export const MAX_ATTACHED_FILES = 10 export const TASK_STATUS_INT = { cancelled: 5,