Skip to content

feat: support multimodal chat #602

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lukasdotcom wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: support multimodal chat #602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions lib/Listener/BeforeTemplateRenderedListener.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ public function handle(Event $event): void {
$this->initialStateService->provideInitialState('contextChatIndexingComplete', $indexingComplete);
$this->initialStateService->provideInitialState('contextAgentToolSources', $this->assistantService->informationSources);
$this->initialStateService->provideInitialState('audio_chat_available', $this->assistantService->isAudioChatAvailable());
$multimodalChatAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes());
$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
$autoplayAudioChat = $this->config->getUserValue($this->userId, Application::APP_ID, 'autoplay_audio_chat', '1') === '1';
$this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat);
$agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes());
Expand Down
68 changes: 66 additions & 2 deletions lib/Service/ChatService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,13 +433,27 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
$taskId = $this->scheduleAudioChatTask($userId, $fileId, $systemPrompt, $history, $sessionId, $lastUserMessage->getId());
} else {
// for a text chat task, let's only use text in the history
$history = array_map(static function (Message $message) {
$historyMessages = array_map(static function (Message $message) {
return json_encode([
'role' => $message->getRole(),
'content' => $message->getContent(),
]);
}, $history);
$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $history, $sessionId);
if (count($lastAttachments) > 0 && $this->isMultimodalChatAvailable()) {
// for a multimodal chat task, let's use the attachments in the history as we don't know the structure in history
$attachmentsHistory = array_map(static function (Message $message) {
return $message->jsonSerialize()['attachments'];
}, $history);
// Just add all the attachments in the history as attachments for the latest message as we can't know the structure in history
$attachmentsHistory = array_merge($lastAttachments, ...$attachmentsHistory);
$attachmentsHistory = array_map(static function (array $attachment) {
return $attachment['file_id'];
}, $attachmentsHistory);

$taskId = $this->scheduleMultimodalChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId, $attachmentsHistory);
} else {
$taskId = $this->scheduleLLMChatTask($userId, $lastUserMessage->getContent(), $systemPrompt, $historyMessages, $sessionId);
}
}
}
return $taskId;
Expand Down Expand Up @@ -554,6 +568,13 @@ public function isContextAgentAudioAvailable(): bool {
return in_array(\OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
}

public function isMultimodalChatAvailable(): bool {
if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools')) {
return false;
}
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
}


private function getAudioHistory(array $history): array {
// history is a list of JSON strings
Expand Down Expand Up @@ -670,6 +691,49 @@ private function scheduleLLMChatTask(
return $task->getId() ?? 0;
}

/**
* Schedule a Multimodal Chat task
*
* @throws BadRequestException
* @throws InternalException
*/
private function scheduleMultimodalChatTask(
?string $userId,
string $content,
string $systemPrompt,
array $history,
int $sessionId,
array $attachmentsHistory,
): int {
$customId = 'chatty-llm:' . $sessionId;
$this->checkIfSessionIsThinking($userId, $customId);
$input = [
'input' => $content,
'system_prompt' => $systemPrompt,
'history' => $history,
'input_attachments' => $attachmentsHistory,
'tools' => '[]', // Empty tools as there is not a non tools version
'tool_message' => '',
];
/** @psalm-suppress UndefinedClass */
$task = new Task(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $input, Application::APP_ID . ':chatty-llm', $userId, $customId);
/** @psalm-suppress UndefinedMethod */
$task->setPreferStreaming(true);
try {
$this->taskProcessingManager->scheduleTask($task);
} catch (PreConditionNotMetException $e) {
throw new BadRequestException('pre_condition_not_met', previous: $e);
} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
throw new BadRequestException('unauthorized', previous: $e);
} catch (ValidationException $e) {
throw new BadRequestException('validation_failed', previous: $e);
} catch (\OCP\TaskProcessing\Exception\Exception $e) {
$this->logger->error($e->getMessage(), ['exception' => $e]);
throw new InternalException(previous: $e);
}
return $task->getId() ?? 0;
}

/**
* Schedule an agency chat task
*
Expand Down
10 changes: 7 additions & 3 deletions src/components/ChattyLLM/ChattyLLMInputForm.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -665,16 +665,20 @@ export default {
return session.timestamp ? (' ' + moment(session.timestamp * 1000).format('LLL')) : t('assistant', 'Untitled conversation')
},

async handleSubmit(event) {
async handleSubmit(attachedFileIds) {
if (this.chatContent.trim() === '') {
console.debug('empty message')
return
}
const attachments = attachedFileIds.map(fileId => ({ type: SHAPE_TYPE_NAMES.File, file_id: fileId }))

const role = Roles.HUMAN
const content = this.chatContent.trim()
const timestamp = +new Date() / 1000 | 0
console.debug('[Assistant] submit text', content)
if (attachedFileIds.length > 0) {
console.debug('[Assistant] submit attachments', attachments)
}

if (this.active === null) {
await this.newSession()
Expand All @@ -686,10 +690,10 @@ export default {
this.active.agencyAnswered = true
}

this.messages.push({ role, content, timestamp, session_id: this.active.id })
this.messages.push({ role, content, timestamp, session_id: this.active.id, attachments })
this.chatContent = ''
this.scrollToLastMessage()
await this.newMessage(role, content, timestamp, this.active.id)
await this.newMessage(role, content, timestamp, this.active.id, attachments)
},

async handleSubmitAudio(fileId) {
Expand Down
82 changes: 82 additions & 0 deletions src/components/ChattyLLM/DocumentPreviews.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
<!--
- SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
- SPDX-License-Identifier: AGPL-3.0-or-later
-->
<template>
<div class="document-previews">
<div class="document-previews__list">
<div v-for="fileId in fileIds"
:key="fileId"
class="document-previews__item">
<FileDisplay :file-id="fileId"
:task-id="null" />
<NcButton class="document-previews__delete"
variant="tertiary"
:aria-label="t('assistant', 'Remove this media')"
@click="$emit('delete', fileId)">
<template #icon>
<TrashCanOutlineIcon :size="20" />
</template>
</NcButton>
</div>
</div>
</div>
</template>

<script>
import TrashCanOutlineIcon from 'vue-material-design-icons/TrashCanOutline.vue'

import NcButton from '@nextcloud/vue/components/NcButton'

import FileDisplay from '../fields/FileDisplay.vue'

export default {
name: 'DocumentPreviews',

components: {
FileDisplay,
NcButton,
TrashCanOutlineIcon,
},

props: {
fileIds: {
type: Array,
required: true,
},
},

emits: [
'delete',
],
}
</script>

<style lang="scss" scoped>
.document-previews {
overflow-x: auto;

&__list {
display: flex;
flex-direction: row;
gap: 8px;
}

&__item {
position: relative;
padding: 8px;
border-radius: var(--border-radius-large);
background-color: var(--color-main-background);

&:hover {
background-color: var(--color-primary-element-light-hover);
}
}

&__delete {
position: absolute;
right: 0;
bottom: 0;
}
}
</style>
Loading
Loading