*/
@@ -695,8 +688,7 @@ private function buildChatCompletionRequestParams(
?array $extraParams = null,
?string $toolMessage = null,
?array $tools = null,
- ?string $userAudioPromptBase64 = null,
- ?string $userAudioPromptFormat = null,
+ ?array $files = null,
bool $stream = false,
): array {
$modelRequestParam = $model === Application::DEFAULT_MODEL_ID
@@ -737,32 +729,38 @@ private function buildChatCompletionRequestParams(
return $formattedToolCall;
}, $message['tool_calls']);
}
+ // Handle file attachments in the history
+ if (isset($message['content']) && is_array($message['content'])) {
+ $content = [];
+ foreach ($message['content'] as $item) {
+ if ($item['type'] === 'file') {
+ $content[] = $this->openAiFileService->buildFileContentFromId($item['file_id'], $userId);
+ } else {
+ $content[] = $item;
+ }
+ }
+ $message['content'] = $content;
+ }
$messages[] = $message;
}
}
- if ($userAudioPromptBase64 !== null) {
- // if there is audio, use the new message format (content is a list of objects)
- $message = [
- 'role' => 'user',
- 'content' => [
- [
- 'type' => 'input_audio',
- 'input_audio' => [
- 'data' => $userAudioPromptBase64,
- 'format' => $userAudioPromptFormat ?? 'wav',
- ],
- ],
- ],
- ];
+ // Attach all files when necessary
+ if ($files !== null && count($files) > 0) {
+ if (count($files) > 500) {
+ throw new UserFacingProcessingException($this->l10n->t('Too many files. Max is 500'), Http::STATUS_BAD_REQUEST);
+ }
+ $content = array_map([$this->openAiFileService, 'buildFileContentFromFile'], $files);
if ($userPrompt !== null) {
- $message['content'][] = [
+ $content[] = [
'type' => 'text',
'text' => $userPrompt,
];
}
- $messages[] = $message;
+ $messages[] = [
+ 'role' => 'user',
+ 'content' => $content,
+ ];
} elseif ($userPrompt !== null) {
- // if there is only text, use the old message format (content is a string)
$messages[] = [
'role' => 'user',
'content' => $userPrompt,
@@ -1442,8 +1440,10 @@ private function normalizeChatCompletionResponse(array $response): array {
'reasoning_messages' => [],
'tool_calls' => [],
'audio_messages' => [],
+ 'images' => [],
];
+
foreach ($response['choices'] as $choice) {
if (!is_array($choice)) {
continue;
@@ -1483,6 +1483,11 @@ private function normalizeChatCompletionResponse(array $response): array {
if (isset($choice['message']['audio'], $choice['message']['audio']['data']) && is_string($choice['message']['audio']['data'])) {
$completions['audio_messages'][] = $choice['message'];
}
+ if (isset($choice['message']['images']) && is_array($choice['message']['images'])) {
+ foreach ($choice['message']['images'] as $image) {
+ $completions['images'][] = $image;
+ }
+ }
}
return $completions;
@@ -1566,7 +1571,6 @@ public function autoDetectFeatures(): array {
$config['stt_provider_enabled'] = $this->isSTTAvailable();
$config['tts_provider_enabled'] = $this->isTTSAvailable();
$this->openAiSettingsService->setAdminConfig($config);
- $config['analyze_image_provider_enabled'] = $this->openAiSettingsService->getAnalyzeImageProviderEnabled();
return $config;
}
}
diff --git a/lib/Service/OpenAiFileService.php b/lib/Service/OpenAiFileService.php
new file mode 100644
index 00000000..beb72039
--- /dev/null
+++ b/lib/Service/OpenAiFileService.php
@@ -0,0 +1,257 @@
+ 'mp3',
+ 'audio/mpeg' => 'mp3',
+ 'audio/wav' => 'wav',
+ 'audio/x-wav' => 'wav',
+ ];
+
+ private const VALID_TEXT_MIME_TYPES = [
+ 'application/javascript',
+ 'application/typescript',
+ 'message/rfc822',
+ 'application/x-sql',
+ 'application/x-scala',
+ 'application/x-rust',
+ 'application/x-powershell',
+ 'application/x-patch',
+ 'application/x-php',
+ 'application/x-httpd-php',
+ 'application/x-httpd-php-source',
+ 'application/json',
+ 'application/x-bash',
+ 'application/x-protobuf',
+ 'application/x-terraform',
+ 'application/x-toml',
+ 'application/graphql',
+ 'application/x-graphql',
+ 'application/x-ndjson',
+ 'application/json5',
+ 'application/x-json5',
+ 'application/toml',
+ 'application/x-yaml',
+ 'application/yaml',
+ 'application/x-awk',
+ 'application/x-subrip',
+ 'application/csv'
+ ];
+
+ public function __construct(
+ private IL10N $l10n,
+ private OpenAiSettingsService $openAiSettingsService,
+ private IRootFolder $rootFolder,
+ ) {
+ }
+
+ /**
+ * Builds file content from a file ID within a given user folder.
+ *
+ * @param int $fileId The ID of the file to build content from.
+ * @param string $userId The user ID.
+ * @return array Content array suitable for OpenAI API or other handlers.
+ * @throws ProcessingException
+ * @throws UserFacingProcessingException
+ */
+ public function buildFileContentFromId(int $fileId, string $userId): array {
+ $userFolder = $this->rootFolder->getUserFolder($userId);
+ $file = $userFolder->getFirstNodeById($fileId);
+ return $this->buildFileContentFromFile($file);
+ }
+
+ /**
+ * Builds file content from a File object.
+ *
+ * @param ?File $file The file to build content from.
+ * @return array Content array suitable for OpenAI API or other handlers.
+ * @throws ProcessingException
+ * @throws UserFacingProcessingException
+ */
+ public function buildFileContentFromFile(?File $file): array {
+ if (!$file instanceof File || !$file->isReadable()) {
+ throw new ProcessingException('File is not readable');
+ }
+ // Maximum file size for openai is 50MB.
+ if ($this->isUsingOpenAi() && $file->getSize() > self::MAX_FILE_SIZE_BYTES) {
+ throw new UserFacingProcessingException(
+ 'Filesize of input files too large. Max is 50MB',
+ 0,
+ null,
+ $this->l10n->t('The size of the input file is too large. A maximum of 50MB is allowed.'),
+ );
+ }
+
+ $fileType = $file->getMimeType();
+ if (str_starts_with($fileType, 'image/')) {
+ return $this->buildImageContent($file);
+ // OpenAI only supports this for very specific models and support is not that common
+ } elseif (str_starts_with($fileType, 'audio/')) {
+ return $this->buildAudioContent($file);
+ // OpenAI does not currently support video attachments
+ } elseif (str_starts_with($fileType, 'video/')) {
+ return $this->buildVideoContent($file);
+ } elseif ($fileType === 'application/pdf') {
+ return $this->buildDocumentContent($file);
+ } else {
+ return $this->buildTextContent($file);
+ }
+ }
+
+
+ /**
+ * @return array{type: string, image_url: array{url: string}}
+ */
+ private function buildImageContent(File $file): array {
+ if (!$this->openAiSettingsService->getMultimodalImageEnabled()) {
+ throw new UserFacingProcessingException(
+ 'Image attachments are disabled',
+ 0,
+ null,
+ $this->l10n->t('Image attachments are unsupported.'),
+ );
+ }
+ $fileType = $file->getMimeType();
+ if ($this->isUsingOpenAi() && !in_array($fileType, self::VALID_IMAGE_MIME_TYPES, true)) {
+ throw new UserFacingProcessingException(
+ 'Invalid input file type for OpenAI ' . $fileType,
+ 0,
+ null,
+ $this->l10n->t('Invalid input file type "%1$s".', [$fileType]),
+ );
+ }
+ return [
+ 'type' => 'image_url',
+ 'image_url' => [
+ 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))),
+ ],
+ ];
+ }
+
+ /**
+ * @return array{type: string, input_audio: array{data: string, format: string}}
+ */
+ private function buildAudioContent(File $file): array {
+ if (!$this->openAiSettingsService->getMultimodalAudioEnabled()) {
+ throw new UserFacingProcessingException(
+ 'Audio attachments are disabled',
+ 0,
+ null,
+ $this->l10n->t('Audio attachments are unsupported.'),
+ );
+ }
+ $fileType = $file->getMimeType();
+
+ if (!array_key_exists($fileType, self::SUPPORTED_INPUT_AUDIO_FORMATS)) {
+ throw new UserFacingProcessingException(
+ 'Invalid input file type for OpenAI ' . $fileType,
+ 0,
+ null,
+ $this->l10n->t('Invalid input file type "%1$s".', [$fileType]),
+ );
+ }
+ $format = self::SUPPORTED_INPUT_AUDIO_FORMATS[$fileType];
+ return [
+ 'type' => 'input_audio',
+ 'input_audio' => [
+ 'data' => base64_encode(stream_get_contents($file->fopen('rb'))),
+ 'format' => $format,
+ ],
+ ];
+ }
+
+ /**
+ * @return array{type: string, video_url: array{url: string}}
+ */
+ private function buildVideoContent(File $file): array {
+ if (!$this->openAiSettingsService->getMultimodalVideoEnabled()) {
+ throw new UserFacingProcessingException(
+ 'Video attachments are disabled',
+ 0,
+ null,
+ $this->l10n->t('Video attachments are unsupported.'),
+ );
+ }
+ $fileType = $file->getMimeType();
+ return [
+ 'type' => 'video_url',
+ 'video_url' => [
+ 'url' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))),
+ ],
+ ];
+ }
+
+ /**
+ * @return array{type: string, file: array{filename: string, file_data: string}}
+ */
+ private function buildDocumentContent(File $file): array {
+ if (!$this->openAiSettingsService->getMultimodalDocumentEnabled()) {
+ throw new UserFacingProcessingException(
+ 'Document attachments are disabled',
+ 0,
+ null,
+ $this->l10n->t('Document attachments are unsupported.'),
+ );
+ }
+ $fileType = $file->getMimeType();
+ return [
+ 'type' => 'file',
+ 'file' => [
+ 'filename' => $file->getName(),
+ 'file_data' => 'data:' . $fileType . ';base64,' . base64_encode(stream_get_contents($file->fopen('rb'))),
+ ],
+ ];
+ }
+
+ /**
+ * @return array{type: string, text: string}
+ */
+ private function buildTextContent(File $file): array {
+ $fileType = $file->getMimeType();
+ // Sanity check that this isn't a binary
+ if (!str_starts_with($fileType, 'text/') && !in_array($fileType, self::VALID_TEXT_MIME_TYPES, true)) {
+ throw new UserFacingProcessingException(
+ 'Invalid input file type: ' . $fileType,
+ 0,
+ null,
+ $this->l10n->t('Invalid input file type: "%1$s".', [$fileType]),
+ );
+ }
+ return [
+ 'type' => 'text',
+ 'text' => 'Filename:' . $file->getName() . "\nContent:\n" . stream_get_contents($file->fopen('rb')),
+ ];
+ }
+
+ private function isUsingOpenAi(): bool {
+ $serviceUrl = $this->openAiSettingsService->getServiceUrl();
+ return $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL;
+ }
+}
diff --git a/lib/Service/OpenAiSettingsService.php b/lib/Service/OpenAiSettingsService.php
index 30661239..f63705c1 100644
--- a/lib/Service/OpenAiSettingsService.php
+++ b/lib/Service/OpenAiSettingsService.php
@@ -43,7 +43,10 @@ class OpenAiSettingsService {
't2i_provider_enabled' => 'boolean',
'stt_provider_enabled' => 'boolean',
'tts_provider_enabled' => 'boolean',
- 'analyze_image_provider_enabled' => 'boolean',
+ 'multimodal_image_enabled' => 'boolean',
+ 'multimodal_audio_enabled' => 'boolean',
+ 'multimodal_video_enabled' => 'boolean',
+ 'multimodal_document_enabled' => 'boolean',
'chat_endpoint_enabled' => 'boolean',
'basic_user' => 'string',
'basic_password' => 'string',
@@ -587,7 +590,10 @@ public function getAdminConfig(): array {
't2i_provider_enabled' => $this->getT2iProviderEnabled(),
'stt_provider_enabled' => $this->getSttProviderEnabled(),
'tts_provider_enabled' => $this->getTtsProviderEnabled(),
- 'analyze_image_provider_enabled' => $this->getAnalyzeImageProviderEnabled(),
+ 'multimodal_image_enabled' => $this->getMultimodalImageEnabled(),
+ 'multimodal_audio_enabled' => $this->getMultimodalAudioEnabled(),
+ 'multimodal_video_enabled' => $this->getMultimodalVideoEnabled(),
+ 'multimodal_document_enabled' => $this->getMultimodalDocumentEnabled(),
'chat_endpoint_enabled' => $this->getChatEndpointEnabled(),
'basic_user' => $this->getAdminBasicUser(),
'basic_password' => $this->getAdminBasicPassword(),
@@ -699,14 +705,29 @@ public function getTtsProviderEnabled(): bool {
/**
* @return bool
*/
- public function getAnalyzeImageProviderEnabled(): bool {
- $config = $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled');
- if ($config === '') {
- $serviceUrl = $this->getServiceUrl();
- $isUsingOpenAI = $serviceUrl === '' || $serviceUrl === Application::OPENAI_API_BASE_URL;
- return $isUsingOpenAI;
- }
- return $config === '1';
+ public function getMultimodalImageEnabled(): bool {
+ return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1') === '1';
+ }
+
+ /**
+ * @return bool
+ */
+ public function getMultimodalAudioEnabled(): bool {
+ return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_audio_enabled', '1') === '1';
+ }
+
+ /**
+ * @return bool
+ */
+ public function getMultimodalVideoEnabled(): bool {
+ return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_video_enabled', '0') === '1';
+ }
+
+ /**
+ * @return bool
+ */
+ public function getMultimodalDocumentEnabled(): bool {
+ return $this->appConfig->getValueString(Application::APP_ID, 'multimodal_document_enabled', '1') === '1';
}
////////////////////////////////////////////
@@ -1268,12 +1289,21 @@ public function setAdminConfig(array $adminConfig): void {
if (isset($adminConfig['tts_provider_enabled'])) {
$this->setTtsProviderEnabled($adminConfig['tts_provider_enabled']);
}
- if (isset($adminConfig['analyze_image_provider_enabled'])) {
- $this->setAnalyzeImageProviderEnabled($adminConfig['analyze_image_provider_enabled']);
- }
if (isset($adminConfig['default_tts_voice'])) {
$this->setAdminDefaultTtsVoice($adminConfig['default_tts_voice']);
}
+ if (isset($adminConfig['multimodal_image_enabled'])) {
+ $this->setMultimodalImageEnabled($adminConfig['multimodal_image_enabled']);
+ }
+ if (isset($adminConfig['multimodal_audio_enabled'])) {
+ $this->setMultimodalAudioEnabled($adminConfig['multimodal_audio_enabled']);
+ }
+ if (isset($adminConfig['multimodal_video_enabled'])) {
+ $this->setMultimodalVideoEnabled($adminConfig['multimodal_video_enabled']);
+ }
+ if (isset($adminConfig['multimodal_document_enabled'])) {
+ $this->setMultimodalDocumentEnabled($adminConfig['multimodal_document_enabled']);
+ }
if (isset($adminConfig['chat_endpoint_enabled'])) {
$this->setChatEndpointEnabled($adminConfig['chat_endpoint_enabled']);
}
@@ -1444,10 +1474,30 @@ public function setTtsProviderEnabled(bool $enabled): void {
/**
* @param bool $enabled
- * @return void
*/
- public function setAnalyzeImageProviderEnabled(bool $enabled): void {
- $this->appConfig->setValueString(Application::APP_ID, 'analyze_image_provider_enabled', $enabled ? '1' : '0');
+ public function setMultimodalImageEnabled(bool $enabled): void {
+ $this->appConfig->setValueString(Application::APP_ID, 'multimodal_image_enabled', $enabled ? '1' : '0');
+ }
+
+ /**
+ * @param bool $enabled
+ */
+ public function setMultimodalAudioEnabled(bool $enabled): void {
+ $this->appConfig->setValueString(Application::APP_ID, 'multimodal_audio_enabled', $enabled ? '1' : '0');
+ }
+
+ /**
+ * @param bool $enabled
+ */
+ public function setMultimodalVideoEnabled(bool $enabled): void {
+ $this->appConfig->setValueString(Application::APP_ID, 'multimodal_video_enabled', $enabled ? '1' : '0');
+ }
+
+ /**
+ * @param bool $enabled
+ */
+ public function setMultimodalDocumentEnabled(bool $enabled): void {
+ $this->appConfig->setValueString(Application::APP_ID, 'multimodal_document_enabled', $enabled ? '1' : '0');
}
/**
diff --git a/lib/Service/StreamingService.php b/lib/Service/StreamingService.php
index 4bda5392..1d528131 100644
--- a/lib/Service/StreamingService.php
+++ b/lib/Service/StreamingService.php
@@ -176,6 +176,15 @@ private function parseSseEvent(string $event, bool &$done, ?array &$usage, array
$choices[$index]['message']['audio'] = $choice['message']['audio'];
}
+ if (isset($choice['delta']['images']) && is_array($choice['delta']['images'])) {
+ $existingImages = $choices[$index]['message']['images'] ?? [];
+ if (!is_array($existingImages)) {
+ $existingImages = [];
+ }
+ $choices[$index]['message']['images'] = array_merge($existingImages, $choice['delta']['images']);
+ } elseif (isset($choice['message']['images']) && is_array($choice['message']['images'])) {
+ $choices[$index]['message']['images'] = $choice['message']['images'];
+ }
// TODO decide if we stream the tool_calls
if (isset($choice['delta']['tool_calls']) && is_array($choice['delta']['tool_calls'])) {
diff --git a/lib/TaskProcessing/AnalyzeImagesProvider.php b/lib/TaskProcessing/AnalyzeImagesProvider.php
index 65934143..df822c12 100644
--- a/lib/TaskProcessing/AnalyzeImagesProvider.php
+++ b/lib/TaskProcessing/AnalyzeImagesProvider.php
@@ -12,7 +12,6 @@
use OCA\OpenAi\AppInfo\Application;
use OCA\OpenAi\Service\OpenAiAPIService;
use OCA\OpenAi\Service\OpenAiSettingsService;
-use OCP\Files\File;
use OCP\IL10N;
use OCP\TaskProcessing\EShapeType;
use OCP\TaskProcessing\Exception\ProcessingException;
@@ -120,68 +119,7 @@ public function process(
if (!isset($input['images']) || !is_array($input['images'])) {
throw new ProcessingException('Invalid file list');
}
- // Maximum file count for openai is 500. Seems reasonable enough to enforce for all apis though (https://platform.openai.com/docs/guides/images-vision?api-mode=responses&format=url#image-input-requirements)
- if (count($input['images']) > 500) {
- throw new UserFacingProcessingException(
- 'Too many files given. Max is 500',
- 0,
- null,
- $this->l->t('Too many files given. A maximum of 500 files is allowed.'),
- );
- }
- $fileSize = 0;
- foreach ($input['images'] as $image) {
- if (!$image instanceof File || !$image->isReadable()) {
- throw new ProcessingException('Invalid input file');
- }
- $fileSize += intval($image->getSize());
- // Maximum file size for openai is 50MB. Seems reasonable enough to enforce for all apis though. (https://platform.openai.com/docs/guides/images-vision?api-mode=responses&format=url#image-input-requirements)
- if ($fileSize > 50 * 1000 * 1000) {
- throw new UserFacingProcessingException(
- 'Filesize of input files too large. Max is 50MB',
- 0,
- null,
- $this->l->t('The total size of the input files is too large. A maximum of 50MB is allowed.'),
- );
- }
- $inputFile = base64_encode(stream_get_contents($image->fopen('rb')));
- $fileType = $image->getMimeType();
- if (!str_starts_with($fileType, 'image/')) {
- throw new UserFacingProcessingException(
- 'Invalid input file type ' . $fileType,
- 0,
- null,
- $this->l->t('Invalid input file type "%1$s". Only image files are supported.', [$fileType]),
- );
- }
- if ($this->openAiAPIService->isUsingOpenAi()) {
- $validFileTypes = [
- 'image/jpeg',
- 'image/png',
- 'image/gif',
- 'image/webp',
- ];
- if (!in_array($fileType, $validFileTypes)) {
- throw new UserFacingProcessingException(
- 'Invalid input file type for OpenAI ' . $fileType,
- 0,
- null,
- $this->l->t('Invalid input file type "%1$s". Only JPEG, PNG, GIF and WebP images are supported.', [$fileType]),
- );
- }
- }
- $history[] = json_encode([
- 'role' => 'user',
- 'content' => [
- [
- 'type' => 'image_url',
- 'image_url' => [
- 'url' => 'data:' . $fileType . ';base64,' . $inputFile,
- ],
- ],
- ],
- ]);
- }
+ $images = $input['images'];
if (!isset($input['input']) || !is_string($input['input'])) {
throw new ProcessingException('Invalid prompt');
@@ -202,7 +140,7 @@ public function process(
try {
$systemPrompt = 'Take the user\'s question and answer it based on the provided images. Ensure that the answer matches the language of the user\'s question.';
if ($preferStreaming) {
- $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens);
+ $chunks = $this->openAiAPIService->createStreamedChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens, null, null, null, $images);
$time = microtime(true);
$streamedOutput = '';
$streamedReasoning = '';
@@ -240,7 +178,7 @@ public function process(
$completion = $returnValue['messages'];
$reasoning = $returnValue['reasoning_messages'];
} else {
- $returnValue = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens);
+ $returnValue = $this->openAiAPIService->createChatCompletion($userId, $model, $prompt, $systemPrompt, $history, 1, $maxTokens, null, null, null, $images);
$completion = $returnValue['messages'];
$reasoning = $returnValue['reasoning_messages'];
}
diff --git a/lib/TaskProcessing/AudioToAudioChatProvider.php b/lib/TaskProcessing/AudioToAudioChatProvider.php
index dd34027b..c7f36194 100644
--- a/lib/TaskProcessing/AudioToAudioChatProvider.php
+++ b/lib/TaskProcessing/AudioToAudioChatProvider.php
@@ -125,7 +125,7 @@ public function getOptionalInputShapeDefaults(): array {
$isUsingOpenAi = $this->openAiAPIService->isUsingOpenAi();
$adminVoice = $this->appConfig->getValueString(Application::APP_ID, 'default_speech_voice', lazy: true) ?: Application::DEFAULT_SPEECH_VOICE;
$adminLlmModel = $isUsingOpenAi
- ? 'gpt-4o-audio-preview'
+ ? 'gpt-audio'
: $this->openAiSettingsService->getAdminDefaultCompletionModelId();
$defaults = [
'voice' => $adminVoice,
@@ -234,9 +234,6 @@ private function oneStep(
string $sttModel, string $llmModel, string $ttsModel, float $speed, string $serviceName,
): array {
$result = [];
- $audioInputMimetype = mime_content_type($inputFile->fopen('rb'));
- $audioInputFormat = self::SUPPORTED_INPUT_AUDIO_FORMATS[$audioInputMimetype] ?? 'wav';
- $b64Audio = base64_encode($inputFile->getContent());
$extraParams = [
'modalities' => ['text', 'audio'],
'audio' => ['voice' => $outputVoice, 'format' => 'mp3'],
@@ -244,7 +241,7 @@ private function oneStep(
$systemPrompt .= ' Producing text responses will break the user interface. Important: You have multimodal voice capability, and you use voice exclusively to respond.';
$completion = $this->openAiAPIService->createChatCompletion(
$userId, $llmModel, null, $systemPrompt, $history, 1, 1000,
- $extraParams, null, null, $b64Audio, $audioInputFormat
+ $extraParams, null, null, [$inputFile]
);
$message = array_pop($completion['audio_messages']);
// TODO find a way to force the model to answer with audio when there is only text in the history
diff --git a/lib/TaskProcessing/MultimodalChatWithToolsProvider.php b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php
new file mode 100644
index 00000000..3c6385df
--- /dev/null
+++ b/lib/TaskProcessing/MultimodalChatWithToolsProvider.php
@@ -0,0 +1,228 @@
+openAiAPIService->getServiceName();
+ }
+
+ public function getTaskTypeId(): string {
+ return MultimodalChatWithTools::ID;
+ }
+
+ public function getExpectedRuntime(): int {
+ return $this->openAiAPIService->getExpTextProcessingTime();
+ }
+
+ public function getInputShapeEnumValues(): array {
+ return [];
+ }
+
+ public function getInputShapeDefaults(): array {
+ return [];
+ }
+
+ public function getOptionalInputShape(): array {
+ return [
+ 'max_tokens' => new ShapeDescriptor(
+ $this->l->t('Maximum output words'),
+ $this->l->t('The maximum number of words/tokens that can be generated in the completion.'),
+ EShapeType::Number
+ ),
+ ];
+ }
+
+ public function getOptionalInputShapeEnumValues(): array {
+ return [];
+ }
+
+ public function getOptionalInputShapeDefaults(): array {
+ return [];
+ }
+
+ public function getOutputShapeEnumValues(): array {
+ return [];
+ }
+
+ public function getOptionalOutputShape(): array {
+ return [
+ 'reasoning' => new ShapeDescriptor(
+ $this->l->t('Reasoning content'),
+ $this->l->t('The model reasoning behind the output'),
+ EShapeType::Text,
+ ),
+ ];
+ }
+
+ public function getOptionalOutputShapeEnumValues(): array {
+ return [];
+ }
+
+ public function process(
+ ?string $userId, array $input, callable $reportProgress, SynchronousProviderOptions $options = new SynchronousProviderOptions(),
+ ): array {
+ $reportOutput = $options->getReportIntermediateOutput();
+ $preferStreaming = $options->getPreferStreaming();
+ $startTime = time();
+ $adminModel = $this->openAiSettingsService->getAdminDefaultCompletionModelId();
+
+ if (!isset($input['input']) || !is_string($input['input'])) {
+ throw new ProcessingException('Invalid input');
+ }
+ $userPrompt = $input['input'];
+ if ($userPrompt === '') {
+ $userPrompt = null;
+ }
+
+ if (!isset($input['system_prompt']) || !is_string($input['system_prompt'])) {
+ throw new ProcessingException('Invalid system_prompt');
+ }
+ $systemPrompt = $input['system_prompt'];
+
+ if (!isset($input['tool_message']) || !is_string($input['tool_message'])) {
+ throw new ProcessingException('Invalid tool_message');
+ }
+ $toolMessage = $input['tool_message'];
+ if ($toolMessage === '') {
+ $toolMessage = null;
+ }
+
+ if (!isset($input['tools']) || !is_string($input['tools'])) {
+ throw new ProcessingException('Invalid tools');
+ }
+ $tools = json_decode($input['tools']);
+ if (!is_array($tools) || !\array_is_list($tools)) {
+ throw new ProcessingException('Invalid JSON tools');
+ }
+
+ if (!isset($input['history']) || !is_array($input['history']) || !\array_is_list($input['history'])) {
+ throw new ProcessingException('Invalid history');
+ }
+ $history = $input['history'];
+
+ if (!isset($input['input_attachments']) || !is_array($input['input_attachments'])) {
+ throw new ProcessingException('Invalid input_attachments');
+ }
+ $inputAttachments = $input['input_attachments'];
+
+ $maxTokens = null;
+ if (isset($input['max_tokens']) && is_int($input['max_tokens'])) {
+ $maxTokens = $input['max_tokens'];
+ }
+ try {
+ if ($preferStreaming) {
+ $chunks = $this->openAiAPIService->createStreamedChatCompletion(
+ $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments
+ );
+ $time = microtime(true);
+ $streamedOutput = '';
+ $streamedReasoning = '';
+ foreach ($chunks as $chunk) {
+ if (!in_array($chunk['kind'] ?? null, ['content', 'reasoning_content'], true)) {
+ continue;
+ }
+ if ($chunk['kind'] === 'reasoning_content') {
+ $streamedReasoning .= $chunk['text'];
+ } elseif ($chunk['kind'] === 'content') {
+ $streamedOutput .= $chunk['text'];
+ }
+ // we don't report more often than every 250ms
+ if (microtime(true) - $time >= 0.25) {
+ $running = $reportOutput([
+ 'output' => $streamedOutput,
+ 'reasoning' => $streamedReasoning,
+ ]);
+ if (!$running) {
+ throw new ProcessingException('OpenAI/LocalAI task cancelled');
+ }
+ $time = microtime(true);
+ }
+ }
+ if ($streamedOutput !== '' || $streamedReasoning !== '') {
+ $running = $reportOutput([
+ 'output' => $streamedOutput,
+ 'reasoning' => $streamedReasoning,
+ ]);
+ if (!$running) {
+ throw new ProcessingException('OpenAI/LocalAI task cancelled');
+ }
+ }
+ $returnValue = $chunks->getReturn();
+ } else {
+ $returnValue = $this->openAiAPIService->createChatCompletion(
+ $userId, $adminModel, $userPrompt, $systemPrompt, $history, 1, $maxTokens, null, $toolMessage, $tools, $inputAttachments
+ );
+ }
+ } catch (UserFacingProcessingException $e) {
+ throw $e;
+ } catch (\Throwable $e) {
+ throw new ProcessingException('OpenAI/LocalAI request failed: ' . $e->getMessage());
+ }
+ if (count($returnValue['messages']) > 0 || count($returnValue['tool_calls']) > 0 || count($returnValue['images']) > 0 || count($returnValue['audio_messages']) > 0) {
+ $endTime = time();
+ $this->openAiAPIService->updateExpTextProcessingTime($endTime - $startTime);
+ $attachments = [];
+
+ // Handle image output
+ foreach ($returnValue['images'] as $image) {
+ if ($image['type'] === 'image_url') {
+ $url = $image['image_url']['url'];
+ $base64Str = explode(',', $url)[1] ?? '';
+ $image = base64_decode($base64Str);
+ $image = $this->watermarkingService->markImage($image);
+ $attachments[] = $image;
+ } else {
+ $this->logger->warning('Encountered an unknown image type in multimodal chat: ' . $image['type']);
+ }
+ }
+
+ return [
+ 'output' => array_pop($returnValue['messages']) ?? '',
+ 'output_attachments' => $attachments,
+ 'reasoning' => count($returnValue['reasoning_messages']) > 0 ? array_pop($returnValue['reasoning_messages']) : '',
+ 'tool_calls' => array_pop($returnValue['tool_calls']) ?? '',
+ ];
+ }
+
+ throw new ProcessingException('No result in OpenAI/LocalAI response.');
+ }
+}
diff --git a/psalm.xml b/psalm.xml
index 36b2c87e..4e84d9f1 100644
--- a/psalm.xml
+++ b/psalm.xml
@@ -47,6 +47,7 @@
+
diff --git a/src/components/AdminSettings.vue b/src/components/AdminSettings.vue
index 01551409..73f99bba 100644
--- a/src/components/AdminSettings.vue
+++ b/src/components/AdminSettings.vue
@@ -326,6 +326,34 @@
{{ t('integration_openai', 'Use "{newParam}" parameter instead of the deprecated "{deprecatedParam}"', { newParam: 'max_completion_tokens', deprecatedParam: 'max_tokens' }) }}
+
+ {{ t('integration_openai', 'Multimodal LLM Support') }}
+
+
+ {{ t('integration_openai', 'Multimodal LLM Support allows you to enable or disable the use of images, audio, video and document attachments in the LLM.') }}
+
+
+
+ {{ t('integration_openai', 'Image attachments') }}
+
+
+ {{ t('integration_openai', 'Audio attachments') }}
+
+
+ {{ t('integration_openai', 'Video attachments') }}
+
+
+ {{ t('integration_openai', 'Document attachments') }}
+
+
@@ -633,11 +661,6 @@
@update:model-value="onCheckboxChanged($event, 'tts_provider_enabled', false)">
{{ t('integration_openai', 'Text-to-speech provider') }}
-
- {{ t('integration_openai', 'Analyze image provider') }}
-
diff --git a/tests/unit/Providers/OpenAiProviderTest.php b/tests/unit/Providers/OpenAiProviderTest.php
index 6e31e2a6..7be7200f 100644
--- a/tests/unit/Providers/OpenAiProviderTest.php
+++ b/tests/unit/Providers/OpenAiProviderTest.php
@@ -16,6 +16,7 @@
use OCA\OpenAi\Db\QuotaUsageMapper;
use OCA\OpenAi\Service\ChunkService;
use OCA\OpenAi\Service\OpenAiAPIService;
+use OCA\OpenAi\Service\OpenAiFileService;
use OCA\OpenAi\Service\OpenAiSettingsService;
use OCA\OpenAi\Service\QuotaRuleService;
use OCA\OpenAi\Service\StreamingService;
@@ -96,6 +97,10 @@ protected function setUp(): void {
\OCP\Server::get(QuotaUsageMapper::class),
$this->openAiSettingsService,
$this->streamingService,
+ new OpenAiFileService(
+ $this->createMock(\OCP\IL10N::class),
+ $this->openAiSettingsService,
+ ),
$this->createMock(\OCP\Notification\IManager::class),
\OCP\Server::get(QuotaRuleService::class),
$clientService,
diff --git a/tests/unit/Quota/QuotaTest.php b/tests/unit/Quota/QuotaTest.php
index f4c9cb48..86082b48 100644
--- a/tests/unit/Quota/QuotaTest.php
+++ b/tests/unit/Quota/QuotaTest.php
@@ -16,6 +16,7 @@
use OCA\OpenAi\Db\EntityType;
use OCA\OpenAi\Db\QuotaUsageMapper;
use OCA\OpenAi\Service\OpenAiAPIService;
+use OCA\OpenAi\Service\OpenAiFileService;
use OCA\OpenAi\Service\OpenAiSettingsService;
use OCA\OpenAi\Service\QuotaRuleService;
use OCA\OpenAi\Service\StreamingService;
@@ -87,6 +88,10 @@ protected function setUp(): void {
new StreamingService(
$this->createMock(IL10N::class),
),
+ new OpenAiFileService(
+ $this->createMock(IL10N::class),
+ $this->openAiSettingsService,
+ ),
$this->notificationManager,
\OCP\Server::get(QuotaRuleService::class),
\OCP\Server::get(IClientService::class),
diff --git a/tests/unit/Service/ServiceOverrideTest.php b/tests/unit/Service/ServiceOverrideTest.php
index 9fc7fa72..d73205ce 100644
--- a/tests/unit/Service/ServiceOverrideTest.php
+++ b/tests/unit/Service/ServiceOverrideTest.php
@@ -16,6 +16,7 @@
use OCA\OpenAi\Db\QuotaUsageMapper;
use OCA\OpenAi\Service\ChunkService;
use OCA\OpenAi\Service\OpenAiAPIService;
+use OCA\OpenAi\Service\OpenAiFileService;
use OCA\OpenAi\Service\OpenAiSettingsService;
use OCA\OpenAi\Service\QuotaRuleService;
use OCA\OpenAi\Service\StreamingService;
@@ -90,6 +91,10 @@ protected function setUp(): void {
new StreamingService(
$this->createMock(\OCP\IL10N::class),
),
+ new OpenAiFileService(
+ $this->createMock(\OCP\IL10N::class),
+ $this->openAiSettingsService,
+ ),
$this->createMock(\OCP\Notification\IManager::class),
\OCP\Server::get(QuotaRuleService::class),
$clientService,