Add MultimodalChatWithToolsProvider

appinfo/info.xml

@@ -101,7 +101,7 @@ Negative:
 
 Learn more about the Nextcloud Ethical AI Rating [in our blog](https://nextcloud.com/blog/nextcloud-ethical-ai-rating/).
 ]]>	</description>
-	<version>4.5.1</version>
+	<version>4.6.0-dev</version>
 	<licence>agpl</licence>
 	<author>Julien Veyssier</author>
 	<namespace>OpenAi</namespace>

lib/AppInfo/Application.php

@@ -143,11 +143,12 @@ public function register(IRegistrationContext $context): void {
 			$context->registerTaskProcessingProvider(EmojiProvider::class);
 			$context->registerTaskProcessingProvider(ChangeToneProvider::class);
 			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\TextToTextChatWithToolsProvider::class);
+			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\MultimodalChatWithToolsProvider::class);
 			$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ProofreadProvider::class);
 			if (class_exists('OCP\\TaskProcessing\\TaskTypes\\TextToTextReformatParagraphs')) {
 				$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\ReformatParagraphsProvider::class);
 			}
-			if ($isUsingOpenAI || $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled') === '1') {
+			if ($this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', '1', lazy: true) === '1') {
 				$context->registerTaskProcessingProvider(\OCA\OpenAi\TaskProcessing\AnalyzeImagesProvider::class);
 			}
 		}

lib/Migration/Version040600Date20260708151000.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\OpenAi\Migration;
+
+use Closure;
+use OCA\OpenAi\AppInfo\Application;
+use OCP\IAppConfig;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version040600Date20260708151000 extends SimpleMigrationStep {
+
+	public function __construct(
+		private IAppConfig $appConfig,
+	) {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure
+	 * @param array $options
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void {
+		$unset = '__unset__';
+		$newValue = $this->appConfig->getValueString(Application::APP_ID, 'multimodal_image_enabled', $unset);
+		if ($newValue !== $unset) {
+			return;
+		}
+
+		$oldValue = $this->appConfig->getValueString(Application::APP_ID, 'analyze_image_provider_enabled', $unset);
+		if (!in_array($oldValue, ['0', '1'], true)) {
+			return;
+		}
+
+		$this->appConfig->setValueString(Application::APP_ID, 'multimodal_image_enabled', $oldValue);
+	}
+}

lib/Service/OpenAiAPIService.php

@@ -51,6 +51,7 @@ public function __construct(
 		private QuotaUsageMapper $quotaUsageMapper,
 		private OpenAiSettingsService $openAiSettingsService,
 		private StreamingService $streamingService,
+		private OpenAiFileService $openAiFileService,
 		private INotificationManager $notificationManager,
 		private QuotaRuleService $quotaRuleService,
 		IClientService $clientService,
@@ -531,8 +532,7 @@ public function createStreamedChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 	): \Generator {
 		if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) {
 			throw new Exception($this->l10n->t('Text generation quota exceeded'), Http::STATUS_TOO_MANY_REQUESTS);
@@ -549,8 +549,7 @@ public function createStreamedChatCompletion(
 			$extraParams,
 			$toolMessage,
 			$tools,
-			$userAudioPromptBase64,
-			$userAudioPromptFormat,
+			$files,
 			true,
 		);
 
@@ -591,13 +590,11 @@ public function createChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 	): array {
 		$response = $this->requestChatCompletion(
 			$userId, $model, $userPrompt, $systemPrompt, $history,
-			$n, $maxTokens, $extraParams, $toolMessage, $tools,
-			$userAudioPromptBase64, $userAudioPromptFormat,
+			$n, $maxTokens, $extraParams, $toolMessage, $tools, $files,
 			false,
 		);
 
@@ -625,8 +622,7 @@ public function createChatCompletion(
 	 * @param array|null $extraParams
 	 * @param string|null $toolMessage JSON string with role, content, tool_call_id
 	 * @param array|null $tools
-	 * @param string|null $userAudioPromptBase64
-	 * @param string|null $userAudioPromptFormat
+	 * @param array|null $files Array of File objects
 	 * @return array{messages?: array<string>, tool_calls?: array<string>, audio_messages?: list<array<string, mixed>>, usage?: array<string, mixed>}
 	 * @throws Exception
 	 */
@@ -641,8 +637,7 @@ public function requestChatCompletion(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 		bool $stream = false,
 	): array {
 		if ($this->isQuotaExceeded($userId, Application::QUOTA_TYPE_TEXT)) {
@@ -660,8 +655,7 @@ public function requestChatCompletion(
 			$extraParams,
 			$toolMessage,
 			$tools,
-			$userAudioPromptBase64,
-			$userAudioPromptFormat,
+			$files,
 			$stream,
 		);
 
@@ -679,8 +673,7 @@ public function requestChatCompletion(
 	 * @param array|null $extraParams
 	 * @param string|null $toolMessage
 	 * @param array|null $tools
-	 * @param string|null $userAudioPromptBase64
-	 * @param string|null $userAudioPromptFormat
+	 * @param array|null $files Array of File objects
 	 * @param bool $stream
 	 * @return array<string, mixed>
 	 */
@@ -695,8 +688,7 @@ private function buildChatCompletionRequestParams(
 		?array $extraParams = null,
 		?string $toolMessage = null,
 		?array $tools = null,
-		?string $userAudioPromptBase64 = null,
-		?string $userAudioPromptFormat = null,
+		?array $files = null,
 		bool $stream = false,
 	): array {
 		$modelRequestParam = $model === Application::DEFAULT_MODEL_ID
@@ -740,29 +732,20 @@ private function buildChatCompletionRequestParams(
 				$messages[] = $message;
 			}
 		}
-		if ($userAudioPromptBase64 !== null) {
-			// if there is audio, use the new message format (content is a list of objects)
-			$message = [
-				'role' => 'user',
-				'content' => [
-					[
-						'type' => 'input_audio',
-						'input_audio' => [
-							'data' => $userAudioPromptBase64,
-							'format' => $userAudioPromptFormat ?? 'wav',
-						],
-					],
-				],
-			];
+		// Attach all files when necessary
+		if ($files !== null) {
+			$content = $this->openAiFileService->buildFileContents($files);
 			if ($userPrompt !== null) {
-				$message['content'][] = [
+				$content[] = [
 					'type' => 'text',
 					'text' => $userPrompt,
 				];
 			}
-			$messages[] = $message;
+			$messages[] = [
+				'role' => 'user',
+				'content' => $content,
+			];
 		} elseif ($userPrompt !== null) {
-			// if there is only text, use the old message format (content is a string)
 			$messages[] = [
 				'role' => 'user',
 				'content' => $userPrompt,
@@ -1442,8 +1425,10 @@ private function normalizeChatCompletionResponse(array $response): array {
 			'reasoning_messages' => [],
 			'tool_calls' => [],
 			'audio_messages' => [],
+			'images' => [],
 		];
 
+
 		foreach ($response['choices'] as $choice) {
 			if (!is_array($choice)) {
 				continue;
@@ -1483,6 +1468,11 @@ private function normalizeChatCompletionResponse(array $response): array {
 			if (isset($choice['message']['audio'], $choice['message']['audio']['data']) && is_string($choice['message']['audio']['data'])) {
 				$completions['audio_messages'][] = $choice['message'];
 			}
+			if (isset($choice['message']['images']) && is_array($choice['message']['images'])) {
+				foreach ($choice['message']['images'] as $image) {
+					$completions['images'][] = $image;
+				}
+			}
 		}
 
 		return $completions;
@@ -1566,7 +1556,6 @@ public function autoDetectFeatures(): array {
 		$config['stt_provider_enabled'] = $this->isSTTAvailable();
 		$config['tts_provider_enabled'] = $this->isTTSAvailable();
 		$this->openAiSettingsService->setAdminConfig($config);
-		$config['analyze_image_provider_enabled'] = $this->openAiSettingsService->getAnalyzeImageProviderEnabled();
 		return $config;
 	}
 }