Feat: Add MultimodalChatWithTools and MultimodalContextAgentInteraction

lib/private/TaskProcessing/Manager.php

@@ -76,6 +76,7 @@
 use OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles;
 use OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction;
 use OCP\TaskProcessing\TaskTypes\ContextAgentInteraction;
+use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction;
 use OCP\TaskProcessing\TaskTypes\ContextWrite;
 use OCP\TaskProcessing\TaskTypes\GenerateEmoji;
 use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition;
@@ -84,6 +85,7 @@
 use OCP\TaskProcessing\TaskTypes\TextToText;
 use OCP\TaskProcessing\TaskTypes\TextToTextChangeTone;
 use OCP\TaskProcessing\TaskTypes\TextToTextChat;
+use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools;
 use OCP\TaskProcessing\TaskTypes\TextToTextChatWithTools;
 use OCP\TaskProcessing\TaskTypes\TextToTextFormalization;
 use OCP\TaskProcessing\TaskTypes\TextToTextHeadline;
@@ -696,13 +698,15 @@ private function _getTaskTypes(): array {
 			GenerateEmoji::ID => Server::get(GenerateEmoji::class),
 			TextToTextChangeTone::ID => Server::get(TextToTextChangeTone::class),
 			TextToTextChatWithTools::ID => Server::get(TextToTextChatWithTools::class),
+			MultimodalChatWithTools::ID => Server::get(MultimodalChatWithTools::class),
 			ContextAgentInteraction::ID => Server::get(ContextAgentInteraction::class),
 			TextToTextProofread::ID => Server::get(TextToTextProofread::class),
 			TextToTextReformatParagraphs::ID => Server::get(TextToTextReformatParagraphs::class),
 			TextToSpeech::ID => Server::get(TextToSpeech::class),
 			AudioToAudioChat::ID => Server::get(AudioToAudioChat::class),
 			AudioToAudioTranslate::ID => Server::get(AudioToAudioTranslate::class),
 			ContextAgentAudioInteraction::ID => Server::get(ContextAgentAudioInteraction::class),
+			MultimodalContextAgentInteraction::ID => Server::get(MultimodalContextAgentInteraction::class),
 			AnalyzeImages::ID => Server::get(AnalyzeImages::class),
 			ImageToTextOpticalCharacterRecognition::ID => Server::get(ImageToTextOpticalCharacterRecognition::class),
 		];

lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php

@@ -0,0 +1,136 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCP\TaskProcessing\TaskTypes;
+
+use OCP\IL10N;
+use OCP\L10N\IFactory;
+use OCP\TaskProcessing\EShapeType;
+use OCP\TaskProcessing\IInternalTaskType;
+use OCP\TaskProcessing\ITaskType;
+use OCP\TaskProcessing\ShapeDescriptor;
+
+/**
+ * This is the task processing task type for invoking multimodal Chat-enabled LLMs with tool call support
+ * @since 35.0.0
+ */
+class MultimodalChatWithTools implements ITaskType {
+	/**
+	 * @since 35.0.0
+	 */
+	public const ID = 'core:text2text:multimodal-chatwithtools';
+
+	private IL10N $l;
+
+	/**
+	 * @param IFactory $l10nFactory
+	 * @since 35.0.0
+	 */
+	public function __construct(
+		IFactory $l10nFactory,
+	) {
+		$this->l = $l10nFactory->get('lib');
+	}
+
+	/**
+	 * @inheritDoc
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getName(): string {
+		// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
+		return $this->l->t('Chat with tools using attachments');
+	}
+
+	/**
+	 * @inheritDoc
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getDescription(): string {
+		// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
+		return $this->l->t('Chat with the language model with tool calling support and attachments.');
+	}
+
+	/**
+	 * @return string
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getId(): string {
+		return self::ID;
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getInputShape(): array {
+		return [
+			'system_prompt' => new ShapeDescriptor(
+				$this->l->t('System prompt'),
+				$this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'),
+				EShapeType::Text
+			),
+			'input' => new ShapeDescriptor(
+				$this->l->t('Chat message'),
+				$this->l->t('Describe a task that you want the assistant to do or ask a question'),
+				EShapeType::Text
+			),
+			'input_attachments' => new ShapeDescriptor(
+				$this->l->t('Input attachments'),
+				$this->l->t('Files to send along with the chat message.'),
+				EShapeType::ListOfFiles
+			),
+			'tool_message' => new ShapeDescriptor(
+				$this->l->t('Tool message'),
+				$this->l->t('The result of tool calls in the last interaction'),
+				EShapeType::Text
+			),
+			'history' => new ShapeDescriptor(
+				$this->l->t('Chat history'),
+				$this->l->t('The history of chat messages before the current message, starting with a message by the user'),
+				EShapeType::ListOfTexts
+			),
+			// See https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools for the format
+			'tools' => new ShapeDescriptor(
+				// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
+				$this->l->t('Available tools'),
+				$this->l->t('The available tools in JSON format'),
+				EShapeType::Text
+			),
+		];
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getOutputShape(): array {
+		return [
+			'output' => new ShapeDescriptor(
+				$this->l->t('Generated response'),
+				$this->l->t('The response from the chat model'),
+				EShapeType::Text
+			),
+			'output_attachments' => new ShapeDescriptor(
+				$this->l->t('Output attachments'),
+				$this->l->t('Files generated by the model.'),
+				EShapeType::ListOfFiles
+			),
+			'tool_calls' => new ShapeDescriptor(
+				$this->l->t('Tool calls'),
+				$this->l->t('Tools call instructions from the model in JSON format'),
+				EShapeType::Text
+			),
+		];
+	}
+}

lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php

@@ -0,0 +1,126 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCP\TaskProcessing\TaskTypes;
+
+use OCP\IL10N;
+use OCP\L10N\IFactory;
+use OCP\TaskProcessing\EShapeType;
+use OCP\TaskProcessing\IInternalTaskType;
+use OCP\TaskProcessing\ShapeDescriptor;
+
+/**
+ * This is the task processing task type for multimodal Context Agent interaction
+ * @since 35.0.0
+ */
+class MultimodalContextAgentInteraction implements IInternalTaskType {
+	/**
+	 * @since 35.0.0
+	 */
+	public const ID = 'core:contextagent:multimodal-interaction';
+
+	private IL10N $l;
+
+	/**
+	 * @param IFactory $l10nFactory
+	 * @since 35.0.0
+	 */
+	public function __construct(
+		IFactory $l10nFactory,
+	) {
+		$this->l = $l10nFactory->get('lib');
+	}
+
+	/**
+	 * @inheritDoc
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getName(): string {
+		return 'ContextAgent multimodal'; // We do not translate this
+	}
+
+	/**
+	 * @inheritDoc
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getDescription(): string {
+		return $this->l->t('Chat with an agent using attachments');
+	}
+
+	/**
+	 * @return string
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getId(): string {
+		return self::ID;
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getInputShape(): array {
+		return [
+			'input' => new ShapeDescriptor(
+				$this->l->t('Chat message'),
+				$this->l->t('A chat message to send to the agent.'),
+				EShapeType::Text
+			),
+			'input_attachments' => new ShapeDescriptor(
+				$this->l->t('Input attachments'),
+				$this->l->t('Files to send along with the chat message.'),
+				EShapeType::ListOfFiles
+			),
+			'confirmation' => new ShapeDescriptor(
+				$this->l->t('Confirmation'),
+				$this->l->t('Whether to confirm previously requested actions: 0 for denial and 1 for confirmation.'),
+				EShapeType::Number
+			),
+			'conversation_token' => new ShapeDescriptor(
+				$this->l->t('Conversation token'),
+				$this->l->t('A token representing the conversation.'),
+				EShapeType::Text
+			),
+		];
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 35.0.0
+	 */
+	#[\Override]
+	public function getOutputShape(): array {
+		return [
+			'output' => new ShapeDescriptor(
+				$this->l->t('Generated response'),
+				$this->l->t('The response from the chat model.'),
+				EShapeType::Text
+			),
+			'output_attachments' => new ShapeDescriptor(
+				$this->l->t('Output attachments'),
+				$this->l->t('Files generated by the agent.'),
+				EShapeType::ListOfFiles
+			),
+			'conversation_token' => new ShapeDescriptor(
+				$this->l->t('The new conversation token'),
+				$this->l->t('Send this along with the next interaction.'),
+				EShapeType::Text
+			),
+			'actions' => new ShapeDescriptor(
+				$this->l->t('Requested actions by the agent'),
+				$this->l->t('Actions that the agent would like to carry out in JSON format.'),
+				EShapeType::Text
+			),
+		];
+	}
+}