From fc6b6ed104de25de5b14e9ad540320a3d33516f9 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 8 Jul 2026 16:41:07 -0400 Subject: [PATCH] Feat: Add MultimodalChatWithTools and MultimodalContextAgentInteraction Signed-off-by: Lukas Schaefer --- lib/private/TaskProcessing/Manager.php | 4 + .../TaskTypes/MultimodalChatWithTools.php | 136 ++++++++++++++++++ .../MultimodalContextAgentInteraction.php | 126 ++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php create mode 100644 lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 608856fd27017..15920e2f5050d 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -76,6 +76,7 @@ use OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles; use OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction; use OCP\TaskProcessing\TaskTypes\ContextAgentInteraction; +use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction; use OCP\TaskProcessing\TaskTypes\ContextWrite; use OCP\TaskProcessing\TaskTypes\GenerateEmoji; use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition; @@ -84,6 +85,7 @@ use OCP\TaskProcessing\TaskTypes\TextToText; use OCP\TaskProcessing\TaskTypes\TextToTextChangeTone; use OCP\TaskProcessing\TaskTypes\TextToTextChat; +use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools; use OCP\TaskProcessing\TaskTypes\TextToTextChatWithTools; use OCP\TaskProcessing\TaskTypes\TextToTextFormalization; use OCP\TaskProcessing\TaskTypes\TextToTextHeadline; @@ -696,6 +698,7 @@ private function _getTaskTypes(): array { GenerateEmoji::ID => Server::get(GenerateEmoji::class), TextToTextChangeTone::ID => Server::get(TextToTextChangeTone::class), TextToTextChatWithTools::ID => Server::get(TextToTextChatWithTools::class), + MultimodalChatWithTools::ID => Server::get(MultimodalChatWithTools::class), ContextAgentInteraction::ID => Server::get(ContextAgentInteraction::class), TextToTextProofread::ID => Server::get(TextToTextProofread::class), TextToTextReformatParagraphs::ID => Server::get(TextToTextReformatParagraphs::class), @@ -703,6 +706,7 @@ private function _getTaskTypes(): array { AudioToAudioChat::ID => Server::get(AudioToAudioChat::class), AudioToAudioTranslate::ID => Server::get(AudioToAudioTranslate::class), ContextAgentAudioInteraction::ID => Server::get(ContextAgentAudioInteraction::class), + MultimodalContextAgentInteraction::ID => Server::get(MultimodalContextAgentInteraction::class), AnalyzeImages::ID => Server::get(AnalyzeImages::class), ImageToTextOpticalCharacterRecognition::ID => Server::get(ImageToTextOpticalCharacterRecognition::class), ]; diff --git a/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php b/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php new file mode 100644 index 0000000000000..2ec2f367125d9 --- /dev/null +++ b/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php @@ -0,0 +1,136 @@ +l = $l10nFactory->get('lib'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getName(): string { + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + return $this->l->t('Chat with tools using attachments'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getDescription(): string { + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + return $this->l->t('Chat with the language model with tool calling support and attachments.'); + } + + /** + * @return string + * @since 35.0.0 + */ + #[\Override] + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getInputShape(): array { + return [ + 'system_prompt' => new ShapeDescriptor( + $this->l->t('System prompt'), + $this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'), + EShapeType::Text + ), + 'input' => new ShapeDescriptor( + $this->l->t('Chat message'), + $this->l->t('Describe a task that you want the assistant to do or ask a question'), + EShapeType::Text + ), + 'input_attachments' => new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to send along with the chat message.'), + EShapeType::ListOfFiles + ), + 'tool_message' => new ShapeDescriptor( + $this->l->t('Tool message'), + $this->l->t('The result of tool calls in the last interaction'), + EShapeType::Text + ), + 'history' => new ShapeDescriptor( + $this->l->t('Chat history'), + $this->l->t('The history of chat messages before the current message, starting with a message by the user'), + EShapeType::ListOfTexts + ), + // See https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools for the format + 'tools' => new ShapeDescriptor( + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + $this->l->t('Available tools'), + $this->l->t('The available tools in JSON format'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Generated response'), + $this->l->t('The response from the chat model'), + EShapeType::Text + ), + 'output_attachments' => new ShapeDescriptor( + $this->l->t('Output attachments'), + $this->l->t('Files generated by the model.'), + EShapeType::ListOfFiles + ), + 'tool_calls' => new ShapeDescriptor( + $this->l->t('Tool calls'), + $this->l->t('Tools call instructions from the model in JSON format'), + EShapeType::Text + ), + ]; + } +} diff --git a/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php new file mode 100644 index 0000000000000..dfc71dc0367a8 --- /dev/null +++ b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php @@ -0,0 +1,126 @@ +l = $l10nFactory->get('lib'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getName(): string { + return 'ContextAgent multimodal'; // We do not translate this + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getDescription(): string { + return $this->l->t('Chat with an agent using attachments'); + } + + /** + * @return string + * @since 35.0.0 + */ + #[\Override] + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Chat message'), + $this->l->t('A chat message to send to the agent.'), + EShapeType::Text + ), + 'input_attachments' => new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to send along with the chat message.'), + EShapeType::ListOfFiles + ), + 'confirmation' => new ShapeDescriptor( + $this->l->t('Confirmation'), + $this->l->t('Whether to confirm previously requested actions: 0 for denial and 1 for confirmation.'), + EShapeType::Number + ), + 'conversation_token' => new ShapeDescriptor( + $this->l->t('Conversation token'), + $this->l->t('A token representing the conversation.'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Generated response'), + $this->l->t('The response from the chat model.'), + EShapeType::Text + ), + 'output_attachments' => new ShapeDescriptor( + $this->l->t('Output attachments'), + $this->l->t('Files generated by the agent.'), + EShapeType::ListOfFiles + ), + 'conversation_token' => new ShapeDescriptor( + $this->l->t('The new conversation token'), + $this->l->t('Send this along with the next interaction.'), + EShapeType::Text + ), + 'actions' => new ShapeDescriptor( + $this->l->t('Requested actions by the agent'), + $this->l->t('Actions that the agent would like to carry out in JSON format.'), + EShapeType::Text + ), + ]; + } +}