diff --git a/src/components/Messages/AnnotationContent/index.tsx b/src/components/Messages/AnnotationContent/index.tsx new file mode 100644 index 0000000..2acebed --- /dev/null +++ b/src/components/Messages/AnnotationContent/index.tsx @@ -0,0 +1,37 @@ +import { AnnotationContent as IAnnotationContent, Role } from "@types"; +import type { FC } from "react"; +import { useState } from "react"; +import { TextFile } from "@icons"; +import style from "./style.module.css"; + +export interface AnnotationContentProps { + role: Extract; + content: IAnnotationContent; +} + +export const AnnotationContent: FC = ({ content, role }) => { + const [expanded, setExpanded] = useState(false); + const rawContent = content.content.content; + + return ( +
setExpanded(!expanded)} + > +
+ + {expanded ? "Annotation" : "Show more"} +
+ {expanded && ( +
+ )} +
+ ); +}; diff --git a/src/components/Messages/AnnotationContent/style.module.css b/src/components/Messages/AnnotationContent/style.module.css new file mode 100644 index 0000000..2a20289 --- /dev/null +++ b/src/components/Messages/AnnotationContent/style.module.css @@ -0,0 +1,58 @@ +.annotationContent { + background-color: var(--clover-ai-colors-accentAlt); + border-radius: 0.5rem; + color: var(--clover-ai-colors-secondary); + cursor: pointer; + padding: 0.75rem; + width: fit-content; +} + +.header { + align-items: center; + display: flex; + gap: 0.5rem; +} + +.header svg { + height: 1rem; + width: 1rem; +} + +.label { + font-size: 0.75rem; +} + +.body { + margin-top: 0.5rem; + overflow: hidden; +} + +.body p { + margin: 0; +} + +.annotationContent:not(.expanded) { + padding: 0.5rem; +} + +.annotationContent:not(.expanded) .header { + gap: 0.25rem; +} + +.annotationContent:not(.expanded) .header svg { + height: 0.875rem; + width: 0.875rem; +} + +.annotationContent.expanded .body { + animation: fadeIn 100ms ease-in; +} + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} diff --git a/src/components/Messages/Message/index.tsx b/src/components/Messages/Message/index.tsx index 9c4731f..e6d2206 100644 --- a/src/components/Messages/Message/index.tsx +++ b/src/components/Messages/Message/index.tsx @@ -1,5 +1,6 @@ import type { Message as IMessage } from "@types"; import { forwardRef } from "react"; +import { AnnotationContent } from "../AnnotationContent"; import { MediaContent } from "../MediaContent"; import { TextContent } from "../TextContent"; import style from "./style.module.css"; @@ -19,11 +20,18 @@ export const Message = forwardRef(({ message }, re )} {message.role === "user" && message.content.map((content, index) => { - return content.type === "text" ? ( - - ) : content.type === "media" ? ( - - ) : null; + switch (content.type) { + case "text": + return ( + + ); + case "media": + return ; + case "annotation": + return ; + default: + return null; + } })}
); diff --git a/src/icons/TextFile.tsx b/src/icons/TextFile.tsx new file mode 100644 index 0000000..39604a9 --- /dev/null +++ b/src/icons/TextFile.tsx @@ -0,0 +1,18 @@ +export const TextFile: React.FC> = () => { + return ( + + + + ); +}; diff --git a/src/icons/index.ts b/src/icons/index.ts index 0f77f3e..2d61479 100644 --- a/src/icons/index.ts +++ b/src/icons/index.ts @@ -4,3 +4,4 @@ export { BulletList } from "./BulletList"; export { Clear } from "./Clear"; export { Close } from "./Close"; export { Gear } from "./Gear"; +export { TextFile } from "./TextFile"; diff --git a/src/plugin/Panel/ChatInput/SelectedAnnotation/index.tsx b/src/plugin/Panel/ChatInput/SelectedAnnotation/index.tsx new file mode 100644 index 0000000..efa969b --- /dev/null +++ b/src/plugin/Panel/ChatInput/SelectedAnnotation/index.tsx @@ -0,0 +1,43 @@ +import { Button } from "@components"; +import { usePlugin } from "@context"; +import { Close, TextFile } from "@icons"; +import type { Annotation } from "@types"; +import style from "./style.module.css"; + +export interface SelectedAnnotationProps { + annotation: Annotation; +} + +export const SelectedAnnotation: React.FC = ({ annotation }) => { + const { dispatch, state } = usePlugin(); + + function handleClick(id: Annotation["id"]) { + const newContent = state.selectedContent.filter( + (item) => !(item.type === "annotation" && item.content.id === id), + ); + dispatch({ type: "setSelectedContent", selectedContent: newContent }); + } + + const truncatedContent = + annotation.content.length > 15 + ? annotation.content.substring(0, 15) + "..." + : annotation.content; + + return ( +
+ + + + +
+ ); +}; diff --git a/src/plugin/Panel/ChatInput/SelectedAnnotation/style.module.css b/src/plugin/Panel/ChatInput/SelectedAnnotation/style.module.css new file mode 100644 index 0000000..5cf0c87 --- /dev/null +++ b/src/plugin/Panel/ChatInput/SelectedAnnotation/style.module.css @@ -0,0 +1,21 @@ +.selectedAnnotation { + display: grid; + width: 30px; + height: 30px; + + > * { + grid-area: 1/1; + } + + .annotationContent svg { + color: var(--clover-ai-colors-accentAlt); + } + + > button { + width: fit-content; + height: fit-content; + transform-origin: bottom right; + translate: 40% 40%; + transform: scale(0.6); + } +} diff --git a/src/plugin/Panel/ChatInput/SelectedMedia/index.tsx b/src/plugin/Panel/ChatInput/SelectedMedia/index.tsx index 84ac1e9..af156d2 100644 --- a/src/plugin/Panel/ChatInput/SelectedMedia/index.tsx +++ b/src/plugin/Panel/ChatInput/SelectedMedia/index.tsx @@ -11,8 +11,10 @@ export const SelectedMedia: React.FC = ({ media }) => { const { dispatch, state } = usePlugin(); function handleClick(id: Media["id"]) { - const newMedia = state.selectedMedia.filter((m) => m.id !== id); - dispatch({ type: "setSelectedMedia", selectedMedia: newMedia }); + const newContent = state.selectedContent.filter( + (item) => !(item.type === "media" && item.content.id === id), + ); + dispatch({ type: "setSelectedContent", selectedContent: newContent }); } return (
diff --git a/src/plugin/Panel/ChatInput/index.test.tsx b/src/plugin/Panel/ChatInput/index.test.tsx deleted file mode 100644 index b3131ff..0000000 --- a/src/plugin/Panel/ChatInput/index.test.tsx +++ /dev/null @@ -1,90 +0,0 @@ -import * as context from "@context"; -import "@testing-library/jest-dom/vitest"; -import { cleanup, render, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import React from "react"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { ChatInput } from "./index"; - -vi.mock("@components", async (importOriginal) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const original = (await importOriginal()) as Record; - return { - ...original, - Textarea: ({ - onChange, - children, - error, - }: { - children: React.ReactNode; - onChange: (value: string) => void; - error?: string; - }) => { - const [value, setValue] = React.useState(""); - return ( -
- { - setValue(e.target.value); - onChange(e.target.value); - }} - /> - {error &&
{error}
} - {children} -
- ); - }, - }; -}); - -describe("ChatInput", () => { - afterEach(() => { - vi.clearAllMocks(); - cleanup(); - }); - - it("shows an error on empty submission and clears it on valid input", async () => { - const user = userEvent.setup(); - const dispatch = vi.fn(); - const provider = { - send_messages: vi.fn().mockResolvedValue(undefined), - }; - - vi.spyOn(context, "usePlugin").mockImplementation(() => ({ - dispatch, - state: { - selectedMedia: [], - messages: [], - provider, - } as any, // eslint-disable-line @typescript-eslint/no-explicit-any, - })); - - render(); - - const submitButton = screen.getByLabelText("Submit question"); - const textarea = screen.getByRole("textbox"); - - // 1. Submit with empty input - await user.click(submitButton); - - // 2. Check for error message - let errorMessage = await screen.findByText("Please enter a message."); - expect(errorMessage).toBeInTheDocument(); - - // 3. Type only whitespace - await user.clear(textarea); - await user.type(textarea, " "); - errorMessage = await screen.findByText("Please enter a message."); - expect(errorMessage).toBeInTheDocument(); - - // 4. Type valid input - await user.clear(textarea); - await user.type(textarea, "hello"); - - // 5. Error message should disappear - expect(screen.queryByText("Please enter a message.")).not.toBeInTheDocument(); - }); -}); diff --git a/src/plugin/Panel/ChatInput/index.tsx b/src/plugin/Panel/ChatInput/index.tsx index cf5c790..cca1801 100644 --- a/src/plugin/Panel/ChatInput/index.tsx +++ b/src/plugin/Panel/ChatInput/index.tsx @@ -1,9 +1,10 @@ import { Button, PromptInput } from "@components"; import { usePlugin } from "@context"; import { Add, ArrowUp, Clear } from "@icons"; -import { MediaContent, UserMessage } from "@types"; +import { UserMessage } from "@types"; import type { FC } from "react"; import { useState } from "react"; +import { SelectedAnnotation } from "./SelectedAnnotation"; import { SelectedMedia } from "./SelectedMedia"; export const ChatInput: FC = () => { @@ -43,13 +44,9 @@ export const ChatInput: FC = () => { }, }; - if (state.selectedMedia.length) { - const mediaContent: MediaContent[] = state.selectedMedia.map((media) => ({ - type: "media", - content: media, - })); - userMessage.content.push(...mediaContent); - dispatch({ type: "setSelectedMedia", selectedMedia: [] }); // Clear selected media after sending + if (state.selectedContent.length) { + userMessage.content.push(...state.selectedContent); + dispatch({ type: "setSelectedContent", selectedContent: [] }); } // Add user message immediately @@ -101,12 +98,17 @@ export const ChatInput: FC = () => { } }} > -
- {state.selectedMedia.length ? ( - state.selectedMedia.map((media, index) => ) - ) : ( - <> - )} +
+ {state.selectedContent.map((item, index) => { + switch (item.type) { + case "media": + return ; + case "annotation": + return ; + default: + return <>; + } + })}
{PromptInputButtons && } @@ -121,17 +123,18 @@ export const ChatInput: FC = () => { + +
+
+ {selectedTab === "media" && ( +
+ + + {canvas.placeholderCanvas && ( + + )} + {canvas.thumbnail?.length && } +
+ )} + {selectedTab === "annotations" && ( +
+ +
+ )}
diff --git a/src/plugin/Panel/MediaDialog/style.module.css b/src/plugin/Panel/MediaDialog/style.module.css index 31ec4bf..ffd10c9 100644 --- a/src/plugin/Panel/MediaDialog/style.module.css +++ b/src/plugin/Panel/MediaDialog/style.module.css @@ -10,6 +10,36 @@ font-style: italic; } +.tabsContainer { + display: grid; + grid-template-columns: auto 1fr; +} + +.tabsList { + display: flex; + flex-direction: column; + gap: var(--clover-ai-space-2); + padding-inline-end: var(--clover-ai-space-4); +} + +.tabButton { + width: 100%; + justify-content: flex-start; +} + +.tabDetails { + display: flex; + flex-direction: column; + gap: var(--clover-ai-space-4); + border-left: 1px solid var(--clover-ai-colors-primary); + padding-inline: var(--clover-ai-space-4); + overflow-wrap: anywhere; + + * { + margin: 0; + } +} + .contentContainer { container-name: content-container; container-type: inline-size; @@ -24,3 +54,51 @@ --figure-caption-font-size: var(--clover-ai-sizes-3); } } + +.emptyMessage { + color: var(--clover-ai-colors-text-secondary); + font-style: italic; +} + +.annotationsList { + display: flex; + flex-direction: column; + gap: var(--clover-ai-space-3); +} + +.annotationItem { + display: flex; + flex-direction: column; + gap: var(--clover-ai-space-2); +} + +.annotationLabel { + display: flex; + align-items: flex-start; + gap: var(--clover-ai-space-2); + cursor: pointer; + + input[type="checkbox"] { + margin-top: var(--clover-ai-space-1); + } +} + +.annotationPreview { + display: flex; + flex-direction: column; + gap: var(--clover-ai-space-1); +} + +.annotationContent { + font-size: var(--clover-ai-sizes-3); + line-height: 1.4; + + p { + margin: 0; + } +} + +.annotationRegion { + font-size: var(--clover-ai-sizes-2); + color: var(--clover-ai-colors-text-secondary); +} diff --git a/src/plugin/Panel/index.test.tsx b/src/plugin/Panel/index.test.tsx index 09f2b4c..7f9cd9d 100644 --- a/src/plugin/Panel/index.test.tsx +++ b/src/plugin/Panel/index.test.tsx @@ -168,13 +168,13 @@ describe("Plugin > Panel", () => { const { container } = render(); // 1. Open the media dialog - const addMediaButton = screen.getByRole("button", { name: "Add media" }); + const addMediaButton = screen.getByRole("button", { name: "Add content" }); await user.click(addMediaButton); // 2. Dialog is open const dialog = screen.getByRole("dialog"); expect(dialog).toBeInTheDocument(); - expect(screen.getByRole("heading", { name: "Add media" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Add content" })).toBeInTheDocument(); // 3. Select an image const imageSelect = await screen.findByRole("button", { diff --git a/src/plugin/context/plugin-context.test.tsx b/src/plugin/context/plugin-context.test.tsx index 3199779..1161709 100644 --- a/src/plugin/context/plugin-context.test.tsx +++ b/src/plugin/context/plugin-context.test.tsx @@ -1,6 +1,6 @@ import type { Plugin as CloverIIIF } from "@samvera/clover-iiif"; import { fireEvent, render, screen } from "@testing-library/react"; -import type { Message } from "@types"; +import type { Message, SelectedContent } from "@types"; import { loadMessagesFromStorage } from "@utils"; import { describe, expect, it, type Mock, vi } from "vitest"; import { @@ -29,7 +29,7 @@ const mockInitialState: PluginContextStore = { messages: [], openSeaDragonViewer: undefined, provider: undefined, - selectedMedia: [], + selectedContent: [], systemPrompt: "", // eslint-disable-next-line @typescript-eslint/no-explicit-any vault: {} as any, @@ -160,13 +160,15 @@ describe("pluginReducer", () => { expect(newState.mediaDialogState).toBe(state); }); - it("should handle setSelectedMedia", () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const selectedMedia = [{ type: "image", url: "http://example.com/img.png" }] as any; - const action: PluginContextActions = { type: "setSelectedMedia", selectedMedia }; + it("should handle setSelectedContent", () => { + const selectedContent: SelectedContent[] = [ + { type: "media", content: { type: "image" as const, id: "media-1", src: "http://example.com/img.png" } }, + { type: "annotation", content: { id: "annotation-1", content: "

Test annotation

" } }, + ]; + const action: PluginContextActions = { type: "setSelectedContent", selectedContent }; const newState = pluginReducer(mockInitialState, action); - expect(newState.selectedMedia).toEqual(selectedMedia); + expect(newState.selectedContent).toEqual(selectedContent); }); it("should handle setOpenSeaDragonViewer", () => { @@ -187,12 +189,12 @@ describe("pluginReducer", () => { expect(newState.vault).toEqual(vault); }); - it("should throw an error for an unknown action type", () => { + it("should handle unknown action type by returning state unchanged", () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const action: PluginContextActions = { type: "UNKNOWN_ACTION" } as any; - expect(() => pluginReducer(mockInitialState, action)).toThrow( - "Unknown action type: UNKNOWN_ACTION", - ); + const newState = pluginReducer(mockInitialState, action); + + expect(newState).toEqual(mockInitialState); }); }); diff --git a/src/plugin/context/plugin-context.tsx b/src/plugin/context/plugin-context.tsx index 2985605..66dcd4e 100644 --- a/src/plugin/context/plugin-context.tsx +++ b/src/plugin/context/plugin-context.tsx @@ -1,7 +1,7 @@ import type { Vault } from "@iiif/helpers"; import type { CanvasNormalized, ManifestNormalized } from "@iiif/presentation-3-normalized"; import type { Plugin as CloverIIIF } from "@samvera/clover-iiif"; -import type { ConversationState, Media, Message } from "@types"; +import type { ConversationState, Message, SelectedContent } from "@types"; import { loadMessagesFromStorage, setMessagesToStorage } from "@utils"; import type { Viewer } from "openseadragon"; import type { Dispatch } from "react"; @@ -16,7 +16,7 @@ export interface PluginContextStore { messages: Message[]; openSeaDragonViewer: Viewer | undefined; provider: BaseProvider | undefined; - selectedMedia: Media[]; + selectedContent: SelectedContent[]; systemPrompt: string; vault: Vault; } @@ -56,9 +56,9 @@ interface SetOSDViewerAction { type: "setOpenSeaDragonViewer"; } -interface SetSelectedMediaAction { - selectedMedia: Media[]; - type: "setSelectedMedia"; +interface SetSelectedContentAction { + selectedContent: SelectedContent[]; + type: "setSelectedContent"; } interface SetVaultAction { @@ -89,7 +89,7 @@ export type PluginContextActions = | SetManifestAction | SetMediaDialogStateAction | SetOSDViewerAction - | SetSelectedMediaAction + | SetSelectedContentAction | SetSystemPromptAction | SetVaultAction | UpdateProviderAction @@ -104,7 +104,7 @@ const defaultPluginContextStore: InitPluginContextStore = { messages: [], openSeaDragonViewer: undefined, provider: undefined, - selectedMedia: [], + selectedContent: [], systemPrompt: "", }; @@ -159,15 +159,14 @@ export function pluginReducer( return { ...state, conversationState: action.conversationState }; case "setMediaDialogState": return { ...state, mediaDialogState: action.state }; - case "setSelectedMedia": - return { ...state, selectedMedia: action.selectedMedia }; + case "setSelectedContent": + return { ...state, selectedContent: action.selectedContent }; case "setOpenSeaDragonViewer": return { ...state, openSeaDragonViewer: action.openSeaDragonViewer }; case "setVault": return { ...state, vault: action.vault }; default: - //@ts-expect-error - this is a catch-all for unknown action types - throw new Error(`Unknown action type: ${action.type}`); + return state; } } diff --git a/src/providers/mediaPipeProvider/index.tsx b/src/providers/mediaPipeProvider/index.tsx index d34f4cb..f28c4d9 100644 --- a/src/providers/mediaPipeProvider/index.tsx +++ b/src/providers/mediaPipeProvider/index.tsx @@ -79,10 +79,21 @@ export class MediaPipeProvider extends BaseProvider { sequence.push("user\n"); for (const content of message.content) { - if (content.type === "text") { - sequence.push(content.content); - } else if (content.type === "media" && content.content.src) { - sequence.push({ imageSource: content.content.src }); + switch (content.type) { + case "text": + sequence.push(content.content); + break; + case "annotation": + sequence.push(` + ## Annotation Content + The user has shared the following annotation content: + ${content.content.content}`); + break; + case "media": + if (content.content.src) { + sequence.push({ imageSource: content.content.src }); + } + break; } } diff --git a/src/providers/userTokenProvider/index.tsx b/src/providers/userTokenProvider/index.tsx index 16d5463..6e791cc 100644 --- a/src/providers/userTokenProvider/index.tsx +++ b/src/providers/userTokenProvider/index.tsx @@ -105,6 +105,16 @@ export class UserTokenProvider extends BaseProvider { return { type: "image", image: c.content.src }; } + if (c.type === "annotation") { + return { + type: "text", + text: dedent` + ## Annotation Content + The user has shared the following annotation content: + ${c.content.content}`, + }; + } + const prevMessages = messages.slice(0, index); const lastUserMessage = prevMessages.findLast((m) => m.role === "user"); let context = ""; @@ -122,25 +132,6 @@ export class UserTokenProvider extends BaseProvider { serializeConfigPresentation3, ); - const annotationTexts: string[] = []; - const traverse = new Traverse({ - annotation: [ - (a) => { - if ( - a.body && - typeof a.body === "object" && - "type" in a.body && - a.body.type === "TextualBody" && - a.body.value - ) { - annotationTexts.push(a.body.value); - } - }, - ], - }); - - traverse.traverseCanvas(canvas); - // prettier-ignore context = dedent.withOptions({ alignValues: true })` ## Context @@ -148,8 +139,7 @@ export class UserTokenProvider extends BaseProvider { Use this information if possible to inform your answer. ### Canvas${canvas.label ? ` - - Label: ${getLabelByUserLanguage(canvas.label)[0]}` : ""}${annotationTexts.length ? ` - - Annotations: ${annotationTexts.join(", ")}` : ""} + - Label: ${getLabelByUserLanguage(canvas.label)[0]}` : ""} `; } diff --git a/src/types.d.ts b/src/types.d.ts index 24cc98f..26a8d7f 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -10,6 +10,18 @@ export type Media = { caption?: string; }; +export type Annotation = { + content: string; + id: string; + region?: { + height: number; + width: number; + x: number; + y: number; + }; + target?: string; +}; + export interface TextContent { content: string; type: "text"; @@ -20,6 +32,13 @@ export interface MediaContent { type: "media"; } +export interface AnnotationContent { + content: Annotation; + type: "annotation"; +} + +export type SelectedContent = AnnotationContent | MediaContent; + export interface ToolContent extends TextContent { tool_name: string; } @@ -39,7 +58,7 @@ export type AssistantMessage = { } & (Response | ToolCall); export interface UserMessage { - content: (TextContent | MediaContent)[]; + content: (TextContent | MediaContent | AnnotationContent)[]; /** Context that can be added to user messages when generating a response */ context: { canvas: CanvasNormalized;