diff --git a/integrations/ai-sdk/CHANGELOG.md b/integrations/ai-sdk/CHANGELOG.md new file mode 100644 index 00000000..a75130bc --- /dev/null +++ b/integrations/ai-sdk/CHANGELOG.md @@ -0,0 +1 @@ +CHANGELOG \ No newline at end of file diff --git a/integrations/ai-sdk/package.json b/integrations/ai-sdk/package.json new file mode 100644 index 00000000..38fd847e --- /dev/null +++ b/integrations/ai-sdk/package.json @@ -0,0 +1,26 @@ +{ + "name": "ai-provider", + "version": "0.0.1", + "description": "Humanloop provider for the AI SDK by Vercel", + "main": "index.js", + "scripts": { + "test": "jest" + }, + "repository": { + "type": "git", + "url": "https://github.com/humanloop/humanloop-node/integrations/ai-sdk" + }, + "keywords": [ + "ai", + "ai-sdk", + "vercel-ai-sdk" + ], + "author": "", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "^1.0.11", + "@ai-sdk/provider-utils": "^2.1.13", + "tsup": "^8.4.0", + "zod": "^3.24.2" + } +} diff --git a/integrations/ai-sdk/src/convert-to-humanloop-chat-messages.ts b/integrations/ai-sdk/src/convert-to-humanloop-chat-messages.ts new file mode 100644 index 00000000..edd4ff24 --- /dev/null +++ b/integrations/ai-sdk/src/convert-to-humanloop-chat-messages.ts @@ -0,0 +1,127 @@ +import { + LanguageModelV1Prompt, + UnsupportedFunctionalityError, +} from '@ai-sdk/provider'; +import { convertUint8ArrayToBase64 } from '@ai-sdk/provider-utils'; +import { HumanloopChatPrompt } from './humanloop-api-types'; + + export function convertToHumanloopChatMessages( + prompt: LanguageModelV1Prompt, + ): HumanloopChatPrompt { + const messages: HumanloopChatPrompt = []; + + for (const { role, content } of prompt) { + // Skip empty messages, i.e. when prompt == "" + if (content === "") { + continue; + } + + switch (role) { + case 'system': { + messages.push({ role: 'system', content }); + break; + } + + case 'user': { + if (content.length === 1 && content[0].type === 'text') { + messages.push({ role: 'user', content: content[0].text }); + break; + } + + messages.push({ + role: 'user', + content: content.map(part => { + switch (part.type) { + case 'text': { + return { type: 'text', text: part.text }; + } + case 'image': { + return { + type: 'image_url', + imageUrl: { + url: + part.image instanceof URL + ? part.image.toString() + : `data:${ + part.mimeType ?? 'image/jpeg' + };base64,${convertUint8ArrayToBase64(part.image)}`, + }, + }; + } + case 'file': { + throw new UnsupportedFunctionalityError({ + functionality: 'File content parts in user messages', + }); + } + } + }), + }); + + break; + } + + case 'assistant': { + let text = ''; + const toolCalls: Array<{ + id: string; + type: 'function'; + function: { name: string; arguments: string }; + }> = []; + + for (const part of content) { + switch (part.type) { + case 'text': { + text += part.text; + break; + } + case 'redacted-reasoning': + case 'reasoning': { + break; // ignored + } + case 'tool-call': { + toolCalls.push({ + id: part.toolCallId, + type: 'function', + function: { + name: part.toolName, + arguments: JSON.stringify(part.args), + }, + }); + break; + } + default: { + const _exhaustiveCheck: never = part; + throw new Error(`Unsupported part: ${_exhaustiveCheck}`); + } + } + } + + messages.push({ + role: 'assistant', + content: text, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + }); + + break; + } + + case 'tool': { + for (const toolResponse of content) { + messages.push({ + role: 'tool', + toolCallId: toolResponse.toolCallId, + content: JSON.stringify(toolResponse.result), + }); + } + break; + } + + default: { + const _exhaustiveCheck: never = role; + throw new Error(`Unsupported role: ${_exhaustiveCheck}`); + } + } + } + + return messages; + } \ No newline at end of file diff --git a/integrations/ai-sdk/src/get-response-metadata.ts b/integrations/ai-sdk/src/get-response-metadata.ts new file mode 100644 index 00000000..90227dbc --- /dev/null +++ b/integrations/ai-sdk/src/get-response-metadata.ts @@ -0,0 +1,15 @@ +export function getResponseMetadata({ + id, + model, + createdAt, +}: { + id?: string | undefined | null; + createdAt?: Date | undefined | null; + model?: string | undefined | null; +}) { + return { + id: id ?? undefined, + modelId: model ?? undefined, + timestamp: createdAt ?? undefined, + }; +} \ No newline at end of file diff --git a/integrations/ai-sdk/src/humanloop-api-types.ts b/integrations/ai-sdk/src/humanloop-api-types.ts new file mode 100644 index 00000000..b79ed60a --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-api-types.ts @@ -0,0 +1,16 @@ +import { + ChatMessage, + PromptsCallRequest, + PromptsCallStreamRequest, + ProviderApiKeys +} from "../../../src/api"; + +export type HumanloopProviderApiKeys = ProviderApiKeys +export type HumanloopChatPrompt = Array; +export type HumanloopGenerateArgs = PromptsCallRequest | PromptsCallStreamRequest; +export type HumanloopTools = + | PromptsCallRequest["prompt"]["tools"] + | PromptsCallStreamRequest["prompt"]["tools"]; +export type HumanloopToolChoice = + | PromptsCallRequest["toolChoice"] + | PromptsCallStreamRequest["toolChoice"]; \ No newline at end of file diff --git a/integrations/ai-sdk/src/humanloop-chat-language-model.ts b/integrations/ai-sdk/src/humanloop-chat-language-model.ts new file mode 100644 index 00000000..078164d2 --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-chat-language-model.ts @@ -0,0 +1,584 @@ +import { + LanguageModelV1, + LanguageModelV1CallWarning, + LanguageModelV1FinishReason, + LanguageModelV1ProviderMetadata, + LanguageModelV1StreamPart +} from "@ai-sdk/provider"; +import { + FetchFunction, + ParseResult, + combineHeaders, + createEventSourceResponseHandler, + createJsonResponseHandler, + generateId, postJsonToApi +} from "@ai-sdk/provider-utils"; +import { z } from "zod"; + +import { convertToHumanloopChatMessages } from "./convert-to-humanloop-chat-messages"; +import { getResponseMetadata } from "./get-response-metadata"; +import { HumanloopGenerateArgs } from "./humanloop-api-types"; +import { HumanloopChatModelId, HumanloopChatSettings } from "./humanloop-chat-settings"; +import { + HumanloopErrorData, humanloopFailedResponseHandler +} from "./humanloop-error"; +import { prepareTools } from "./humanloop-prepare-tools"; +import { mapHumanloopFinishReason } from "./map-humanloop-finish-reason"; +import { HumanloopProviderMetadata } from "./humanloop-provider"; +import path = require("path"); + +type HumanloopChatConfig = { + provider: string; + headers: () => Record; + url: (options: { path: string }) => string; + fetch?: FetchFunction; +}; + +export class HumanloopChatLanguageModel implements LanguageModelV1 { + readonly specificationVersion = "v1"; + + readonly supportsStructuredOutputs = false; + readonly defaultObjectGenerationMode = "json"; + + readonly modelId: HumanloopChatModelId; + readonly settings: HumanloopChatSettings; + + private readonly config: HumanloopChatConfig; + + constructor( + modelId: HumanloopChatModelId, + settings: HumanloopChatSettings, + config: HumanloopChatConfig, + ) { + this.modelId = modelId; + this.settings = settings; + this.config = config; + } + + get provider(): string { + return this.config.provider; + } + + get supportsImageUrls(): boolean { + // image urls can be sent if downloadImages is disabled (default): + // return !this.settings.downloadImages; + // TODO should this be true? + return true; + } + + private getArgs({ + mode, + prompt, + maxTokens, + temperature, + topP, + topK, + frequencyPenalty, + presencePenalty, + stopSequences, + responseFormat, + seed, + stream, + providerMetadata, + }: Parameters[0] & { + stream: boolean; + }): { args: HumanloopGenerateArgs; warnings: LanguageModelV1CallWarning[] } { + const type = mode.type; + + const warnings: LanguageModelV1CallWarning[] = []; + + if (topK != null) { + warnings.push({ + type: "unsupported-setting", + setting: "topK", + }); + } + + const { path, id, prompt: promptProviderMetadata, ...restProviderMetadata } = (providerMetadata?.humanloop as HumanloopProviderMetadata) ?? {}; + + if (!path && !id) { + throw new Error("Either path or id is required in providerOptions.humanloop.") + } + + // prompt hyperparameters: + const promptArgs: HumanloopGenerateArgs["prompt"] = { + ...(promptProviderMetadata ?? {}), + model: this.modelId, + maxTokens, + temperature, + topP, + frequencyPenalty, + presencePenalty, + stop: stopSequences, + seed, + + // TODO: is response format supported for streaming? + responseFormat: + stream === false && responseFormat?.type === "json" + ? { + type: "json_object", + jsonSchema: responseFormat?.schema as + | Record + | undefined, + } + : undefined, + }; + + // TODO Can we allow empty messages? With prompt.call you should be able to only pass inputs. + const messages = convertToHumanloopChatMessages(prompt); + const callArgs: Partial = { + user: this.settings.user, // overridden by providerMetadata if passed at the function level + ...restProviderMetadata, + path, + id, + messages, + }; + + switch (type) { + case "regular": { + const { tools, toolChoice, toolWarnings } = prepareTools({ + tools: mode.tools, + toolChoice: mode.toolChoice, + }); + return { + args: { + ...callArgs, + prompt: { + ...promptArgs, + tools, + }, + toolChoice, + }, + warnings: [...warnings, ...toolWarnings], + }; + } + + case "object-json": { + return { + args: { + ...callArgs, + prompt: { + ...promptArgs, + // json object response format is not supported for streaming: + responseFormat: + stream === false + ? { + type: "json_object", + jsonSchema: mode.schema as + | Record + | undefined, + } + : undefined, + }, + }, + warnings, + }; + } + + case "object-tool": { + return { + args: { + ...callArgs, + prompt: { + ...promptArgs, + tools: [ + { + name: mode.tool.name, + description: mode.tool.description, + parameters: mode.tool.parameters as Record< + string, + unknown + >, + }, + ], + }, + toolChoice: { + type: "function", + function: { name: mode.tool.name }, + }, + }, + warnings, + }; + } + + default: { + const _exhaustiveCheck: never = type; + throw new Error(`Unsupported type: ${_exhaustiveCheck}`); + } + } + } + + async doGenerate( + options: Parameters[0], + ): Promise>> { + const { args, warnings } = this.getArgs({ ...options, stream: false }); + + const body = JSON.stringify(args); + + const { + responseHeaders, + value: response, + rawValue: rawResponse, + } = await postJsonToApi({ + url: this.config.url({ + path: "/prompts/call", + }), + headers: combineHeaders(this.config.headers(), options.headers), + body: args, + failedResponseHandler: humanloopFailedResponseHandler, + successfulResponseHandler: createJsonResponseHandler( + humanloopChatResponseSchema, + ), + abortSignal: options.abortSignal, + fetch: this.config.fetch, + }); + + const { messages: rawPrompt, ...rawSettings } = args; + const log = response.logs[0]; + + // TODO: Should we sum these? I think it might be better to have usage + // at the top level return instead of on each sample? + const promptTokens = response.logs.some((l) => l.promptTokens) + ? response.logs.reduce((s, l) => s + (l.promptTokens ?? 0), 0) + : NaN; + const completionTokens = response.logs.some((l) => l.outputTokens) + ? response.logs.reduce((s, l) => s + (l.outputTokens ?? 0), 0) + : NaN; + + return { + text: log.output ?? undefined, + toolCalls: log.outputMessage.toolCalls?.map((toolCall) => ({ + toolCallType: "function", + toolCallId: toolCall.id ?? generateId(), + toolName: toolCall.function.name, + args: toolCall.function.arguments || "", + })), + finishReason: mapHumanloopFinishReason(log.finishReason), + usage: { + promptTokens, + completionTokens, + }, + rawCall: { rawPrompt, rawSettings }, + rawResponse: { headers: responseHeaders, body: rawResponse }, + response: getResponseMetadata(response), + warnings, + request: { body }, + }; + } + + async doStream( + options: Parameters[0], + ): Promise>> { + const { args, warnings } = this.getArgs({ ...options, stream: true }); + + const body = JSON.stringify({ ...args, stream: true }); + + const { responseHeaders, value: response } = await postJsonToApi({ + url: this.config.url({ + path: "/prompts/call", + }), + headers: combineHeaders(this.config.headers(), options.headers), + body: args, + failedResponseHandler: humanloopFailedResponseHandler, + successfulResponseHandler: createEventSourceResponseHandler( + humanloopChatChunkSchema, + ), + abortSignal: options.abortSignal, + fetch: this.config.fetch, + }); + + const { messages: rawPrompt, ...rawSettings } = args; + + const toolCalls: Array<{ + id: string; + type: "function"; + function: { + name: string; + arguments: string; + }; + hasFinished: boolean; + }> = []; + + let finishReason: LanguageModelV1FinishReason = "unknown"; + let usage: { + promptTokens: number | undefined; + completionTokens: number | undefined; + } = { + promptTokens: undefined, + completionTokens: undefined, + }; + let isFirstChunk = true; + + let providerMetadata: LanguageModelV1ProviderMetadata | undefined; + return { + stream: response.pipeThrough( + new TransformStream< + ParseResult>, + LanguageModelV1StreamPart + >({ + transform(chunk, controller) { + // handle failed chunk parsing / validation: + if (chunk.success == false) { + finishReason = "error"; + controller.enqueue({ type: "error", error: chunk.error }); + return; + } + + // handle error chunks: + if (!("id" in chunk.value)) { + finishReason = "error"; + controller.enqueue({ type: "error", error: (chunk.value as HumanloopErrorData).message }); + return; + } + + + const value: z.infer = chunk.value; + + if (isFirstChunk) { + isFirstChunk = false; + + controller.enqueue({ + type: "response-metadata", + ...getResponseMetadata({ + id: value.id, + // no modelId or createdAt on stream response + }), + }); + } + + if (value.finishReason != null) { + finishReason = mapHumanloopFinishReason(value.finishReason); + } + + // TODO handle outputMessage + if (value.output != null) { + controller.enqueue({ + type: "text-delta", + textDelta: value.output, + }); + } + + // TODO: confused -- don't see any tool calls/requests in prompts call response + // if (delta.tool_calls != null) { + // for (const toolCallDelta of delta.tool_calls) { + // const index = toolCallDelta.index; + + // if (toolCalls[index] == null) { + // if (toolCallDelta.type !== "function") { + // throw new InvalidResponseDataError({ + // data: toolCallDelta, + // message: `Expected 'function' type.`, + // }); + // } + + // if (toolCallDelta.id == null) { + // throw new InvalidResponseDataError({ + // data: toolCallDelta, + // message: `Expected 'id' to be a string.`, + // }); + // } + + // if (toolCallDelta.function?.name == null) { + // throw new InvalidResponseDataError({ + // data: toolCallDelta, + // message: `Expected 'function.name' to be a string.`, + // }); + // } + + // toolCalls[index] = { + // id: toolCallDelta.id, + // type: "function", + // function: { + // name: toolCallDelta.function.name, + // arguments: + // toolCallDelta.function.arguments ?? "", + // }, + // hasFinished: false, + // }; + + // const toolCall = toolCalls[index]; + + // if ( + // toolCall.function?.name != null && + // toolCall.function?.arguments != null + // ) { + // // send delta if the argument text has already started: + // if (toolCall.function.arguments.length > 0) { + // controller.enqueue({ + // type: "tool-call-delta", + // toolCallType: "function", + // toolCallId: toolCall.id, + // toolName: toolCall.function.name, + // argsTextDelta: + // toolCall.function.arguments, + // }); + // } + + // // check if tool call is complete + // // (some providers send the full tool call in one chunk): + // if ( + // isParsableJson(toolCall.function.arguments) + // ) { + // controller.enqueue({ + // type: "tool-call", + // toolCallType: "function", + // toolCallId: toolCall.id ?? generateId(), + // toolName: toolCall.function.name, + // args: toolCall.function.arguments, + // }); + // toolCall.hasFinished = true; + // } + // } + + // continue; + // } + + // // existing tool call, merge if not finished + // const toolCall = toolCalls[index]; + + // if (toolCall.hasFinished) { + // continue; + // } + + // if (toolCallDelta.function?.arguments != null) { + // toolCall.function!.arguments += + // toolCallDelta.function?.arguments ?? ""; + // } + + // // send delta + // controller.enqueue({ + // type: "tool-call-delta", + // toolCallType: "function", + // toolCallId: toolCall.id, + // toolName: toolCall.function.name, + // argsTextDelta: + // toolCallDelta.function.arguments ?? "", + // }); + + // // check if tool call is complete + // if ( + // toolCall.function?.name != null && + // toolCall.function?.arguments != null && + // isParsableJson(toolCall.function.arguments) + // ) { + // controller.enqueue({ + // type: "tool-call", + // toolCallType: "function", + // toolCallId: toolCall.id ?? generateId(), + // toolName: toolCall.function.name, + // args: toolCall.function.arguments, + // }); + // toolCall.hasFinished = true; + // } + // } + // } + }, + + flush(controller) { + controller.enqueue({ + type: "finish", + finishReason, + usage: { + promptTokens: usage.promptTokens ?? NaN, + completionTokens: usage.completionTokens ?? NaN, + }, + ...(providerMetadata != null ? { providerMetadata } : {}), + }); + }, + }), + ), + rawCall: { rawPrompt, rawSettings }, + rawResponse: { headers: responseHeaders }, + warnings, + request: { body }, + }; + } +} + +// limited version of the schema, focussed on what is needed for the implementation +// this approach limits breakages when the API changes and increases efficiency +const humanloopChatMessageSchema = z + .object({ + name: z.string().nullish(), + toolCallId: z.string().nullish(), + role: z.literal("assistant"), + content: z + .union([ + z.string(), + z.array( + z.union([ + z.object({ + type: z.literal("text"), + text: z.string(), + }), + z.object({ + type: z.literal("image_url"), + imageUrl: z.object({ + url: z.string(), + detail: z.union([ + z.literal("high"), + z.literal("low"), + z.literal("auto"), + ]), + }), + }), + ]), + ), + ]) + .nullish(), + toolCalls: z + .array( + z.object({ + id: z.string().nullish(), + type: z.literal("function"), + function: z.object({ + name: z.string(), + arguments: z.string().nullish(), + }), + }), + ) + .nullish(), + }) +const humanloopChatResponseSchema = z.object({ + id: z.string(), + traceId: z.string().nullish(), + startTime: z.date().nullish(), + endTime: z.date().nullish(), + prompt: z.object({ + id: z.string(), + versionId: z.string(), + }), + logs: z.array( + z.object({ + output: z.string().nullish(), + createdAt: z.date().nullish(), + error: z.string().nullish(), + outputMessage: humanloopChatMessageSchema.nullish(), + finishReason: z.string().nullish(), + index: z.number(), + providerLatency: z.number().nullish(), + promptTokens: z.number().nullish(), + reasoningTokens: z.number().nullish(), + outputTokens: z.number().nullish(), + promptCost: z.number().nullish(), + outputCost: z.number().nullish(), + }), + ), +}); + +// limited version of the schema, focussed on what is needed for the implementation +// this approach limits breakages when the API changes and increases efficiency +const humanloopChatChunkSchema = z.union([ + z.object({ + id: z.string(), + index: z.number(), + promptId: z.string(), + versionId: z.string(), + output: z.string().nullish(), + outputMessage: humanloopChatMessageSchema.nullish(), + finishReason: z.string().nullish(), + }), + z.object({ + error: z.string(), + }), +]); diff --git a/integrations/ai-sdk/src/humanloop-chat-settings.ts b/integrations/ai-sdk/src/humanloop-chat-settings.ts new file mode 100644 index 00000000..f1f129c1 --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-chat-settings.ts @@ -0,0 +1,163 @@ +// https://humanloop.com/docs/reference/supported-models +// Supported models +export type HumanloopChatModelId = + | "deepseek-r1-distill-llama-70b" + | "gemma2-9b-it" + | "gemma-7b-it" + | "llama-3.3-70b-versatile" + | "llama-3.1-8b-instant" + | "llama-guard-3-8b" + | "llama3-70b-8192" + | "llama3-8b-8192" + | "mixtral-8x7b-32768" + | "deepseek-reasoner" + | "deepseek-chat" + | "o1-preview" + | "o1-preview-2024-09-12" + | "o3-mini" + | "o1-mini" + | "o1" + | "o1-2024-12-17" + | "o1-preview" + | "o1-preview-2024-09-12" + | "o3-mini" + | "o1-mini" + | "o1" + | "o1-2024-12-17" + | "gpt-4.5-preview" + | "gpt-4" + | "gpt-4o-64k-output-alpha" + | "gpt-4o" + | "gpt-4o-mini" + | "gpt-4o-mini-2024-07-18" + | "gpt-4-turbo" + | "gpt-4-turbo-2024-04-09" + | "gpt-4-0" + | "gpt-4-0" + | "gpt-4-32k" + | "gpt-4-1106-preview" + | "gpt-4-0125-preview" + | "gpt-4-vision" + | "gpt-4-1106-vision-preview" + | "gpt-3.5-turbo" + | "gpt-3.5-turbo-instruct" + | "babbage-002" + | "davinci-002" + | "ft:gpt-3.5-turbo" + | "ft:gpt-4o" + | "ft:davinci-002" + | "claude-3-opus-20240229" + | "claude-3-sonnet-20240229" + | "claude-3-7-sonnet" + | "claude-3-5-sonnet-20241022" + | "claude-3-5-sonnet-20240620" + | "claude-3-haiku-20240307" + | "claude-3-5-haiku-20241022" + | "claude-2.1" + | "claude-2" + | "claude-instant-1.2" + | "claude-instant-1" + | "mock-model-007" + | "gpt-4" + | "gpt-3.5-turbo" + | "llama-3.3-70b-versatile" + | "llama-3.2-1b-preview" + | "llama-3.2-3b-preview" + | "llama-3.2-11b-vision-preview" + | "llama-3.2-90b-vision-preview" + | "llama-3.1-70b-versatile" + | "llama-3.1-8b-instant" + | "llama3-8b-8192" + | "llama3-70b-8192" + | "deepseek-r1-distill-llama-70b" + | "mixtral-8x7b-32768" + | "gemma2-9b-it" + | "gemma-7b-it" + | "llama-3-70b-instruct" + | "llama-3-70b" + | "llama-3-8b-instruct" + | "llama-3-8b" + | "llama-2-70b" + | "llama70b-v2" + | "mixtral-8x7b" + | "gpt-4o" + | "gpt-4o-mini" + | "gpt-4o-2024-05-13" + | "gpt-4-turbo-2024-04-09" + | "gpt-4" + | "gpt-4-0314" + | "gpt-dv-duo" + | "gpt-dv-duo" + | "gpt-4-32k" + | "gpt-4-0125" + | "gpt-4-1106" + | "GPT-4-1106" + | "gpt-4-0613" + | "gpt-4-turbo" + | "gpt-4-turbo-vision" + | "gpt-4-vision" + | "gpt-35-turbo-1106" + | "gpt-35-turbo-0125" + | "gpt-35-turbo-16k" + | "gpt-35-turbo" + | "gpt-3.5-turbo-instruct" + | "gpt-35-turbo-instruct" + | "command-r" + | "command-light" + | "command-r-plus" + | "command-nightly" + | "command" + | "command-medium-beta" + | "command-xlarge-beta" + | "gemini-pro-vision" + | "gemini-1.0-pro-vision" + | "gemini-pro" + | "gemini-1.0-pro" + | "gemini-2.0-flash" + | "gemini-1.5-flash" + | "gemini-1.5-pro" + | "gemini-exp-1206" + | "gemini-exp-1114" + | "gemini-exp-1121" + | "gemini-2.0-flash-exp" + | "gemini-experimental" + | "anthropic.claude-3-7-sonnet-20250219-v1:0" + | "anthropic.claude-3-5-sonnet-20241022-v2:0" + | "anthropic.claude-3-5-haiku-20241022-v1:0" + | "anthropic.claude-3-sonnet-20240229-v1:0" + | "anthropic.claude-3-haiku-20240307-v1:0" + | "anthropic.claude-3-opus-20240229-v1:0" + | "anthropic.claude-3-5-sonnet-20240620-v1:0" + | "anthropic.claude-v2" + | "anthropic.claude-v2:1" + | "anthropic.claude-instant-v1" + | "meta.llama3-3-70b-instruct-v1:0" + | "meta.llama3-1-405b-instruct-v1:0" + | "meta.llama3-1-70b-instruct-v1:0" + | "meta.llama3-1-8b-instruct-v1:0" + | "meta.llama3-70b-instruct-v1:0" + | "meta.llama3-8b-instruct-v1:0" + | "mock" + | (string & {}); + +export interface HumanloopChatSettings { + /** +Whether to enable parallel function calling during tool use. Default to true. + */ + // parallelToolCalls?: boolean; + + /** +A unique identifier representing your end-user, which can help OpenAI to +monitor and detect abuse. Learn more. +*/ + user?: string; + + /** +Automatically download images and pass the image as data to the model. +Humanloop supports image URLs for public models, so this is only needed for +private models or when the images are not publicly accessible. + +Defaults to `false`. + */ + // downloadImages?: boolean; +} diff --git a/integrations/ai-sdk/src/humanloop-error.ts b/integrations/ai-sdk/src/humanloop-error.ts new file mode 100644 index 00000000..b6f2239f --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-error.ts @@ -0,0 +1,18 @@ +import { createJsonErrorResponseHandler } from '@ai-sdk/provider-utils'; +import { z } from 'zod'; + +// TODO: validate this is our error schema for API +export const humanloopErrorDataSchema = z.object({ + object: z.literal('error'), + message: z.string(), + type: z.string(), + param: z.string().nullable(), + code: z.string().nullable(), +}); + +export type HumanloopErrorData = z.infer; + +export const humanloopFailedResponseHandler = createJsonErrorResponseHandler({ + errorSchema: humanloopErrorDataSchema, + errorToMessage: data => data.message, +}); \ No newline at end of file diff --git a/integrations/ai-sdk/src/humanloop-prepare-tools.ts b/integrations/ai-sdk/src/humanloop-prepare-tools.ts new file mode 100644 index 00000000..2966579a --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-prepare-tools.ts @@ -0,0 +1,71 @@ +import { + LanguageModelV1CallWarning, + LanguageModelV1FunctionTool, + LanguageModelV1ProviderDefinedTool, + LanguageModelV1ToolChoice, + UnsupportedFunctionalityError +} from '@ai-sdk/provider'; +import { HumanloopToolChoice, HumanloopTools } from './humanloop-api-types'; + + export function prepareTools({ + tools, + toolChoice = { type: 'auto' }, + }: { + tools?: Array, + toolChoice: LanguageModelV1ToolChoice, + } + ): { + tools: HumanloopTools | undefined + toolChoice: HumanloopToolChoice | undefined + toolWarnings: LanguageModelV1CallWarning[]; + } { + // when the tools array is empty, change it to undefined to prevent errors: + const toolWarnings: LanguageModelV1CallWarning[] = []; + + if (!tools || !tools.length) { + return { tools: undefined, toolChoice: undefined, toolWarnings }; + } + + const humanloopTools: HumanloopTools = []; + + for (const tool of tools) { + if (tool.type === 'provider-defined') { + toolWarnings.push({ type: 'unsupported-tool', tool }); + } else { + humanloopTools.push({ + name: tool.name, + description: tool.description, + parameters: tool.parameters as Record, + }); + } + } + + if (toolChoice == null) { + return { tools: humanloopTools, toolChoice: undefined, toolWarnings }; + } + + const type = toolChoice.type; + + switch (type) { + case 'auto': + case 'none': + case 'required': + return { tools: humanloopTools, toolChoice: type, toolWarnings }; + + case 'tool': + return { + tools: humanloopTools, + toolChoice: { + type: 'function', + function: { name: toolChoice.toolName } + }, + toolWarnings, + } + default: { + const _exhaustiveCheck: never = type; + throw new UnsupportedFunctionalityError({ + functionality: `Unsupported tool choice type: ${_exhaustiveCheck}`, + }); + } + } + } \ No newline at end of file diff --git a/integrations/ai-sdk/src/humanloop-provider.ts b/integrations/ai-sdk/src/humanloop-provider.ts new file mode 100644 index 00000000..2cd07c3e --- /dev/null +++ b/integrations/ai-sdk/src/humanloop-provider.ts @@ -0,0 +1,132 @@ +import { LanguageModelV1, NoSuchModelError, ProviderV1 } from "@ai-sdk/provider"; +import { + FetchFunction, + loadApiKey, + withoutTrailingSlash, +} from "@ai-sdk/provider-utils"; + +import { HumanloopGenerateArgs } from "./humanloop-api-types"; +import { HumanloopChatLanguageModel } from "./humanloop-chat-language-model"; +import { HumanloopChatModelId, HumanloopChatSettings } from "./humanloop-chat-settings"; + +export interface HumanloopProvider extends ProviderV1 { + /** +Creates a model for text generation. +*/ + (modelId: HumanloopChatModelId, settings?: HumanloopChatSettings): LanguageModelV1; + + /** +Creates an Humanloop chat model for text generation. + */ + languageModel( + modelId: HumanloopChatModelId, + settings?: HumanloopChatSettings, + ): LanguageModelV1; +} + +export interface HumanloopProviderSettings { + /** +Base URL for the Humanloop API calls. + */ + baseUrl?: string; + + /** +API key for authenticating requests. + */ + apiKey?: string; + + /** +Custom headers to include in the requests. + */ + headers?: Record; + + /** +Custom fetch implementation. You can use it as a middleware to intercept requests, +or to provide a custom fetch implementation for e.g. testing. + */ + fetch?: FetchFunction; +} + +export type HumanloopProviderMetadata = Omit< + HumanloopGenerateArgs, + "messages" | "toolChoice" | "prompt" +> & { + // Common prompt hyperparams are passed through the AI SDK as LanguageModelV1CallSettings + prompt?: Omit< + HumanloopGenerateArgs["prompt"], + | "maxTokens" + | "temperature" + | "stopSequences" + | "topP" + | "topK" + | "presencePenalty" + | "frequencyPenalty" + | "responseFormat" + | "seed" + | "abortSignal" + | "headers" + >; +}; + +/** +Create an Humanloop provider instance. + */ +export function createHumanloop( + options: HumanloopProviderSettings = {}, +): HumanloopProvider { + const baseURL = + withoutTrailingSlash(options.baseUrl) ?? "https://api.humanloop.com/v5"; + + const getHeaders = () => ({ + "X-API-KEY": ` ${loadApiKey({ + apiKey: options.apiKey, + environmentVariableName: "HUMANLOOP_API_KEY", + description: "Humanloop", + })}`, + ...options.headers, + }); + + const createChatModel = ( + modelId: HumanloopChatModelId, + settings: HumanloopChatSettings = {}, + ) => + new HumanloopChatLanguageModel(modelId, settings, { + provider: "humanloop", + url: ({ path }) => `${baseURL}${path}`, + headers: getHeaders, + fetch: options.fetch, + }); + + const createLanguageModel = ( + modelId: HumanloopChatModelId, + settings?: HumanloopChatSettings, + ) => { + if (new.target) { + throw new Error( + "The Humanloop model function cannot be called with the new keyword.", + ); + } + + return createChatModel(modelId, settings); + }; + + const provider = function ( + modelId: HumanloopChatModelId, + settings?: HumanloopChatSettings, + ) { + return createLanguageModel(modelId, settings); + }; + + provider.languageModel = createLanguageModel; + provider.chat = createChatModel; + provider.textEmbeddingModel = (modelId: string) => { + throw new NoSuchModelError({ modelId, modelType: "textEmbeddingModel" }); + }; + + return provider; +} + +/** +Default Humanloop provider instance. + */ +export const humanloop = createHumanloop(); diff --git a/integrations/ai-sdk/src/map-humanloop-finish-reason.ts b/integrations/ai-sdk/src/map-humanloop-finish-reason.ts new file mode 100644 index 00000000..235a8073 --- /dev/null +++ b/integrations/ai-sdk/src/map-humanloop-finish-reason.ts @@ -0,0 +1,20 @@ +import { LanguageModelV1FinishReason } from '@ai-sdk/provider'; + +// TODO see what provider finish reasons are and map +export function mapHumanloopFinishReason( + finishReason: string | null | undefined, +): LanguageModelV1FinishReason { + switch (finishReason) { + case 'stop': + return 'stop'; + case 'length': + case 'model_length': + return 'length'; + case 'tool_calls': + return 'tool-calls'; + case 'content-filter': + return 'content-filter'; + default: + return 'unknown'; + } +} \ No newline at end of file diff --git a/integrations/ai-sdk/tsconfig.json b/integrations/ai-sdk/tsconfig.json new file mode 100644 index 00000000..b6073401 --- /dev/null +++ b/integrations/ai-sdk/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "./node_modules/@vercel/ai-tsconfig/ts-library.json", + "include": ["."], + "exclude": ["*/dist", "dist", "build", "node_modules"] +} diff --git a/integrations/ai-sdk/tsup.config.ts b/integrations/ai-sdk/tsup.config.ts new file mode 100644 index 00000000..9480ab1b --- /dev/null +++ b/integrations/ai-sdk/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig([ + { + entry: ['src/index.ts'], + format: ['cjs', 'esm'], + dts: true, + sourcemap: true, + }, +]); \ No newline at end of file diff --git a/integrations/ai-sdk/turbo.json b/integrations/ai-sdk/turbo.json new file mode 100644 index 00000000..a2a029f9 --- /dev/null +++ b/integrations/ai-sdk/turbo.json @@ -0,0 +1,8 @@ +{ + "extends": ["//"], + "tasks": { + "build": { + "outputs": ["**/dist/**"] + } + } +} diff --git a/integrations/ai-sdk/vitest.edge.config.js b/integrations/ai-sdk/vitest.edge.config.js new file mode 100644 index 00000000..6b50643d --- /dev/null +++ b/integrations/ai-sdk/vitest.edge.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from "vite"; + +// https://vitejs.dev/config/ +export default defineConfig({ + test: { + environment: "edge-runtime", + globals: true, + include: ["**/*.test.ts", "**/*.test.tsx"], + }, +}); diff --git a/integrations/ai-sdk/vitest.node.config.js b/integrations/ai-sdk/vitest.node.config.js new file mode 100644 index 00000000..e0c9013d --- /dev/null +++ b/integrations/ai-sdk/vitest.node.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from "vite"; + +// https://vitejs.dev/config/ +export default defineConfig({ + test: { + environment: "node", + globals: true, + include: ["**/*.test.ts", "**/*.test.tsx"], + }, +});