Add Claudev3 model support

extension/package.json

@@ -78,6 +78,7 @@
   "activationEvents": [],
   "dependencies": {
     "@ai-zen/node-fetch-event-source": "^2.1.2",
+    "@anthropic-ai/sdk": "^0.20.1",
     "gpt-tokenizer": "^2.1.2",
     "llama-tokenizer-js": "^1.1.3",
     "p-timeout": "^6.1.2"

extension/scripts/gitpodup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Run setup commands within gitpod for a quick start
+cd /workspace/wingman/webview/ && npm install
+npm run dev &
+cd /workspace/wingman/extension/ && npm install
+npm run build:watch &
+# Git lets you set your email arbitrarily. Might as well use this one, even if you're someone else.
+NOREPLY_EMAIL="37577626+fredxfred@users.noreply.github.com"
+git config --global user.email $NOREPLY_EMAIL

extension/src/extension.ts

@@ -72,18 +72,30 @@ export const createDefaultPresetsForAllModes = () => {
         },
         {
           id: generateId(),
-          name: "LM Studio",
-          provider: "OpenAI",
-          format: "OpenAI",
-          tokenizer: "OpenAI",
-          url: "http://localhost:1234/v1/chat/completions",
+          name: "Claude v3.5 Sonnet",
+          provider: "ClaudeV3",
+          format: "Anthropic",
+          tokenizer: "Anthropic",
+          url: "https://api.anthropic.com/v1/messages",
           system: systems.get(mode.id),
           completionParams: {
-            ...getProviderCompletionParamDefaults("OpenAI") as any,
-            model: null,
-            stop: null,
+            ...getProviderCompletionParamDefaults("ClaudeV3") as any,
+            model: "claude-3-5-sonnet-20240620",
           },
-        }
+        },
+        {
+          id: generateId(),
+          name: "Claude v3 Opus",
+          provider: "ClaudeV3",
+          format: "Anthropic",
+          tokenizer: "Anthropic",
+          url: "https://api.anthropic.com/v1/messages",
+          system: systems.get(mode.id),
+          completionParams: {
+            ...getProviderCompletionParamDefaults("ClaudeV3") as any,
+            model: "claude-3-opus-20240229",
+          },
+        },
       ] as Preset[]);
       State.set(`${mode.id}-activePreset`, State.get(`${mode.id}-presets`)[0]);
     });

extension/src/providers/anthropic.ts

@@ -3,6 +3,7 @@ import { PreparedCommand } from "../dispatcher";
 import { AnthropicClient, CompletionResponse } from "./clients/anthropic";
 import { APIProvider, applyFormat } from "./common";
 
+// Note that this provider is only for Anthropic's legacy v1 and v2 Claude APIs.
 export class AnthropicProvider implements APIProvider {
   webviewView: WebviewView;
   command: PreparedCommand;

extension/src/providers/claudev3.ts

@@ -0,0 +1,65 @@
+import { WebviewView } from "vscode";
+import { PreparedCommand } from "../dispatcher";
+import { ClaudeV3Client } from "./clients/claudev3";
+import { APIProvider, applyFormat, ClaudeOpenAIMessage, PartialResponse } from "./common";
+
+// This file exists because Anthropic introduced breaking changes in their APIs between Claude's 2nd and 3rd generations.
+// This Provider, and to a lesser extent the ClaudeV3 client, are very similar to the OpenAI versions.
+// They can probably be combined.
+export class ClaudeV3Provider implements APIProvider {
+  webviewView: WebviewView;
+  command: PreparedCommand;
+  onProgressCallback: (text: string) => void;
+  client: ClaudeV3Client;
+  abortController: AbortController;
+  messages: Array<ClaudeOpenAIMessage>;
+
+  constructor(viewProvider: WebviewView, command: PreparedCommand, onProgressCallback?: (text: string) => void) {
+    this.webviewView = viewProvider;
+    this.command = command;
+    this.abortController = new AbortController();
+    this.onProgressCallback = onProgressCallback;
+    this.client = new ClaudeV3Client(this.command);
+  }
+
+  async send(message: string = undefined): Promise<string> {
+    const fmt = applyFormat("ClaudeV3", this.command);
+
+    if (message === undefined || !this.messages || this.messages.length == 0) {
+      this.messages = [{"role": "user", "content": fmt.user}];
+    } else {
+      this.messages.push({"role": "user", "content": fmt.user});
+    }
+
+    // @ts-ignore
+    this.command.completionParams.messages = this.messages;
+    // @ts-ignore
+    this.command.completionParams.system = fmt.system;
+
+    try {
+      const response = await this.client.stream(
+        this.abortController.signal,
+        (data: PartialResponse) => {
+          this.onProgressCallback?.(data.text);
+        },
+      );
+
+      this.messages.push({"role": "assistant", "content": response.text});
+
+      return response.text;
+    } catch (error) {
+      throw new Error(error);
+    }
+  }
+
+  abort() {
+    try {
+      if (this.abortController) {
+        this.abortController.abort();
+      }
+      this.abortController = new AbortController();
+    } catch (error) {
+      console.error(error);
+    }
+  }
+}

extension/src/providers/clients/anthropic.ts

@@ -15,6 +15,7 @@ export type CompletionResponse = {
 const ANTHROPIC_SDK = "anthropic-typescript/0.4.4";
 const ANTHROPIC_VERSION = "2023-01-01";
 
+// Note that this client is only for Anthropic's legacy v1 and v2 Claude APIs.
 export class AnthropicClient {
   private key: string;
   private command: PreparedCommand;

extension/src/providers/clients/claudev3.ts

@@ -0,0 +1,86 @@
+import { fetchEventSource } from "@ai-zen/node-fetch-event-source";
+import { PreparedCommand } from "../../dispatcher";
+import { getCurrentProviderAPIKey } from "../../utils";
+import pTimeout from "p-timeout";
+import Anthropic from '@anthropic-ai/sdk';
+import { PartialResponse } from "../common";
+
+// This file exists because Anthropic introduced breaking changes in their APIs between Claude's 2nd and 3rd generations.
+// This is very similar to OpenAI's Client now. They can potentially be combined, if the Anthropic SDK gets
+// replaced with general SSE logic like in clients/openai.ts
+export class ClaudeV3Client {
+  private key: string;
+  private command: PreparedCommand;
+  private client: Anthropic;
+  private created: boolean = false;
+
+  constructor(command: PreparedCommand) {
+    this.command = command;
+  }
+
+  async create() {
+    // The Svelte UI assumes that each Provider has its own API Key.
+    // Both legacy and v3 Anthropic APIs use the same keys, but differently enough to justify separate Providers.
+    // Svelte should probably be updated to not assume a 1:1 Provider:API key relationship, but in the meantime,
+    // to avoid user confusion, fall back to an Anthropic API key if a ClaudeV3 key is not present.
+    try {
+      this.key = await getCurrentProviderAPIKey();
+      if (!this.key) {
+        throw new Error("Empty ClaudeV3 API key");
+      }
+    } catch (e: unknown) {
+      this.key = await getCurrentProviderAPIKey("Anthropic");
+    }
+    this.client = new Anthropic({
+      apiKey: this.key,
+    });
+    this.created = true;
+  }
+
+  async stream(abortSignal: AbortSignal, onProgress?: (PartialResponse: PartialResponse) => void): Promise<PartialResponse> {
+    if (!this.created) {
+      await this.create();
+    }
+
+    const abortController = new AbortController();
+    const result: PartialResponse = {
+      role: "assistant",
+      id: "1",
+      text: "",
+      delta: "",
+      detail: {},
+    };
+
+    const responseP = new Promise<PartialResponse>((resolve, reject) => {
+      abortSignal.addEventListener("abort", (event) => {
+        abortController.abort(event);
+        reject(new Error("Caller aborted completion stream."));
+      });
+
+      // @ts-ignore
+      this.client.messages.stream(this.command.completionParams).on('text', (delta, snapshot) => {
+        result.text += delta;
+        result.delta = delta;
+        result.detail = snapshot;
+        onProgress?.(result);
+      }).on('error', (error) => {
+        abortController.abort();
+        return reject(error);
+      }).on('abort', (error) => {
+        abortController.abort();
+        return reject(error);
+      }).on('end', () => {
+        return resolve(result);
+      });
+    });
+
+    pTimeout(responseP, {
+      milliseconds: 10 * 60 * 1000,
+      message: "Completion stream timed out.",
+    }).catch(() => {
+      abortController.abort();
+    });
+
+    return responseP;
+  }
+}

extension/src/providers/clients/openai.ts

@@ -2,17 +2,7 @@ import { PreparedCommand } from "../../dispatcher";
 import { fetchEventSource } from "@ai-zen/node-fetch-event-source";
 import pTimeout from "p-timeout";
 import { getCurrentProviderAPIKey } from "../../utils";
-
-export type PartialResponse = {
-  id: string;
-  text: string;
-  role: "user" | "system" | "assistant" | "function";
-  name?: string;
-  delta?: string;
-  detail?: any;
-  parentMessageId?: string;
-  conversationId?: string;
-};
+import { PartialResponse } from "../common";
 
 export class OpenAIClient {
   private key: string;

extension/src/providers/common.ts

@@ -3,6 +3,7 @@ import { PreparedCommand } from "../dispatcher";
 import { OpenAITokenizer } from "../tokenizers/openai";
 import { AnthropicProvider } from "./anthropic";
 import { OpenAIProvider } from "./openai";
+import { ClaudeV3Provider } from "./claudev3";
 
 interface Format {
   system: string;
@@ -24,6 +25,12 @@ export const formats: { [key: string]: Format } = {
     first: "{system}{user}",
     stops: ["Human:"],
   },
+  ClaudeV3: {
+    system: "{system_message}",
+    user: "{user_message}",
+    first: "{system}{user}",
+    stops: [],
+  },
   Alpaca: {
     system: "{system_message}",
     user: "### Instruction: {user_message}\n\n### Response:",
@@ -86,6 +93,7 @@ export const providers = {
       { name: "stop", default: [] },
     ],
   },
+  // Legacy API
   Anthropic: {
     instance: AnthropicProvider,
     // https://docs.anthropic.com/claude/reference/complete_post
@@ -98,6 +106,19 @@ export const providers = {
       // { name: "stop_sequence", default: ["\\n\\nHuman:"] },
     ],
   },
+  ClaudeV3: {
+    instance: ClaudeV3Provider,
+    // https://docs.anthropic.com/claude/reference/messages_post
+    completionParams: [
+      // The current max configurable value across all Anthropic models is 4096
+      // Source: https://web.archive.org/web/20240402111826/https://docs.anthropic.com/claude/docs/models-overview
+      { name: "max_tokens", default: 4096 },
+      { name: "model", default: "claude-3-5-sonnet-20240620" },
+      // Anthropic recommends only setting top_k and top_p for special use cases, and says temperature is usually sufficient.
+      { name: "temperature", default: 0.0 },
+      // { name: "stop_sequence", default: ["\\n\\nHuman:"] },
+    ],
+  },
 };
 
 export const tokenizers = {
@@ -110,6 +131,9 @@ export const tokenizers = {
   Anthropic: {
     instance: OpenAITokenizer,
   },
+  ClaudeV3: {
+    instance: OpenAITokenizer,
+  },
 };
 
 /**
@@ -137,3 +161,18 @@ export interface APIProvider {
 }
 
 export const EXTENSION_SCHEME = "wingman";
+
+export type ClaudeOpenAIMessage = {
+  role: string;
+  content: string;
+};
+export type PartialResponse = {
+  id: string;
+  text: string;
+  role: "user" | "system" | "assistant" | "function";
+  name?: string;
+  delta?: string;
+  detail?: any;
+  parentMessageId?: string;
+  conversationId?: string;
+};

extension/src/providers/openai.ts

@@ -1,15 +1,15 @@
 import { WebviewView } from "vscode";
 import { PreparedCommand } from "../dispatcher";
-import { OpenAIClient, PartialResponse } from "./clients/openai";
-import { APIProvider } from "./common";
+import { OpenAIClient } from "./clients/openai";
+import { APIProvider, ClaudeOpenAIMessage, PartialResponse } from "./common";
 
 export class OpenAIProvider implements APIProvider {
   webviewView: WebviewView;
   command: PreparedCommand;
   onProgressCallback: (text: string) => void;
   client: OpenAIClient;
   abortController: AbortController;
-  messages: any[] = [];
+  messages: Array<ClaudeOpenAIMessage>;
 
   constructor(viewProvider: WebviewView, command: PreparedCommand, onProgressCallback?: (text: string) => void) {
     this.webviewView = viewProvider;
@@ -23,8 +23,8 @@ export class OpenAIProvider implements APIProvider {
    * @param message Only provided when this is a follow-up message. Otherwise, the command message is used.
    */
   async send(message: string = undefined): Promise<string> {
-    if (this.messages.length === 0) {
-      this.messages.push({ role: "system", content: this.command.system });
+    if (message === undefined || !this.messages || this.messages.length === 0) {
+      this.messages = [{ role: "system", content: this.command.system }];
     }
 
     if (message === undefined) {