test baseten responses endpoint against lingua testcases

payloads/cases/advanced.ts

@@ -538,6 +538,35 @@ export const advancedCases: TestCaseCollection = {
       ],
       tool_choice: "auto",
     },
+    "baseten-responses": {
+      model: BASETEN_MODEL,
+      input: [
+        {
+          role: "user",
+          content:
+            "Before doing anything else, reply with one short sentence telling me you are about to look up the weather. Then call the get_weather tool for San Francisco, CA.",
+        },
+      ],
+      tools: [
+        {
+          type: "function",
+          name: "get_weather",
+          description: "Get the current weather for a location",
+          parameters: {
+            type: "object",
+            properties: {
+              location: {
+                type: "string",
+                description: "The city and state, e.g. San Francisco, CA",
+              },
+            },
+            required: ["location"],
+          },
+          strict: false,
+        },
+      ],
+      tool_choice: "auto",
+    },
   },
 
   parallelToolCallsRequest: {

payloads/cases/types.ts

@@ -84,6 +84,8 @@ export interface TestCase {
   "vertex-anthropic"?: AnthropicMessageCreateParams | null;
   // Baseten serves OSS models via an OpenAI-compatible chat-completions API.
   baseten?: ChatCompletionCreateParams | null;
+  // Baseten's OpenAI-compatible Responses API.
+  "baseten-responses"?: OpenAI.Responses.ResponseCreateParams | null;
   // Optional expectations for proxy compatibility tests
   expect?: TestExpectation;
 }
@@ -104,4 +106,5 @@ export const PROVIDER_TYPES = [
   "bedrock",
   "bedrock-anthropic",
   "vertex-anthropic",
+  "baseten-responses",
 ] as const;

payloads/scripts/capture.ts

@@ -15,6 +15,7 @@ import { bedrockExecutor } from "./providers/bedrock";
 import { bedrockAnthropicExecutor } from "./providers/bedrock-anthropic";
 import { vertexAnthropicExecutor } from "./providers/vertex-anthropic";
 import { basetenExecutor } from "./providers/baseten";
+import { basetenResponsesExecutor } from "./providers/baseten-responses";
 import { type ProviderExecutor } from "./types";
 
 // Update provider names to be more descriptive
@@ -27,6 +28,7 @@ const allProviders = [
   bedrockAnthropicExecutor,
   vertexAnthropicExecutor,
   basetenExecutor,
+  basetenResponsesExecutor,
 ] as const;
 
 interface CaptureOptions {

payloads/scripts/providers/baseten-responses.ts

@@ -0,0 +1,70 @@
+import OpenAI from "openai";
+import { CaptureResult, ExecuteOptions, ProviderExecutor } from "../types";
+import {
+  allTestCases,
+  getCaseNames,
+  getCaseForProvider,
+  hasExpectation,
+} from "../../cases";
+import { BASETEN_BASE_URL, BASETEN_MODEL } from "../../cases/models";
+import {
+  executeOpenAIResponses,
+  openaiResponsesExecutor,
+} from "./openai-responses";
+
+type BasetenResponsesRequest = OpenAI.Responses.ResponseCreateParams;
+type BasetenResponsesResponse = OpenAI.Responses.Response;
+
+export const basetenResponsesCases: Record<string, BasetenResponsesRequest> =
+  {};
+
+getCaseNames(allTestCases).forEach((caseName) => {
+  if (hasExpectation(allTestCases, caseName)) {
+    return;
+  }
+
+  const explicitCase = getCaseForProvider(
+    allTestCases,
+    caseName,
+    "baseten-responses"
+  );
+  if (explicitCase) {
+    basetenResponsesCases[caseName] = explicitCase;
+    return;
+  }
+
+  const responsesCase = getCaseForProvider(allTestCases, caseName, "responses");
+  if (responsesCase) {
+    basetenResponsesCases[caseName] = {
+      ...responsesCase,
+      model: BASETEN_MODEL,
+    };
+  }
+});
+
+const BASETEN_HOST = BASETEN_BASE_URL.replace(/\/v1\/?$/, "");
+
+export function executeBasetenResponses(
+  caseName: string,
+  payload: BasetenResponsesRequest,
+  options?: ExecuteOptions
+): Promise<
+  CaptureResult<BasetenResponsesRequest, BasetenResponsesResponse, unknown>
+> {
+  return executeOpenAIResponses(caseName, payload, {
+    ...options,
+    baseURL: BASETEN_HOST,
+    apiKey: process.env.BASETEN_API_KEY,
+  });
+}
+
+export const basetenResponsesExecutor: ProviderExecutor<
+  BasetenResponsesRequest,
+  BasetenResponsesResponse,
+  unknown
+> = {
+  name: "baseten-responses",
+  cases: basetenResponsesCases,
+  execute: executeBasetenResponses,
+  ignoredFields: openaiResponsesExecutor.ignoredFields,
+};

payloads/scripts/providers/baseten.ts

@@ -6,7 +6,7 @@ import {
   getCaseForProvider,
   hasExpectation,
 } from "../../cases";
-import { BASETEN_BASE_URL } from "../../cases/models";
+import { BASETEN_BASE_URL, BASETEN_MODEL } from "../../cases/models";
 import { executeOpenAI, openaiExecutor } from "./openai";
 
 // Baseten serves OSS models behind an OpenAI-compatible chat-completions API, so the
@@ -24,6 +24,19 @@ getCaseNames(allTestCases).forEach((caseName) => {
   const caseData = getCaseForProvider(allTestCases, caseName, "baseten");
   if (caseData) {
     basetenCases[caseName] = caseData;
+    return;
+  }
+
+  const chatCompletionsCase = getCaseForProvider(
+    allTestCases,
+    caseName,
+    "chat-completions"
+  );
+  if (chatCompletionsCase) {
+    basetenCases[caseName] = {
+      ...chatCompletionsCase,
+      model: BASETEN_MODEL,
+    };
   }
 });
 

payloads/scripts/transforms/capture-transforms.ts

@@ -342,9 +342,13 @@ export async function captureTransforms(
             targetModel
           ) as Record<string, unknown>;
 
-          const streamResponse = await callProvider(captureProvider, streamRequest, {
-            stream: true,
-          });
+          const streamResponse = await callProvider(
+            captureProvider,
+            streamRequest,
+            {
+              stream: true,
+            }
+          );
           const chunks = await collectStreamChunks(streamResponse);
 
           writeFileSync(streamingPath, JSON.stringify(chunks, null, 2));