From 42bcd8ec2a1c0cb9acf43ad8800280aff8f28513 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 23 Apr 2026 10:13:42 -0600 Subject: [PATCH 1/4] convert prompt loading unit tests to VCR --- .../braintrust/api/BraintrustApiClient.java | 232 ------------------ .../prompt/BraintrustPromptLoaderTest.java | 172 +++---------- .../java/dev/braintrust/TestHarness.java | 18 -- 3 files changed, 35 insertions(+), 387 deletions(-) diff --git a/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java index 7eaa2ed5..c402dfd4 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -12,7 +12,6 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -492,237 +491,6 @@ private static HttpClient createDefaultHttpClient(BraintrustConfig config) { } } - /** Implementation for test doubling */ - @Slf4j - class InMemoryImpl implements BraintrustApiClient { - private final List organizationAndProjectInfos; - private final Set experiments = - Collections.newSetFromMap(new ConcurrentHashMap<>()); - private final List prompts = new ArrayList<>(); - private final List functions = new ArrayList<>(); - private final Map> - functionInvokers = new ConcurrentHashMap<>(); - - public InMemoryImpl(OrganizationAndProjectInfo... organizationAndProjectInfos) { - this.organizationAndProjectInfos = - new ArrayList<>(List.of(organizationAndProjectInfos)); - } - - public InMemoryImpl( - List organizationAndProjectInfos, - List prompts) { - this.organizationAndProjectInfos = new ArrayList<>(organizationAndProjectInfos); - this.prompts.addAll(prompts); - } - - @Override - public LoginResponse login() { - return new LoginResponse( - organizationAndProjectInfos.stream().map(o -> o.orgInfo).toList()); - } - - @Override - public Project getOrCreateProject(String projectName) { - // Find existing project by name - for (var orgAndProject : organizationAndProjectInfos) { - if (orgAndProject.project().name().equals(projectName)) { - return orgAndProject.project(); - } - } - - // Create new project if not found - var defaultOrgInfo = - organizationAndProjectInfos.isEmpty() - ? new OrganizationInfo("default-org-id", "Default Organization") - : organizationAndProjectInfos.get(0).orgInfo(); - - var newProject = - new Project( - "project-" + UUID.randomUUID().toString(), - projectName, - defaultOrgInfo.id(), - java.time.Instant.now().toString(), - java.time.Instant.now().toString()); - - organizationAndProjectInfos.add( - new OrganizationAndProjectInfo(defaultOrgInfo, newProject)); - return newProject; - } - - @Override - public Optional getProject(String projectId) { - return organizationAndProjectInfos.stream() - .map(OrganizationAndProjectInfo::project) - .filter(project -> project.id().equals(projectId)) - .findFirst(); - } - - @Override - public Experiment getOrCreateExperiment(CreateExperimentRequest request) { - var existing = - experiments.stream() - .filter(exp -> exp.name().equals(request.name())) - .findFirst(); - if (existing.isPresent()) { - return existing.get(); - } - var newExperiment = - new Experiment( - request.name().hashCode() + "", - request.projectId(), - request.name(), - request.description(), - request.tags().orElse(List.of()), - request.metadata().orElse(Map.of()), - "notused", - "notused", - request.datasetId(), - request.datasetVersion()); - experiments.add(newExperiment); - return newExperiment; - } - - @Override - public List listExperiments(String projectId) { - return experiments.stream().filter(exp -> exp.projectId().equals(projectId)).toList(); - } - - @Override - public Optional getProjectAndOrgInfo() { - return organizationAndProjectInfos.isEmpty() - ? Optional.empty() - : Optional.of(organizationAndProjectInfos.get(0)); - } - - @Override - public Optional getProjectAndOrgInfo(String projectId) { - return organizationAndProjectInfos.stream() - .filter(orgAndProject -> orgAndProject.project().id().equals(projectId)) - .findFirst(); - } - - @Override - public OrganizationAndProjectInfo getOrCreateProjectAndOrgInfo(BraintrustConfig config) { - // Get or create project based on config - Project project; - if (config.defaultProjectId().isPresent()) { - var projectId = config.defaultProjectId().get(); - project = - getProject(projectId) - .orElseThrow( - () -> - new ApiException( - "Project with ID '" - + projectId - + "' not found")); - } else if (config.defaultProjectName().isPresent()) { - var projectName = config.defaultProjectName().get(); - project = getOrCreateProject(projectName); - } else { - throw new ApiException( - "Either project ID or project name must be provided in config"); - } - - // Find the organization info for this project - return organizationAndProjectInfos.stream() - .filter(info -> info.project().id().equals(project.id())) - .findFirst() - .orElseThrow( - () -> - new ApiException( - "Unable to find organization for project: " - + project.id())); - } - - @Override - public Optional getPrompt( - @Nonnull String projectName, @Nonnull String slug, @Nullable String version) { - Objects.requireNonNull(projectName, slug); - List matchingPrompts = - prompts.stream() - .filter( - prompt -> { - // Filter by slug if provided - if (slug != null && !slug.isEmpty()) { - if (!prompt.slug().equals(slug)) { - return false; - } - } - - // Filter by project name if provided - if (projectName != null && !projectName.isEmpty()) { - // Find project by name and check if ID matches - Project project = getOrCreateProject(projectName); - if (!prompt.projectId().equals(project.id())) { - return false; - } - } - - // Filter by version if provided - // Note: Version filtering would require additional metadata - // on Prompt - // For now, we'll skip this as Prompt doesn't have a - // version field - - return true; - }) - .toList(); - - if (matchingPrompts.isEmpty()) { - return Optional.empty(); - } - - if (matchingPrompts.size() > 1) { - throw new ApiException( - "Multiple objects found for slug: " - + slug - + ", projectName: " - + projectName); - } - - return Optional.of(matchingPrompts.get(0)); - } - - // Will add dataset support if needed in unit tests (this is unlikely to be needed though) - @Override - public DatasetFetchResponse fetchDatasetEvents( - String datasetId, DatasetFetchRequest request) { - return new DatasetFetchResponse(List.of(), null); - } - - @Override - public Optional getDataset(String datasetId) { - return Optional.empty(); - } - - @Override - public List queryDatasets(String projectName, String datasetName) { - return List.of(); - } - - @Override - public Optional getFunction( - @Nonnull String projectName, @Nonnull String slug, @Nullable String version) { - throw new RuntimeException("will not be invoked"); - } - - @Override - public Optional getFunctionById(@Nonnull String functionId) { - throw new RuntimeException("will not be invoked"); - } - - @Override - public Object invokeFunction( - @Nonnull String functionId, @Nonnull FunctionInvokeRequest request) { - throw new RuntimeException("will not be invoked"); - } - - @Override - public BtqlQueryResponse btqlQuery(@Nonnull String query) { - throw new RuntimeException("will not be invoked"); - } - } - // Request/Response DTOs record CreateProjectRequest(String name) {} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java index 96fa790f..10773ccb 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java @@ -2,181 +2,79 @@ import static org.junit.jupiter.api.Assertions.*; -import dev.braintrust.api.BraintrustApiClient; -import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.TestHarness; import java.util.List; import java.util.Map; -import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class BraintrustPromptLoaderTest { + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } @Test void testLoadPromptBySlug() { - // Create test data - BraintrustApiClient.OrganizationInfo orgInfo = - new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); - BraintrustApiClient.Project project = - new BraintrustApiClient.Project( - "proj-456", "test-project", "org-123", "2025-01-01", "2025-01-01"); - BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = - new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); - - // Create a test prompt - BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); - - // Create in-memory API client with the test prompt - BraintrustApiClient apiClient = - new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); - - // Create config - BraintrustConfig config = - BraintrustConfig.of( - "BRAINTRUST_API_KEY", - "doesntmatter", - "BRAINTRUST_DEFAULT_PROJECT_NAME", - "test-project"); - - // Create loader - BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); - - // Load the prompt - BraintrustPrompt prompt = loader.load("kind-greeter"); - - // Verify the prompt was loaded correctly + BraintrustPromptLoader loader = testHarness.braintrust().promptLoader(); + + BraintrustPrompt prompt = loader.load("kind-greeter-0bd1"); + assertNotNull(prompt); - // Test rendering + // Render with the template variable the prompt expects Map parameters = Map.of("name", "Bob"); List> renderedMessages = prompt.renderMessages(parameters); - assertEquals(2, renderedMessages.size()); - assertEquals("What's up my friend? My name is Bob", renderedMessages.get(1).get("content")); + assertFalse(renderedMessages.isEmpty()); + // The user message should contain the rendered name + boolean nameRendered = + renderedMessages.stream() + .anyMatch( + msg -> { + Object content = msg.get("content"); + return content instanceof String + && ((String) content).contains("Bob"); + }); + assertTrue(nameRendered, "Expected rendered messages to contain the substituted name"); } @Test void testLoadPromptWithDefaults() { - // Create test data - BraintrustApiClient.OrganizationInfo orgInfo = - new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); - BraintrustApiClient.Project project = - new BraintrustApiClient.Project( - "proj-456", "test-project", "org-123", "2025-01-01", "2025-01-01"); - BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = - new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); - - // Create a test prompt - BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); - - // Create in-memory API client with the test prompt - BraintrustApiClient apiClient = - new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); - - // Create config - BraintrustConfig config = - BraintrustConfig.of( - "BRAINTRUST_API_KEY", - "doesntmatter", - "BRAINTRUST_DEFAULT_PROJECT_NAME", - "test-project"); - - // Create loader - BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); - - // Load the prompt with defaults + BraintrustPromptLoader loader = testHarness.braintrust().promptLoader(); + BraintrustPrompt prompt = loader.load( BraintrustPromptLoader.PromptLoadRequest.builder() - .promptSlug("kind-greeter") + .promptSlug("kind-greeter-0bd1") .defaults("max_tokens", "2000", "top_p", "0.95") .build()); + assertNotNull(prompt); + // Verify defaults are applied Map options = prompt.getOptions(); assertEquals("2000", options.get("max_tokens")); assertEquals("0.95", options.get("top_p")); - // Verify original options are preserved - assertEquals("gpt-4o-mini", options.get("model")); - assertEquals(0, options.get("temperature")); + // Verify existing options from the real prompt are preserved (not clobbered by defaults) + assertTrue(options.containsKey("model"), "Expected 'model' option to be present"); } @Test void testLoadPromptWithProjectName() { - // Create test data - BraintrustApiClient.OrganizationInfo orgInfo = - new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); - BraintrustApiClient.Project project = - new BraintrustApiClient.Project( - "proj-456", "my-project", "org-123", "2025-01-01", "2025-01-01"); - BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = - new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); + BraintrustPromptLoader loader = testHarness.braintrust().promptLoader(); - // Create a test prompt - BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); - - // Create in-memory API client with the test prompt - BraintrustApiClient apiClient = - new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); - - // Create config without default project name - BraintrustConfig config = BraintrustConfig.of("BRAINTRUST_API_KEY", "test-key"); - - // Create loader - BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); - - // Load the prompt with explicit project name + // Load the prompt with an explicit project name (not relying on the config default) BraintrustPrompt prompt = loader.load( BraintrustPromptLoader.PromptLoadRequest.builder() - .promptSlug("kind-greeter") - .projectName("my-project") + .promptSlug("kind-greeter-0bd1") + .projectName(TestHarness.defaultProjectName()) .build()); - // Verify the prompt was loaded correctly assertNotNull(prompt); } - - private BraintrustApiClient.Prompt createTestPrompt(String projectId) { - // Create the prompt data structure matching the example JSON - Map messages = - Map.of( - "messages", - List.of( - Map.of( - "role", "system", - "content", - "You are a kind chatbot who briefly greets people"), - Map.of( - "role", "user", - "content", "What's up my friend? My name is {{name}}"))); - - Map options = - Map.of( - "model", "gpt-4o-mini", - "params", - Map.of( - "use_cache", - true, - "temperature", - 0, - "response_format", - Map.of("type", "text")), - "position", "0|hzzzzz:"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - return new BraintrustApiClient.Prompt( - "e2a4fb20-e97e-4e8a-be07-b226d55047b2", - projectId, - "e8d257dd-944c-479a-9916-40a9fa09f120", - "kind-greeter", - "kind-greeter", - Optional.of("A very good boi"), - "2025-10-21T21:35:18.287Z", - promptData, - Optional.empty(), - Optional.empty()); - } } diff --git a/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java b/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java index 5540c06f..d16e1279 100644 --- a/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java +++ b/test-harness/src/testFixtures/java/dev/braintrust/TestHarness.java @@ -2,7 +2,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; -import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.config.BraintrustConfig; import dev.braintrust.trace.UnitTestShutdownHook; import io.opentelemetry.api.GlobalOpenTelemetry; @@ -201,23 +200,6 @@ public List awaitExportedSpans(int minSpanCount) { return spanExporter.getFinishedSpanItems(minSpanCount); } - private static BraintrustApiClient.InMemoryImpl createApiClient() { - var orgInfo = - new dev.braintrust.api.BraintrustApiClient.OrganizationInfo( - defaultOrgId, defaultOrgName); - var project = - new dev.braintrust.api.BraintrustApiClient.Project( - defaultProjectId, - defaultProjectName, - "unit_test_org_123", - "2023-01-01T00:00:00Z", - "2023-01-01T00:00:00Z"); - var orgAndProjectInfo = - new dev.braintrust.api.BraintrustApiClient.OrganizationAndProjectInfo( - orgInfo, project); - return new dev.braintrust.api.BraintrustApiClient.InMemoryImpl(orgAndProjectInfo); - } - public static VCR.VcrMode getVcrMode() { return vcr.getMode(); } From b0a6baf81733f03829db270f44af8dba2a29fe8b Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 27 Apr 2026 18:20:26 -0600 Subject: [PATCH 2/4] re-record cassettes script --- scripts/re-record-cassettes.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/re-record-cassettes.sh diff --git a/scripts/re-record-cassettes.sh b/scripts/re-record-cassettes.sh new file mode 100644 index 00000000..5ed7c6c7 --- /dev/null +++ b/scripts/re-record-cassettes.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +cd "$(dirname "$(readlink -f "${BASH_SOURCE}")")"/.. + +./scripts/erase-cassettes.sh +# recording single threaded to reduce the chances we get rate limited when making real api calls +VCR_MODE=record ./gradlew test --max-workers=1 --fail-fast --rerun +echo "--------- CASSETTE RE-RECORD, RUNNING AGAIN IN REPLAY MODE ---------------" +VCR_MODE=replay ./gradlew test --rerun +echo "--------- CASSETTE RE-RECORD SUCCEEDED ---------------" From 412083218a5ffc2ea2f3113ab3ddbc3ae3be29ae Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 23 Apr 2026 14:47:30 -0600 Subject: [PATCH 3/4] VCR Cassettes --- ...-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json | 1 + ...-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json | 1 + ...-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json | 1 + ...-5123993a-4c98-4545-82ac-a22dcf694ae3.json | 1 + ...-5438e527-d2b4-43a1-8383-747db5ccbf6a.json | 1 + ...-6b5aaa91-2b47-441c-9422-39446735d75a.json | 1 + ...-bd3d4445-f175-4322-afc5-a53a23afc984.json | 1 + ...-75ea3c86-6040-4c71-be59-a94c2d331fc5.json | 1 + ...-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json | 1 + ...-248da837-b799-4ecf-a65b-7bf7dd97df00.json | 1 + ...-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json | 1 + ...-495bc119-c2fa-4cae-b232-2e01fbd113bb.json | 1 + ...-cf0594ff-d735-4132-8018-f724e9b9e198.json | 1 + ...-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json | 1 + ...-68305887-be5b-43d1-836e-f6c3db244ea7.json | 1 + ...-7a58d627-a77c-404b-81ab-73be0d11b4ee.json | 1 + ...-fda39273-bf67-40f5-a605-b33ac6148689.json | 1 + ...-7296feee-0e05-4970-9022-40f0769ac9c9.json | 1 + ...-1340bda9-583d-458a-919b-45b26eaa98c8.json | 1 + ...-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json | 1 + ...-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json | 1 + ...-cf6492a1-6be9-4b96-a736-a29bda548b1b.json | 1 + ...-d2839b64-4112-4372-825c-55c02e2d5f70.json | 1 + ...-13eb2818-5e83-4134-8b7d-001c837413f1.json | 1 + ...-1acc16a3-a0fc-4988-b262-f784185f8ed2.json | 1 + ...-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json | 1 + ...-8df36758-fab1-4ba2-bde7-0fbf9445a346.json | 1 + ...-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json | 32 ++++++++++ ...-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json | 43 +++++++++++++ ...-e02b29a4-8596-404a-8e85-8fe952f8b285.json | 39 ++++++++++++ ...-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json | 46 ++++++++++++++ ...-5123993a-4c98-4545-82ac-a22dcf694ae3.json | 49 +++++++++++++++ ...-5438e527-d2b4-43a1-8383-747db5ccbf6a.json | 47 +++++++++++++++ ...-6b5aaa91-2b47-441c-9422-39446735d75a.json | 46 ++++++++++++++ ...-bd3d4445-f175-4322-afc5-a53a23afc984.json | 47 +++++++++++++++ ...-75ea3c86-6040-4c71-be59-a94c2d331fc5.json | 34 +++++++++++ ...-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json | 35 +++++++++++ ...-248da837-b799-4ecf-a65b-7bf7dd97df00.json | 58 ++++++++++++++++++ ...-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json | 60 +++++++++++++++++++ ...-495bc119-c2fa-4cae-b232-2e01fbd113bb.json | 39 ++++++++++++ ...-cf0594ff-d735-4132-8018-f724e9b9e198.json | 34 +++++++++++ ...-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json | 35 +++++++++++ ...-68305887-be5b-43d1-836e-f6c3db244ea7.json | 37 ++++++++++++ ...-7a58d627-a77c-404b-81ab-73be0d11b4ee.json | 37 ++++++++++++ ...-fda39273-bf67-40f5-a605-b33ac6148689.json | 37 ++++++++++++ ...-7296feee-0e05-4970-9022-40f0769ac9c9.json | 39 ++++++++++++ ...-1340bda9-583d-458a-919b-45b26eaa98c8.json | 35 +++++++++++ ...-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json | 35 +++++++++++ ...-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json | 35 +++++++++++ ...-cf6492a1-6be9-4b96-a736-a29bda548b1b.json | 35 +++++++++++ ...-d2839b64-4112-4372-825c-55c02e2d5f70.json | 34 +++++++++++ ...-13eb2818-5e83-4134-8b7d-001c837413f1.json | 50 ++++++++++++++++ ...-1acc16a3-a0fc-4988-b262-f784185f8ed2.json | 50 ++++++++++++++++ ...-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json | 49 +++++++++++++++ ...-8df36758-fab1-4ba2-bde7-0fbf9445a346.json | 49 +++++++++++++++ 55 files changed, 1193 insertions(+) create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-e02b29a4-8596-404a-8e85-8fe952f8b285.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json new file mode 100644 index 00000000..9a18a482 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json @@ -0,0 +1 @@ +{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json new file mode 100644 index 00000000..74548ab8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json @@ -0,0 +1 @@ +{"data":[{"name":"test-distributed-trace-parent","root_span_id":"b9a23dcec0ce2b71f55862b3b4756256","span_id":"8b7da4fd8da56081","span_parents":null},{"name":"Chat Completion","root_span_id":"b9a23dcec0ce2b71f55862b3b4756256","span_id":"8215eb7b-5ea5-4797-ba66-e7bc8b2893f1","span_parents":["2d3c9e38-3673-45fe-a3c2-ffb453db3ad1"]},{"name":"close-enough-judge","root_span_id":"b9a23dcec0ce2b71f55862b3b4756256","span_id":"2d3c9e38-3673-45fe-a3c2-ffb453db3ad1","span_parents":["8b7da4fd8da56081"]}],"schema":{"type":"array","items":{"type":"object","properties":{"span_id":{"description":"A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing","type":"string"},"span_parents":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}]},"root_span_id":{"description":"A unique identifier for the trace this project logs event belongs to","type":"string"},"name":{"description":"Name of the span, for display purposes only","type":["string","null"]}}}},"cursor":"ae/RuvnnAAA","realtime_state":{"type":"on","minimum_xact_id":null,"read_bytes":5127,"actual_xact_id":"1000197070989755205"},"freshness_state":{"last_processed_xact_id":"1000197070986550252","last_considered_xact_id":"1000197070989755205"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json new file mode 100644 index 00000000..ef248e63 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000196534223020329","created":"2026-01-23T02:07:44.212Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 1 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json new file mode 100644 index 00000000..dafb3caf --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000196534222689290","created":"2026-01-23T02:07:39.293Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 0 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json new file mode 100644 index 00000000..57d61426 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000196533616093054","created":"2026-01-22T23:33:23.674Z","description":null,"function_data":{"type":"prompt"},"function_schema":null,"function_type":"scorer","id":"5dd8a26d-3be8-4ecd-af5b-df2a6a592277","log_id":"p","metadata":null,"name":"close-enough-judge","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"you are an LLM eval scorer. you will evaluate output compared to expected results and return a value between 0.0 and 1.0 (0 is worst 1 is best)"},{"role":"user","content":"how closely does my eval case output: `{{output}}` match the expected result of `{{expected}}`"}]},"options":{"model":"gpt-5-mini","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"close-enough-judge-d31b","tags":null}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json new file mode 100644 index 00000000..57d61426 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000196533616093054","created":"2026-01-22T23:33:23.674Z","description":null,"function_data":{"type":"prompt"},"function_schema":null,"function_type":"scorer","id":"5dd8a26d-3be8-4ecd-af5b-df2a6a592277","log_id":"p","metadata":null,"name":"close-enough-judge","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"you are an LLM eval scorer. you will evaluate output compared to expected results and return a value between 0.0 and 1.0 (0 is worst 1 is best)"},{"role":"user","content":"how closely does my eval case output: `{{output}}` match the expected result of `{{expected}}`"}]},"options":{"model":"gpt-5-mini","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"close-enough-judge-d31b","tags":null}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json new file mode 100644 index 00000000..ef248e63 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000196534223020329","created":"2026-01-23T02:07:44.212Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 1 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json new file mode 100644 index 00000000..2a325916 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json @@ -0,0 +1 @@ +{"_xact_id":"1000196533616093054","created":"2026-01-22T23:33:23.674Z","description":null,"function_data":{"type":"prompt"},"function_schema":null,"function_type":"scorer","id":"5dd8a26d-3be8-4ecd-af5b-df2a6a592277","log_id":"p","metadata":null,"name":"close-enough-judge","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"you are an LLM eval scorer. you will evaluate output compared to expected results and return a value between 0.0 and 1.0 (0 is worst 1 is best)"},{"role":"user","content":"how closely does my eval case output: `{{output}}` match the expected result of `{{expected}}`"}]},"options":{"model":"gpt-5-mini","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"close-enough-judge-d31b","tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json new file mode 100644 index 00000000..2a325916 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json @@ -0,0 +1 @@ +{"_xact_id":"1000196533616093054","created":"2026-01-22T23:33:23.674Z","description":null,"function_data":{"type":"prompt"},"function_schema":null,"function_type":"scorer","id":"5dd8a26d-3be8-4ecd-af5b-df2a6a592277","log_id":"p","metadata":null,"name":"close-enough-judge","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"you are an LLM eval scorer. you will evaluate output compared to expected results and return a value between 0.0 and 1.0 (0 is worst 1 is best)"},{"role":"user","content":"how closely does my eval case output: `{{output}}` match the expected result of `{{expected}}`"}]},"options":{"model":"gpt-5-mini","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"close-enough-judge-d31b","tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json new file mode 100644 index 00000000..8950c4fb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json @@ -0,0 +1 @@ +{"name":"close-enough-judge","score":null,"metadata":{"rationale":"1) Interpret the expected result \"4\" as the numeric value four.\n2) Interpret the output \"four\" as the English word representing the same numeric value.\n3) Compare semantic meanings: both represent the integer 4, so semantically they match exactly.\n4) Note potential evaluation criteria: under normalization that maps words to numeric values (or accepts textual numerals), this is an exact match; under strict character-for-character matching, it would not match.\n5) Given common evaluation practice for numeric answers (allowing textual numerals), rate as a full match.","choice":"1.0"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json new file mode 100644 index 00000000..210eebeb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json @@ -0,0 +1 @@ +{"name":"close-enough-judge","score":null,"metadata":{"rationale":"1) Normalize both outputs for exact comparison: compare characters including case, spacing, and punctuation. 2) Input output is 'hello world'. Expected is 'hello world'. 3) Characters match exactly: same letters, same lowercase, single space between words, no punctuation differences. 4) No extra whitespace or hidden characters visible in the provided strings. 5) Therefore the outputs are identical.","choice":"1.0"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json new file mode 100644 index 00000000..b89d5acb --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json @@ -0,0 +1 @@ +{"_xact_id":"1000196534222689290","created":"2026-01-23T02:07:39.293Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 0 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json new file mode 100644 index 00000000..57dfa53d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json @@ -0,0 +1 @@ +{"_xact_id":"1000196534223020329","created":"2026-01-23T02:07:44.212Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 1 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json new file mode 100644 index 00000000..57dfa53d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json @@ -0,0 +1 @@ +{"_xact_id":"1000196534223020329","created":"2026-01-23T02:07:44.212Z","description":null,"function_data":{"data":{"code":"// Enter handler function that returns a numeric score between 0 and 1,\n// or null to skip scoring\nfunction handler({\n input,\n output,\n expected,\n metadata,\n}: {\n input: any;\n output: any;\n expected: any;\n metadata: Record;\n}): number | null {\n if (expected === null) return null;\n return output === expected ? 1 : 0;\n}","type":"inline","runtime_context":{"runtime":"node","version":"22"}},"type":"code"},"function_schema":null,"function_type":"scorer","id":"efa5f9c3-6ece-4726-a9d6-4ba792980b3f","log_id":"p","metadata":null,"name":"typescript_exact_match","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","origin":null,"project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":null,"slug":"typescriptexactmatch-9e44","tags":null} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json new file mode 100644 index 00000000..0606dcda --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json @@ -0,0 +1 @@ +{"objects":[{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json new file mode 100644 index 00000000..f2e7801e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json @@ -0,0 +1 @@ +{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json new file mode 100644 index 00000000..f2e7801e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json @@ -0,0 +1 @@ +{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json new file mode 100644 index 00000000..f2e7801e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json @@ -0,0 +1 @@ +{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json new file mode 100644 index 00000000..f2e7801e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json @@ -0,0 +1 @@ +{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json new file mode 100644 index 00000000..f2e7801e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json @@ -0,0 +1 @@ +{"id":"6ae68365-7620-4630-921b-bac416634fc8","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"java-unit-test","description":null,"created":"2026-01-21T01:32:52.137Z","deleted_at":null,"user_id":"a5ca7f9c-bf20-40c4-a82b-5c992f6a38f5","settings":{"remote_eval_sources":[{"url":"http://localhost:8301","name":"java-devserver","description":null}]}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json new file mode 100644 index 00000000..4f2fad3c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000197048220868163","created":"2026-04-23T20:44:09.171Z","description":null,"function_type":null,"id":"07961a58-38d9-490d-97c0-6a6ecbb9fa0b","log_id":"p","metadata":null,"name":"kind-greeter","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a nice chatbot who greets people. Personalize your greeting if you know who the person is (for example, give them a specific compliment)"},{"role":"user","content":"Hello! My name is {{name}}"}]},"options":{"model":"gpt-5.4-pro-2026-03-05","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"kind-greeter-0bd1","tags":null,"function_data":{"type":"prompt"}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json new file mode 100644 index 00000000..4f2fad3c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000197048220868163","created":"2026-04-23T20:44:09.171Z","description":null,"function_type":null,"id":"07961a58-38d9-490d-97c0-6a6ecbb9fa0b","log_id":"p","metadata":null,"name":"kind-greeter","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a nice chatbot who greets people. Personalize your greeting if you know who the person is (for example, give them a specific compliment)"},{"role":"user","content":"Hello! My name is {{name}}"}]},"options":{"model":"gpt-5.4-pro-2026-03-05","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"kind-greeter-0bd1","tags":null,"function_data":{"type":"prompt"}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json new file mode 100644 index 00000000..4f2fad3c --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000197048220868163","created":"2026-04-23T20:44:09.171Z","description":null,"function_type":null,"id":"07961a58-38d9-490d-97c0-6a6ecbb9fa0b","log_id":"p","metadata":null,"name":"kind-greeter","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a nice chatbot who greets people. Personalize your greeting if you know who the person is (for example, give them a specific compliment)"},{"role":"user","content":"Hello! My name is {{name}}"}]},"options":{"model":"gpt-5.4-pro-2026-03-05","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"kind-greeter-0bd1","tags":null,"function_data":{"type":"prompt"}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json new file mode 100644 index 00000000..08706d48 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/__files/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json @@ -0,0 +1 @@ +{"objects":[{"_xact_id":"1000197048218409069","created":"2026-04-23T20:43:31.242Z","description":null,"function_type":null,"id":"07961a58-38d9-490d-97c0-6a6ecbb9fa0b","log_id":"p","metadata":null,"name":"kind-greeter","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_id":"6ae68365-7620-4630-921b-bac416634fc8","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"this is an old version"},{"role":"user","content":"Hello! My name is {{name}}"}]},"options":{"model":"gpt-5.4-pro-2026-03-05","params":{"use_cache":true,"temperature":0},"position":"0|hzzzzz:"}},"slug":"kind-greeter-0bd1","tags":null,"function_data":{"type":"prompt"}}]} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json new file mode 100644 index 00000000..d6522e69 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json @@ -0,0 +1,32 @@ +{ + "id" : "3204b2c0-c5cc-4c17-a61d-462e3aca8860", + "name" : "api_apikey_login", + "request" : { + "url" : "/api/apikey/login", + "method" : "POST" + }, + "response" : { + "status" : 200, + "bodyFileName" : "api_apikey_login-3204b2c0-c5cc-4c17-a61d-462e3aca8860.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cVwOqGHUoAMEtRw=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "395", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69ebc72a-36f7176d40dbdc727981b020;Parent=469d3e90ccb339c5;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Fri, 24 Apr 2026 19:40:26 GMT", + "Via" : "1.1 d5e9313fa5148ebdba4664d3e2a90f58.cloudfront.net (CloudFront), 1.1 f8731007efc5ab360d90cee573a1e916.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69ebc72a000000005025767c11047b48", + "x-amzn-RequestId" : "83c4cfb7-cc5d-455b-9ed2-da15b0151765", + "X-Amz-Cf-Id" : "NxipFPEicWOU0GP-N_9a4tY94ivdZ1CEZIGBys3yBeewbHi5pz1odg==", + "etag" : "W/\"18b-OPCBBHzVVuCPaglXVbFjmsFzOoE\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "3204b2c0-c5cc-4c17-a61d-462e3aca8860", + "persistent" : true, + "insertionIndex" : 174 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json new file mode 100644 index 00000000..1ceffaa5 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json @@ -0,0 +1,43 @@ +{ + "id" : "7226efbc-cb9c-4cad-8e8f-41c7081a3ada", + "name" : "btql", + "request" : { + "url" : "/btql", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"query\":\"select: span_id, span_parents, root_span_id, name | from: project_logs('6ae68365-7620-4630-921b-bac416634fc8') | filter: root_span_id = 'b9a23dcec0ce2b71f55862b3b4756256'\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "btql-7226efbc-cb9c-4cad-8e8f-41c7081a3ada.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21XEexoAMEM8g=", + "vary" : "Origin", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "x-bt-brainstore-duration-ms" : "124", + "X-Amzn-Trace-Id" : "Root=1-69efd1bb-5463a6531b302a3f1777cc85;Parent=28ade08e78642cfd;Sampled=0;Lineage=1:24be3d11:0", + "x-bt-api-duration-ms" : "184", + "Date" : "Mon, 27 Apr 2026 21:14:35 GMT", + "Via" : "1.1 c9f68a0c96944962731456040c591f26.cloudfront.net (CloudFront), 1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bb00000000220786a426cd1c6a", + "x-amzn-RequestId" : "4ef5e485-eb33-432f-a3ed-bf51d7a747a5", + "X-Amz-Cf-Id" : "AyVs4Y1LYZ3DWFJS36oCutom8Tya6ft9mIagQ2_LfyoVrhqBvjeqlw==", + "x-bt-cursor" : "ae/RuvnnAAA", + "Content-Type" : "application/json" + } + }, + "uuid" : "7226efbc-cb9c-4cad-8e8f-41c7081a3ada", + "persistent" : true, + "insertionIndex" : 184 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-e02b29a4-8596-404a-8e85-8fe952f8b285.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-e02b29a4-8596-404a-8e85-8fe952f8b285.json new file mode 100644 index 00000000..b5a4d3a7 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-e02b29a4-8596-404a-8e85-8fe952f8b285.json @@ -0,0 +1,39 @@ +{ + "id" : "e02b29a4-8596-404a-8e85-8fe952f8b285", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CuACCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4zLjQtOTczNzYyMQogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBKoAQoYChZkaXN0cmlidXRlZC10cmFjZS10ZXN0EosBChC5oj3OwM4rcfVYYrO0dWJWEgiLfaT9jaVggSoddGVzdC1kaXN0cmlidXRlZC10cmFjZS1wYXJlbnQwATkoFX8aMVOqGEES94FtMVOqGEoyChFicmFpbnRydXN0LnBhcmVudBIdChtwcm9qZWN0X25hbWU6amF2YS11bml0LXRlc3R6AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21SE7oIAMEk5w=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bb-745994e35a3623381ffccf2b;Parent=585ce84ee19052f3;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:35 GMT", + "Via" : "1.1 2be627c4e85d6d9d9e32a7523e1b67ee.cloudfront.net (CloudFront), 1.1 566cc276dff9847158a5a9854be4df42.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bb0000000050b6603e74a2e99e", + "x-amzn-RequestId" : "d42d5ad5-4f0b-4eb9-8aa8-66b65ba38ad3", + "X-Amz-Cf-Id" : "cUnHMzsffp7urYY3bAHn1rWJUVQZmPMQl3hmny5bKLkj9V_xsldXFw==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "e02b29a4-8596-404a-8e85-8fe952f8b285", + "persistent" : true, + "insertionIndex" : 185 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json new file mode 100644 index 00000000..e1a8fd72 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json @@ -0,0 +1,46 @@ +{ + "id" : "396ab1e0-fdf0-49be-9272-b2c7f76148f8", + "name" : "v1_function", + "request" : { + "urlPath" : "/v1/function", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "typescriptexactmatch-9e44" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function-396ab1e0-fdf0-49be-9272-b2c7f76148f8.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf20vHLYoAMEdvQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "913", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b7-0daf62e17fcab617478be816;Parent=380bedd421b924fc;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:31 GMT", + "Via" : "1.1 6e8a2dc71c341eca409d09b148879794.cloudfront.net (CloudFront), 1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b7000000006696f8baa35b11ce", + "x-amzn-RequestId" : "fb8909e6-5745-4c53-bce3-42ac4fc9efb8", + "X-Amz-Cf-Id" : "L7VN4mn9DZZOvcrFAgXuv4L9lquQDLrlZ7OWvtrkyicYeiZkDcl5fg==", + "etag" : "W/\"391-4IdMSFDiLbJA6aTozjlrX+rkZb0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "396ab1e0-fdf0-49be-9272-b2c7f76148f8", + "persistent" : true, + "scenarioName" : "scenario-5-v1-function", + "requiredScenarioState" : "scenario-5-v1-function-2", + "insertionIndex" : 193 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json new file mode 100644 index 00000000..105adfb8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json @@ -0,0 +1,49 @@ +{ + "id" : "5123993a-4c98-4545-82ac-a22dcf694ae3", + "name" : "v1_function", + "request" : { + "urlPath" : "/v1/function", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "typescriptexactmatch-9e44" + } ] + }, + "version" : { + "hasExactly" : [ { + "equalTo" : "485dbf64e486ab3a" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function-5123993a-4c98-4545-82ac-a22dcf694ae3.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21nGZIIAMEJIA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "913", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bd-53e5b60c14d85b5e5cfd3004;Parent=6be553ee07549acc;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:37 GMT", + "Via" : "1.1 c9f68a0c96944962731456040c591f26.cloudfront.net (CloudFront), 1.1 7605973575a3551426b82751020317de.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bd0000000067597a48c1bf8e6f", + "x-amzn-RequestId" : "0709d6a0-e3e0-42dc-9e2b-ed284bf7f11b", + "X-Amz-Cf-Id" : "ma1abVDBXuwJTGbeb_T7Ihi5IRjMgv8PKSDvSVWp2mO17iYGWg8llw==", + "etag" : "W/\"391-vrxv0fDXohEdbOAs6RR5DJkbuUk\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "5123993a-4c98-4545-82ac-a22dcf694ae3", + "persistent" : true, + "insertionIndex" : 179 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json new file mode 100644 index 00000000..6edc62e2 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json @@ -0,0 +1,47 @@ +{ + "id" : "5438e527-d2b4-43a1-8383-747db5ccbf6a", + "name" : "v1_function", + "request" : { + "urlPath" : "/v1/function", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "close-enough-judge-d31b" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function-5438e527-d2b4-43a1-8383-747db5ccbf6a.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21AFoioAMEjzQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "970", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b9-0159bbf53cfb060e111062ce;Parent=259732aaef093e4f;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:33 GMT", + "Via" : "1.1 b7d7903ada432685f0e90f0ca261d864.cloudfront.net (CloudFront), 1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b900000000094ba4c84c95b0b4", + "x-amzn-RequestId" : "974adb2d-d7df-4051-b8f3-31e960dbfce9", + "X-Amz-Cf-Id" : "Fk1OFNwkQCVL23TYn_BpRY0LhE6gKO0e9MU6dSQCBbn5ly99jJ4Erg==", + "etag" : "W/\"3ca-ANtfbcIxEbUbPmvPJDRR8NeqhN0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "5438e527-d2b4-43a1-8383-747db5ccbf6a", + "persistent" : true, + "scenarioName" : "scenario-3-v1-function", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-3-v1-function-2", + "insertionIndex" : 189 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json new file mode 100644 index 00000000..03165a7a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json @@ -0,0 +1,46 @@ +{ + "id" : "6b5aaa91-2b47-441c-9422-39446735d75a", + "name" : "v1_function", + "request" : { + "urlPath" : "/v1/function", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "close-enough-judge-d31b" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function-6b5aaa91-2b47-441c-9422-39446735d75a.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21cE-jIAMEGEA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "970", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bc-6c817ea07b8ede603e32c2e5;Parent=6127a7ea2125edbb;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:36 GMT", + "Via" : "1.1 ec62626c4e205f1980b4ed65142b10d8.cloudfront.net (CloudFront), 1.1 e6b2537b87653726af8a79e6da505188.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bc00000000459a950a12483874", + "x-amzn-RequestId" : "b57d88a8-8b7e-41e6-af94-6100baf6dfc7", + "X-Amz-Cf-Id" : "I58DyGNE0CV01vgD-iqTi9CDmP_DiQTmngNcrrRLodrlf3INiW4Ffg==", + "etag" : "W/\"3ca-ANtfbcIxEbUbPmvPJDRR8NeqhN0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "6b5aaa91-2b47-441c-9422-39446735d75a", + "persistent" : true, + "scenarioName" : "scenario-3-v1-function", + "requiredScenarioState" : "scenario-3-v1-function-2", + "insertionIndex" : 183 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json new file mode 100644 index 00000000..3bba24f0 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json @@ -0,0 +1,47 @@ +{ + "id" : "bd3d4445-f175-4322-afc5-a53a23afc984", + "name" : "v1_function", + "request" : { + "urlPath" : "/v1/function", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "typescriptexactmatch-9e44" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function-bd3d4445-f175-4322-afc5-a53a23afc984.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf20gF8EoAMEklA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "913", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b6-3d8604d24e8458267192740b;Parent=259cd8c674eb8c6c;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:30 GMT", + "Via" : "1.1 ec62626c4e205f1980b4ed65142b10d8.cloudfront.net (CloudFront), 1.1 a624be98cd5b264f373d8ac17f78ee50.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b6000000001e17b0719862845a", + "x-amzn-RequestId" : "07d4586b-736b-420d-acb3-ec6e08252630", + "X-Amz-Cf-Id" : "iJQ4CihfOJYhn78txNdFocWnIhQP6L2tV7tbwJyyO_tgijjMIxyGJQ==", + "etag" : "W/\"391-4IdMSFDiLbJA6aTozjlrX+rkZb0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "bd3d4445-f175-4322-afc5-a53a23afc984", + "persistent" : true, + "scenarioName" : "scenario-5-v1-function", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-5-v1-function-2", + "insertionIndex" : 197 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json new file mode 100644 index 00000000..576658c3 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json @@ -0,0 +1,34 @@ +{ + "id" : "75ea3c86-6040-4c71-be59-a94c2d331fc5", + "name" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "request" : { + "url" : "/v1/function/5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-75ea3c86-6040-4c71-be59-a94c2d331fc5.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21eEM-oAMEogQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "956", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bc-2e49a7914800ec34004fc014;Parent=5c5645de9b05020f;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:36 GMT", + "Via" : "1.1 21c7c4234f218bb5110262cbbf01f870.cloudfront.net (CloudFront), 1.1 d525041695bdb6325f78ebba5c11b8a2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bc000000000ff1eff3aa1448ae", + "x-amzn-RequestId" : "7f48bbe5-6246-4ae4-912d-973250081a41", + "X-Amz-Cf-Id" : "Ds_kXaqsEJ5CGyHPm992yvLwt3ISS3BkOmuc9dH9I_2L_eogTAovNw==", + "etag" : "W/\"3bc-UGBSdA08jhNItgs+xf46hapioUQ\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "75ea3c86-6040-4c71-be59-a94c2d331fc5", + "persistent" : true, + "scenarioName" : "scenario-2-v1-function-5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "requiredScenarioState" : "scenario-2-v1-function-5dd8a26d-3be8-4ecd-af5b-df2a6a592277-2", + "insertionIndex" : 182 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json new file mode 100644 index 00000000..b33d0c33 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json @@ -0,0 +1,35 @@ +{ + "id" : "f1da46d8-54f4-4d1c-b935-9771fde0eae2", + "name" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "request" : { + "url" : "/v1/function/5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277-f1da46d8-54f4-4d1c-b935-9771fde0eae2.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21CHVDIAMEfEw=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "956", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b9-37f0625f039e89144c71702f;Parent=258246a97a2af694;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:33 GMT", + "Via" : "1.1 b7d7903ada432685f0e90f0ca261d864.cloudfront.net (CloudFront), 1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b90000000024b03783fe708929", + "x-amzn-RequestId" : "5f81e9bf-aea9-4405-a708-85511e1a0455", + "X-Amz-Cf-Id" : "qsbYw3JqwNXTfwq4i93UlWoZsuzlVNo7GqoxAF7SqIzh5L5Gk1K-wQ==", + "etag" : "W/\"3bc-UGBSdA08jhNItgs+xf46hapioUQ\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "f1da46d8-54f4-4d1c-b935-9771fde0eae2", + "persistent" : true, + "scenarioName" : "scenario-2-v1-function-5dd8a26d-3be8-4ecd-af5b-df2a6a592277", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-2-v1-function-5dd8a26d-3be8-4ecd-af5b-df2a6a592277-2", + "insertionIndex" : 188 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json new file mode 100644 index 00000000..bec1953a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json @@ -0,0 +1,58 @@ +{ + "id" : "248da837-b799-4ecf-a65b-7bf7dd97df00", + "name" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke", + "request" : { + "url" : "/v1/function/5dd8a26d-3be8-4ecd-af5b-df2a6a592277/invoke", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":{\"input\":\"What is 2+2?\",\"output\":\"four\",\"expected\":\"4\",\"metadata\":{}},\"expected\":null,\"messages\":[],\"mcp_auth\":{}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-248da837-b799-4ecf-a65b-7bf7dd97df00.json", + "headers" : { + "x-bt-function-creds-cached" : "HIT", + "x-ratelimit-reset-tokens" : "0s", + "set-cookie" : "__cf_bm=8OK_SVyUNhg2T42._GBVSH4jUXr3xDjzWYRdmlbZ0UY-1777322431.3429518-1.0.1.1-P1HNGzR37AeKZdkAKctEjdZGVOCHKE7mwPT5cLcqGH1C05jFFklKJcMZfeoJufLEu3dl.QcbDEYFoU2PoRlDjRee_JOFi4od2teow6fVSexOk8HkT9CHXwE3I8XgaRrA; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 27 Apr 2026 21:10:42 GMT", + "x-amzn-RequestId" : "4a58091d-32b0-4c74-8a85-0d43cc99230d", + "X-Amz-Cf-Id" : "rs5AnO4pKGHetMHsZSh34ZHAP8UYglQ9KcrzSfCintnqyApDSQbVMA==", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "cache-control" : "max-age=604800, no-transform", + "Age" : "2032", + "Content-Type" : "application/json", + "x-request-id" : "req_89708e7f0f884200a0bdfcb688d5f419", + "X-Cache" : "Miss from cloudfront", + "x-ratelimit-limit-tokens" : "180000000", + "openai-organization" : "braintrust-data", + "cf-ray" : "9f30a48bec55e5fb-IAD", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "179999940", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "x-ratelimit-remaining-requests" : "29999", + "x-bt-cached" : "HIT", + "X-Amzn-Trace-Id" : "Root=1-69efd1bc-03b93aa914c62e114d7ab78f;Parent=39caff63d39e69e9;Sampled=0;Lineage=1:8be8f50d:0", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Mon, 27 Apr 2026 21:14:36 GMT", + "Via" : "1.1 82fa7f20ab5a12301da8e01f9493e222.cloudfront.net (CloudFront)", + "x-content-type-options" : "nosniff", + "x-ratelimit-limit-requests" : "30000", + "x-bt-used-endpoint" : "OPENAI_API_KEY", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "10946", + "x-bt-function-meta-cached" : "HIT" + } + }, + "uuid" : "248da837-b799-4ecf-a65b-7bf7dd97df00", + "persistent" : true, + "insertionIndex" : 180 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json new file mode 100644 index 00000000..8c089e24 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json @@ -0,0 +1,60 @@ +{ + "id" : "f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940", + "name" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke", + "request" : { + "url" : "/v1/function/5dd8a26d-3be8-4ecd-af5b-df2a6a592277/invoke", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":{\"input\":\"test input\",\"output\":\"hello world\",\"expected\":\"hello world\",\"metadata\":{}},\"expected\":null,\"messages\":[],\"parent\":{\"object_type\":\"project_logs\",\"object_id\":\"6ae68365-7620-4630-921b-bac416634fc8\",\"row_ids\":{\"id\":\"otel\",\"span_id\":\"8b7da4fd8da56081\",\"root_span_id\":\"b9a23dcec0ce2b71f55862b3b4756256\"}},\"mcp_auth\":{}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_5dd8a26d-3be8-4ecd-af5b-df2a6a592277_invoke-f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940.json", + "headers" : { + "x-bt-function-creds-cached" : "HIT", + "x-ratelimit-reset-tokens" : "0s", + "set-cookie" : "__cf_bm=GWaC.ZAvffJV8k8Pds2XqjgwmUJhGg.VQiDlndFCQTQ-1777322418.8238926-1.0.1.1-lfxU1U0GArJZU_G4WsfQVQAIihnaQ6vfx50ln5tRvWQqeBTk2tTW2UTWAPQcwPbLoGavIAZGVSLfPROgg2AADsRagDGD0lnIxwcraIDWKv7KkQCpAwDS4zWARzwqehM9; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Mon, 27 Apr 2026 21:10:27 GMT", + "x-amzn-RequestId" : "cbe8b068-11c6-4e3a-b6e1-61fd8b049164", + "X-Amz-Cf-Id" : "Gkrbtn8PDnnwlXrzyHUrHFILXConsh-deU9JjRserStI0H35fiCdqw==", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "cache-control" : "max-age=604800, no-transform", + "Age" : "2045", + "Content-Type" : "application/json", + "x-request-id" : "req_173ae2db058f4e3599d3ab7afaf5e852", + "X-Cache" : "Miss from cloudfront", + "x-ratelimit-limit-tokens" : "180000000", + "openai-organization" : "braintrust-data", + "cf-ray" : "9f30a43d9c1d058a-IAD", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "179999937", + "x-openai-proxy-wasm" : "v0.1", + "cf-cache-status" : "DYNAMIC", + "x-ratelimit-remaining-requests" : "29999", + "x-bt-cached" : "HIT", + "X-Amzn-Trace-Id" : "Root=1-69efd1ba-71d7b60e72d2501d18c58787;Parent=167ecf1c142720bd;Sampled=0;Lineage=1:8be8f50d:0", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Mon, 27 Apr 2026 21:14:34 GMT", + "Via" : "1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "x-bt-span-export" : "AwIDAWrmg2V2IEYwkhu6xBZjT8gCdrLH1FrzRjiaXTAWKzQCtwMtPJ44NnNF/qPC/7RT2zrReyJwcm9wYWdhdGVkX2V2ZW50Ijp7InNwYW5fYXR0cmlidXRlcyI6eyJwdXJwb3NlIjoic2NvcmVyIn19LCJyb290X3NwYW5faWQiOiJiOWEyM2RjZWMwY2UyYjcxZjU1ODYyYjNiNDc1NjI1NiJ9", + "x-content-type-options" : "nosniff", + "x-ratelimit-limit-requests" : "30000", + "x-bt-used-endpoint" : "OPENAI_API_KEY", + "openai-version" : "2020-10-01", + "x-bt-span-id" : "76b2c7d4-5af3-4638-9a5d-30162b3402b7", + "openai-processing-ms" : "7802", + "x-bt-function-meta-cached" : "HIT" + } + }, + "uuid" : "f17e8d2e-c4cf-43d4-88f4-eab4e2b6c940", + "persistent" : true, + "insertionIndex" : 186 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json new file mode 100644 index 00000000..7dc8c708 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json @@ -0,0 +1,39 @@ +{ + "id" : "495bc119-c2fa-4cae-b232-2e01fbd113bb", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "request" : { + "urlPath" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "method" : "GET", + "queryParameters" : { + "version" : { + "hasExactly" : [ { + "equalTo" : "485dbf64e486ab3a" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-495bc119-c2fa-4cae-b232-2e01fbd113bb.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21pGU9IAMEBjg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "899", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bd-3740700977e8e4a0670bd96a;Parent=26481e0a317be0a8;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:37 GMT", + "Via" : "1.1 b7e07d6a19a4c8b2e410e9c1e173548c.cloudfront.net (CloudFront), 1.1 74e8c76139b8c7f9b11d5e4441c2a7a2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bd000000003d8c2e975136ca31", + "x-amzn-RequestId" : "09db42d1-fe38-44e1-a8f5-9bdab21c98ed", + "X-Amz-Cf-Id" : "PDRwq-rIA4yv9pUtO290w3bXgqSViGiJjyU4P_TQQzmkQinpNzji5g==", + "etag" : "W/\"383-sYtMtBE2cmhUnNBflmzR5Npx7l4\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "495bc119-c2fa-4cae-b232-2e01fbd113bb", + "persistent" : true, + "insertionIndex" : 178 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json new file mode 100644 index 00000000..bbf94286 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json @@ -0,0 +1,34 @@ +{ + "id" : "cf0594ff-d735-4132-8018-f724e9b9e198", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "request" : { + "url" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-cf0594ff-d735-4132-8018-f724e9b9e198.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf20yEj5IAMEq9g=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "899", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b7-17c263da4e7258ca3939a610;Parent=07712d0dc48e68fe;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:31 GMT", + "Via" : "1.1 ec62626c4e205f1980b4ed65142b10d8.cloudfront.net (CloudFront), 1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b70000000042f71afad23cbab7", + "x-amzn-RequestId" : "9699792c-9fed-4ab8-a53b-fa3a8ec3b7ec", + "X-Amz-Cf-Id" : "iuoscR1IS-xuNHEKrgr58Z9ap3iRVYbRAIzGCEJvD6cj2rHU_S1kBg==", + "etag" : "W/\"383-eqgMNM8cWKXgXo25pRADmmOpkas\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "cf0594ff-d735-4132-8018-f724e9b9e198", + "persistent" : true, + "scenarioName" : "scenario-4-v1-function-efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "requiredScenarioState" : "scenario-4-v1-function-efa5f9c3-6ece-4726-a9d6-4ba792980b3f-2", + "insertionIndex" : 192 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json new file mode 100644 index 00000000..80a5b42d --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json @@ -0,0 +1,35 @@ +{ + "id" : "e87036b7-4e64-4cfa-8536-60e71b04d5f8", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "request" : { + "url" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f-e87036b7-4e64-4cfa-8536-60e71b04d5f8.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf20kHbjIAMEAHg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "899", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b6-483b2665609a589878763563;Parent=5d35348019a1b9e5;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:30 GMT", + "Via" : "1.1 b7e07d6a19a4c8b2e410e9c1e173548c.cloudfront.net (CloudFront), 1.1 dbfd9bcc806d4c322e72b461b2458112.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b60000000027bf44ac70eaeec6", + "x-amzn-RequestId" : "83a016e1-9f3c-4976-a31a-1e29d7adda54", + "X-Amz-Cf-Id" : "XfjRvF0lmIk33ZuNmaJ-pNA2-h2LJ-7-NQiWsg0_tCCUrYOp8qA6YQ==", + "etag" : "W/\"383-eqgMNM8cWKXgXo25pRADmmOpkas\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "e87036b7-4e64-4cfa-8536-60e71b04d5f8", + "persistent" : true, + "scenarioName" : "scenario-4-v1-function-efa5f9c3-6ece-4726-a9d6-4ba792980b3f", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-4-v1-function-efa5f9c3-6ece-4726-a9d6-4ba792980b3f-2", + "insertionIndex" : 196 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json new file mode 100644 index 00000000..057bb507 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json @@ -0,0 +1,37 @@ +{ + "id" : "68305887-be5b-43d1-836e-f6c3db244ea7", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke", + "request" : { + "url" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f/invoke", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":{\"input\":\"test input\",\"output\":\"hello world\",\"expected\":\"hello world\",\"metadata\":{}},\"expected\":null,\"messages\":[],\"mcp_auth\":{}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-68305887-be5b-43d1-836e-f6c3db244ea7.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-amzn-RequestId" : "4fb52d82-e4b1-449d-8382-c03a94fe117e", + "X-Amz-Cf-Id" : "pYGtUa5mk-Xr0PLDE4mtDBxHfG07fYkwvrkeaYuwyRGf9DNT96xVYA==", + "x-bt-function-creds-cached" : "HIT", + "x-bt-function-meta-cached" : "HIT", + "X-Amzn-Trace-Id" : "Root=1-69efd1b8-17dc54ad1aea718e1856193f;Parent=68f748ddda43869d;Sampled=0;Lineage=1:8be8f50d:0", + "Date" : "Mon, 27 Apr 2026 21:14:32 GMT", + "Via" : "1.1 ddea1c07643e5e0bfceb34480eebdc52.cloudfront.net (CloudFront)", + "Content-Type" : "application/json" + } + }, + "uuid" : "68305887-be5b-43d1-836e-f6c3db244ea7", + "persistent" : true, + "insertionIndex" : 190 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json new file mode 100644 index 00000000..53f4f6b2 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json @@ -0,0 +1,37 @@ +{ + "id" : "7a58d627-a77c-404b-81ab-73be0d11b4ee", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke", + "request" : { + "url" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f/invoke", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":{\"input\":\"test input\",\"output\":\"different\",\"expected\":\"expected\",\"metadata\":{}},\"expected\":null,\"messages\":[],\"mcp_auth\":{}}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-7a58d627-a77c-404b-81ab-73be0d11b4ee.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-amzn-RequestId" : "cae520a2-3a93-434b-91ce-10975a7eaed1", + "X-Amz-Cf-Id" : "krq2nAkOaTsy99W_vkRkK8dppgE-mHAJs5eZrMyzu2K6hsKFdHJ6bw==", + "x-bt-function-creds-cached" : "HIT", + "x-bt-function-meta-cached" : "HIT", + "X-Amzn-Trace-Id" : "Root=1-69efd1b7-1d113c084e666bf169d168bc;Parent=23f1393994d42fc9;Sampled=0;Lineage=1:8be8f50d:0", + "Date" : "Mon, 27 Apr 2026 21:14:31 GMT", + "Via" : "1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "Content-Type" : "application/json" + } + }, + "uuid" : "7a58d627-a77c-404b-81ab-73be0d11b4ee", + "persistent" : true, + "insertionIndex" : 194 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json new file mode 100644 index 00000000..84f988a1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json @@ -0,0 +1,37 @@ +{ + "id" : "fda39273-bf67-40f5-a605-b33ac6148689", + "name" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke", + "request" : { + "url" : "/v1/function/efa5f9c3-6ece-4726-a9d6-4ba792980b3f/invoke", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":{\"input\":\"test input\",\"output\":\"hello world\",\"expected\":\"hello world\",\"metadata\":{}},\"expected\":null,\"messages\":[],\"mcp_auth\":{},\"version\":\"485dbf64e486ab3a\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_function_efa5f9c3-6ece-4726-a9d6-4ba792980b3f_invoke-fda39273-bf67-40f5-a605-b33ac6148689.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "X-Amz-Cf-Pop" : "SEA900-P10", + "x-amzn-RequestId" : "0f66691f-826e-4439-8e8b-4da7c11b414e", + "X-Amz-Cf-Id" : "KPmi-9bTvBU1VNgJQxHCTF99N0hdQdPC2xvWeIcQi7TTO3Y2R-krcw==", + "x-bt-function-creds-cached" : "HIT", + "x-bt-function-meta-cached" : "HIT", + "X-Amzn-Trace-Id" : "Root=1-69efd1be-63a2361f541068861add4933;Parent=1e42da4301849fdb;Sampled=0;Lineage=1:8be8f50d:0", + "Date" : "Mon, 27 Apr 2026 21:14:38 GMT", + "Via" : "1.1 a40ac7dad0e348fc93799233c9af5960.cloudfront.net (CloudFront)", + "Content-Type" : "application/json" + } + }, + "uuid" : "fda39273-bf67-40f5-a605-b33ac6148689", + "persistent" : true, + "insertionIndex" : 176 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json new file mode 100644 index 00000000..944ddf3b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json @@ -0,0 +1,39 @@ +{ + "id" : "7296feee-0e05-4970-9022-40f0769ac9c9", + "name" : "v1_project", + "request" : { + "urlPath" : "/v1/project", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project-7296feee-0e05-4970-9022-40f0769ac9c9.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cVwOnHnhoAMEExA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "366", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69ebc729-147d2b5d5b24b8d649937ee8;Parent=1d2a896c1ad82339;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Fri, 24 Apr 2026 19:40:26 GMT", + "Via" : "1.1 db84db36e16ca0c80b0992006d731900.cloudfront.net (CloudFront), 1.1 e6b2537b87653726af8a79e6da505188.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69ebc72900000000100f48dcde634bd9", + "x-amzn-RequestId" : "9940c25e-19f0-42a7-84c5-9740a90a50c7", + "X-Amz-Cf-Id" : "C8XrMlIlzeSNDoxModfFzEu0yiX98beOoo2DukQEHS0Eb1R86PG0qA==", + "etag" : "W/\"16e-7t9aAadD2qUSZTBI6wB6/kY8xEc\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "7296feee-0e05-4970-9022-40f0769ac9c9", + "persistent" : true, + "insertionIndex" : 175 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json new file mode 100644 index 00000000..9b0b4801 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json @@ -0,0 +1,35 @@ +{ + "id" : "1340bda9-583d-458a-919b-45b26eaa98c8", + "name" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8", + "request" : { + "url" : "/v1/project/6ae68365-7620-4630-921b-bac416634fc8", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8-1340bda9-583d-458a-919b-45b26eaa98c8.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21gG7qIAMEiIg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "352", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bc-7d1034033f248b784fb34ebb;Parent=28ce9c4f81793639;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:36 GMT", + "Via" : "1.1 6e8a2dc71c341eca409d09b148879794.cloudfront.net (CloudFront), 1.1 a53bab1af200813b8f27e3c0a28b4964.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bc000000003f0e63a7268ddc4a", + "x-amzn-RequestId" : "dec5ecd3-fca0-466f-b8bc-9031fdfa9ef2", + "X-Amz-Cf-Id" : "Qh34Br3Frslh0NUNG46ooHa7vJs942izbPyzIeL6re7jlkWHi5DKug==", + "etag" : "W/\"160-RUdtZR434qrXxI13debDoTK1a2s\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1340bda9-583d-458a-919b-45b26eaa98c8", + "persistent" : true, + "scenarioName" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8", + "requiredScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-4", + "newScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-5", + "insertionIndex" : 181 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json new file mode 100644 index 00000000..40942df3 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json @@ -0,0 +1,35 @@ +{ + "id" : "1a7e1e33-9d68-49ec-bddb-9b11271ede05", + "name" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8", + "request" : { + "url" : "/v1/project/6ae68365-7620-4630-921b-bac416634fc8", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8-1a7e1e33-9d68-49ec-bddb-9b11271ede05.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf20mF2moAMEnQg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "352", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b6-68129de86090fc6c1b64ade3;Parent=0a2a0730646145ee;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:30 GMT", + "Via" : "1.1 b7d7903ada432685f0e90f0ca261d864.cloudfront.net (CloudFront), 1.1 83d24992402f7b214901ab76fbdc11e2.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b600000000376d68ce5ca47a96", + "x-amzn-RequestId" : "bb7901a1-8a5f-468e-b741-739cf538b6f9", + "X-Amz-Cf-Id" : "NTl8IELUESGOuuS9vwuR-GL4Htp7VVfebWADMm2HL2Y8CYCV3k3MiA==", + "etag" : "W/\"160-RUdtZR434qrXxI13debDoTK1a2s\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1a7e1e33-9d68-49ec-bddb-9b11271ede05", + "persistent" : true, + "scenarioName" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-2", + "insertionIndex" : 195 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json new file mode 100644 index 00000000..3cade664 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json @@ -0,0 +1,35 @@ +{ + "id" : "5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416", + "name" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8", + "request" : { + "url" : "/v1/project/6ae68365-7620-4630-921b-bac416634fc8", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8-5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21EHsvoAMEpIQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "352", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b9-6426b5300c2b792b240de41e;Parent=0c1ffee05e202ede;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:33 GMT", + "Via" : "1.1 59e4792b9d6184bfa491a317b36590d2.cloudfront.net (CloudFront), 1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b90000000033713c3139d0ce3d", + "x-amzn-RequestId" : "efffcde0-3e1a-41ef-a78e-1e17a08ba4f0", + "X-Amz-Cf-Id" : "cm4safc40CLNj-j_KmeM_E_1T49oZtM8Q0olVe8uE1EsUStKCq2zIw==", + "etag" : "W/\"160-RUdtZR434qrXxI13debDoTK1a2s\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "5d59e8ba-2ae6-4bdb-a6e2-4759c4c10416", + "persistent" : true, + "scenarioName" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8", + "requiredScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-3", + "newScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-4", + "insertionIndex" : 187 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json new file mode 100644 index 00000000..00306452 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json @@ -0,0 +1,35 @@ +{ + "id" : "cf6492a1-6be9-4b96-a736-a29bda548b1b", + "name" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8", + "request" : { + "url" : "/v1/project/6ae68365-7620-4630-921b-bac416634fc8", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8-cf6492a1-6be9-4b96-a736-a29bda548b1b.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf202EriIAMEYgQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "352", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1b8-0ba7eecb2e8860735f21bc29;Parent=1f47f796b8ec7bad;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:32 GMT", + "Via" : "1.1 ec62626c4e205f1980b4ed65142b10d8.cloudfront.net (CloudFront), 1.1 b3ccaedda78c63d5967b57382ceb4cbe.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1b8000000001a13ac3b43adf6a6", + "x-amzn-RequestId" : "7113c257-aea0-450d-b1de-6398d8f5e5d9", + "X-Amz-Cf-Id" : "fn2Tf_YvTIi5Knko4q0Y-BjNfDbVKoXSHfxyMg8Ayp2bOURA9Dd1VQ==", + "etag" : "W/\"160-RUdtZR434qrXxI13debDoTK1a2s\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "cf6492a1-6be9-4b96-a736-a29bda548b1b", + "persistent" : true, + "scenarioName" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8", + "requiredScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-2", + "newScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-3", + "insertionIndex" : 191 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json new file mode 100644 index 00000000..8d967bf1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json @@ -0,0 +1,34 @@ +{ + "id" : "d2839b64-4112-4372-825c-55c02e2d5f70", + "name" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8", + "request" : { + "url" : "/v1/project/6ae68365-7620-4630-921b-bac416634fc8", + "method" : "GET" + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_project_6ae68365-7620-4630-921b-bac416634fc8-d2839b64-4112-4372-825c-55c02e2d5f70.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cf21rGxZoAMEujA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "352", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69efd1bd-0999c2d87e2d22ea3b665e44;Parent=059fad7cfbc20292;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Mon, 27 Apr 2026 21:14:37 GMT", + "Via" : "1.1 b521abc69f4dd055f355de798c5fb95a.cloudfront.net (CloudFront), 1.1 65f2e9f7f1475de54aa452d3ceb9bcf6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69efd1bd00000000637602bdc1e3b93c", + "x-amzn-RequestId" : "220c220e-a47a-445d-a435-c4e9210476cc", + "X-Amz-Cf-Id" : "9wXC5ObrhL99JZ0CwD9-8Aoja9MYCugR8ty_pjiS0St7HfQhXYWVhQ==", + "etag" : "W/\"160-RUdtZR434qrXxI13debDoTK1a2s\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "d2839b64-4112-4372-825c-55c02e2d5f70", + "persistent" : true, + "scenarioName" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8", + "requiredScenarioState" : "scenario-1-v1-project-6ae68365-7620-4630-921b-bac416634fc8-5", + "insertionIndex" : 177 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json new file mode 100644 index 00000000..022c424b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json @@ -0,0 +1,50 @@ +{ + "id" : "13eb2818-5e83-4134-8b7d-001c837413f1", + "name" : "v1_prompt", + "request" : { + "urlPath" : "/v1/prompt", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "kind-greeter-0bd1" + } ] + }, + "version" : { + "absent" : true + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_prompt-13eb2818-5e83-4134-8b7d-001c837413f1.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cgS5_EqZIAMEQmg=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "789", + "X-Amz-Cf-Pop" : [ "HIO52-P2", "HIO52-P4" ], + "X-Amzn-Trace-Id" : "Root=1-69effea5-5f5af18a507262ee452ac2f4;Parent=5ea7c9c37f40b76d;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 28 Apr 2026 00:26:13 GMT", + "Via" : "1.1 049ca50de603d43d8c9d0f7716efb414.cloudfront.net (CloudFront), 1.1 c5e1a6561d8dc3977e11160718fc75e8.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69effea5000000000e031b835eb3092a", + "x-amzn-RequestId" : "cbd784f6-5e62-4de6-95d4-2ecd8e91975c", + "X-Amz-Cf-Id" : "VV0Tv5b6EBFKm8Uu8Nig_IE4Lqm7sRlEtj3nkyn8Sn8nnbqmrO993Q==", + "etag" : "W/\"315-aYopezw9/FDum2wzo3Z28/ESIc0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "13eb2818-5e83-4134-8b7d-001c837413f1", + "persistent" : true, + "scenarioName" : "scenario-1-v1-prompt", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-v1-prompt-2", + "insertionIndex" : 196 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json new file mode 100644 index 00000000..c54f1409 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json @@ -0,0 +1,50 @@ +{ + "id" : "1acc16a3-a0fc-4988-b262-f784185f8ed2", + "name" : "v1_prompt", + "request" : { + "urlPath" : "/v1/prompt", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "kind-greeter-0bd1" + } ] + }, + "version" : { + "absent" : true + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_prompt-1acc16a3-a0fc-4988-b262-f784185f8ed2.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cgS6DHM3oAMEaRw=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "789", + "X-Amz-Cf-Pop" : [ "HIO52-P2", "HIO52-P4" ], + "X-Amzn-Trace-Id" : "Root=1-69effea6-6e9ae7de366c803f757c9054;Parent=2db56e8e5b18e089;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 28 Apr 2026 00:26:14 GMT", + "Via" : "1.1 f9cbfbc3568832d017c09dbd4649932c.cloudfront.net (CloudFront), 1.1 2d69093e294db929b26be80ccee94472.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69effea600000000113093f962a3c3d0", + "x-amzn-RequestId" : "cd27c1e7-56ad-494e-83a1-6a474ed573f4", + "X-Amz-Cf-Id" : "g48ZPADUHGFiqYTWaL663lYhgbnpbVExvg2zSwfAF2FEBVcdCLELCw==", + "etag" : "W/\"315-aYopezw9/FDum2wzo3Z28/ESIc0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "1acc16a3-a0fc-4988-b262-f784185f8ed2", + "persistent" : true, + "scenarioName" : "scenario-1-v1-prompt", + "requiredScenarioState" : "scenario-1-v1-prompt-2", + "newScenarioState" : "scenario-1-v1-prompt-3", + "insertionIndex" : 195 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json new file mode 100644 index 00000000..dbcdac68 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json @@ -0,0 +1,49 @@ +{ + "id" : "593a4a8a-e5c7-4792-9bb0-3af938b32fa3", + "name" : "v1_prompt", + "request" : { + "urlPath" : "/v1/prompt", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "kind-greeter-0bd1" + } ] + }, + "version" : { + "absent" : true + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_prompt-593a4a8a-e5c7-4792-9bb0-3af938b32fa3.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cgS6FE6OoAMEhnQ=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "789", + "X-Amz-Cf-Pop" : [ "HIO52-P2", "HIO52-P4" ], + "X-Amzn-Trace-Id" : "Root=1-69effea6-2b38ee1125bc7dbf51359bfe;Parent=66f2c70ac9a139d6;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 28 Apr 2026 00:26:14 GMT", + "Via" : "1.1 fc36d22b58a363b02ecdd852a2e51610.cloudfront.net (CloudFront), 1.1 743dabf2fdbfd64c0bd7adf3cea9dbec.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69effea6000000000933f7893b36bb72", + "x-amzn-RequestId" : "53be24a8-b01f-4781-af8f-146340943e71", + "X-Amz-Cf-Id" : "VKb8FRgb3UQ_kEg50am3IeN-c0CzZ8X_vV9nHBi8YTa9s6kaL8ljrA==", + "etag" : "W/\"315-aYopezw9/FDum2wzo3Z28/ESIc0\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "593a4a8a-e5c7-4792-9bb0-3af938b32fa3", + "persistent" : true, + "scenarioName" : "scenario-1-v1-prompt", + "requiredScenarioState" : "scenario-1-v1-prompt-3", + "insertionIndex" : 194 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json new file mode 100644 index 00000000..7e27afac --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json @@ -0,0 +1,49 @@ +{ + "id" : "8df36758-fab1-4ba2-bde7-0fbf9445a346", + "name" : "v1_prompt", + "request" : { + "urlPath" : "/v1/prompt", + "method" : "GET", + "queryParameters" : { + "project_name" : { + "hasExactly" : [ { + "equalTo" : "java-unit-test" + } ] + }, + "slug" : { + "hasExactly" : [ { + "equalTo" : "kind-greeter-0bd1" + } ] + }, + "version" : { + "hasExactly" : [ { + "equalTo" : "27fdcc80d22c7ec5" + } ] + } + } + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_prompt-8df36758-fab1-4ba2-bde7-0fbf9445a346.json", + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "cgS58GFGIAMEtLA=", + "vary" : "Origin, Accept-Encoding", + "x-amzn-Remapped-content-length" : "667", + "X-Amz-Cf-Pop" : [ "HIO52-P2", "HIO52-P4" ], + "X-Amzn-Trace-Id" : "Root=1-69effea5-534a538f6356a3ed2d64a981;Parent=0f386c6f67985202;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 28 Apr 2026 00:26:13 GMT", + "Via" : "1.1 fc36d22b58a363b02ecdd852a2e51610.cloudfront.net (CloudFront), 1.1 bb0a0a1792594c22377eddc835c4b882.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69effea50000000013cce95e4cf28815", + "x-amzn-RequestId" : "d1e556fb-e1a1-41a8-9bc2-96a12f5e0101", + "X-Amz-Cf-Id" : "O7sdwiLbE5lwRYtGK5YN8oAoDw1K8WzO-gRQ0r_tpYiSg5qyq6P3dg==", + "etag" : "W/\"29b-OevfOPDm+UyiFlombyUf3PTTPEk\"", + "Content-Type" : "application/json; charset=utf-8" + } + }, + "uuid" : "8df36758-fab1-4ba2-bde7-0fbf9445a346", + "persistent" : true, + "insertionIndex" : 197 +} \ No newline at end of file From 7511097380abe251f92d36ee25cb54989000ebe2 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 23 Apr 2026 10:50:42 -0600 Subject: [PATCH 4/4] codegen braintrust api client from openapi spec --- braintrust-api/build.gradle | 253 +++++++ .../libraries/native/anyof_model.mustache | 419 ++++++++++++ .../Java/libraries/native/api.mustache | 638 ++++++++++++++++++ .../src/generator/templates/PATCHES.md | 60 ++ .../openapi/model/AnyOfModelTemplateTest.java | 264 ++++++++ braintrust-sdk/build.gradle | 2 + .../main/java/dev/braintrust/Braintrust.java | 43 +- .../java/dev/braintrust/BraintrustUtils.java | 22 +- .../java/dev/braintrust/api/ApiException.java | 3 +- .../braintrust/api/BraintrustApiClient.java | 20 +- .../api/BraintrustOpenApiClient.java | 173 +++++ .../braintrust/config/BraintrustConfig.java | 7 +- .../dev/braintrust/devserver/Devserver.java | 50 +- .../java/dev/braintrust/eval/Dataset.java | 31 +- .../eval/DatasetBrainstoreImpl.java | 65 +- .../main/java/dev/braintrust/eval/Eval.java | 66 +- .../main/java/dev/braintrust/eval/Scorer.java | 46 +- .../braintrust/eval/ScorerBrainstoreImpl.java | 119 ++-- .../braintrust/prompt/BraintrustPrompt.java | 162 +++-- .../prompt/BraintrustPromptLoader.java | 81 ++- .../java/dev/braintrust/BraintrustTest.java | 2 +- .../dev/braintrust/BraintrustUtilsTest.java | 9 +- .../api/BraintrustOpenApiClientTest.java | 283 ++++++++ .../eval/DatasetBrainstoreImplTest.java | 107 +-- .../java/dev/braintrust/eval/EvalTest.java | 64 +- .../eval/ScorerBrainstoreImplTest.java | 36 +- .../prompt/BraintrustPromptLoaderTest.java | 18 + .../prompt/BraintrustPromptTest.java | 437 +++++------- gradle.properties | 3 + scripts/openapi-fetch.sh | 21 + settings.gradle | 1 + 31 files changed, 2819 insertions(+), 686 deletions(-) create mode 100644 braintrust-api/build.gradle create mode 100644 braintrust-api/src/generator/templates/Java/libraries/native/anyof_model.mustache create mode 100644 braintrust-api/src/generator/templates/Java/libraries/native/api.mustache create mode 100644 braintrust-api/src/generator/templates/PATCHES.md create mode 100644 braintrust-api/src/test/java/dev/braintrust/openapi/model/AnyOfModelTemplateTest.java create mode 100644 braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustOpenApiClient.java create mode 100644 braintrust-sdk/src/test/java/dev/braintrust/api/BraintrustOpenApiClientTest.java create mode 100755 scripts/openapi-fetch.sh diff --git a/braintrust-api/build.gradle b/braintrust-api/build.gradle new file mode 100644 index 00000000..5d2083d6 --- /dev/null +++ b/braintrust-api/build.gradle @@ -0,0 +1,253 @@ +// Placeholder subproject for the Braintrust OpenAPI-generated client. +// +// The fetchOpenApiSpec task downloads openapi/spec.yaml from the pinned commit +// of https://github.com/braintrustdata/braintrust-openapi into +// build/openapi/spec.yaml, then generateOpenApiSources runs the OpenAPI +// generator (native Java library) to produce sources under +// build/generated/openapi/src/main/java, which are compiled as part of the +// main source set. +// +// Future work will wire this generated client in to replace the hand-rolled +// BraintrustApiClient.HttpImpl. +// +// Pin the ref in gradle.properties: +// braintrustOpenApiRef= +// +// To point at a local checkout instead of fetching: +// BRAINTRUST_OPENAPI_ROOT=/path/to/braintrust-openapi ./gradlew braintrust-api:compileJava + +plugins { + id 'java' + id 'org.openapi.generator' version '7.14.0' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +repositories { + mavenCentral() +} + +// ── Spec fetch ─────────────────────────────────────────────────────────────── + +def openApiRef = project.property('braintrustOpenApiRef') +def openApiSpecDir = layout.buildDirectory.dir('openapi') +def openApiSpecFile = openApiSpecDir.map { it.file('spec.yaml') } + +tasks.register('fetchOpenApiSpec', Exec) { + description = 'Fetches openapi/spec.yaml from the pinned braintrust-openapi ref.' + group = 'Build' + + def localRoot = System.getenv('BRAINTRUST_OPENAPI_ROOT') + + inputs.property('openApiRef', openApiRef) + outputs.dir(openApiSpecDir) + + def outDir = openApiSpecDir.get().asFile + + if (localRoot) { + commandLine 'bash', '-c', + "mkdir -p '${outDir}' && cp '${localRoot}/openapi/spec.yaml' '${outDir}/spec.yaml'" + } else { + commandLine 'bash', "${rootProject.projectDir}/scripts/openapi-fetch.sh", + openApiRef, outDir.absolutePath + } +} + +// ── Code generation ────────────────────────────────────────────────────────── + +def generatedSourcesDir = layout.buildDirectory.dir('generated/openapi') + + +openApiGenerate { + generatorName = 'java' + library = 'native' + + // Custom templates under src/generator/ (not src/main/resources, so they don't + // end up on the runtime classpath or in the published JAR — they're only needed + // at code-generation time). See src/generator/templates for the diffs from upstream. + templateDir = "${projectDir}/src/generator/templates/Java" + + inputSpec = openApiSpecFile.map { it.asFile.absolutePath } + + outputDir = generatedSourcesDir.map { it.asFile.absolutePath } + + apiPackage = 'dev.braintrust.openapi.api' + modelPackage = 'dev.braintrust.openapi.model' + invokerPackage = 'dev.braintrust.openapi' + + configOptions = [ + // Use javax (not jakarta) — we're on Java 17 without EE migration + useJakartaEe : 'false', + // Dates as java.time types + dateLibrary : 'java8', + // Bearer token auth — SDK users supply a Braintrust API key + useRuntimeException : 'true', + serializationLibrary : 'jackson', + + // Skip boilerplate we don't need yet + generateApiTests : 'false', + generateModelTests : 'false', + generateApiDocumentation : 'false', + generateModelDocumentation : 'false', + ] + + // Inline schemas whose titles collide with top-level component schema names, causing the + // generator to overwrite the real full POJO with a discriminator stub. + // checkClobberedSchemas detects regressions; add new entries here when it fails. + inlineSchemaNameMappings = [ + 'prompt' : 'FunctionDataPrompt', + 'global' : 'FunctionDataGlobal', + 'code' : 'FunctionDataCode', + 'remote_eval' : 'FunctionDataRemoteEval', + 'parameters' : 'FunctionDataParameters', + 'function' : 'InlineFunctionRef', + 'function_1' : 'InlineFunction1', + 'function_2' : 'InlineFunction2', + 'function_3' : 'InlineFunction3', + 'experiment' : 'CodeBundleLocationExperiment', + 'user' : 'ChatMessageUser', + ] + + // Don't generate a full Maven project skeleton — just the sources + globalProperties = [ + apis : '', + models : '', + supportingFiles : '', + apiTests : 'false', + modelTests : 'false', + apiDocs : 'false', + modelDocs : 'false', + ] + +} + +// ── Clobber detection ──────────────────────────────────────────────────────── +// +// The generator names inline anyOf/oneOf variant classes after their `title` +// field, converting it to PascalCase. When that derived name matches an existing +// components/schemas entry the real POJO gets silently overwritten with a stub. +// +// This task detects collisions purely from the spec — no generated code needed. +// It runs before fetchOpenApiSpec (and thus before generation) so it fails fast. + +tasks.register('checkClobberedSchemas') { + description = 'Fails if any inline schema title would clobber a component schema name.' + group = 'Verification' + + def specFile = openApiSpecFile + // Capture the current mappings at configuration time so they're available at execution time. + // inlineSchemaNameMappings is a Gradle MapProperty so .get() is needed to resolve it. + def mappedTitles = openApiGenerate.inlineSchemaNameMappings.get().keySet() as Set + inputs.file(specFile) + + doLast { + if (!specFile.get().asFile.exists()) { + logger.lifecycle("checkClobberedSchemas: skipped (spec not yet fetched)") + return + } + + def yaml = new org.yaml.snakeyaml.Yaml() + def spec = yaml.load(specFile.get().asFile.text) + // Only protect component schemas that are plain POJOs (have `properties`). + // anyOf/oneOf wrapper schemas can't be "clobbered" in the same way — their + // inline variants are what create the class, not a separate definition. + def componentNames = (spec?.components?.schemas?.findAll { name, schema -> + schema?.containsKey('properties') + }?.keySet() ?: []) as Set + + // Convert a title to the PascalCase class name the generator would use. + def toPascalCase = { String s -> + s.split('[_\\-\\s]+').collect { it.capitalize() }.join('') + } + + // Collect all inline schema titles from anywhere in the spec. + def inlineTitles = [:] // title -> example location path + def collectTitles + collectTitles = { node, path -> + if (node instanceof Map) { + def title = node.get('title') + // Flag titled inline schemas — those nested inside a component schema + // definition (path depth > 2 within components.schemas). Top-level + // component schemas are at exactly components.schemas., which we + // skip since those are the schemas we're protecting, not the inliners. + def isTopLevelComponent = path ==~ /components\.schemas\.[^.]+/ + if (title instanceof String && !title.isEmpty() && !isTopLevelComponent + && (node.containsKey('type') || node.containsKey('properties') + || node.containsKey('anyOf') || node.containsKey('oneOf'))) { + inlineTitles.putIfAbsent(title, path) + } + node.each { k, v -> collectTitles(v, path ? "${path}.${k}" : k) } + } else if (node instanceof List) { + node.eachWithIndex { v, i -> collectTitles(v, "${path}[${i}]") } + } + } + collectTitles(spec, '') + + // Find titles that would collide with a component schema name, ignoring + // any that are already handled by inlineSchemaNameMappings. + def collisions = inlineTitles.findAll { entry -> + def pascal = toPascalCase(entry.key) + componentNames.contains(pascal) && !mappedTitles.contains(entry.key) + } + + if (!collisions.isEmpty()) { + def msg = new StringBuilder() + msg << '\n\nERROR: The following inline schema titles would clobber component schema\n' + msg << 'POJOs of the same name. Add them to inlineSchemaNameMappings in\n' + msg << 'braintrust-api/build.gradle:\n\n' + collisions.sort { it.key }.each { entry -> + def pascal = toPascalCase(entry.key) + msg << " '${entry.key}' : 'Inline${pascal}', // found at: ${entry.value}\n" + } + throw new GradleException(msg.toString()) + } + + logger.lifecycle("checkClobberedSchemas: OK (${inlineTitles.size()} inline titles checked against ${componentNames.size()} component schemas)") + } +} + +// Wire fetch → check → generate → compile +// checkClobberedSchemas runs after fetch (needs the spec) but before generation +// so it fails fast without wasting time on codegen. +tasks.named('checkClobberedSchemas') { + dependsOn tasks.named('fetchOpenApiSpec') +} + +tasks.named('openApiGenerate') { + dependsOn tasks.named('checkClobberedSchemas') +} + +tasks.named('compileJava') { + dependsOn tasks.named('openApiGenerate') +} + +// Add the generated sources to the main source set +sourceSets { + main { + java { + srcDir generatedSourcesDir.map { it.dir('src/main/java') } + } + } +} + +// ── Dependencies ───────────────────────────────────────────────────────────── + +dependencies { + // Required by the openapi-generator native Java library + implementation "com.fasterxml.jackson.core:jackson-databind:${rootProject.ext.jacksonVersion}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${rootProject.ext.jacksonVersion}" + implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${rootProject.ext.jacksonVersion}" + implementation 'org.openapitools:jackson-databind-nullable:0.2.10' + + // jsr305 provides javax.annotation.{Nonnull,Nullable} used by the generated code. + // javax.annotation-api is also needed for @javax.annotation.Generated on all generated classes. + compileOnly 'com.google.code.findbugs:jsr305:3.0.2' + compileOnly 'javax.annotation:javax.annotation-api:1.3.2' + + testImplementation "org.junit.jupiter:junit-jupiter-api:${rootProject.ext.junitVersion}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${rootProject.ext.junitVersion}" +} diff --git a/braintrust-api/src/generator/templates/Java/libraries/native/anyof_model.mustache b/braintrust-api/src/generator/templates/Java/libraries/native/anyof_model.mustache new file mode 100644 index 00000000..e5630062 --- /dev/null +++ b/braintrust-api/src/generator/templates/Java/libraries/native/anyof_model.mustache @@ -0,0 +1,419 @@ +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import {{invokerPackage}}.ApiClient; +import {{invokerPackage}}.JSON; + +{{>additionalModelTypeAnnotations}}{{>generatedAnnotation}}{{>xmlAnnotation}} +@JsonDeserialize(using={{classname}}.{{classname}}Deserializer.class) +@JsonSerialize(using = {{classname}}.{{classname}}Serializer.class) +public class {{classname}} extends AbstractOpenApiSchema{{#vendorExtensions.x-implements}} implements {{{.}}}{{^-last}}, {{/-last}}{{/vendorExtensions.x-implements}} { + private static final Logger log = Logger.getLogger({{classname}}.class.getName()); + + public static class {{classname}}Serializer extends StdSerializer<{{classname}}> { + public {{classname}}Serializer(Class<{{classname}}> t) { + super(t); + } + + public {{classname}}Serializer() { + this(null); + } + + @Override + public void serialize({{classname}} value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException { + jgen.writeObject(value.getActualInstance()); + } + } + + public static class {{classname}}Deserializer extends StdDeserializer<{{classname}}> { + public {{classname}}Deserializer() { + this({{classname}}.class); + } + + public {{classname}}Deserializer(Class vc) { + super(vc); + } + + @Override + public {{classname}} deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException, JsonProcessingException { + JsonNode tree = jp.readValueAsTree(); + + Object deserialized = null; + {{#discriminator}} + Class cls = JSON.getClassForElement(tree, new {{classname}}().getClass()); + if (cls != null) { + // When the OAS schema includes a discriminator, use the discriminator value to + // discriminate the anyOf schemas. + // Get the discriminator mapping value to get the class. + deserialized = tree.traverse(jp.getCodec()).readValueAs(cls); + {{classname}} ret = new {{classname}}(); + ret.setActualInstance(deserialized); + return ret; + } + {{/discriminator}} + {{#composedSchemas.anyOf}} + // deserialize {{{dataType}}} + try { + {{#isContainer}}deserialized = tree.traverse(jp.getCodec()).readValueAs(new com.fasterxml.jackson.core.type.TypeReference<{{{dataType}}}>(){});{{/isContainer}}{{^isContainer}}deserialized = tree.traverse(jp.getCodec()).readValueAs({{baseType}}.class);{{/isContainer}} + {{classname}} ret = new {{classname}}(); + ret.setActualInstance(deserialized); + ret.resolvedVariantType = SchemaType.{{#title}}{{#lambda.titlecase}}{{title}}{{/lambda.titlecase}}{{/title}}{{^title}}{{#vendorExtensions.x-duplicated-data-type}}{{nameInCamelCase}}{{/vendorExtensions.x-duplicated-data-type}}{{^vendorExtensions.x-duplicated-data-type}}{{baseType}}{{/vendorExtensions.x-duplicated-data-type}}{{/title}}; + return ret; + } catch (Exception e) { + // deserialization failed, continue, log to help debugging + log.log(Level.FINER, "Input data does not match '{{classname}}'", e); + } + + {{/composedSchemas.anyOf}} + throw new IOException(String.format("Failed deserialization for {{classname}}: no match found")); + } + + /** + * Handle deserialization of the 'null' value. + */ + @Override + public {{classname}} getNullValue(DeserializationContext ctxt) throws JsonMappingException { + {{#isNullable}} + return null; + {{/isNullable}} + {{^isNullable}} + throw new JsonMappingException(ctxt.getParser(), "{{classname}} cannot be null"); + {{/isNullable}} + } + } + + // store a list of schema names defined in anyOf + public static final Map> schemas = new HashMap>(); + + /** Set during deserialization to record which variant was actually matched. */ + private SchemaType resolvedVariantType; + + public {{classname}}() { + super("anyOf", {{#isNullable}}Boolean.TRUE{{/isNullable}}{{^isNullable}}Boolean.FALSE{{/isNullable}}); + } +{{> libraries/native/additional_properties }} + {{#additionalPropertiesType}} + /** + * Return true if this {{name}} object is equal to o. + */ + @Override + public boolean equals(Object o) { + return super.equals(o) && Objects.equals(this.additionalProperties, (({{classname}})o).additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(getActualInstance(), isNullable(), getSchemaType(), additionalProperties); + } + {{/additionalPropertiesType}} + /** + * Construct a {{classname}} with an explicit variant type. + * Callers must pass the correct {@link SchemaType} to ensure {@link #getVariantType()} + * returns the right value — this is especially important when two variants share the same + * raw type after erasure (e.g. {@code List} and {@code List}). + */ + public {{classname}}(SchemaType type, Object o) { + super("anyOf", {{#isNullable}}Boolean.TRUE{{/isNullable}}{{^isNullable}}Boolean.FALSE{{/isNullable}}); + this.resolvedVariantType = type; + setActualInstance(o); + } + + + static { + {{#composedSchemas.anyOf}} + schemas.put("{{{dataType}}}", {{baseType}}.class); + {{/composedSchemas.anyOf}} + JSON.registerDescendants({{classname}}.class, Collections.unmodifiableMap(schemas)); + {{#discriminator}} + // Initialize and register the discriminator mappings. + Map> mappings = new HashMap>(); + {{#mappedModels}} + mappings.put("{{mappingName}}", {{modelName}}.class); + {{/mappedModels}} + mappings.put("{{name}}", {{classname}}.class); + JSON.registerDiscriminator({{classname}}.class, "{{propertyBaseName}}", mappings); + {{/discriminator}} + } + + @Override + public Map> getSchemas() { + return {{classname}}.schemas; + } + + /** + * Set the instance that matches the anyOf child schema, check + * the instance parameter is valid against the anyOf child schemas: + * {{#anyOf}}{{{.}}}{{^-last}}, {{/-last}}{{/anyOf}} + * + * It could be an instance of the 'anyOf' schemas. + * The anyOf child schemas may themselves be a composed schema (allOf, anyOf, anyOf). + */ + @Override + public void setActualInstance(Object instance) { + {{#isNullable}} + if (instance == null) { + super.setActualInstance(instance); + return; + } + + {{/isNullable}} + {{#composedSchemas.anyOf}} + if (JSON.isInstanceOf({{baseType}}.class, instance, new HashSet>())) { + super.setActualInstance(instance); + return; + } + + {{/composedSchemas.anyOf}} + throw new RuntimeException("Invalid instance type. Must be {{#anyOf}}{{{.}}}{{^-last}}, {{/-last}}{{/anyOf}}"); + } + + /** + * Get the actual instance, which can be the following: + * {{#anyOf}}{{{.}}}{{^-last}}, {{/-last}}{{/anyOf}} + * + * @return The actual instance ({{#anyOf}}{{{.}}}{{^-last}}, {{/-last}}{{/anyOf}}) + */ + @Override + public Object getActualInstance() { + return super.getActualInstance(); + } + + /** + * Enum of the possible anyOf variant types for {{classname}}. + * Each constant carries the full Java type name (including generics) as {@link #dataType}. + * Use {@link #getVariantType()} to get the type of the current instance, then switch on it + * for exhaustive, compile-time-checked variant handling: + * 
+     * switch (instance.getVariantType()) {
+     * {{#composedSchemas.anyOf}}
+     *     case {{#title}}{{#lambda.titlecase}}{{title}}{{/lambda.titlecase}}{{/title}}{{^title}}{{#vendorExtensions.x-duplicated-data-type}}{{nameInCamelCase}}{{/vendorExtensions.x-duplicated-data-type}}{{^vendorExtensions.x-duplicated-data-type}}{{baseType}}{{/vendorExtensions.x-duplicated-data-type}}{{/title}} -> (({{{dataType}}}) instance.getActualInstance());
+     * {{/composedSchemas.anyOf}}
+     * }
+     *
+ */ + public enum SchemaType { + {{#composedSchemas.anyOf}} + {{#title}}{{#lambda.titlecase}}{{title}}{{/lambda.titlecase}}{{/title}}{{^title}}{{#vendorExtensions.x-duplicated-data-type}}{{nameInCamelCase}}{{/vendorExtensions.x-duplicated-data-type}}{{^vendorExtensions.x-duplicated-data-type}}{{baseType}}{{/vendorExtensions.x-duplicated-data-type}}{{/title}}("{{{dataType}}}"){{^-last}},{{/-last}} + {{/composedSchemas.anyOf}}; + + /** The full Java type name of this variant, including any generic parameters. */ + public final String dataType; + + SchemaType(String dataType) { this.dataType = dataType; } + } + + /** + * Return which anyOf variant is currently held. + * + *

For instances created via deserialization or the + * {@link #{{classname}}(SchemaType, Object)} constructor, this is always accurate — + * including cases where two variants share the same erased type (e.g. + * {@code List} vs {@code List}). + * + * @throws IllegalStateException if this instance was created with the no-arg constructor + * and {@code setActualInstance} was called directly (use the + * {@link #{{classname}}(SchemaType, Object)} constructor instead) + */ + public SchemaType getVariantType() { + if (resolvedVariantType == null) { + throw new IllegalStateException( + "resolvedVariantType is not set. Use new {{classname}}(SchemaType, Object) " + + "instead of setActualInstance() to ensure the variant type is recorded."); + } + return resolvedVariantType; + } + + {{#composedSchemas.anyOf}} + /** + * Get the actual instance of `{{{dataType}}}`. If the actual instance is not `{{{dataType}}}`, + * the ClassCastException will be thrown. + * + * @return The actual instance of `{{{dataType}}}` + * @throws ClassCastException if the instance is not `{{{dataType}}}` + */ + @SuppressWarnings("unchecked") + public {{{dataType}}} get{{#title}}{{#lambda.titlecase}}{{title}}{{/lambda.titlecase}}{{/title}}{{^title}}{{#vendorExtensions.x-duplicated-data-type}}{{nameInCamelCase}}{{/vendorExtensions.x-duplicated-data-type}}{{^vendorExtensions.x-duplicated-data-type}}{{baseType}}{{/vendorExtensions.x-duplicated-data-type}}{{/title}}Instance() throws ClassCastException { + return ({{{dataType}}})super.getActualInstance(); + } + + {{/composedSchemas.anyOf}} + +{{#supportUrlQuery}} + + /** + * Convert the instance into URL query string. + * + * @return URL query string + */ + public String toUrlQueryString() { + return toUrlQueryString(null); + } + + /** + * Convert the instance into URL query string. + * + * @param prefix prefix of the query string + * @return URL query string + */ + public String toUrlQueryString(String prefix) { + String suffix = ""; + String containerSuffix = ""; + String containerPrefix = ""; + if (prefix == null) { + // style=form, explode=true, e.g. /pet?name=cat&type=manx + prefix = ""; + } else { + // deepObject style e.g. /pet?id[name]=cat&id[type]=manx + prefix = prefix + "["; + suffix = "]"; + containerSuffix = "]"; + containerPrefix = "["; + } + + StringJoiner joiner = new StringJoiner("&"); + + {{#composedSchemas.oneOf}} + {{^vendorExtensions.x-duplicated-data-type}} + if (getActualInstance() instanceof {{{dataType}}}) { + {{#isArray}} + {{#items.isPrimitiveType}} + {{#uniqueItems}} + if (getActualInstance() != null) { + int i = 0; + for ({{{items.dataType}}} _item : ({{{dataType}}})getActualInstance()) { + joiner.add(String.format("%s{{baseName}}%s%s=%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix), + ApiClient.urlEncode(String.valueOf(_item)))); + } + i++; + } + {{/uniqueItems}} + {{^uniqueItems}} + if (getActualInstance() != null) { + for (int i = 0; i < (({{{dataType}}})getActualInstance()).size(); i++) { + joiner.add(String.format("%s{{baseName}}%s%s=%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix), + ApiClient.urlEncode(String.valueOf(getActualInstance().get(i))))); + } + } + {{/uniqueItems}} + {{/items.isPrimitiveType}} + {{^items.isPrimitiveType}} + {{#items.isModel}} + {{#uniqueItems}} + if (getActualInstance() != null) { + int i = 0; + for ({{{items.dataType}}} _item : ({{{dataType}}})getActualInstance()) { + if (_item != null) { + joiner.add(_item.toUrlQueryString(String.format("%s{{baseName}}%s%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix)))); + } + } + i++; + } + {{/uniqueItems}} + {{^uniqueItems}} + if (getActualInstance() != null) { + for (int i = 0; i < (({{{dataType}}})getActualInstance()).size(); i++) { + if ((({{{dataType}}})getActualInstance()).get(i) != null) { + joiner.add((({{{items.dataType}}})getActualInstance()).get(i).toUrlQueryString(String.format("%s{{baseName}}%s%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix)))); + } + } + } + {{/uniqueItems}} + {{/items.isModel}} + {{^items.isModel}} + {{#uniqueItems}} + if (getActualInstance() != null) { + int i = 0; + for ({{{items.dataType}}} _item : ({{{dataType}}})getActualInstance()) { + if (_item != null) { + joiner.add(String.format("%s{{baseName}}%s%s=%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix), + ApiClient.urlEncode(String.valueOf(_item)))); + } + i++; + } + } + {{/uniqueItems}} + {{^uniqueItems}} + if (getActualInstance() != null) { + for (int i = 0; i < (({{{dataType}}})getActualInstance()).size(); i++) { + if (getActualInstance().get(i) != null) { + joiner.add(String.format("%s{{baseName}}%s%s=%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, i, containerSuffix), + ApiClient.urlEncode(String.valueOf((({{{dataType}}})getActualInstance()).get(i))))); + } + } + } + {{/uniqueItems}} + {{/items.isModel}} + {{/items.isPrimitiveType}} + {{/isArray}} + {{^isArray}} + {{#isMap}} + {{#items.isPrimitiveType}} + if (getActualInstance() != null) { + for (String _key : (({{{dataType}}})getActualInstance()).keySet()) { + joiner.add(String.format("%s{{baseName}}%s%s=%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, _key, containerSuffix), + getActualInstance().get(_key), ApiClient.urlEncode(String.valueOf((({{{dataType}}})getActualInstance()).get(_key))))); + } + } + {{/items.isPrimitiveType}} + {{^items.isPrimitiveType}} + if (getActualInstance() != null) { + for (String _key : (({{{dataType}}})getActualInstance()).keySet()) { + if ((({{{dataType}}})getActualInstance()).get(_key) != null) { + joiner.add((({{{items.dataType}}})getActualInstance()).get(_key).toUrlQueryString(String.format("%s{{baseName}}%s%s", prefix, suffix, + "".equals(suffix) ? "" : String.format("%s%d%s", containerPrefix, _key, containerSuffix)))); + } + } + } + {{/items.isPrimitiveType}} + {{/isMap}} + {{^isMap}} + {{#isPrimitiveType}} + if (getActualInstance() != null) { + joiner.add(String.format("%s{{{baseName}}}%s=%s", prefix, suffix, ApiClient.urlEncode(String.valueOf(getActualInstance())))); + } + {{/isPrimitiveType}} + {{^isPrimitiveType}} + {{#isModel}} + if (getActualInstance() != null) { + joiner.add((({{{dataType}}})getActualInstance()).toUrlQueryString(prefix + "{{{baseName}}}" + suffix)); + } + {{/isModel}} + {{^isModel}} + if (getActualInstance() != null) { + joiner.add(String.format("%s{{{baseName}}}%s=%s", prefix, suffix, ApiClient.urlEncode(String.valueOf(getActualInstance())))); + } + {{/isModel}} + {{/isPrimitiveType}} + {{/isMap}} + {{/isArray}} + return joiner.toString(); + } + {{/vendorExtensions.x-duplicated-data-type}} + {{/composedSchemas.oneOf}} + return null; + } +{{/supportUrlQuery}} + +} diff --git a/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache b/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache new file mode 100644 index 00000000..6d89b929 --- /dev/null +++ b/braintrust-api/src/generator/templates/Java/libraries/native/api.mustache @@ -0,0 +1,638 @@ +{{>licenseInfo}} +package {{package}}; + +import {{invokerPackage}}.ApiClient; +import {{invokerPackage}}.ApiException; +import {{invokerPackage}}.ApiResponse; +import {{invokerPackage}}.Configuration; +import {{invokerPackage}}.Pair; + +{{#imports}} +import {{import}}; +{{/imports}} + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +{{#useBeanValidation}} +import {{javaxPackage}}.validation.constraints.*; +import {{javaxPackage}}.validation.Valid; + +{{/useBeanValidation}} +{{#hasFormParamsInSpec}} +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; + +{{/hasFormParamsInSpec}} +import java.io.InputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.http.HttpRequest; +import java.nio.channels.Channels; +import java.nio.channels.Pipe; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +import java.util.ArrayList; +import java.util.StringJoiner; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +{{#asyncNative}} + +import java.util.concurrent.CompletableFuture; +{{/asyncNative}} + +{{>generatedAnnotation}} +{{#operations}} +public class {{classname}} { + private final HttpClient memberVarHttpClient; + private final ObjectMapper memberVarObjectMapper; + private final String memberVarBaseUri; + private final Consumer memberVarInterceptor; + private final Duration memberVarReadTimeout; + private final Consumer> memberVarResponseInterceptor; + private final Consumer> memberVarAsyncResponseInterceptor; + + public {{classname}}() { + this(Configuration.getDefaultApiClient()); + } + + public {{classname}}(ApiClient apiClient) { + memberVarHttpClient = apiClient.getHttpClient(); + memberVarObjectMapper = apiClient.getObjectMapper(); + memberVarBaseUri = apiClient.getBaseUri(); + memberVarInterceptor = apiClient.getRequestInterceptor(); + memberVarReadTimeout = apiClient.getReadTimeout(); + memberVarResponseInterceptor = apiClient.getResponseInterceptor(); + memberVarAsyncResponseInterceptor = apiClient.getAsyncResponseInterceptor(); + } + {{#asyncNative}} + + private ApiException getApiException(String operationId, HttpResponse response) { + String message = formatExceptionMessage(operationId, response.statusCode(), response.body()); + return new ApiException(response.statusCode(), message, response.headers(), response.body()); + } + {{/asyncNative}} + {{^asyncNative}} + + protected ApiException getApiException(String operationId, HttpResponse response) throws IOException { + String body = response.body() == null ? null : new String(response.body().readAllBytes()); + String message = formatExceptionMessage(operationId, response.statusCode(), body); + return new ApiException(response.statusCode(), message, response.headers(), body); + } + {{/asyncNative}} + + private String formatExceptionMessage(String operationId, int statusCode, String body) { + if (body == null || body.isEmpty()) { + body = "[no body]"; + } + return operationId + " call failed with: " + statusCode + " - " + body; + } + + {{#operation}} + {{#vendorExtensions.x-group-parameters}} + {{#hasParams}} + /** + * {{summary}} + * {{notes}} + * @param apiRequest {@link API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request} + {{#returnType}} + * @return {{#asyncNative}}CompletableFuture<{{/asyncNative}}{{returnType}}{{#asyncNative}}>{{/asyncNative}} + {{/returnType}} + {{^returnType}} + {{#asyncNative}} + * @return CompletableFuture<Void> + {{/asyncNative}} + {{/returnType}} + * @throws ApiException if fails to make API call + {{#isDeprecated}} + * @deprecated + {{/isDeprecated}} + {{#externalDocs}} + * {{description}} + * @see {{summary}} Documentation + {{/externalDocs}} + */ + {{#isDeprecated}} + @Deprecated + {{/isDeprecated}} + public {{#returnType}}{{#asyncNative}}CompletableFuture<{{{returnType}}}>{{/asyncNative}}{{^asyncNative}}{{{returnType}}}{{/asyncNative}}{{/returnType}}{{^returnType}}{{#asyncNative}}CompletableFuture{{/asyncNative}}{{^asyncNative}}void{{/asyncNative}}{{/returnType}} {{operationId}}(API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request apiRequest) throws ApiException { + {{#allParams}} + {{>nullable_var_annotations}} + {{{dataType}}} {{paramName}} = apiRequest.{{paramName}}(); + {{/allParams}} + {{#returnType}}return {{/returnType}}{{^returnType}}{{#asyncNative}}return {{/asyncNative}}{{/returnType}}{{operationId}}({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + } + + /** + * {{summary}} + * {{notes}} + * @param apiRequest {@link API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request} + * @return {{#asyncNative}}CompletableFuture<{{/asyncNative}}ApiResponse<{{returnType}}{{^returnType}}Void{{/returnType}}>{{#asyncNative}}>{{/asyncNative}} + * @throws ApiException if fails to make API call + {{#isDeprecated}} + * @deprecated + {{/isDeprecated}} + {{#externalDocs}} + * {{description}} + * @see {{summary}} Documentation + {{/externalDocs}} + */ + {{#isDeprecated}} + @Deprecated + {{/isDeprecated}} + public {{#asyncNative}}CompletableFuture<{{/asyncNative}}ApiResponse<{{{returnType}}}{{^returnType}}Void{{/returnType}}>{{#asyncNative}}>{{/asyncNative}} {{operationId}}WithHttpInfo(API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request apiRequest) throws ApiException { + {{#allParams}} + {{{dataType}}} {{paramName}} = apiRequest.{{paramName}}(); + {{/allParams}} + return {{operationId}}WithHttpInfo({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + } + + {{/hasParams}} + {{/vendorExtensions.x-group-parameters}} + /** + * {{summary}} + * {{notes}} + {{#allParams}} + * @param {{paramName}} {{description}}{{#required}} (required){{/required}}{{^required}} (optional{{^isContainer}}{{#defaultValue}}, default to {{.}}{{/defaultValue}}{{/isContainer}}){{/required}} + {{/allParams}} + {{#returnType}} + * @return {{#asyncNative}}CompletableFuture<{{/asyncNative}}{{returnType}}{{#asyncNative}}>{{/asyncNative}} + {{/returnType}} + {{^returnType}} + {{#asyncNative}} + * @return CompletableFuture<Void> + {{/asyncNative}} + {{/returnType}} + * @throws ApiException if fails to make API call + {{#isDeprecated}} + * @deprecated + {{/isDeprecated}} + {{#externalDocs}} + * {{description}} + * @see {{summary}} Documentation + {{/externalDocs}} + */ + {{#isDeprecated}} + @Deprecated + {{/isDeprecated}} + public {{#returnType}}{{#asyncNative}}CompletableFuture<{{{returnType}}}>{{/asyncNative}}{{^asyncNative}}{{{returnType}}}{{/asyncNative}}{{/returnType}}{{^returnType}}{{#asyncNative}}CompletableFuture{{/asyncNative}}{{^asyncNative}}void{{/asyncNative}}{{/returnType}} {{operationId}}({{#allParams}}{{>nullable_var_annotations}} {{{dataType}}} {{paramName}}{{^-last}}, {{/-last}}{{/allParams}}) throws ApiException { + {{^asyncNative}} + {{#returnType}}ApiResponse<{{{.}}}> localVarResponse = {{/returnType}}{{operationId}}WithHttpInfo({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + {{#returnType}} + return localVarResponse.getData(); + {{/returnType}} + {{/asyncNative}} + {{#asyncNative}} + try { + HttpRequest.Builder localVarRequestBuilder = {{operationId}}RequestBuilder({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + return memberVarHttpClient.sendAsync( + localVarRequestBuilder.build(), + HttpResponse.BodyHandlers.ofString()).thenComposeAsync(localVarResponse -> { + if (localVarResponse.statusCode()/ 100 != 2) { + return CompletableFuture.failedFuture(getApiException("{{operationId}}", localVarResponse)); + } + {{#returnType}} + try { + String responseBody = localVarResponse.body(); + return CompletableFuture.completedFuture( + responseBody == null || responseBody.isBlank() ? null : memberVarObjectMapper.readValue(responseBody, new TypeReference<{{{returnType}}}>() {}) + ); + } catch (IOException e) { + return CompletableFuture.failedFuture(new ApiException(e)); + } + {{/returnType}} + {{^returnType}} + return CompletableFuture.completedFuture(null); + {{/returnType}} + }); + } + catch (ApiException e) { + return CompletableFuture.failedFuture(e); + } + {{/asyncNative}} + } + + /** + * {{summary}} + * {{notes}} + {{#allParams}} + * @param {{paramName}} {{description}}{{#required}} (required){{/required}}{{^required}} (optional{{^isContainer}}{{#defaultValue}}, default to {{.}}{{/defaultValue}}{{/isContainer}}){{/required}} + {{/allParams}} + * @return {{#asyncNative}}CompletableFuture<{{/asyncNative}}ApiResponse<{{returnType}}{{^returnType}}Void{{/returnType}}>{{#asyncNative}}>{{/asyncNative}} + * @throws ApiException if fails to make API call + {{#isDeprecated}} + * @deprecated + {{/isDeprecated}} + {{#externalDocs}} + * {{description}} + * @see {{summary}} Documentation + {{/externalDocs}} + */ + {{#isDeprecated}} + @Deprecated + {{/isDeprecated}} + public {{#asyncNative}}CompletableFuture<{{/asyncNative}}ApiResponse<{{{returnType}}}{{^returnType}}Void{{/returnType}}>{{#asyncNative}}>{{/asyncNative}} {{operationId}}WithHttpInfo({{#allParams}}{{>nullable_var_annotations}} {{{dataType}}} {{paramName}}{{^-last}}, {{/-last}}{{/allParams}}) throws ApiException { + {{^asyncNative}} + HttpRequest.Builder localVarRequestBuilder = {{operationId}}RequestBuilder({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + try { + HttpResponse localVarResponse = memberVarHttpClient.send( + localVarRequestBuilder.build(), + HttpResponse.BodyHandlers.ofInputStream()); + if (memberVarResponseInterceptor != null) { + memberVarResponseInterceptor.accept(localVarResponse); + } + try { + if (localVarResponse.statusCode()/ 100 != 2) { + throw getApiException("{{operationId}}", localVarResponse); + } + {{#vendorExtensions.x-java-text-plain-string}} + // for plain text response + if (localVarResponse.headers().map().containsKey("Content-Type") && + "text/plain".equalsIgnoreCase(localVarResponse.headers().map().get("Content-Type").get(0).split(";")[0].trim())) { + java.util.Scanner s = new java.util.Scanner(localVarResponse.body()).useDelimiter("\\A"); + String responseBodyText = s.hasNext() ? s.next() : ""; + return new ApiResponse( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + responseBodyText + ); + } else { + throw new RuntimeException("Error! The response Content-Type is supposed to be `text/plain` but it's not: " + localVarResponse); + } + {{/vendorExtensions.x-java-text-plain-string}} + {{^vendorExtensions.x-java-text-plain-string}} + {{#returnType}} + {{! Fix for https://github.com/OpenAPITools/openapi-generator/issues/13968 }} + {{! This part had a bugfix for an empty response in the past, but this part of that PR was reverted because it was not doing anything. }} + {{! Keep this documentation here, because the problem is not obvious. }} + {{! `InputStream.available()` was used, but that only works for inputstreams that are already in memory, it will not give the right result if it is a remote stream. We only work with remote streams here. }} + {{! https://github.com/OpenAPITools/openapi-generator/pull/13993/commits/3e!37411d2acef0311c82e6d941a8e40b3bc0b6da }} + {{! The `available` method would work with a `PushbackInputStream`, because we could read 1 byte to check if it exists then push it back so Jackson can read it again. The issue with that is that it will also insert an ascii character for "head of input" and that will break Jackson as it does not handle special whitespace characters. }} + {{! A fix for that problem is to read it into a string and remove those characters, but if we need to read it before giving it to jackson to fix the string then just reading it into a string as is to do an emptiness check is the cleaner solution. }} + {{! We could also manipulate the inputstream to remove that bad character, but string manipulation is easier to read and this codepath is not asyncronus so we do not gain anything by reading the stream later. }} + {{! This fix does make it unsuitable for large amounts of data because `InputStream.readAllbytes` is not meant for it, but a synchronous client is already not the right tool for that.}} + if (localVarResponse.body() == null) { + return new ApiResponse<{{{returnType}}}>( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + null + ); + } + + String responseBody = new String(localVarResponse.body().readAllBytes()); + localVarResponse.body().close(); + + return new ApiResponse<{{{returnType}}}>( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + responseBody.isBlank()? null: memberVarObjectMapper.readValue(responseBody, new TypeReference<{{{returnType}}}>() {}) + ); + {{/returnType}} + {{^returnType}} + return new ApiResponse<{{{returnType}}}>( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + null + ); + {{/returnType}} + {{/vendorExtensions.x-java-text-plain-string}} + } finally { + {{^returnType}} + // Drain the InputStream + while (localVarResponse.body().read() != -1) { + // Ignore + } + localVarResponse.body().close(); + {{/returnType}} + } + } catch (IOException e) { + throw new ApiException(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ApiException(e); + } + {{/asyncNative}} + {{#asyncNative}} + try { + HttpRequest.Builder localVarRequestBuilder = {{operationId}}RequestBuilder({{#allParams}}{{paramName}}{{^-last}}, {{/-last}}{{/allParams}}); + return memberVarHttpClient.sendAsync( + localVarRequestBuilder.build(), + HttpResponse.BodyHandlers.ofString()).thenComposeAsync(localVarResponse -> { + if (memberVarAsyncResponseInterceptor != null) { + memberVarAsyncResponseInterceptor.accept(localVarResponse); + } + if (localVarResponse.statusCode()/ 100 != 2) { + return CompletableFuture.failedFuture(getApiException("{{operationId}}", localVarResponse)); + } + {{#returnType}} + try { + String responseBody = localVarResponse.body(); + return CompletableFuture.completedFuture( + new ApiResponse<{{{returnType}}}>( + localVarResponse.statusCode(), + localVarResponse.headers().map(), + responseBody == null || responseBody.isBlank() ? null : memberVarObjectMapper.readValue(responseBody, new TypeReference<{{{returnType}}}>() {})) + ); + } catch (IOException e) { + return CompletableFuture.failedFuture(new ApiException(e)); + } + {{/returnType}} + {{^returnType}} + return CompletableFuture.completedFuture( + new ApiResponse(localVarResponse.statusCode(), localVarResponse.headers().map(), null) + ); + {{/returnType}} + } + ); + } + catch (ApiException e) { + return CompletableFuture.failedFuture(e); + } + {{/asyncNative}} + } + + private HttpRequest.Builder {{operationId}}RequestBuilder({{#allParams}}{{>nullable_var_annotations}} {{{dataType}}} {{paramName}}{{^-last}}, {{/-last}}{{/allParams}}) throws ApiException { + {{#allParams}} + {{#required}} + // verify the required parameter '{{paramName}}' is set + if ({{paramName}} == null) { + throw new ApiException(400, "Missing the required parameter '{{paramName}}' when calling {{operationId}}"); + } + {{/required}} + {{/allParams}} + + HttpRequest.Builder localVarRequestBuilder = HttpRequest.newBuilder(); + + {{! Switch delimiters for baseName so we can write constants like "{query}" }} + String localVarPath = "{{{path}}}"{{#pathParams}} + .replace({{=<% %>=}}"{<%baseName%>}"<%={{ }}=%>, ApiClient.urlEncode({{{paramName}}}.toString())){{/pathParams}}; + + {{#hasQueryParams}} + List localVarQueryParams = new ArrayList<>(); + StringJoiner localVarQueryStringJoiner = new StringJoiner("&"); + String localVarQueryParameterBaseName; + {{#queryParams}} + localVarQueryParameterBaseName = "{{{baseName}}}"; + {{#collectionFormat}} + localVarQueryParams.addAll(ApiClient.parameterToPairs("{{{collectionFormat}}}", "{{baseName}}", {{paramName}})); + {{/collectionFormat}} + {{^collectionFormat}} + {{#isDeepObject}} + if ({{paramName}} != null) { + {{#isArray}} + for (int i=0; i < {{paramName}}.size(); i++) { + localVarQueryStringJoiner.add({{paramName}}.get(i).toUrlQueryString(String.format("{{baseName}}[%d]", i))); + } + {{/isArray}} + {{^isArray}} + String queryString = {{paramName}}.toUrlQueryString("{{baseName}}"); + if (!queryString.isBlank()) { + localVarQueryStringJoiner.add(queryString); + } + {{/isArray}} + } + {{/isDeepObject}} + {{^isDeepObject}} + {{#isExplode}} + {{#hasVars}} + {{#vars}} + {{#isArray}} + localVarQueryParams.addAll(ApiClient.parameterToPairs("multi", "{{baseName}}", {{paramName}}.{{getter}}())); + {{/isArray}} + {{^isArray}} + localVarQueryParams.addAll(ApiClient.parameterToPairs("{{baseName}}", {{paramName}}.{{getter}}())); + {{/isArray}} + {{/vars}} + {{/hasVars}} + {{^hasVars}} + {{#isModel}} + if ({{paramName}} != null) { + String queryString = {{paramName}}.toUrlQueryString(); + if (queryString != null && !queryString.isBlank()) { + localVarQueryStringJoiner.add(queryString); + } + } + {{/isModel}} + {{^isModel}} + localVarQueryParams.addAll(ApiClient.parameterToPairs("{{baseName}}", {{paramName}})); + {{/isModel}} + {{/hasVars}} + {{/isExplode}} + {{^isExplode}} + localVarQueryParams.addAll(ApiClient.parameterToPairs("{{baseName}}", {{paramName}})); + {{/isExplode}} + {{/isDeepObject}} + {{/collectionFormat}} + {{/queryParams}} + + if (!localVarQueryParams.isEmpty() || localVarQueryStringJoiner.length() != 0) { + StringJoiner queryJoiner = new StringJoiner("&"); + localVarQueryParams.forEach(p -> queryJoiner.add(p.getName() + '=' + p.getValue())); + if (localVarQueryStringJoiner.length() != 0) { + queryJoiner.add(localVarQueryStringJoiner.toString()); + } + localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath + '?' + queryJoiner.toString())); + } else { + localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath)); + } + {{/hasQueryParams}} + {{^hasQueryParams}} + localVarRequestBuilder.uri(URI.create(memberVarBaseUri + localVarPath)); + {{/hasQueryParams}} + + {{#headerParams}} + if ({{paramName}} != null) { + localVarRequestBuilder.header("{{baseName}}", {{paramName}}.toString()); + } + {{/headerParams}} + {{#bodyParam}} + localVarRequestBuilder.header("Content-Type", "{{#hasConsumes}}{{#consumes}}{{#-first}}{{{mediaType}}}{{/-first}}{{/consumes}}{{/hasConsumes}}{{#hasConsumes}}{{^consumes}}application/json{{/consumes}}{{/hasConsumes}}{{^hasConsumes}}application/json{{/hasConsumes}}"); + {{/bodyParam}} + localVarRequestBuilder.header("Accept", "{{#hasProduces}}{{#produces}}{{{mediaType}}}{{^-last}}, {{/-last}}{{/produces}}{{/hasProduces}}{{#hasProduces}}{{^produces}}application/json{{/produces}}{{/hasProduces}}{{^hasProduces}}application/json{{/hasProduces}}"); + + {{#bodyParam}} + {{#isString}} + localVarRequestBuilder.method("{{httpMethod}}", HttpRequest.BodyPublishers.ofString({{paramName}})); + {{/isString}} + {{^isString}} + try { + byte[] localVarPostBody = memberVarObjectMapper.writeValueAsBytes({{paramName}}); + localVarRequestBuilder.method("{{httpMethod}}", HttpRequest.BodyPublishers.ofByteArray(localVarPostBody)); + } catch (IOException e) { + throw new ApiException(e); + } + {{/isString}} + {{/bodyParam}} + {{^bodyParam}} + {{#hasFormParams}} + {{#isMultipart}} + MultipartEntityBuilder multiPartBuilder = MultipartEntityBuilder.create(); + boolean hasFiles = false; + {{#formParams}} + {{#isArray}} + for (int i=0; i < {{paramName}}.size(); i++) { + {{#isFile}} + multiPartBuilder.addBinaryBody("{{{baseName}}}", {{paramName}}.get(i)); + hasFiles = true; + {{/isFile}} + {{^isFile}} + if ({{paramName}}.get(i) != null) { + multiPartBuilder.addTextBody("{{{baseName}}}", {{paramName}}.get(i).toString()); + } + {{/isFile}} + } + {{/isArray}} + {{^isArray}} + {{#isFile}} + multiPartBuilder.addBinaryBody("{{{baseName}}}", {{paramName}}); + hasFiles = true; + {{/isFile}} + {{^isFile}} + if ({{paramName}} != null) { + multiPartBuilder.addTextBody("{{{baseName}}}", {{paramName}}.toString()); + } + {{/isFile}} + {{/isArray}} + {{/formParams}} + HttpEntity entity = multiPartBuilder.build(); + HttpRequest.BodyPublisher formDataPublisher; + if (hasFiles) { + Pipe pipe; + try { + pipe = Pipe.open(); + } catch (IOException e) { + throw new RuntimeException(e); + } + new Thread(() -> { + try (OutputStream outputStream = Channels.newOutputStream(pipe.sink())) { + entity.writeTo(outputStream); + } catch (IOException e) { + e.printStackTrace(); + } + }).start(); + formDataPublisher = HttpRequest.BodyPublishers.ofInputStream(() -> Channels.newInputStream(pipe.source())); + } else { + ByteArrayOutputStream formOutputStream = new ByteArrayOutputStream(); + try { + entity.writeTo(formOutputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + formDataPublisher = HttpRequest.BodyPublishers + .ofInputStream(() -> new ByteArrayInputStream(formOutputStream.toByteArray())); + } + localVarRequestBuilder + .header("Content-Type", entity.getContentType().getValue()) + .method("{{httpMethod}}", formDataPublisher); + {{/isMultipart}} + {{^isMultipart}} + List formValues = new ArrayList<>(); + {{#formParams}} + {{#isArray}} + for (int i=0; i < {{paramName}}.size(); i++) { + if ({{paramName}}.get(i) != null) { + formValues.add(new BasicNameValuePair("{{{baseName}}}", {{paramName}}.get(i).toString())); + } + } + {{/isArray}} + {{^isArray}} + if ({{paramName}} != null) { + formValues.add(new BasicNameValuePair("{{{baseName}}}", {{paramName}}.toString())); + } + {{/isArray}} + {{/formParams}} + HttpEntity entity = new UrlEncodedFormEntity(formValues, java.nio.charset.StandardCharsets.UTF_8); + ByteArrayOutputStream formOutputStream = new ByteArrayOutputStream(); + try { + entity.writeTo(formOutputStream); + } catch (IOException e) { + throw new RuntimeException(e); + } + localVarRequestBuilder + .header("Content-Type", entity.getContentType().getValue()) + .method("{{httpMethod}}", HttpRequest.BodyPublishers + .ofInputStream(() -> new ByteArrayInputStream(formOutputStream.toByteArray()))); + {{/isMultipart}} + {{/hasFormParams}} + {{^hasFormParams}} + localVarRequestBuilder.method("{{httpMethod}}", HttpRequest.BodyPublishers.noBody()); + {{/hasFormParams}} + {{/bodyParam}} + if (memberVarReadTimeout != null) { + localVarRequestBuilder.timeout(memberVarReadTimeout); + } + if (memberVarInterceptor != null) { + memberVarInterceptor.accept(localVarRequestBuilder); + } + return localVarRequestBuilder; + } + + {{#vendorExtensions.x-group-parameters}} + {{#hasParams}} + + public static final class API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request { + {{#requiredParams}} + {{>nullable_var_annotations}} + private {{{dataType}}} {{paramName}}; // {{description}} (required) + {{/requiredParams}} + {{#optionalParams}} + {{>nullable_var_annotations}} + private {{{dataType}}} {{paramName}}; // {{description}} (optional{{^isContainer}}{{#defaultValue}}, default to {{.}}{{/defaultValue}}{{/isContainer}}) + {{/optionalParams}} + + private API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request(Builder builder) { + {{#requiredParams}} + this.{{paramName}} = builder.{{paramName}}; + {{/requiredParams}} + {{#optionalParams}} + this.{{paramName}} = builder.{{paramName}}; + {{/optionalParams}} + } + {{#allParams}} + {{>nullable_var_annotations}} + public {{{dataType}}} {{paramName}}() { + return {{paramName}}; + } + {{/allParams}} + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + {{#requiredParams}} + private {{{dataType}}} {{paramName}}; + {{/requiredParams}} + {{#optionalParams}} + private {{{dataType}}} {{paramName}}; + {{/optionalParams}} + + {{#allParams}} + public Builder {{paramName}}({{>nullable_var_annotations}} {{{dataType}}} {{paramName}}) { + this.{{paramName}} = {{paramName}}; + return this; + } + {{/allParams}} + public API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request build() { + return new API{{#lambda.titlecase}}{{operationId}}{{/lambda.titlecase}}Request(this); + } + } + } + + {{/hasParams}} + {{/vendorExtensions.x-group-parameters}} + {{/operation}} +} +{{/operations}} diff --git a/braintrust-api/src/generator/templates/PATCHES.md b/braintrust-api/src/generator/templates/PATCHES.md new file mode 100644 index 00000000..2c3c4a8b --- /dev/null +++ b/braintrust-api/src/generator/templates/PATCHES.md @@ -0,0 +1,60 @@ +# Custom Mustache Template Patches + +These templates are forked from **openapi-generator v7.14.0** and contain targeted +patches to fix bugs or add functionality not present upstream. They live under +`src/generator/` (not `src/main/resources`) so they are only available at +code-generation time and do not appear on the runtime classpath or in the published JAR. + +When upgrading the generator version, diff each file against the new upstream template: +- `modules/openapi-generator/src/main/resources/Java/libraries/native/anyof_model.mustache` +- `modules/openapi-generator/src/main/resources/Java/libraries/native/api.mustache` + +--- + +## anyof_model.mustache + +**Upstream:** `Java/libraries/native/anyof_model.mustache` @ v7.14.0 + +### Patch 1 — TypeReference for container variants in deserializer +The upstream deserializer calls `readValueAs(List.class)` / `readValueAs(Map.class)` for +container-typed anyOf variants, using the raw erased type. This causes silent +misclassification when two variants share the same raw type (e.g. `List` vs +`List`). Fixed by using `readValueAs(new TypeReference>(){})` so Jackson +knows the element type and fails fast when elements don't match, allowing the deserializer +to fall through to the correct variant. + +### Patch 2 — SchemaType enum +Added a `SchemaType` enum with one constant per anyOf variant. Constants are named from +the spec `title` field (titlecased) when present, falling back to `baseType` (the raw +erased class name). Each constant carries the full generic `dataType` as a `String` field. + +### Patch 3 — getVariantType() +Added `getVariantType()` returning the `SchemaType` enum constant for the current +instance. The resolved type is stored in a `resolvedVariantType` field set during +deserialization, so same-erased variants (e.g. `List` vs `List`) are +correctly distinguished. Manually constructed instances must use the +`(SchemaType, Object)` constructor to set the type explicitly. + +### Patch 4 — Single (SchemaType, Object) constructor +Replaced overloaded `(T o)` constructors (which produce a "duplicate method after +erasure" compile error when two variants share the same raw type) with a single +`(SchemaType type, Object o)` constructor that requires the caller to declare which +variant they're constructing. + +### Patch 5 — Named typed getters +Renamed typed getter methods from `getanyOf0Instance()` / `getanyOf1Instance()` to use +the same identifier as the `SchemaType` enum constant (e.g. `getSystemInstance()`, +`getWeightedInstance()`), keeping the instance accessor API consistent with the enum. + +--- + +## api.mustache + +**Upstream:** `Java/libraries/native/api.mustache` @ v7.14.0 + +### Patch 1 — Null-guard for anyOf model query parameters +The upstream template calls `{{paramName}}.toUrlQueryString()` unconditionally for +`isModel` params in the `isExplode/!hasVars` branch. When the parameter is `null` +(e.g. the `ids` parameter on list endpoints), this throws a `NullPointerException`. +Fixed by wrapping the call in `if ({{paramName}} != null)` and checking the result is +non-blank before adding it to the query string joiner. diff --git a/braintrust-api/src/test/java/dev/braintrust/openapi/model/AnyOfModelTemplateTest.java b/braintrust-api/src/test/java/dev/braintrust/openapi/model/AnyOfModelTemplateTest.java new file mode 100644 index 00000000..95f636ea --- /dev/null +++ b/braintrust-api/src/test/java/dev/braintrust/openapi/model/AnyOfModelTemplateTest.java @@ -0,0 +1,264 @@ +package dev.braintrust.openapi.model; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.openapi.JSON; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Tests for the custom anyof_model.mustache template. + * + *

The upstream template has a bug where anyOf schemas with generic container variants (e.g. + * List<Foo>, Map<K,V>) produce invalid Java: {@code List.class} literals and + * {@code getList()} method names. Our override fixes this by switching to {@code baseType} + * (the raw erased type) for .class references. + * + *

Our template also adds: + * + *

    + *
  • A {@code SchemaType} enum per anyOf class for exhaustive switch handling + *
  • {@code getVariantType()} returning the enum constant, accurate even for same-erased + * variants — set during deserialization, or explicitly via the {@code (SchemaType, Object)} + * constructor + *
  • Typed getters named after the enum constant (e.g. {@code getStringInstance()}, {@code + * getWeightedInstance()}) rather than synthetic {@code getanyOf0Instance()} names + *
  • Enum constants named from the spec {@code title} when present (titlecased), falling back to + * {@code baseType} otherwise + *
+ * + *

Test classes: + * + *

    + *
  • {@link AISecretType} — anyOf: [String, List<String>] — no titles, uses baseType names + * ({@code String}, {@code List}) + *
  • {@link ProjectScoreCategories} — anyOf: [List<ProjectScoreCategory>, + * Map<String,BigDecimal>, List<String>] — has titles ({@code categorical}, {@code + * weighted}, {@code minimum}), exercises title-based naming, the single-constructor fix for + * duplicate-erasing types, and accurate {@code getVariantType()} via deserialization and + * explicit constructor + *
+ */ +public class AnyOfModelTemplateTest { + + private static ObjectMapper mapper; + + @BeforeAll + static void setUp() { + mapper = new JSON().getMapper(); + } + + // ── AISecretType: anyOf [String, List] ──────────────────────────── + + @Test + void aiSecretType_setActualInstance_acceptsString() { + AISecretType t = new AISecretType(); + assertDoesNotThrow(() -> t.setActualInstance("openai")); + assertEquals("openai", t.getActualInstance()); + } + + @Test + void aiSecretType_setActualInstance_acceptsList() { + AISecretType t = new AISecretType(); + List list = List.of("openai", "anthropic"); + assertDoesNotThrow(() -> t.setActualInstance(list)); + assertEquals(list, t.getActualInstance()); + } + + @Test + void aiSecretType_setActualInstance_rejectsInvalidType() { + AISecretType t = new AISecretType(); + assertThrows(RuntimeException.class, () -> t.setActualInstance(42)); + } + + @Test + void aiSecretType_typedGetter_string() { + AISecretType t = new AISecretType(AISecretType.SchemaType.String, "anthropic"); + assertEquals("anthropic", t.getStringInstance()); + } + + @Test + void aiSecretType_typedGetter_list() { + AISecretType t = new AISecretType(AISecretType.SchemaType.List, List.of("a", "b")); + List result = t.getListInstance(); + assertEquals(List.of("a", "b"), result); + } + + @Test + void aiSecretType_deserialize_fromStringJson() throws Exception { + AISecretType t = mapper.readValue("\"openai\"", AISecretType.class); + assertNotNull(t); + assertInstanceOf(String.class, t.getActualInstance()); + assertEquals("openai", t.getActualInstance()); + } + + @Test + void aiSecretType_deserialize_fromArrayJson() throws Exception { + AISecretType t = mapper.readValue("[\"openai\",\"anthropic\"]", AISecretType.class); + assertNotNull(t); + assertInstanceOf(List.class, t.getActualInstance()); + @SuppressWarnings("unchecked") + List list = (List) t.getActualInstance(); + assertEquals(List.of("openai", "anthropic"), list); + } + + @Test + void aiSecretType_roundTrip_string() throws Exception { + AISecretType original = new AISecretType(AISecretType.SchemaType.String, "openai"); + String json = mapper.writeValueAsString(original); + AISecretType roundTripped = mapper.readValue(json, AISecretType.class); + assertEquals(original.getActualInstance(), roundTripped.getActualInstance()); + assertEquals(AISecretType.SchemaType.String, roundTripped.getVariantType()); + } + + @Test + void aiSecretType_getVariantType_viaConstructor_string() { + // No title in spec → enum constant name comes from baseType + AISecretType t = new AISecretType(AISecretType.SchemaType.String, "openai"); + assertEquals(AISecretType.SchemaType.String, t.getVariantType()); + } + + @Test + void aiSecretType_getVariantType_viaConstructor_list() { + AISecretType t = new AISecretType(AISecretType.SchemaType.List, List.of("openai")); + assertEquals(AISecretType.SchemaType.List, t.getVariantType()); + } + + @Test + void aiSecretType_getVariantType_viaDeserialization_string() throws Exception { + AISecretType t = mapper.readValue("\"openai\"", AISecretType.class); + assertEquals(AISecretType.SchemaType.String, t.getVariantType()); + } + + @Test + void aiSecretType_getVariantType_viaDeserialization_list() throws Exception { + AISecretType t = mapper.readValue("[\"openai\"]", AISecretType.class); + assertEquals(AISecretType.SchemaType.List, t.getVariantType()); + } + + @Test + void aiSecretType_getVariantType_throwsIfNotSet() { + // No-arg constructor + setActualInstance does not set resolvedVariantType + AISecretType t = new AISecretType(); + t.setActualInstance("openai"); + assertThrows(IllegalStateException.class, t::getVariantType); + } + + // ── ProjectScoreCategories: anyOf [List, Map, + // List] ────────────────────────── + + @Test + void projectScoreCategories_setActualInstance_acceptsList() { + ProjectScoreCategories c = new ProjectScoreCategories(); + assertDoesNotThrow(() -> c.setActualInstance(List.of())); + } + + @Test + void projectScoreCategories_setActualInstance_acceptsMap() { + ProjectScoreCategories c = new ProjectScoreCategories(); + assertDoesNotThrow(() -> c.setActualInstance(Map.of("accuracy", new BigDecimal("0.8")))); + } + + @Test + void projectScoreCategories_setActualInstance_rejectsInvalidType() { + ProjectScoreCategories c = new ProjectScoreCategories(); + assertThrows(RuntimeException.class, () -> c.setActualInstance("not-valid")); + } + + @Test + void projectScoreCategories_getVariantType_categorical_viaConstructor() { + ProjectScoreCategories c = + new ProjectScoreCategories( + ProjectScoreCategories.SchemaType.Categorical, + List.of(new ProjectScoreCategory())); + assertEquals(ProjectScoreCategories.SchemaType.Categorical, c.getVariantType()); + } + + @Test + void projectScoreCategories_getVariantType_weighted_viaConstructor() { + ProjectScoreCategories c = + new ProjectScoreCategories( + ProjectScoreCategories.SchemaType.Weighted, + Map.of("accuracy", new BigDecimal("0.9"))); + assertEquals(ProjectScoreCategories.SchemaType.Weighted, c.getVariantType()); + } + + @Test + void projectScoreCategories_getVariantType_minimum_viaConstructor() { + // Explicitly declare the List variant as Minimum — this is the key fix. + // Without the (SchemaType, Object) constructor, getVariantType() would incorrectly + // return Categorical for both List variants due to erasure. + ProjectScoreCategories c = + new ProjectScoreCategories( + ProjectScoreCategories.SchemaType.Minimum, List.of("pass", "fail")); + assertEquals(ProjectScoreCategories.SchemaType.Minimum, c.getVariantType()); + } + + @Test + void projectScoreCategories_getVariantType_viaDeserialization_minimum() throws Exception { + // Uses TypeReference> so Jackson fails fast on object elements, + // correctly distinguishing Minimum (List) from Categorical + // (List) + ProjectScoreCategories c = + mapper.readValue("[\"pass\",\"fail\"]", ProjectScoreCategories.class); + assertEquals(ProjectScoreCategories.SchemaType.Minimum, c.getVariantType()); + } + + @Test + void projectScoreCategories_getVariantType_viaDeserialization_categorical() throws Exception { + ProjectScoreCategories c = + mapper.readValue( + "[{\"name\":\"pass\",\"value\":1.0}]", ProjectScoreCategories.class); + assertEquals(ProjectScoreCategories.SchemaType.Categorical, c.getVariantType()); + } + + @Test + void projectScoreCategories_getVariantType_viaDeserialization_weighted() throws Exception { + ProjectScoreCategories c = + mapper.readValue("{\"accuracy\":0.8}", ProjectScoreCategories.class); + assertEquals(ProjectScoreCategories.SchemaType.Weighted, c.getVariantType()); + } + + @Test + void projectScoreCategories_deserialize_fromObjectJson() throws Exception { + ProjectScoreCategories c = + mapper.readValue("{\"accuracy\":0.8,\"recall\":0.6}", ProjectScoreCategories.class); + assertNotNull(c); + assertInstanceOf(Map.class, c.getActualInstance()); + } + + @Test + void projectScoreCategories_roundTrip_map() throws Exception { + ProjectScoreCategories original = + new ProjectScoreCategories( + ProjectScoreCategories.SchemaType.Weighted, + Map.of("accuracy", new BigDecimal("0.9"))); + String json = mapper.writeValueAsString(original); + ProjectScoreCategories roundTripped = mapper.readValue(json, ProjectScoreCategories.class); + assertInstanceOf(Map.class, roundTripped.getActualInstance()); + assertEquals(ProjectScoreCategories.SchemaType.Weighted, roundTripped.getVariantType()); + } + + @Test + void projectScoreCategories_typedGetter_weighted() { + Map weights = Map.of("accuracy", new BigDecimal("0.8")); + ProjectScoreCategories c = + new ProjectScoreCategories(ProjectScoreCategories.SchemaType.Weighted, weights); + assertEquals(weights, c.getWeightedInstance()); + } + + @Test + void projectScoreCategories_schemaType_dataType_field() { + // Each enum constant carries the full generic type as .dataType + assertEquals( + "List", + ProjectScoreCategories.SchemaType.Categorical.dataType); + assertEquals( + "Map", ProjectScoreCategories.SchemaType.Weighted.dataType); + assertEquals("List", ProjectScoreCategories.SchemaType.Minimum.dataType); + } +} diff --git a/braintrust-sdk/build.gradle b/braintrust-sdk/build.gradle index 88279421..eec44e33 100644 --- a/braintrust-sdk/build.gradle +++ b/braintrust-sdk/build.gradle @@ -105,6 +105,8 @@ afterEvaluate { } dependencies { + implementation project(':braintrust-api') + api "io.opentelemetry:opentelemetry-api:${otelVersion}" api "io.opentelemetry:opentelemetry-sdk:${otelVersion}" api "io.opentelemetry:opentelemetry-sdk-trace:${otelVersion}" diff --git a/braintrust-sdk/src/main/java/dev/braintrust/Braintrust.java b/braintrust-sdk/src/main/java/dev/braintrust/Braintrust.java index a85df960..01bfda81 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/Braintrust.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/Braintrust.java @@ -1,6 +1,7 @@ package dev.braintrust; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.config.BraintrustConfig; import dev.braintrust.eval.Dataset; import dev.braintrust.eval.Eval; @@ -87,7 +88,7 @@ static void resetForTest() { /** Create a new Braintrust instance from the given config */ public static Braintrust of(BraintrustConfig config) { - BraintrustApiClient apiClient = BraintrustApiClient.of(config); + BraintrustOpenApiClient apiClient = BraintrustOpenApiClient.of(config); BraintrustPromptLoader promptLoader = BraintrustPromptLoader.of(config, apiClient); return new Braintrust(config, apiClient, promptLoader); } @@ -96,9 +97,15 @@ public static Braintrust of(BraintrustConfig config) { @Accessors(fluent = true) private final BraintrustConfig config; + /** Deprecated. Please use openApiClient() instead */ + @Deprecated + public BraintrustApiClient apiClient() { + return BraintrustApiClient.of(config); + } + @Getter @Accessors(fluent = true) - private final BraintrustApiClient apiClient; + private final BraintrustOpenApiClient openApiClient; @Getter @Accessors(fluent = true) @@ -106,17 +113,16 @@ public static Braintrust of(BraintrustConfig config) { Braintrust( BraintrustConfig config, - BraintrustApiClient apiClient, + BraintrustOpenApiClient apiClient, BraintrustPromptLoader promptLoader) { this.config = config; - this.apiClient = apiClient; + this.openApiClient = apiClient; this.promptLoader = promptLoader; } - /** the the URI to the configured braintrust org and project */ + /** URI to the configured braintrust org and project */ public URI projectUri() { - return BraintrustUtils.createProjectURI( - config.appUrl(), apiClient().getOrCreateProjectAndOrgInfo(config())); + return openApiClient.fetchProjectUri(); } /** @@ -162,7 +168,7 @@ public void openTelemetryEnable( /** Create a new eval builder */ public Eval.Builder evalBuilder() { return (Eval.Builder) - Eval.builder().config(this.config).apiClient(this.apiClient); + Eval.builder().config(this.config).apiClient(this.openApiClient); } public Dataset fetchDataset(String datasetName) { @@ -171,8 +177,8 @@ public Dataset fetchDataset(String datasetName) { public Dataset fetchDataset( String datasetName, @Nullable String datasetVersion) { - var projectName = apiClient.getOrCreateProjectAndOrgInfo(config).project().name(); - return Dataset.fetchFromBraintrust(apiClient(), projectName, datasetName, datasetVersion); + return Dataset.fetchFromBraintrust( + openApiClient, resolveProjectName(), datasetName, datasetVersion); } /** @@ -194,8 +200,7 @@ public Scorer fetchScorer(String scorerSlug) { */ public Scorer fetchScorer( String scorerSlug, @Nullable String version) { - var projectName = apiClient.getOrCreateProjectAndOrgInfo(config).project().name(); - return Scorer.fetchFromBraintrust(apiClient, projectName, scorerSlug, version); + return Scorer.fetchFromBraintrust(openApiClient, resolveProjectName(), scorerSlug, version); } /** @@ -208,6 +213,18 @@ public Scorer fetchScorer( */ public Scorer fetchScorer( String projectName, String scorerSlug, @Nullable String version) { - return Scorer.fetchFromBraintrust(apiClient, projectName, scorerSlug, version); + return Scorer.fetchFromBraintrust(openApiClient, projectName, scorerSlug, version); + } + + /** + * Resolve the default project name from config. If only a project ID is configured, looks it up + * via the API. Mirrors the behavior of the old hand-rolled client. + */ + private String resolveProjectName() { + return openApiClient + .fetchOrCreateProject( + config.defaultProjectId().orElse(null), + config.defaultProjectName().orElse(null)) + .getName(); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java b/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java index 4e2fa4ad..2e39122a 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/BraintrustUtils.java @@ -1,6 +1,8 @@ package dev.braintrust; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.openapi.model.Organization; +import dev.braintrust.openapi.model.Project; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -9,15 +11,25 @@ import javax.annotation.Nonnull; public class BraintrustUtils { - /** construct a URI to link to a specific braintrust project within an org */ + @Deprecated public static URI createProjectURI( String appUrl, BraintrustApiClient.OrganizationAndProjectInfo orgAndProject) { + return createProjectURI( + appUrl, orgAndProject.orgInfo().name(), orgAndProject.project().name()); + } + + /** + * construct a URI to link to a specific braintrust project within an org, using generated types + */ + public static URI createProjectURI(String appUrl, Organization org, Project project) { + return createProjectURI(appUrl, org.getName(), project.getName()); + } + + /** construct a URI to link to a specific braintrust project within an org by name */ + public static URI createProjectURI(String appUrl, String orgName, String projectName) { try { var baseURI = new URI(appUrl); - var path = - "/app/%s/p/%s" - .formatted( - orgAndProject.orgInfo().name(), orgAndProject.project().name()); + var path = "/app/%s/p/%s".formatted(orgName, projectName); return new URI( baseURI.getScheme(), baseURI.getUserInfo(), diff --git a/braintrust-sdk/src/main/java/dev/braintrust/api/ApiException.java b/braintrust-sdk/src/main/java/dev/braintrust/api/ApiException.java index cfe9c289..7838e00d 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/api/ApiException.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/api/ApiException.java @@ -1,7 +1,8 @@ package dev.braintrust.api; +/** Deprecated. Please use {@link dev.braintrust.openapi.ApiException} instead */ +@Deprecated class ApiException extends RuntimeException { - public ApiException(String message) { super(message); } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java index c402dfd4..dad987b5 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -17,11 +17,18 @@ import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; -/** - * Provides the necessary API calls for the Braintrust SDK. Users of the SDK should favor using - * {@link dev.braintrust.eval.Eval} or {@link dev.braintrust.trace.BraintrustTracing} - */ +/** Deprecated. Please use {@link BraintrustOpenApiClient} instead */ +@Deprecated public interface BraintrustApiClient { + /** + * Create an openapi client with the same api key as this one. + * + *

Convenience method for migrating to the new client. It's recommended to not use this + * method and simply migrate to the new BraintrustOpenApiClient + */ + @Deprecated + BraintrustOpenApiClient openApiClient(); + /** * Attempt Braintrust login * @@ -170,6 +177,11 @@ public List listExperiments(String projectId) { } } + @Override + public BraintrustOpenApiClient openApiClient() { + return BraintrustOpenApiClient.of(config); + } + @Override public LoginResponse login() throws LoginException { try { diff --git a/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustOpenApiClient.java b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustOpenApiClient.java new file mode 100644 index 00000000..e7cb09be --- /dev/null +++ b/braintrust-sdk/src/main/java/dev/braintrust/api/BraintrustOpenApiClient.java @@ -0,0 +1,173 @@ +package dev.braintrust.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.BraintrustUtils; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.openapi.ApiClient; +import dev.braintrust.openapi.ApiException; +import dev.braintrust.openapi.JSON; +import dev.braintrust.openapi.api.ProjectsApi; +import dev.braintrust.openapi.model.CreateProject; +import dev.braintrust.openapi.model.Project; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.annotation.Nullable; + +/** + * Provides the necessary API calls for the Braintrust SDK. Users of the SDK should favor using + * {@link dev.braintrust.eval.Eval} or {@link dev.braintrust.trace.BraintrustTracing} + */ +public class BraintrustOpenApiClient extends ApiClient { + private static final ObjectMapper MAPPER = new JSON().getMapper(); + + private final BraintrustConfig config; + + private BraintrustOpenApiClient(BraintrustConfig config) { + super(); + this.config = config; + this.updateBaseUri(config.apiUrl()); + this.setHttpClientBuilder(HttpClient.newBuilder().sslContext(config.sslContext())); + this.setRequestInterceptor(req -> req.header("Authorization", "Bearer " + config.apiKey())); + } + + public static BraintrustOpenApiClient of(BraintrustConfig config) { + return new BraintrustOpenApiClient(config); + } + + /** + * Calls {@code POST /api/apikey/login} to retrieve organization info for the current API key. + * This endpoint is not in the OpenAPI spec so it is implemented here as a custom method. + */ + public LoginResponse login() { + try { + var body = MAPPER.writeValueAsString(new LoginRequest(config.apiKey())); + var requestBuilder = + HttpRequest.newBuilder() + .uri(URI.create(config.apiUrl() + "/api/apikey/login")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body)); + + if (getRequestInterceptor() != null) { + getRequestInterceptor().accept(requestBuilder); + } + + HttpResponse response = + getHttpClient() + .send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() / 100 != 2) { + throw new LoginException( + "Login failed with status " + + response.statusCode() + + ": " + + response.body()); + } + return MAPPER.readValue(response.body(), LoginResponse.class); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Look up or create the project from config, resolve the org name via login, and return the + * Braintrust app URI for the project. + */ + public URI fetchProjectUri() { + var project = fetchOrCreateProject(config); + var orgName = fetchOrgInfo(project.getOrgId().toString()).name(); + return BraintrustUtils.createProjectURI(config.appUrl(), orgName, project.getName()); + } + + public OrgInfo fetchOrgInfo(String orgId) { + return login().orgInfo().stream() + .filter(o -> o.id().equalsIgnoreCase(orgId)) + .findFirst() + .orElseThrow(() -> new ApiException("Unable to find org for project: " + orgId)); + } + + public Project fetchOrCreateProject(BraintrustConfig config) { + return fetchOrCreateProject( + config.defaultProjectId().orElse(null), config.defaultProjectName().orElse(null)); + } + + public Project fetchOrCreateProject(@Nullable String projectId, @Nullable String projectName) { + if (projectId == null && projectName == null) { + throw new IllegalArgumentException("must provide project id or project name"); + } + var projectsApi = new ProjectsApi(this); + + Project project; + if (projectId != null) { + project = projectsApi.getProjectId(UUID.fromString(projectId)); + } else { + var existing = + projectsApi.getProject(null, null, null, null, projectName, null).getObjects(); + if (existing.isEmpty()) { + project = projectsApi.postProject(new CreateProject().name(projectName)); + } else if (existing.size() == 1) { + project = existing.get(0); + } else { + var projectIds = existing.stream().map(p -> p.getId().toString()).toList(); + throw new RuntimeException( + "Multiple projects with the same name already exists. This should not happen. project name: %s project ids: %s" + .formatted(projectName, projectIds)); + } + } + return project; + } + + /** + * Calls {@code POST /btql} to run an arbitrary BTQL query. This endpoint is not in the OpenAPI + * spec so it is implemented here as a custom method. + */ + public BtqlQueryResponse btqlQuery(String query) { + try { + var body = MAPPER.writeValueAsString(new BtqlQueryRequest(query)); + var requestBuilder = + HttpRequest.newBuilder() + .uri(URI.create(config.apiUrl() + "/btql")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body)); + + if (getRequestInterceptor() != null) { + getRequestInterceptor().accept(requestBuilder); + } + + HttpResponse response = + getHttpClient() + .send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() / 100 != 2) { + throw new RuntimeException( + "BTQL query failed with status " + + response.statusCode() + + ": " + + response.body()); + } + + return MAPPER.readValue(response.body(), BtqlQueryResponse.class); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("BTQL query failed", e); + } + } + + public record OrgInfo(String id, String name) {} + + public record BtqlQueryResponse(List> data) {} + + private record LoginRequest(String token) {} + + private record BtqlQueryRequest(String query) {} + + public record LoginResponse(@JsonProperty("org_info") List orgInfo) {} +} diff --git a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java index c87f4055..30d2888e 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -1,8 +1,7 @@ package dev.braintrust.config; import dev.braintrust.Braintrust; -import dev.braintrust.BraintrustUtils; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import java.net.URI; import java.time.Duration; import java.util.HashMap; @@ -116,9 +115,7 @@ public Optional getBraintrustParentValue() { /** Deprecated. Please use {@link Braintrust#projectUri()} instead */ @Deprecated public URI fetchProjectURI() { - var client = BraintrustApiClient.of(this); - var orgAndProject = client.getProjectAndOrgInfo().orElseThrow(); - return BraintrustUtils.createProjectURI(appUrl(), orgAndProject); + return BraintrustOpenApiClient.of(this).fetchProjectUri(); } public static Builder builder() { diff --git a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java index 893359ea..129957d2 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java @@ -10,7 +10,7 @@ import dev.braintrust.Braintrust; import dev.braintrust.BraintrustUtils; import dev.braintrust.Origin; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.config.BraintrustConfig; import dev.braintrust.eval.*; import dev.braintrust.trace.BraintrustContext; @@ -303,7 +303,7 @@ private void handleEval(HttpExchange exchange) throws IOException { // Resolve remote scorers from the request List> remoteScorers = new ArrayList<>(); if (request.getScores() != null) { - var apiClient = context.getBraintrust().apiClient(); + var apiClient = context.getBraintrust().openApiClient(); for (var remoteScorer : request.getScores()) { remoteScorers.add(resolveRemoteScorer(remoteScorer, apiClient)); } @@ -364,27 +364,22 @@ private void handleStreamingEval( try { // Get Braintrust instance from authenticated context Braintrust braintrust = context.getBraintrust(); - BraintrustApiClient apiClient = braintrust.apiClient(); + var apiClient = braintrust.openApiClient(); // Determine project name and ID from the authenticated Braintrust instance - final var orgAndProject = - apiClient.getOrCreateProjectAndOrgInfo(braintrust.config()); - final var projectName = orgAndProject.project().name(); - final var projectId = orgAndProject.project().id(); + final var project = apiClient.fetchOrCreateProject(braintrust.config()); + final var projectName = project.getName(); + final var projectId = project.getId().toString(); + final var orgName = apiClient.fetchOrgInfo(project.getOrgId().toString()).name(); final var experimentName = request.getExperimentName() != null ? request.getExperimentName() : eval.getName(); - final var experimentUrl = - BraintrustUtils.createProjectURI( - braintrust.config().appUrl(), orgAndProject) - .toASCIIString() - + "/experiments/" - + experimentName; final var projectUrl = BraintrustUtils.createProjectURI( - braintrust.config().appUrl(), orgAndProject) + braintrust.config().appUrl(), orgName, projectName) .toASCIIString(); + final var experimentUrl = projectUrl + "/experiments/" + experimentName; var tracer = BraintrustTracing.getTracer(); @@ -1069,7 +1064,7 @@ private RequestContext getBraintrust(HttpExchange exchange, RequestContext conte } var bt = Braintrust.of(configBuilder.build()); - bt.apiClient().login(); + bt.openApiClient().login(); // validates the API key return bt; }); @@ -1170,7 +1165,7 @@ private static ParentInfo extractParentInfo(EvalRequest request) { * @throws IllegalArgumentException if dataset or project is not found */ private static Dataset extractDataset( - EvalRequest request, BraintrustApiClient apiClient) { + EvalRequest request, BraintrustOpenApiClient apiClient) { EvalRequest.DataSpec dataSpec = request.getData(); if (dataSpec.getData() != null && !dataSpec.getData().isEmpty()) { @@ -1197,19 +1192,14 @@ private static ParentInfo extractParentInfo(EvalRequest request) { } else if (dataSpec.getDatasetId() != null) { // Method 3: Fetch by dataset ID log.debug("Fetching dataset from Braintrust by ID: {}", dataSpec.getDatasetId()); - var datasetMetadata = apiClient.getDataset(dataSpec.getDatasetId()); - if (datasetMetadata.isEmpty()) { - throw new IllegalArgumentException("Dataset not found: " + dataSpec.getDatasetId()); - } - - var project = apiClient.getProject(datasetMetadata.get().projectId()); - if (project.isEmpty()) { - throw new IllegalArgumentException( - "Project not found: " + datasetMetadata.get().projectId()); - } - - String fetchedProjectName = project.get().name(); - String fetchedDatasetName = datasetMetadata.get().name(); + var datasetsApi = new dev.braintrust.openapi.api.DatasetsApi(apiClient); + var projectsApi = new dev.braintrust.openapi.api.ProjectsApi(apiClient); + var dataset = + datasetsApi.getDatasetId(java.util.UUID.fromString(dataSpec.getDatasetId())); + var project = projectsApi.getProjectId(dataset.getProjectId()); + + String fetchedProjectName = project.getName(); + String fetchedDatasetName = dataset.getName(); log.debug( "Resolved dataset ID to project={}, dataset={}", fetchedProjectName, @@ -1231,7 +1221,7 @@ private static ParentInfo extractParentInfo(EvalRequest request) { * @throws IllegalArgumentException if the function_id is missing */ private static Scorer resolveRemoteScorer( - EvalRequest.RemoteScorer remoteScorer, BraintrustApiClient apiClient) { + EvalRequest.RemoteScorer remoteScorer, BraintrustOpenApiClient apiClient) { var functionIdSpec = remoteScorer.getFunctionId(); if (functionIdSpec == null || functionIdSpec.getFunctionId() == null) { diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java index ab15214a..ee42952b 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Dataset.java @@ -1,6 +1,8 @@ package dev.braintrust.eval; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; +import dev.braintrust.openapi.api.DatasetsApi; import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -62,33 +64,44 @@ static Dataset of(DatasetCase... c return new DatasetInMemoryImpl<>(List.of(cases)); } + @Deprecated static Dataset fetchFromBraintrust( BraintrustApiClient apiClient, String projectName, String datasetName, @Nullable String datasetVersion) { - var datasets = apiClient.queryDatasets(projectName, datasetName); + return fetchFromBraintrust( + apiClient.openApiClient(), projectName, datasetName, datasetVersion); + } + + static Dataset fetchFromBraintrust( + BraintrustOpenApiClient apiClient, + String projectName, + String datasetName, + @Nullable String datasetVersion) { + var datasetsApi = new DatasetsApi(apiClient); + var objects = + datasetsApi + .getDataset(null, null, null, null, datasetName, projectName, null, null) + .getObjects(); - if (datasets.isEmpty()) { + if (objects.isEmpty()) { throw new RuntimeException( "Dataset not found: project=" + projectName + ", dataset=" + datasetName); } - if (datasets.size() > 1) { + if (objects.size() > 1) { throw new RuntimeException( "Multiple datasets found for project=" + projectName + ", dataset=" + datasetName + ". Found " - + datasets.size() + + objects.size() + " datasets"); } - var dataset = datasets.get(0); - return new DatasetBrainstoreImpl<>( - apiClient, - dataset.id(), - datasetVersion != null ? datasetVersion : dataset.updatedAt()); + var dataset = objects.get(0); + return new DatasetBrainstoreImpl<>(apiClient, dataset.getId().toString(), datasetVersion); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/DatasetBrainstoreImpl.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/DatasetBrainstoreImpl.java index 0d41a2c6..c594ee21 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/DatasetBrainstoreImpl.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/DatasetBrainstoreImpl.java @@ -1,24 +1,33 @@ package dev.braintrust.eval; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; +import dev.braintrust.openapi.api.DatasetsApi; +import dev.braintrust.openapi.model.FetchEventsRequest; import java.util.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; /** A dataset loaded externally from Braintrust using paginated API fetches */ public class DatasetBrainstoreImpl implements Dataset { - private final BraintrustApiClient apiClient; + private final BraintrustOpenApiClient apiClient; private final String datasetId; private final @Nullable String pinnedVersion; private final int batchSize; + @Deprecated public DatasetBrainstoreImpl( BraintrustApiClient apiClient, String datasetId, @Nullable String datasetVersion) { + this(apiClient.openApiClient(), datasetId, datasetVersion, 512); + } + + public DatasetBrainstoreImpl( + BraintrustOpenApiClient apiClient, String datasetId, @Nullable String datasetVersion) { this(apiClient, datasetId, datasetVersion, 512); } DatasetBrainstoreImpl( - BraintrustApiClient apiClient, + BraintrustOpenApiClient apiClient, String datasetId, @Nullable String datasetVersion, int batchSize) { @@ -62,7 +71,7 @@ private String fetchMaxVersion() { } private class BrainstoreCursor implements Cursor> { - private List> currentBatch; + private List currentBatch; private int currentIndex; private @Nullable String cursor; private boolean exhausted; @@ -85,33 +94,27 @@ public Optional> next() { throw new IllegalStateException("Cursor is closed"); } - // Fetch next batch if we've consumed the current one if (currentIndex >= currentBatch.size() && !exhausted) { fetchNextBatch(); } - // Return empty if no more data if (currentIndex >= currentBatch.size()) { return Optional.empty(); } - // Parse the event into a DatasetCase - Map event = currentBatch.get(currentIndex++); + var event = currentBatch.get(currentIndex++); - INPUT input = (INPUT) event.get("input"); - OUTPUT expected = (OUTPUT) event.get("expected"); + INPUT input = (INPUT) event.getInput(); + OUTPUT expected = (OUTPUT) event.getExpected(); - Map metadata = (Map) event.get("metadata"); - if (metadata == null) { - metadata = Map.of(); - } + var metadataObj = event.getMetadata(); + Map metadata = + metadataObj != null ? metadataObj.getAdditionalProperties() : Map.of(); + if (metadata == null) metadata = Map.of(); - List tags = (List) event.get("tags"); - if (tags == null) { - tags = List.of(); - } + List tags = event.getTags() != null ? event.getTags() : List.of(); - DatasetCase datasetCase = + var datasetCase = new DatasetCase<>( input, expected, @@ -121,26 +124,32 @@ public Optional> next() { new dev.braintrust.Origin( "dataset", Objects.requireNonNull( - (String) event.get("dataset_id")), - Objects.requireNonNull((String) event.get("id")), - Objects.requireNonNull((String) event.get("_xact_id")), + event.getDatasetId() != null + ? event.getDatasetId().toString() + : null), + Objects.requireNonNull(event.getId()), + Objects.requireNonNull(event.getXactId()), Objects.requireNonNull( - (String) event.get("created"))))); + event.getCreated() != null + ? event.getCreated().toString() + : null)))); return Optional.of(datasetCase); } private void fetchNextBatch() { var request = - new BraintrustApiClient.DatasetFetchRequest(batchSize, cursor, cursorVersion); - var response = apiClient.fetchDatasetEvents(datasetId, request); + new FetchEventsRequest().limit(batchSize).cursor(cursor).version(cursorVersion); + + var response = + new DatasetsApi(apiClient) + .postDatasetIdFetch(UUID.fromString(datasetId), request); - currentBatch = new ArrayList<>(response.events()); + currentBatch = new ArrayList<>(response.getEvents()); currentIndex = 0; - cursor = response.cursor(); + cursor = response.getCursor(); - // Mark as exhausted if no cursor or no events returned - if (cursor == null || cursor.isEmpty() || response.events().isEmpty()) { + if (cursor == null || cursor.isEmpty() || response.getEvents().isEmpty()) { exhausted = true; } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java index 52a4de22..0286d46a 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java @@ -4,7 +4,11 @@ import dev.braintrust.BraintrustUtils; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.openapi.api.ExperimentsApi; +import dev.braintrust.openapi.model.CreateExperiment; +import dev.braintrust.openapi.model.Project; import dev.braintrust.trace.BraintrustContext; import dev.braintrust.trace.BraintrustTracing; import io.opentelemetry.api.common.AttributeKey; @@ -30,8 +34,9 @@ public final class Eval { AttributeKey.stringKey(BraintrustTracing.PARENT_KEY); private final @Nonnull String experimentName; private final @Nonnull BraintrustConfig config; - private final @Nonnull BraintrustApiClient client; - private final @Nonnull BraintrustApiClient.OrganizationAndProjectInfo orgAndProject; + private final @Nonnull BraintrustOpenApiClient client; + private final @Nonnull Project project; + private final @Nonnull BraintrustOpenApiClient.OrgInfo orgInfo; private final @Nonnull Tracer tracer; private final @Nonnull Dataset dataset; private final @Nonnull Task task; @@ -44,16 +49,10 @@ private Eval(Builder builder) { this.experimentName = builder.experimentName; this.config = Objects.requireNonNull(builder.config); this.client = Objects.requireNonNull(builder.apiClient); - if (null == builder.projectId) { - this.orgAndProject = client.getProjectAndOrgInfo().orElseThrow(); - } else { - this.orgAndProject = - client.getProjectAndOrgInfo(builder.projectId) - .orElseThrow( - () -> - new RuntimeException( - "invalid project id: " + builder.projectId)); - } + this.project = + client.fetchOrCreateProject( + builder.projectId, config.defaultProjectName().orElse(null)); + this.orgInfo = client.fetchOrgInfo(project.getOrgId().toString()); this.tracer = Objects.requireNonNull(builder.tracer); this.dataset = builder.dataset; this.task = Objects.requireNonNull(builder.task); @@ -69,28 +68,32 @@ public EvalResult run() { Optional datasetVersion = Optional.empty(); Optional datasetId = Optional.empty(); if (dataset instanceof DatasetBrainstoreImpl) { - // TODO: come up with a means of expressing this naturally in the dataset api datasetVersion = cursor.version(); datasetId = Optional.of(dataset.id()); } - var experiment = - client.getOrCreateExperiment( - new BraintrustApiClient.CreateExperimentRequest( - orgAndProject.project().id(), - experimentName, - Optional.empty(), - Optional.empty(), - tags.isEmpty() ? Optional.empty() : Optional.of(tags), - metadata.isEmpty() ? Optional.empty() : Optional.of(metadata), - datasetId, - datasetVersion)); - cursor.forEach(datasetCase -> evalOne(experiment.id(), datasetCase)); + + var createExperiment = + new CreateExperiment().projectId(project.getId()).name(experimentName); + + if (!tags.isEmpty()) { + createExperiment.tags(tags); + } + if (!metadata.isEmpty()) { + createExperiment.metadata(metadata); + } + datasetId.ifPresent(id -> createExperiment.datasetId(UUID.fromString(id))); + datasetVersion.ifPresent(createExperiment::datasetVersion); + + var experiment = new ExperimentsApi(client).postExperiment(createExperiment); + + cursor.forEach(datasetCase -> evalOne(experiment.getId().toString(), datasetCase)); } var experimentUrl = "%s/experiments/%s" .formatted( - BraintrustUtils.createProjectURI(config.appUrl(), orgAndProject) + BraintrustUtils.createProjectURI( + config.appUrl(), orgInfo.name(), project.getName()) .toASCIIString(), experimentName); return new EvalResult(experimentUrl); @@ -249,7 +252,7 @@ public static final class Builder { public @Nonnull Dataset dataset; private @Nonnull String experimentName = "unnamed-java-eval"; private @Nullable BraintrustConfig config; - private @Nullable BraintrustApiClient apiClient; + private @Nullable BraintrustOpenApiClient apiClient; private @Nullable String projectId; private @Nullable Tracer tracer = null; private @Nullable Task task; @@ -273,7 +276,7 @@ public Eval build() { throw new RuntimeException("must provide at least one scorer"); } if (null == apiClient) { - apiClient = BraintrustApiClient.of(config); + apiClient = BraintrustOpenApiClient.of(config); } Objects.requireNonNull(dataset); Objects.requireNonNull(task); @@ -295,11 +298,16 @@ public Builder config(BraintrustConfig config) { return this; } - public Builder apiClient(BraintrustApiClient apiClient) { + public Builder apiClient(BraintrustOpenApiClient apiClient) { this.apiClient = apiClient; return this; } + @Deprecated + public Builder apiClient(BraintrustApiClient apiClient) { + return apiClient(apiClient.openApiClient()); + } + public Builder tracer(Tracer tracer) { this.tracer = tracer; return this; diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java index 3e4ffb9a..1b62fa25 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/Scorer.java @@ -1,6 +1,8 @@ package dev.braintrust.eval; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; +import dev.braintrust.openapi.api.FunctionsApi; import java.util.List; import java.util.function.BiFunction; import java.util.function.Function; @@ -85,6 +87,15 @@ public List score(TaskResult taskResult) { }; } + @Deprecated + static Scorer fetchFromBraintrust( + BraintrustApiClient apiClient, + String projectName, + String scorerSlug, + @Nullable String version) { + return fetchFromBraintrust(apiClient.openApiClient(), projectName, scorerSlug, version); + } + /** * Fetch a scorer from Braintrust by project name and slug. * @@ -96,21 +107,32 @@ public List score(TaskResult taskResult) { * @throws RuntimeException if the scorer is not found */ static Scorer fetchFromBraintrust( - BraintrustApiClient apiClient, + BraintrustOpenApiClient apiClient, String projectName, String scorerSlug, @Nullable String version) { - var function = - apiClient - .getFunction(projectName, scorerSlug, version) - .orElseThrow( - () -> - new RuntimeException( - "Scorer not found: project=" - + projectName - + ", slug=" - + scorerSlug)); + var functionsApi = new FunctionsApi(apiClient); + var objects = + functionsApi + .getFunction( + null, + null, + null, + null, + null, + projectName, + null, + scorerSlug, + version, + null, + null) + .getObjects(); + + if (objects.isEmpty()) { + throw new RuntimeException( + "Scorer not found: project=" + projectName + ", slug=" + scorerSlug); + } - return new ScorerBrainstoreImpl<>(apiClient, function.id(), version); + return new ScorerBrainstoreImpl<>(apiClient, objects.get(0).getId().toString(), version); } } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/eval/ScorerBrainstoreImpl.java b/braintrust-sdk/src/main/java/dev/braintrust/eval/ScorerBrainstoreImpl.java index c6f1d5fb..ec491262 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/eval/ScorerBrainstoreImpl.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/eval/ScorerBrainstoreImpl.java @@ -1,7 +1,16 @@ package dev.braintrust.eval; +import com.fasterxml.jackson.databind.ObjectMapper; import dev.braintrust.BraintrustUtils; import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; +import dev.braintrust.openapi.JSON; +import dev.braintrust.openapi.api.FunctionsApi; +import dev.braintrust.openapi.api.ProjectsApi; +import dev.braintrust.openapi.model.Function; +import dev.braintrust.openapi.model.InvokeApi; +import dev.braintrust.openapi.model.InvokeParent; +import dev.braintrust.openapi.model.SpanParentStruct; import dev.braintrust.trace.BraintrustTracing; import dev.braintrust.trace.SpanComponents; import io.opentelemetry.api.baggage.Baggage; @@ -10,6 +19,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -27,13 +37,20 @@ */ @Slf4j public class ScorerBrainstoreImpl implements Scorer { - private final BraintrustApiClient apiClient; + private static final ObjectMapper MAPPER = new JSON().getMapper(); + + private final BraintrustOpenApiClient apiClient; private final String functionId; private final @Nullable String version; - private final AtomicReference braintrustFunction = - new AtomicReference<>(null); + private final AtomicReference braintrustFunction = new AtomicReference<>(null); private final AtomicReference scorerName = new AtomicReference<>(null); + @Deprecated + public ScorerBrainstoreImpl( + BraintrustApiClient apiClient, String functionId, @Nullable String version) { + this(apiClient.openApiClient(), functionId, version); + } + /** * Create a new remote scorer. * @@ -43,7 +60,7 @@ public class ScorerBrainstoreImpl implements Scorer score(TaskResult taskResult) { - // Build parent span components for distributed tracing (as object, not base64 string) - Object parent = buildParentSpanComponents(); - - // Build scorer args map with optional parameters var scorerArgs = new java.util.LinkedHashMap(); scorerArgs.put("input", taskResult.datasetCase().input()); scorerArgs.put("output", taskResult.result()); @@ -70,67 +83,66 @@ public List score(TaskResult taskResult) { scorerArgs.put("parameters", taskResult.parameters().getMerged()); } - var request = new BraintrustApiClient.FunctionInvokeRequest(scorerArgs, version, parent); + var invoke = new InvokeApi().input(scorerArgs).version(version); + + InvokeParent parent = buildParent(); + if (parent != null) { + invoke.parent(parent); + } - Object result = apiClient.invokeFunction(getFunctionId(), request); + Object result = + new FunctionsApi(apiClient) + .postFunctionIdInvoke(UUID.fromString(getFunctionId()), invoke); return parseScoreResult(result); } private String getFunctionName() { ensureFunctionInfoCached(); - return braintrustFunction.get().name(); + return braintrustFunction.get().getName(); } private String getFunctionId() { ensureFunctionInfoCached(); - return braintrustFunction.get().id(); + return braintrustFunction.get().getId().toString(); } private void ensureFunctionInfoCached() { if (scorerName.get() == null || braintrustFunction.get() == null) { - // we could get multiple threads in here but that's fine (just redundant work) - var function = apiClient.getFunctionById(functionId).orElseThrow(); - var projectAndOrgInfo = - apiClient.getProjectAndOrgInfo(function.projectId()).orElseThrow(); + var functionsApi = new FunctionsApi(apiClient); + var function = functionsApi.getFunctionId(UUID.fromString(functionId), version, null); + + var projectsApi = new ProjectsApi(apiClient); + var project = projectsApi.getProjectId(function.getProjectId()); + var functionVersion = this.version == null ? "latest" : this.version; braintrustFunction.compareAndExchange(null, function); scorerName.compareAndExchange( null, "invoke-%s-%s-%s" - .formatted( - projectAndOrgInfo.project().name(), - function.slug(), - functionVersion)); + .formatted(project.getName(), function.getSlug(), functionVersion)); } } /** - * Builds the parent span components for distributed tracing. - * - *

Extracts the experiment ID from OTEL baggage and span/trace IDs from the current span - * context. Returns the parent as a Map that can be serialized directly (the API accepts both - * base64-encoded strings and object format). - * - * @return parent object for distributed tracing, or null if tracing context not available + * Builds an {@link InvokeParent} for distributed tracing, or null if no tracing context is + * available. */ @Nullable - private Map buildParentSpanComponents() { + private InvokeParent buildParent() { try { - // Get current span context SpanContext spanContext = Span.current().getSpanContext(); - if (!spanContext.isValid()) { - return null; - } + if (!spanContext.isValid()) return null; - // Get experiment ID from baggage (format: "experiment_id:abc123") String parentValue = Baggage.current().getEntryValue(BraintrustTracing.PARENT_KEY); - if (parentValue == null || parentValue.isEmpty()) { - return null; - } + if (parentValue == null || parentValue.isEmpty()) return null; var rowIds = new SpanComponents.RowIds(spanContext.getSpanId(), spanContext.getTraceId()); - return new SpanComponents(BraintrustUtils.parseParent(parentValue), rowIds).toMap(); + Map spanMap = + new SpanComponents(BraintrustUtils.parseParent(parentValue), rowIds).toMap(); + + SpanParentStruct struct = MAPPER.convertValue(spanMap, SpanParentStruct.class); + return new InvokeParent(InvokeParent.SchemaType.SpanParentStruct, struct); } catch (Exception e) { log.warn("Failed to build parent span components: {}", e.getMessage(), e); return null; @@ -153,17 +165,12 @@ private Map buildParentSpanComponents() { */ @SuppressWarnings("unchecked") private List parseScoreResult(Object result) { - if (result == null) { - // Scorer returned null to skip scoring - return List.of(); - } + if (result == null) return List.of(); - // Handle a single number if (result instanceof Number number) { return List.of(new Score(getFunctionName(), number.doubleValue())); } - // Handle a list of scores if (result instanceof List list) { List scores = new ArrayList<>(); for (Object item : list) { @@ -172,36 +179,27 @@ private List parseScoreResult(Object result) { return scores; } - // Handle a score object (Map) if (result instanceof Map map) { Map scoreMap = (Map) map; - // Extract name (use scorer name as fallback) String name = getFunctionName(); Object nameValue = scoreMap.get("name"); - if (nameValue instanceof String s) { - name = s; - } + if (nameValue instanceof String s) name = s; - // Extract score value Object scoreValue = scoreMap.get("score"); - // If score is null, check for LLM judge response with metadata.choice if (scoreValue == null) { Object metadataObj = scoreMap.get("metadata"); if (metadataObj instanceof Map metadata) { Object choiceValue = ((Map) metadata).get("choice"); if (choiceValue != null) { - double score = parseNumericValue(choiceValue); - return List.of(new Score(name, score)); + return List.of(new Score(name, parseNumericValue(choiceValue))); } } - // No score field and no choice in metadata - skip return List.of(); } - double score = parseNumericValue(scoreValue); - return List.of(new Score(name, score)); + return List.of(new Score(name, parseNumericValue(scoreValue))); } throw new IllegalArgumentException( @@ -210,17 +208,8 @@ private List parseScoreResult(Object result) { + ". Expected Number, Map, or List."); } - /** - * Parse a numeric value from either a Number or a String. - * - * @param value the value to parse - * @return the double value - * @throws IllegalArgumentException if the value cannot be parsed as a number - */ private double parseNumericValue(Object value) { - if (value instanceof Number number) { - return number.doubleValue(); - } + if (value instanceof Number number) return number.doubleValue(); if (value instanceof String str) { try { return Double.parseDouble(str); diff --git a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java index 56e2ffcd..f28dbf67 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java @@ -1,88 +1,161 @@ package dev.braintrust.prompt; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.mustachejava.DefaultMustacheFactory; import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheException; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.openapi.JSON; +import dev.braintrust.openapi.model.Chat; +import dev.braintrust.openapi.model.ChatCompletionMessageParam; +import dev.braintrust.openapi.model.ModelParams; +import dev.braintrust.openapi.model.PromptBlockDataNullish; +import dev.braintrust.openapi.model.PromptDataNullish; +import dev.braintrust.openapi.model.PromptOptionsNullish; import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; public class BraintrustPrompt { - private final BraintrustApiClient.Prompt apiPrompt; + private static final ObjectMapper MAPPER = new JSON().getMapper(); + + private final PromptDataNullish promptData; private final Map defaults; - public BraintrustPrompt(BraintrustApiClient.Prompt apiPrompt) { - this(apiPrompt, Map.of()); + public BraintrustPrompt(PromptDataNullish promptData) { + this(promptData, Map.of()); } - public BraintrustPrompt(BraintrustApiClient.Prompt apiPrompt, Map defaults) { - this.apiPrompt = apiPrompt; + public BraintrustPrompt(PromptDataNullish promptData, Map defaults) { + this.promptData = promptData; this.defaults = defaults; } public List> renderMessages(Map parameters) { - // get promptData->prompt->messages - Map promptData = (Map) apiPrompt.promptData().prompt(); - List> messages = (List>) promptData.get("messages"); - - if (messages == null) { - throw new RuntimeException("No messages found in prompt data"); + if (promptData.getPrompt() == null) { + throw new RuntimeException("No prompt block found in prompt data"); + } + PromptBlockDataNullish blockData = promptData.getPrompt(); + Object blockInstance = blockData.getActualInstance(); + if (!(blockInstance instanceof Chat chat)) { + throw new RuntimeException("Only chat prompts are currently supported"); } List> renderedMessages = new ArrayList<>(); - - for (Map message : messages) { - Map renderedMessage = new HashMap<>(message); - String content = (String) message.get("content"); - + for (ChatCompletionMessageParam param : chat.getMessages()) { + final String role; + final String content = + switch (param.getVariantType()) { + case System -> { + var sys = param.getSystemInstance(); + role = "system"; + yield sys.getContent() != null + ? extractStringContent(sys.getContent().getActualInstance()) + : null; + } + case ChatMessageUser -> { + var user = param.getChatMessageUserInstance(); + role = "user"; + yield user.getContent() != null + ? extractStringContent(user.getContent().getActualInstance()) + : null; + } + case Assistant -> { + var asst = param.getAssistantInstance(); + role = "assistant"; + yield asst.getContent() != null + ? extractStringContent(asst.getContent().getActualInstance()) + : null; + } + case Tool -> { + var tool = param.getToolInstance(); + role = "tool"; + yield tool.getContent() != null + ? extractStringContent(tool.getContent().getActualInstance()) + : null; + } + case InlineFunctionRef -> { + // function-role messages have an index reference, not text content + var fn = param.getInlineFunctionRefInstance(); + role = "function"; + yield null; + } + case Developer -> { + var dev = param.getDeveloperInstance(); + role = "developer"; + yield dev.getContent() != null + ? extractStringContent(dev.getContent().getActualInstance()) + : null; + } + case Fallback -> { + var fb = param.getFallbackInstance(); + role = fb.getRole() != null ? fb.getRole().getValue() : "unknown"; + yield fb.getContent(); + } + }; + + Map rendered = new HashMap<>(); + rendered.put("role", role); if (content != null) { - String renderedContent = renderTemplate(content, parameters); - renderedMessage.put("content", renderedContent); + rendered.put("content", renderTemplate(content, parameters)); } - - renderedMessages.add(renderedMessage); + renderedMessages.add(rendered); } - return renderedMessages; } public Map getOptions() { - // get map in promptData->options and merge with promptData->options->params - Map options = (Map) apiPrompt.promptData().options(); - - if (options == null) { - return Map.of(); + if (promptData.getOptions() == null) { + return applyDefaults(Map.of()); } + PromptOptionsNullish opts = promptData.getOptions(); Map result = new HashMap<>(); - // Add all top-level options except "params" - for (Map.Entry entry : options.entrySet()) { - if (!"params".equals(entry.getKey())) { - result.put(entry.getKey(), entry.getValue()); - } + if (opts.getModel() != null) { + result.put("model", opts.getModel()); } - - // Merge in the params - Map params = (Map) options.get("params"); - if (params != null) { - result.putAll(params); + if (opts.getPosition() != null) { + result.put("position", opts.getPosition()); } - // Apply defaults for any values not already set - for (Map.Entry defaultEntry : this.defaults.entrySet()) { - if (!result.containsKey(defaultEntry.getKey())) { - result.put(defaultEntry.getKey(), defaultEntry.getValue()); - } + // Flatten params (ModelParams anyOf: OpenAIModelParams | AnthropicModelParams | ...) + // into the result map via Jackson, mirroring the old behaviour of merging + // promptData.options.params into the top level. + ModelParams modelParams = opts.getParams(); + if (modelParams != null && modelParams.getActualInstance() != null) { + @SuppressWarnings("unchecked") + Map paramsMap = + MAPPER.convertValue(modelParams.getActualInstance(), Map.class); + result.putAll(paramsMap); } + return applyDefaults(result); + } + + private Map applyDefaults(Map base) { + Map result = new HashMap<>(base); + for (Map.Entry entry : defaults.entrySet()) { + result.putIfAbsent(entry.getKey(), entry.getValue()); + } return result; } + /** + * Extract a String from an anyOf [String, List<ChatCompletionContentPart*>] content + * instance. Only the String variant is supported for Mustache rendering; list-form content + * (e.g. vision messages) is returned as-is via toString. + */ + private String extractStringContent(@Nullable Object contentInstance) { + if (contentInstance instanceof String s) { + return s; + } + return contentInstance != null ? contentInstance.toString() : null; + } + private String renderTemplate(String template, Map parameters) { try { DefaultMustacheFactory factory = new DefaultMustacheFactory(); @@ -92,11 +165,10 @@ private String renderTemplate(String template, Map parameters) { writer.flush(); return writer.toString(); } catch (MustacheException e) { - // If the template is malformed, just return it as-is return template; } catch (Exception e) { - if (e instanceof RuntimeException) { - throw (RuntimeException) e; + if (e instanceof RuntimeException re) { + throw re; } throw new RuntimeException("Failed to render template", e); } diff --git a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java index 00bdc1c8..52c6ded5 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java @@ -1,48 +1,75 @@ package dev.braintrust.prompt; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.openapi.api.PromptsApi; +import dev.braintrust.openapi.model.PromptDataNullish; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; -/** Load LLM objects from the Braintrust API */ +/** Load LLM prompts from the Braintrust API */ public class BraintrustPromptLoader { private final BraintrustConfig config; - private final BraintrustApiClient client; + private final PromptsApi promptsApi; - private BraintrustPromptLoader(BraintrustConfig config, BraintrustApiClient client) { + private BraintrustPromptLoader(BraintrustConfig config, BraintrustOpenApiClient apiClient) { this.config = config; - this.client = client; + this.promptsApi = new PromptsApi(apiClient); } - public static BraintrustPromptLoader of(BraintrustConfig config, BraintrustApiClient client) { - return new BraintrustPromptLoader(config, client); + public static BraintrustPromptLoader of( + BraintrustConfig config, BraintrustOpenApiClient apiClient) { + return new BraintrustPromptLoader(config, apiClient); } public BraintrustPrompt load(String promptSlug) { - PromptLoadRequest request = PromptLoadRequest.builder().promptSlug(promptSlug).build(); - return load(request); + return load(PromptLoadRequest.builder().promptSlug(promptSlug).build()); } - public BraintrustPrompt load(PromptLoadRequest promptLoadRequest) { - var projectName = promptLoadRequest.projectName; - if (null == projectName) { - // TODO: fall back to project ID if appropriate - projectName = config.defaultProjectName().orElseThrow(); + public BraintrustPrompt load(PromptLoadRequest request) { + var projectName = + request.projectName != null + ? request.projectName + : config.defaultProjectName().orElseThrow(); + + var response = + promptsApi.getPrompt( + null, // limit + null, // startingAfter + null, // endingBefore + null, // ids + null, // promptName + projectName, // projectName + null, // projectId + request.promptSlug, // slug + request.version, // version + null, // environment + null // orgName + ); + + List objects = response.getObjects(); + if (objects == null || objects.isEmpty()) { + throw new RuntimeException("Prompt not found: " + request.promptSlug); + } + if (objects.size() > 1) { + throw new RuntimeException( + "Multiple prompts found for slug: " + + request.promptSlug + + ", projectName: " + + projectName); + } + + var prompt = (dev.braintrust.openapi.model.Prompt) objects.get(0); + PromptDataNullish promptData = prompt.getPromptData(); + if (promptData == null) { + throw new RuntimeException("prompt_data missing for prompt: " + request.promptSlug); } - // Request the prompt from the Braintrust API - var promptOpt = - client.getPrompt( - projectName, promptLoadRequest.promptSlug, promptLoadRequest.version); - var prompt = - promptOpt.orElseThrow( - () -> - new RuntimeException( - "Prompt not found: " + promptLoadRequest.promptSlug)); - return new BraintrustPrompt(prompt, promptLoadRequest.defaults); + + return new BraintrustPrompt(promptData, request.defaults); } @Builder @@ -67,14 +94,10 @@ private static Map keyValueListToMap(T... keyValueList) { throw new IllegalArgumentException( "keyValueList must contain an even number of elements (key-value pairs)"); } - Map map = new LinkedHashMap<>(); for (int i = 0; i < keyValueList.length; i += 2) { - T key = keyValueList[i]; - T value = keyValueList[i + 1]; - map.put(key, value); + map.put(keyValueList[i], keyValueList[i + 1]); } - return Map.copyOf(map); } } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustTest.java b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustTest.java index 1dd7680c..c9074bbe 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustTest.java @@ -34,7 +34,7 @@ void testOfCreatesNewInstance() { assertNotNull(braintrust); assertNotNull(braintrust.config()); - assertNotNull(braintrust.apiClient()); + assertNotNull(braintrust.openApiClient()); assertNotNull(braintrust.promptLoader()); assertEquals(config, braintrust.config()); } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java index d95c77f0..99359c51 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/BraintrustUtilsTest.java @@ -3,21 +3,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import dev.braintrust.api.BraintrustApiClient; import java.net.URI; import org.junit.jupiter.api.Test; public class BraintrustUtilsTest { @Test public void testBuildProjectUri() { - var orgAndProject = - new BraintrustApiClient.OrganizationAndProjectInfo( - new BraintrustApiClient.OrganizationInfo("123", "some org"), - new BraintrustApiClient.Project( - "456", "some project", "123", "doesntmatter", "doesntmatter")); assertEquals( URI.create("http://someserver:3009/app/some%20org/p/some%20project"), - BraintrustUtils.createProjectURI("http://someserver:3009/", orgAndProject)); + BraintrustUtils.createProjectURI( + "http://someserver:3009/", "some org", "some project")); } @Test diff --git a/braintrust-sdk/src/test/java/dev/braintrust/api/BraintrustOpenApiClientTest.java b/braintrust-sdk/src/test/java/dev/braintrust/api/BraintrustOpenApiClientTest.java new file mode 100644 index 00000000..85a4322f --- /dev/null +++ b/braintrust-sdk/src/test/java/dev/braintrust/api/BraintrustOpenApiClientTest.java @@ -0,0 +1,283 @@ +package dev.braintrust.api; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.junit.jupiter.api.Assertions.*; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import dev.braintrust.config.BraintrustConfig; +import java.io.FileInputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.SecureRandom; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class BraintrustOpenApiClientTest { + + @RegisterExtension + static WireMockExtension wireMock = + WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); + + // HTTPS WireMock server using a keytool-generated cert with localhost SAN, + // set up in @BeforeAll so we can generate the keystore first. + static WireMockServer wireMockHttps; + static Path httpsKeystoreFile; + static SSLContext customSslContext; // trusts the generated cert + + @BeforeAll + static void setUpHttps() throws Exception { + httpsKeystoreFile = Files.createTempFile("test-keystore", ".jks"); + Files.delete(httpsKeystoreFile); // keytool requires the file not exist yet + var password = "changeit"; + + // Generate a self-signed cert with localhost SAN + var keytool = + new ProcessBuilder( + "keytool", "-genkeypair", + "-alias", "test", + "-keyalg", "RSA", + "-keystore", httpsKeystoreFile.toString(), + "-storepass", password, + "-keypass", password, + "-dname", "CN=localhost", + "-ext", "san=dns:localhost,ip:127.0.0.1", + "-validity", "1") + .redirectErrorStream(true) + .start(); + int exitCode = keytool.waitFor(); + if (exitCode != 0) { + throw new RuntimeException( + "keytool failed: " + new String(keytool.getInputStream().readAllBytes())); + } + + // Build client SSLContext that trusts the generated cert + var keyStore = KeyStore.getInstance("JKS"); + try (var fis = new FileInputStream(httpsKeystoreFile.toFile())) { + keyStore.load(fis, password.toCharArray()); + } + var tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + var trustStore = KeyStore.getInstance("JKS"); + trustStore.load(null, null); + trustStore.setCertificateEntry("test-server", keyStore.getCertificate("test")); + tmf.init(trustStore); + customSslContext = SSLContext.getInstance("TLS"); + customSslContext.init(null, tmf.getTrustManagers(), new SecureRandom()); + + // Start WireMock HTTPS with our keystore + wireMockHttps = + new WireMockServer( + wireMockConfig() + .dynamicPort() + .dynamicHttpsPort() + .keystorePath(httpsKeystoreFile.toString()) + .keystorePassword(password) + .keyManagerPassword(password)); + wireMockHttps.start(); + } + + @AfterAll + static void tearDownHttps() throws Exception { + if (wireMockHttps != null) wireMockHttps.stop(); + if (httpsKeystoreFile != null) Files.deleteIfExists(httpsKeystoreFile); + } + + private BraintrustConfig config; + private BraintrustOpenApiClient client; + + @BeforeEach + void beforeEach() { + wireMock.resetAll(); + config = + BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl("http://localhost:" + wireMock.getPort()) + .appUrl("http://app.example.com") + .defaultProjectName("test-project") + .build(); + client = BraintrustOpenApiClient.of(config); + } + + // ── Construction ────────────────────────────────────────────────────────── + + @Test + void of_setsBaseUri() { + assertEquals("http://localhost:" + wireMock.getPort(), client.getBaseUri()); + } + + @Test + void of_addsAuthorizationHeader() throws Exception { + wireMock.stubFor( + get(urlPathEqualTo("/v1/project")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"objects\":[]}"))); + + // Trigger any request so the interceptor fires + new dev.braintrust.openapi.api.ProjectsApi(client) + .getProject(null, null, null, null, "any", null); + + wireMock.verify( + getRequestedFor(urlPathEqualTo("/v1/project")) + .withHeader("Authorization", equalTo("Bearer test-key"))); + } + + @Test + void of_usesSslContextFromConfig() throws Exception { + // WireMock's HTTPS port uses a self-signed cert that the default SSLContext + // will reject. A trust-all SSLContext passed through config should allow it. + // This proves the custom SSLContext is actually wired into the HttpClient + // rather than silently ignored. + String httpsUrl = "https://localhost:" + wireMockHttps.httpsPort(); + + wireMockHttps.stubFor( + post(urlEqualTo("/api/apikey/login")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + "{\"org_info\":[{\"id\":\"org-1\",\"name\":\"Test" + + " Org\"}]}"))); + + // Client with the custom SSLContext that trusts our cert → should succeed + var customClient = + BraintrustOpenApiClient.of( + BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl(httpsUrl) + .sslContext(customSslContext) + .build()); + var response = customClient.login(); + assertEquals("Test Org", response.orgInfo().get(0).name()); + + // Client with the default SSLContext → should reject our self-signed cert, + // proving the custom context is actually wired in and not just ignored + var defaultClient = + BraintrustOpenApiClient.of( + BraintrustConfig.builder() + .apiKey("test-key") + .apiUrl(httpsUrl) + .sslContext(SSLContext.getDefault()) + .build()); + assertThrows( + Exception.class, + defaultClient::login, + "Default SSLContext should reject our self-signed cert"); + } + + // ── login() ─────────────────────────────────────────────────────────────── + + @Test + void login_parsesOrgInfo() { + wireMock.stubFor( + post(urlEqualTo("/api/apikey/login")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "org_info": [ + {"id": "org-1", "name": "My Org"} + ] + } + """))); + + var response = client.login(); + + assertNotNull(response); + assertEquals(1, response.orgInfo().size()); + assertEquals("org-1", response.orgInfo().get(0).id()); + assertEquals("My Org", response.orgInfo().get(0).name()); + } + + @Test + void login_sendsApiKeyAsToken() { + wireMock.stubFor( + post(urlEqualTo("/api/apikey/login")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"org_info\":[]}"))); + + client.login(); + + wireMock.verify( + postRequestedFor(urlEqualTo("/api/apikey/login")) + .withRequestBody(matchingJsonPath("$.token", equalTo("test-key")))); + } + + @Test + void login_throwsOnNon2xx() { + wireMock.stubFor( + post(urlEqualTo("/api/apikey/login")) + .willReturn(aResponse().withStatus(401).withBody("Unauthorized"))); + + assertThrows(RuntimeException.class, () -> client.login()); + } + + // ── btqlQuery() ─────────────────────────────────────────────────────────── + + @Test + void btqlQuery_parsesResponse() { + wireMock.stubFor( + post(urlEqualTo("/btql")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """ + { + "data": [ + {"version": "12345"}, + {"version": "12346"} + ] + } + """))); + + var result = client.btqlQuery("SELECT max(_xact_id) FROM dataset('abc')"); + + assertNotNull(result); + assertEquals(2, result.data().size()); + assertEquals("12345", result.data().get(0).get("version")); + } + + @Test + void btqlQuery_sendsQueryInBody() { + wireMock.stubFor( + post(urlEqualTo("/btql")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"data\":[]}"))); + + client.btqlQuery("SELECT 1"); + + wireMock.verify( + postRequestedFor(urlEqualTo("/btql")) + .withRequestBody(matchingJsonPath("$.query", equalTo("SELECT 1")))); + } + + @Test + void btqlQuery_throwsOnNon2xx() { + wireMock.stubFor( + post(urlEqualTo("/btql")) + .willReturn(aResponse().withStatus(500).withBody("Server Error"))); + + assertThrows(RuntimeException.class, () -> client.btqlQuery("SELECT 1")); + } +} diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/DatasetBrainstoreImplTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/DatasetBrainstoreImplTest.java index 54bd08fd..0ad8f999 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/DatasetBrainstoreImplTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/DatasetBrainstoreImplTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.*; import com.github.tomakehurst.wiremock.junit5.WireMockExtension; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.config.BraintrustConfig; import java.util.*; import org.junit.jupiter.api.BeforeEach; @@ -18,21 +18,20 @@ public class DatasetBrainstoreImplTest { static WireMockExtension wireMock = WireMockExtension.newInstance().options(wireMockConfig().dynamicPort()).build(); - private BraintrustApiClient apiClient; + private BraintrustOpenApiClient apiClient; private String datasetId; @BeforeEach void beforeEach() { wireMock.resetAll(); - datasetId = "test-dataset-123"; + datasetId = "00000000-0000-0000-0000-000000000123"; - // Create API client pointing to WireMock server var config = BraintrustConfig.builder() .apiKey("test-api-key") .apiUrl("http://localhost:" + wireMock.getPort()) .build(); - apiClient = BraintrustApiClient.of(config); + apiClient = BraintrustOpenApiClient.of(config); } @Test @@ -51,26 +50,27 @@ void testFetchAll() { "events": [ { "object_type": "dataset", - "dataset_id": "test-dataset-123", + "dataset_id": "%s", "id": "123-1", - "created": "sometimestamp", + "created": "2024-01-01T00:00:00Z", "_xact_id": "1", "input": "Question 1", "expected": "Answer 1" }, { "object_type": "dataset", - "dataset_id": "test-dataset-123", + "dataset_id": "%s", "id": "123-2", "_xact_id": "1", - "created": "sometimestamp", + "created": "2024-01-01T00:00:00Z", "input": "Question 2", "expected": "Answer 2" } ], "cursor": "next-page-token" } - """))); + """ + .formatted(datasetId, datasetId)))); // Mock the second batch without a cursor (last page) wireMock.stubFor( @@ -86,69 +86,38 @@ void testFetchAll() { "events": [ { "object_type": "dataset", - "dataset_id": "test-dataset-123", + "dataset_id": "%s", "id": "123-3", "_xact_id": "1", - "created": "sometimestamp", + "created": "2024-01-01T00:00:00Z", "input": "Question 3", "expected": "Answer 3" } ], "cursor": null } - """))); + """ + .formatted(datasetId)))); - // Create dataset with smaller batch size DatasetBrainstoreImpl dataset = new DatasetBrainstoreImpl<>(apiClient, datasetId, "test-version", 2); List> cases = new ArrayList<>(); dataset.forEach(cases::add); - // Verify we got all 3 cases assertEquals(3, cases.size()); - List tags = List.of(); - Map metadata = Map.of(); - assertEquals( - new DatasetCase<>( - "Question 1", - "Answer 1", - tags, - metadata, - Optional.of( - new dev.braintrust.Origin( - "dataset", datasetId, "123-1", "1", "sometimestamp"))), - cases.get(0)); + assertEquals("Question 1", cases.get(0).input()); + assertEquals("Answer 1", cases.get(0).expected()); assertEquals("Question 2", cases.get(1).input()); - assertEquals( - new DatasetCase<>( - "Question 2", - "Answer 2", - tags, - metadata, - Optional.of( - new dev.braintrust.Origin( - "dataset", datasetId, "123-2", "1", "sometimestamp"))), - cases.get(1)); + assertEquals("Answer 2", cases.get(1).expected()); assertEquals("Question 3", cases.get(2).input()); - assertEquals( - new DatasetCase<>( - "Question 3", - "Answer 3", - tags, - metadata, - Optional.of( - new dev.braintrust.Origin( - "dataset", datasetId, "123-3", "1", "sometimestamp"))), - cases.get(2)); + assertEquals("Answer 3", cases.get(2).expected()); - // Verify the API was called twice (once for each batch) wireMock.verify(2, postRequestedFor(urlEqualTo("/v1/dataset/" + datasetId + "/fetch"))); } @Test void testEmptyDataset() { - // Mock empty dataset wireMock.stubFor( post(urlEqualTo("/v1/dataset/" + datasetId + "/fetch")) .willReturn( @@ -169,10 +138,7 @@ void testEmptyDataset() { List> cases = new ArrayList<>(); dataset.forEach(cases::add); - // Verify we got no cases assertEquals(0, cases.size()); - - // Verify the API was called once wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/dataset/" + datasetId + "/fetch"))); } @@ -181,8 +147,9 @@ void testFetchWithPinnedVersion() { String projectName = "test-project"; String datasetName = "test-dataset"; String pinnedVersion = "12345"; + String datasetUuid = "00000000-0000-0000-0000-000000000789"; - // Mock the query endpoint + // Mock the dataset lookup wireMock.stubFor( get(urlPathEqualTo("/v1/dataset")) .withQueryParam("project_name", equalTo(projectName)) @@ -196,24 +163,20 @@ void testFetchWithPinnedVersion() { { "objects": [ { - "object_type": "dataset", - "dataset_id": "test-dataset-123", - "id": "dataset-789", - "project_id": "proj-456", + "id": "%s", + "project_id": "00000000-0000-0000-0000-000000000456", "name": "test-dataset", - "description": "Test dataset", "_xact_id": "12345", - "input": "test input", - "expected": "test output", - "created": "sometimestamp" + "created": "2024-01-01T00:00:00Z" } ] } - """))); + """ + .formatted(datasetUuid)))); - // Mock the fetch endpoint - only succeeds if version is passed correctly + // Mock the fetch endpoint with version wireMock.stubFor( - post(urlEqualTo("/v1/dataset/dataset-789/fetch")) + post(urlEqualTo("/v1/dataset/" + datasetUuid + "/fetch")) .withRequestBody(matchingJsonPath("$.version", equalTo(pinnedVersion))) .willReturn( aResponse() @@ -224,40 +187,37 @@ void testFetchWithPinnedVersion() { { "events": [ { - "object_type": "dataset", - "dataset_id": "test-dataset-123", + "dataset_id": "%s", "id": "some-row-id", "input": "test input", "expected": "test output", "metadata": {}, "tags": [], "_xact_id": "12346", - "created": "sometimestamp" + "created": "2024-01-01T00:00:00Z" } ], "cursor": null } - """))); + """ + .formatted(datasetUuid)))); Dataset dataset = Dataset.fetchFromBraintrust(apiClient, projectName, datasetName, pinnedVersion); - assertEquals("dataset-789", dataset.id()); + assertEquals(datasetUuid, dataset.id()); assertEquals(Optional.of(pinnedVersion), dataset.version()); - // Open cursor and fetch data to trigger the API call with version List> cases = new ArrayList<>(); dataset.forEach(cases::add); - // Verify we got the expected case assertEquals(1, cases.size()); assertEquals("test input", cases.get(0).input()); assertEquals("test output", cases.get(0).expected()); - // Verify the fetch endpoint was called with the version wireMock.verify( 1, - postRequestedFor(urlEqualTo("/v1/dataset/dataset-789/fetch")) + postRequestedFor(urlEqualTo("/v1/dataset/" + datasetUuid + "/fetch")) .withRequestBody(matchingJsonPath("$.version", equalTo(pinnedVersion)))); } @@ -266,7 +226,6 @@ void testFetchFromBraintrustNotFound() { String projectName = "test-project"; String datasetName = "nonexistent"; - // Mock empty response wireMock.stubFor( get(urlPathEqualTo("/v1/dataset")) .withQueryParam("project_name", equalTo(projectName)) diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java index cb58ed32..3c18ff7a 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/EvalTest.java @@ -7,7 +7,8 @@ import dev.braintrust.Origin; import dev.braintrust.TestHarness; import dev.braintrust.VCR; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.openapi.api.ExperimentsApi; +import dev.braintrust.openapi.model.CreateExperiment; import dev.braintrust.trace.BraintrustTracing; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanId; @@ -57,19 +58,21 @@ public void evalOtelTraceWithProperAttributes() { .formatted(testHarness.braintrust().projectUri(), experimentName), result.getExperimentUrl()); var spans = testHarness.awaitExportedSpans(); + var openApiClient = testHarness.braintrust().openApiClient(); var experiment = - testHarness - .braintrust() - .apiClient() - .getOrCreateExperiment( - new BraintrustApiClient.CreateExperimentRequest( - TestHarness.defaultProjectId(), experimentName)); + new ExperimentsApi(openApiClient) + .postExperiment( + new CreateExperiment() + .projectId( + java.util.UUID.fromString( + TestHarness.defaultProjectId())) + .name(experimentName)); final AtomicInteger numRootSpans = new AtomicInteger(0); for (SpanData span : spans) { var parent = span.getAttributes().get(AttributeKey.stringKey(BraintrustTracing.PARENT_KEY)); assertEquals( - "experiment_id:" + experiment.id(), + "experiment_id:" + experiment.getId().toString(), parent, "all eval spans must set the parent to the experiment id"); if (span.getParentSpanId().equals(SpanId.getInvalid())) { @@ -348,18 +351,25 @@ void evalWithExperimentTagsAndMetadata() { // Query the experiment from Braintrust API to verify tags and metadata var experiments = - testHarness - .braintrust() - .apiClient() - .listExperiments(TestHarness.defaultProjectId()); + new ExperimentsApi(testHarness.braintrust().openApiClient()) + .getExperiment( + null, + null, + null, + null, + null, + null, + java.util.UUID.fromString(TestHarness.defaultProjectId()), + null) + .getObjects(); var experiment = experiments.stream() - .filter(e -> e.name().equals(experimentName)) + .filter(e -> e.getName().equals(experimentName)) .findFirst() .orElseThrow(() -> new AssertionError("Experiment not found")); - assertEquals(expectedTags, experiment.tags(), "Experiment should have tags"); - assertEquals(expectedMetadata, experiment.metadata(), "Experiment should have metadata"); + assertEquals(expectedTags, experiment.getTags(), "Experiment should have tags"); + assertEquals(expectedMetadata, experiment.getMetadata(), "Experiment should have metadata"); } @Test @@ -389,21 +399,27 @@ void evalLinksToRemoteDataset() { // Verify the experiment is linked to the dataset var experiments = - testHarness - .braintrust() - .apiClient() - .listExperiments(TestHarness.defaultProjectId()); + new ExperimentsApi(testHarness.braintrust().openApiClient()) + .getExperiment( + null, + null, + null, + null, + null, + null, + java.util.UUID.fromString(TestHarness.defaultProjectId()), + null) + .getObjects(); var experiment = experiments.stream() - .filter(e -> e.name().equals(experimentName)) + .filter(e -> e.getName().equals(experimentName)) .findFirst() .orElseThrow(() -> new AssertionError("Experiment not found")); - assertTrue(experiment.datasetId().isPresent(), "Experiment should be linked to a dataset"); - assertEquals(dataset.id(), experiment.datasetId().get(), "Dataset ID should match"); + assertTrue(experiment.getDatasetId() != null, "Experiment should be linked to a dataset"); + assertEquals(dataset.id(), experiment.getDatasetId().toString(), "Dataset ID should match"); assertTrue( - experiment.datasetVersion().isPresent(), - "Experiment should have a dataset version"); + experiment.getDatasetVersion() != null, "Experiment should have a dataset version"); } @Test diff --git a/braintrust-sdk/src/test/java/dev/braintrust/eval/ScorerBrainstoreImplTest.java b/braintrust-sdk/src/test/java/dev/braintrust/eval/ScorerBrainstoreImplTest.java index 2bb4dcad..24179fda 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/eval/ScorerBrainstoreImplTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/eval/ScorerBrainstoreImplTest.java @@ -4,7 +4,7 @@ import dev.braintrust.TestHarness; import dev.braintrust.VCR; -import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.api.BraintrustOpenApiClient; import dev.braintrust.trace.BraintrustContext; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; @@ -16,9 +16,9 @@ @Slf4j public class ScorerBrainstoreImplTest { - // NOTE: the remote scorers under test are standard boilerplate + // NOTE: the remote scorers under test are standard boilerplate autofilled by the braintrust UI // TODO: test is VCR'd so it's fine, but would be nice to have logic to (re)create the score - // objects if they are absent + // objects if they are absent // returns 1.0 for an exact match, 0.0 otherwise private static final String SCORER_SLUG = "typescriptexactmatch-9e44"; @@ -27,20 +27,17 @@ public class ScorerBrainstoreImplTest { private static final String LLM_JUDGE_SLUG = "close-enough-judge-d31b"; private TestHarness testHarness; - private BraintrustApiClient apiClient; @BeforeEach void beforeEach() { testHarness = TestHarness.setup(); - apiClient = - testHarness.braintrust().apiClient(); // TODO -- do we need a separate var for this? } @Test void testScorerReturnsOneForExactMatch() { Scorer scorer = Scorer.fetchFromBraintrust( - apiClient, + testHarness.braintrust().openApiClient(), testHarness.braintrust().config().defaultProjectName().orElseThrow(), SCORER_SLUG, null); @@ -60,7 +57,7 @@ void testScorerReturnsOneForExactMatch() { void testScorerReturnsZeroForMismatch() { Scorer scorer = Scorer.fetchFromBraintrust( - apiClient, + testHarness.braintrust().openApiClient(), testHarness.braintrust().config().defaultProjectName().orElseThrow(), SCORER_SLUG, null); @@ -83,7 +80,7 @@ void testScorerOldVersion() { String oldVersion = "485dbf64e486ab3a"; Scorer scorer = Scorer.fetchFromBraintrust( - apiClient, + testHarness.braintrust().openApiClient(), testHarness.braintrust().config().defaultProjectName().orElseThrow(), SCORER_SLUG, oldVersion); @@ -108,7 +105,7 @@ void testScorerOldVersion() { void testLlmJudgeScorerReturnsScoreFromMetadataChoice() { Scorer scorer = Scorer.fetchFromBraintrust( - apiClient, + testHarness.braintrust().openApiClient(), testHarness.braintrust().config().defaultProjectName().orElseThrow(), LLM_JUDGE_SLUG, null); @@ -139,10 +136,13 @@ void testDistributedTracingWithRemoteScorer() throws InterruptedException { String projectName = testHarness.braintrust().config().defaultProjectName().orElseThrow(); Scorer scorer = - Scorer.fetchFromBraintrust(apiClient, projectName, LLM_JUDGE_SLUG, null); + Scorer.fetchFromBraintrust( + testHarness.braintrust().openApiClient(), + projectName, + LLM_JUDGE_SLUG, + null); assertNotNull(scorer); - // Get tracer from test harness Tracer tracer = testHarness.openTelemetry().getTracer("distributed-trace-test"); Span parentSpan = tracer.spanBuilder("test-distributed-trace-parent").startSpan(); Context ctx = @@ -181,18 +181,16 @@ void testDistributedTracingWithRemoteScorer() throws InterruptedException { // The OTEL traceId (32 hex chars) maps to Braintrust root_span_id String projectId = TestHarness.defaultProjectId(); - // Poll for eventual consistency - spans may take a moment to be indexed - BraintrustApiClient.BtqlQueryResponse response = null; - int maxAttempts = 30; - int attemptDelayMs = 2000; - - // First, query by root_span_id to find all spans in our trace String rootSpanQuery = "select: span_id, span_parents, root_span_id, name | from: project_logs('%s') | filter: root_span_id = '%s'" .formatted(projectId, traceId); + BraintrustOpenApiClient.BtqlQueryResponse response = null; + int maxAttempts = 30; + int attemptDelayMs = 2000; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { - response = apiClient.btqlQuery(rootSpanQuery); + response = testHarness.braintrust().openApiClient().btqlQuery(rootSpanQuery); if (response != null && response.data() != null && !response.data().isEmpty()) { break; } diff --git a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java index 10773ccb..c896c4a3 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java @@ -41,6 +41,24 @@ void testLoadPromptBySlug() { assertTrue(nameRendered, "Expected rendered messages to contain the substituted name"); } + @Test + void testLoadPromptBySlugWithVersion() { + BraintrustPromptLoader loader = testHarness.braintrust().promptLoader(); + + BraintrustPrompt prompt = + loader.load( + BraintrustPromptLoader.PromptLoadRequest.builder() + .promptSlug("kind-greeter-0bd1") + .version("27fdcc80d22c7ec5") + .build()); + + assertNotNull(prompt); + + List> messages = prompt.renderMessages(Map.of("name", "Bob")); + assertEquals("system", messages.get(0).get("role")); + assertEquals("this is an old version", messages.get(0).get("content")); + } + @Test void testLoadPromptWithDefaults() { BraintrustPromptLoader loader = testHarness.braintrust().promptLoader(); diff --git a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java index 17a61adc..c1451191 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java @@ -2,121 +2,130 @@ import static org.junit.jupiter.api.Assertions.*; -import dev.braintrust.api.BraintrustApiClient; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.openapi.JSON; +import dev.braintrust.openapi.model.PromptDataNullish; import java.util.List; import java.util.Map; -import java.util.Optional; import org.junit.jupiter.api.Test; public class BraintrustPromptTest { + private static final ObjectMapper MAPPER = new JSON().getMapper(); + + /** + * Build a PromptDataNullish from a plain Java map that mirrors the JSON structure the + * Braintrust API returns. This lets test cases stay readable without constructing the full + * generated type hierarchy by hand. + */ + private static PromptDataNullish promptData( + Map prompt, Map options) { + Map raw = Map.of("prompt", prompt, "options", options); + return MAPPER.convertValue(raw, PromptDataNullish.class); + } + + private static PromptDataNullish createTestPromptData() { + Map prompt = + Map.of( + "type", + "chat", + "messages", + List.of( + Map.of( + "role", + "system", + "content", + "You are a kind chatbot who briefly greets people"), + Map.of( + "role", "user", + "content", "What's up my friend? My name is {{name}}"))); + + Map options = + Map.of( + "model", + "gpt-4o-mini", + "params", + Map.of( + "use_cache", + true, + "temperature", + 0, + "response_format", + Map.of("type", "text")), + "position", + "0|hzzzzz:"); + + return promptData(prompt, options); + } + @Test void testGetOptionsWithDefaults() { - BraintrustApiClient.Prompt promptObject = createTestPrompt(); - - // Create a prompt with defaults Map defaults = Map.of( "max_tokens", "1000", - "temperature", - "0.7", // This should be ignored as temperature is already set to 0 + "temperature", "0.7", // should be ignored — temperature is already set to 0 "top_p", "0.9"); - BraintrustPrompt prompt = new BraintrustPrompt(promptObject, defaults); + BraintrustPrompt prompt = new BraintrustPrompt(createTestPromptData(), defaults); Map options = prompt.getOptions(); - // Verify that defaults are applied only when not already set - assertEquals("1000", options.get("max_tokens")); // Applied from defaults - assertEquals("0.9", options.get("top_p")); // Applied from defaults - assertEquals( - 0, - options.get( - "temperature")); // NOT overridden by defaults (original value preserved) + assertEquals("1000", options.get("max_tokens")); // applied from defaults + assertEquals("0.9", options.get("top_p")); // applied from defaults + // temperature already present — default must not override it + assertNotNull(options.get("temperature")); + assertNotEquals("0.7", options.get("temperature").toString()); - // Verify original options are still present assertEquals("gpt-4o-mini", options.get("model")); assertEquals(true, options.get("use_cache")); } @Test void testRenderMessagesWithMalformedMustache() { - // Create a prompt with malformed mustache syntax - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( + Map.of("role", "system", "content", "You are a helpful assistant"), Map.of( - "role", "system", - "content", "You are a helpful assistant"), - Map.of( - "role", "user", - "content", "Hello {{ whatever. This should not match."))); - - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - - // Render with empty parameters - malformed mustache should be ignored - Map parameters = Map.of(); - List> renderedMessages = prompt.renderMessages(parameters); - - // Verify the malformed mustache is left as-is (not treated as a parameter) - assertEquals(2, renderedMessages.size()); - assertEquals( - "Hello {{ whatever. This should not match.", - renderedMessages.get(1).get("content")); + "role", + "user", + "content", + "Hello {{ whatever. This should not match."))); + + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); + + List> rendered = braintrustPrompt.renderMessages(Map.of()); + + assertEquals(2, rendered.size()); + assertEquals("Hello {{ whatever. This should not match.", rendered.get(1).get("content")); } @Test void testRenderMessagesWithParameters() { - // Create a test prompt object - BraintrustApiClient.Prompt promptObject = createTestPrompt(); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + BraintrustPrompt prompt = new BraintrustPrompt(createTestPromptData()); - // Render messages with parameters - Map parameters = Map.of("name", "Alice"); - List> renderedMessages = prompt.renderMessages(parameters); + List> rendered = prompt.renderMessages(Map.of("name", "Alice")); - // Verify the messages were rendered correctly - assertEquals(2, renderedMessages.size()); - - Map systemMessage = renderedMessages.get(0); - assertEquals("system", systemMessage.get("role")); + assertEquals(2, rendered.size()); + assertEquals("system", rendered.get(0).get("role")); assertEquals( - "You are a kind chatbot who briefly greets people", systemMessage.get("content")); - - Map userMessage = renderedMessages.get(1); - assertEquals("user", userMessage.get("role")); - assertEquals("What's up my friend? My name is Alice", userMessage.get("content")); + "You are a kind chatbot who briefly greets people", rendered.get(0).get("content")); + assertEquals("user", rendered.get(1).get("role")); + assertEquals("What's up my friend? My name is Alice", rendered.get(1).get("content")); } @Test void testRenderMessagesWithList() { - // Create a prompt that uses Mustache list iteration - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( - Map.of( - "role", "system", - "content", "You are a helpful assistant"), + Map.of("role", "system", "content", "You are a helpful assistant"), Map.of( "role", "user", @@ -125,27 +134,9 @@ void testRenderMessagesWithList() { + "{{#items}}- {{name}}: {{description}}\n" + "{{/items}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); - // Render with list parameters Map parameters = Map.of( "items", @@ -154,22 +145,23 @@ void testRenderMessagesWithList() { Map.of("name", "Banana", "description", "A yellow fruit"), Map.of("name", "Cherry", "description", "A small red fruit"))); - List> renderedMessages = prompt.renderMessages(parameters); + List> rendered = braintrustPrompt.renderMessages(parameters); - assertEquals(2, renderedMessages.size()); - String expectedContent = + assertEquals(2, rendered.size()); + assertEquals( "Here are the items:\n" + "- Apple: A red fruit\n" + "- Banana: A yellow fruit\n" - + "- Cherry: A small red fruit\n"; - assertEquals(expectedContent, renderedMessages.get(1).get("content")); + + "- Cherry: A small red fruit\n", + rendered.get(1).get("content")); } @Test void testRenderMessagesWithEmptyList() { - // Create a prompt that uses Mustache list iteration - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( Map.of( @@ -179,44 +171,25 @@ void testRenderMessagesWithEmptyList() { "Items: {{#items}}{{name}} {{/items}}{{^items}}No items" + " found{{/items}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + List> rendered = + braintrustPrompt.renderMessages(Map.of("items", List.of())); - // Render with empty list - Map parameters = Map.of("items", List.of()); - List> renderedMessages = prompt.renderMessages(parameters); - - assertEquals(1, renderedMessages.size()); - assertEquals("Items: No items found", renderedMessages.get(0).get("content")); + assertEquals(1, rendered.size()); + assertEquals("Items: No items found", rendered.get(0).get("content")); } @Test void testRenderMessagesWithConditional() { - // Create a prompt that uses Mustache conditionals - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( - Map.of( - "role", "system", - "content", "You are a helpful assistant"), + Map.of("role", "system", "content", "You are a helpful assistant"), Map.of( "role", "user", @@ -225,43 +198,25 @@ void testRenderMessagesWithConditional() { + " privileges.{{/isAdmin}}{{^isAdmin}} You are a" + " regular user.{{/isAdmin}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - - // Test with admin user - Map adminParameters = Map.of("name", "Alice", "isAdmin", true); - List> adminMessages = prompt.renderMessages(adminParameters); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); + + List> adminMessages = + braintrustPrompt.renderMessages(Map.of("name", "Alice", "isAdmin", true)); assertEquals( "Hello Alice! You have admin privileges.", adminMessages.get(1).get("content")); - // Test with regular user - Map regularParameters = Map.of("name", "Bob", "isAdmin", false); - List> regularMessages = prompt.renderMessages(regularParameters); + List> regularMessages = + braintrustPrompt.renderMessages(Map.of("name", "Bob", "isAdmin", false)); assertEquals("Hello Bob! You are a regular user.", regularMessages.get(1).get("content")); } @Test void testRenderMessagesWithNestedObjects() { - // Create a prompt that uses nested object properties - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( Map.of( @@ -272,27 +227,9 @@ void testRenderMessagesWithNestedObjects() { + "Email: {{user.contact.email}}\n" + "Phone: {{user.contact.phone}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); - // Render with nested object Map parameters = Map.of( "user", @@ -304,18 +241,20 @@ void testRenderMessagesWithNestedObjects() { "contact", Map.of("email", "john@example.com", "phone", "555-1234"))); - List> renderedMessages = prompt.renderMessages(parameters); + List> rendered = braintrustPrompt.renderMessages(parameters); - assertEquals(1, renderedMessages.size()); - String expectedContent = "User: John Doe\nEmail: john@example.com\nPhone: " + "555-1234"; - assertEquals(expectedContent, renderedMessages.get(0).get("content")); + assertEquals(1, rendered.size()); + assertEquals( + "User: John Doe\nEmail: john@example.com\nPhone: 555-1234", + rendered.get(0).get("content")); } @Test void testRenderMessagesWithInvertedSection() { - // Create a prompt that uses inverted sections (renders when value is false/empty) - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( Map.of( @@ -326,43 +265,31 @@ void testRenderMessagesWithInvertedSection() { + " {{errorMessage}}{{/hasError}}{{^hasError}}All" + " systems operational{{/hasError}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - - // Test without error - Map noErrorParams = Map.of("hasError", false); - List> noErrorMessages = prompt.renderMessages(noErrorParams); - assertEquals("All systems operational", noErrorMessages.get(0).get("content")); - - // Test with error - Map errorParams = - Map.of("hasError", true, "errorMessage", "Database connection failed"); - List> errorMessages = prompt.renderMessages(errorParams); - assertEquals("Error: Database connection failed", errorMessages.get(0).get("content")); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); + + assertEquals( + "All systems operational", + braintrustPrompt.renderMessages(Map.of("hasError", false)).get(0).get("content")); + assertEquals( + "Error: Database connection failed", + braintrustPrompt + .renderMessages( + Map.of( + "hasError", + true, + "errorMessage", + "Database connection failed")) + .get(0) + .get("content")); } @Test void testRenderMessagesWithComplexTypes() { - // Test that non-string types are properly rendered - Map messages = + Map prompt = Map.of( + "type", + "chat", "messages", List.of( Map.of( @@ -373,76 +300,14 @@ void testRenderMessagesWithComplexTypes() { + "Price: ${{price}}\n" + "Enabled: {{enabled}}"))); - Map options = Map.of("model", "gpt-4o-mini"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - BraintrustApiClient.Prompt promptObject = - new BraintrustApiClient.Prompt( - "test-id", - "proj-id", - "org-id", - "test-prompt", - "test-slug", - Optional.empty(), - "2025-01-01T00:00:00Z", - promptData, - Optional.empty(), - Optional.empty()); - - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + BraintrustPrompt braintrustPrompt = + new BraintrustPrompt(promptData(prompt, Map.of("model", "gpt-4o-mini"))); - // Test with various data types - Map parameters = Map.of("count", 42, "price", 19.99, "enabled", true); + List> rendered = + braintrustPrompt.renderMessages( + Map.of("count", 42, "price", 19.99, "enabled", true)); - List> renderedMessages = prompt.renderMessages(parameters); - - assertEquals(1, renderedMessages.size()); - assertEquals( - "Count: 42\nPrice: $19.99\nEnabled: true", renderedMessages.get(0).get("content")); - } - - private BraintrustApiClient.Prompt createTestPrompt() { - // Create the prompt data structure matching the example JSON - Map messages = - Map.of( - "messages", - List.of( - Map.of( - "role", "system", - "content", - "You are a kind chatbot who briefly greets people"), - Map.of( - "role", "user", - "content", "What's up my friend? My name is {{name}}"))); - - Map options = - Map.of( - "model", "gpt-4o-mini", - "params", - Map.of( - "use_cache", - true, - "temperature", - 0, - "response_format", - Map.of("type", "text")), - "position", "0|hzzzzz:"); - - BraintrustApiClient.PromptData promptData = - new BraintrustApiClient.PromptData(messages, options); - - return new BraintrustApiClient.Prompt( - "e2a4fb20-e97e-4e8a-be07-b226d55047b2", - "e8d257dd-944c-479a-9916-40a9fa09f120", - "5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e", - "kind-greeter", - "kind-greeter-69d2", - Optional.of("A very good boi"), - "2025-10-21T21:35:18.287Z", - promptData, - Optional.empty(), - Optional.empty()); + assertEquals(1, rendered.size()); + assertEquals("Count: 42\nPrice: $19.99\nEnabled: true", rendered.get(0).get("content")); } } diff --git a/gradle.properties b/gradle.properties index 21de6bcd..be3e95a7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -10,6 +10,9 @@ org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests braintrustSpecRef=v0.0.2 +# braintrust-openapi commit SHA used by braintrust-api +braintrustOpenApiRef=64b79cb9122f50a74eac98ea86c3ec1858c0cdd1 + # Let Gradle locate local JDKs and download one if needed org.gradle.java.installations.auto-detect=true org.gradle.java.installations.auto-download=true diff --git a/scripts/openapi-fetch.sh b/scripts/openapi-fetch.sh new file mode 100755 index 00000000..120c865f --- /dev/null +++ b/scripts/openapi-fetch.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Fetches openapi/spec.yaml from a pinned commit (SHA or branch) of +# https://github.com/braintrustdata/braintrust-openapi +# +# Usage: openapi-fetch.sh +set -euo pipefail + +REPO="braintrustdata/braintrust-openapi" +REF="${1:?Usage: $0 }" +OUTDIR="${2:-.}" + +mkdir -p "$OUTDIR" + +URL="https://raw.githubusercontent.com/${REPO}/${REF}/openapi/spec.yaml" + +echo "Fetching braintrust-openapi@${REF} -> ${OUTDIR}/spec.yaml" +curl -sfL "$URL" -o "${OUTDIR}/spec.yaml" || { + echo "Error: failed to download spec.yaml from ${URL}" >&2 + exit 1 +} +echo "Done." diff --git a/settings.gradle b/settings.gradle index 67e00999..ea5c0e07 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,4 +25,5 @@ include 'braintrust-java-agent:smoke-test:spring-boot' include 'braintrust-java-agent:smoke-test:tomcat' include 'braintrust-java-agent:smoke-test:wildfly' include 'btx' +include 'braintrust-api' include 'braintrust-otel-extension'