From e313508442f787936fc977202697850f6f76238f Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 4 Dec 2025 09:59:15 -0700 Subject: [PATCH 1/3] wip --- README.md | 7 +- examples/build.gradle | 15 + .../examples/ExperimentExample.java | 15 +- .../examples/RemoteEvalExample.java | 74 ++ .../dev/braintrust/devserver/Devserver.java | 892 ++++++++++++++++++ .../dev/braintrust/devserver/EvalRequest.java | 116 +++ .../braintrust/devserver/EvalResponse.java | 62 ++ .../dev/braintrust/devserver/RemoteEval.java | 152 +++ src/main/java/dev/braintrust/eval/Scorer.java | 11 +- .../dev/braintrust/devserver/CorsTest.java | 165 ++++ .../braintrust/devserver/DevserverTest.java | 66 ++ .../devserver/EvalEndpointTest.java | 175 ++++ .../devserver/ListEndpointTest.java | 161 ++++ .../braintrust/devserver/RemoteEvalTest.java | 104 ++ 14 files changed, 1994 insertions(+), 21 deletions(-) create mode 100644 examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java create mode 100644 src/main/java/dev/braintrust/devserver/Devserver.java create mode 100644 src/main/java/dev/braintrust/devserver/EvalRequest.java create mode 100644 src/main/java/dev/braintrust/devserver/EvalResponse.java create mode 100644 src/main/java/dev/braintrust/devserver/RemoteEval.java create mode 100644 src/test/java/dev/braintrust/devserver/CorsTest.java create mode 100644 src/test/java/dev/braintrust/devserver/DevserverTest.java create mode 100644 src/test/java/dev/braintrust/devserver/EvalEndpointTest.java create mode 100644 src/test/java/dev/braintrust/devserver/ListEndpointTest.java create mode 100644 src/test/java/dev/braintrust/devserver/RemoteEvalTest.java diff --git a/README.md b/README.md index 85d34301..0d33aaa4 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,8 @@ var eval = braintrust.evalBuilder() .taskFunction(getFoodType) .scorers( Scorer.of( - "fruit_scorer", - result -> "fruit".equals(result) ? 1.0 : 0.0), - Scorer.of( - "vegetable_scorer", - result -> "vegetable".equals(result) ? 1.0 : 0.0)) + "exact_match", + (expected, result) -> expected.equals(result) ? 1.0 : 0.0)) .build(); var result = eval.run(); System.out.println("\n\n" + result.createReportString()); diff --git a/examples/build.gradle b/examples/build.gradle index 2244681e..d1e747ce 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -141,3 +141,18 @@ task runSpringAI(type: JavaExec) { suspend = false } } + + +task runRemoteEval(type: JavaExec) { + group = 'Braintrust SDK Examples' + description = 'Run the remote eval example' + classpath = sourceSets.main.runtimeClasspath + mainClass = 'dev.braintrust.examples.RemoteEvalExample' + systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel + debugOptions { + enabled = true + port = 5566 + server = true + suspend = false + } +} diff --git a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java index 74b2d1cf..7c3de222 100644 --- a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java +++ b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java @@ -34,10 +34,8 @@ public static void main(String[] args) throws Exception { var eval = braintrust .evalBuilder() - .name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a - // constant, additional runs - // will append new cases to - // the same experiment + // NOTE: pre-existing experiment names will append results + .name("java-eval-x-" + System.currentTimeMillis()) .cases( new DatasetCase<>( "strawberry", @@ -49,14 +47,13 @@ public static void main(String[] args) throws Exception { DatasetCase.of("asparagus", "vegetable"), DatasetCase.of("apple", "fruit"), DatasetCase.of("banana", "fruit")) + // Or, to fetch a remote dataset: + // .dataset(braintrust.fetchDataset("my-dataset-name")) .taskFunction(getFoodType) .scorers( Scorer.of( - "fruit_scorer", - result -> "fruit".equals(result) ? 1.0 : 0.0), - Scorer.of( - "vegetable_scorer", - result -> "vegetable".equals(result) ? 1.0 : 0.0)) + "exact_match", + (expected, result) -> expected.equals(result) ? 1.0 : 0.0)) .build(); var result = eval.run(); System.out.println("\n\n" + result.createReportString()); diff --git a/examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java b/examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java new file mode 100644 index 00000000..5219ca51 --- /dev/null +++ b/examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java @@ -0,0 +1,74 @@ +package dev.braintrust.examples; + +import com.openai.client.okhttp.OpenAIOkHttpClient; +import com.openai.models.ChatModel; +import com.openai.models.chat.completions.ChatCompletionCreateParams; +import dev.braintrust.Braintrust; +import dev.braintrust.devserver.Devserver; +import dev.braintrust.devserver.RemoteEval; +import dev.braintrust.eval.Scorer; +import dev.braintrust.instrumentation.openai.BraintrustOpenAI; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +/** Simple Dev Server for Remote Evals */ +public class RemoteEvalExample { + public static void main(String[] args) throws Exception { + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); + + RemoteEval foodTypeEval = + RemoteEval.builder() + .name("food-type-classifier") + .config(braintrust.config()) + .taskFunction( + food -> { + var request = + ChatCompletionCreateParams.builder() + .model(ChatModel.GPT_4O_MINI) + .addSystemMessage("Return a one word answer") + .addUserMessage( + "What kind of food is " + food + "?") + .maxTokens(50L) + .temperature(0.0) + .build(); + var response = + openAIClient.chat().completions().create(request); + return response.choices() + .get(0) + .message() + .content() + .orElse("") + .toLowerCase(); + }) + .scorers( + List.of( + Scorer.of( + "exact_match", + (expected, result) -> + expected.equals(result) ? 1.0 : 0.0))) + .build(); + + Devserver devserver = + Devserver.builder() + .registerEval(foodTypeEval) + .host("localhost") // set to 0.0.0.0 to bind all interfaces + .port(8300) + .build(); + + System.out.println("Starting Braintrust dev server"); + devserver.start(); + // await shutdown + CountDownLatch latch = new CountDownLatch(1); + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + System.out.println("Shutting down..."); + devserver.stop(); + latch.countDown(); + })); + latch.await(); + } +} diff --git a/src/main/java/dev/braintrust/devserver/Devserver.java b/src/main/java/dev/braintrust/devserver/Devserver.java new file mode 100644 index 00000000..14d759b6 --- /dev/null +++ b/src/main/java/dev/braintrust/devserver/Devserver.java @@ -0,0 +1,892 @@ +package dev.braintrust.devserver; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import dev.braintrust.BraintrustUtils; +import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.eval.Dataset; +import dev.braintrust.eval.DatasetCase; +import dev.braintrust.eval.Score; +import dev.braintrust.trace.BraintrustTracing; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.Executors; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import lombok.Getter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +/** Remote Eval Dev Server */ +@Slf4j +public class Devserver { + // Whitelisted origins for CORS + private static final List WHITELISTED_ORIGINS = + Arrays.asList( + "https://www.braintrust.dev", + "https://www.braintrustdata.com", + System.getenv("WHITELISTED_ORIGIN"), + System.getenv("BRAINTRUST_APP_URL")); + + private static final Pattern PREVIEW_DOMAIN_PATTERN = + Pattern.compile("^https://[^/]+\\.preview\\.braintrust\\.dev$"); + + // Allowed headers for CORS + private static final String ALLOWED_HEADERS = + String.join( + ", ", + "Content-Type", + "X-Amz-Date", + "Authorization", + "X-Api-Key", + "X-Amz-Security-Token", + "x-bt-auth-token", + "x-bt-parent", + "x-bt-org-name", + "x-bt-project-id", + "x-bt-stream-fmt", + "x-bt-use-cache", + "x-stainless-os", + "x-stainless-lang", + "x-stainless-package-version", + "x-stainless-runtime", + "x-stainless-runtime-version", + "x-stainless-arch"); + + private static final String EXPOSED_HEADERS = + "x-bt-cursor, x-bt-found-existing-experiment, x-bt-span-id, x-bt-span-export"; + + private static final AttributeKey PARENT = + AttributeKey.stringKey(BraintrustTracing.PARENT_KEY); + + @Getter + @Accessors(fluent = true) + private final String host; + + @Getter + @Accessors(fluent = true) + private final int port; + + private final @Nullable String orgName; + private final Map> evals; + private @Nullable HttpServer server; + private static final ObjectMapper JSON_MAPPER = + new ObjectMapper() + .enable( + com.fasterxml.jackson.core.JsonParser.Feature + .INCLUDE_SOURCE_IN_LOCATION); + + private Devserver(Builder builder) { + this.host = builder.host; + this.port = builder.port; + this.orgName = builder.orgName; + // Index evals by name for quick lookup + Map> evalMap = new HashMap<>(); + for (RemoteEval eval : builder.evals) { + if (evalMap.containsKey(eval.getName())) { + throw new IllegalArgumentException("Duplicate evaluator name: " + eval.getName()); + } + evalMap.put(eval.getName(), eval); + } + this.evals = Collections.unmodifiableMap(evalMap); + if (orgName != null) { + throw new RuntimeException("TODO: org name filtering not implemented yet"); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** Start the dev server. This method blocks until the server is stopped. */ + public void start() throws IOException { + if (server != null) { + throw new IllegalStateException("Server is already running"); + } + + server = HttpServer.create(new InetSocketAddress(host, port), 0); + server.setExecutor(Executors.newCachedThreadPool()); + + // Health check endpoint with CORS + server.createContext("/", withCors(this::handleHealthCheck)); + + // List evaluators endpoint with CORS + server.createContext("/list", withCors(this::handleList)); + + // Eval execution endpoint with CORS + server.createContext("/eval", withCors(this::handleEval)); + + server.start(); + log.info("Braintrust dev server started on http://{}:{}", host, port); + log.info("Registered {} evaluator(s): {}", evals.size(), evals.keySet()); + } + + /** Stop the dev server. */ + public void stop() { + if (server != null) { + server.stop(0); + server = null; + log.info("Braintrust dev server stopped"); + } + } + + private void handleHealthCheck(HttpExchange exchange) throws IOException { + if (!"GET".equals(exchange.getRequestMethod())) { + sendResponse(exchange, 405, "text/plain", "Method Not Allowed"); + return; + } + sendResponse(exchange, 200, "text/plain", "Hello, world!"); + } + + private void handleList(HttpExchange exchange) throws IOException { + if (!"GET".equals(exchange.getRequestMethod())) { + sendResponse(exchange, 405, "text/plain", "Method Not Allowed"); + return; + } + + // TODO: Add authentication check here + // For now, we'll allow unauthenticated access for local testing + + try { + // Build the response: Map + Map> response = new LinkedHashMap<>(); + + for (Map.Entry> entry : evals.entrySet()) { + String evalName = entry.getKey(); + RemoteEval eval = entry.getValue(); + + Map metadata = new LinkedHashMap<>(); + + // Add parameters + Map> parametersMap = new LinkedHashMap<>(); + for (Map.Entry paramEntry : + eval.getParameters().entrySet()) { + String paramName = paramEntry.getKey(); + RemoteEval.Parameter param = paramEntry.getValue(); + + Map paramMetadata = new LinkedHashMap<>(); + paramMetadata.put("type", param.getType().getValue()); + + if (param.getDescription() != null) { + paramMetadata.put("description", param.getDescription()); + } + + if (param.getDefaultValue() != null) { + paramMetadata.put("default", param.getDefaultValue()); + } + + // Only include schema for data type parameters + if (param.getType() == RemoteEval.ParameterType.DATA + && param.getSchema() != null) { + paramMetadata.put("schema", param.getSchema()); + } + + parametersMap.put(paramName, paramMetadata); + } + metadata.put("parameters", parametersMap); + + // Add scores (list of scorer names) + List> scores = new ArrayList<>(); + for (var scorer : eval.getScorers()) { + Map scoreInfo = new LinkedHashMap<>(); + scoreInfo.put("name", scorer.getName()); + scores.add(scoreInfo); + } + metadata.put("scores", scores); + + response.put(evalName, metadata); + } + + String jsonResponse = JSON_MAPPER.writeValueAsString(response); + sendResponse(exchange, 200, "application/json", jsonResponse); + } catch (Exception e) { + log.error("Error generating /list response", e); + sendResponse(exchange, 500, "text/plain", "Internal Server Error"); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private void handleEval(HttpExchange exchange) throws IOException { + if (!"POST".equals(exchange.getRequestMethod())) { + sendResponse(exchange, 405, "text/plain", "Method Not Allowed"); + return; + } + + // TODO: Add authentication check here + // For now, we'll allow unauthenticated access for local testing + + try { + InputStream requestBody = exchange.getRequestBody(); + var requestBodyString = new String(requestBody.readAllBytes(), StandardCharsets.UTF_8); + EvalRequest request = JSON_MAPPER.readValue(requestBodyString, EvalRequest.class); + + // Validate evaluator exists + RemoteEval eval = evals.get(request.getName()); + if (eval == null) { + sendResponse( + exchange, 404, "text/plain", "Evaluator not found: " + request.getName()); + return; + } + + // Validate dataset specification + if (request.getData() == null) { + sendResponse(exchange, 400, "text/plain", "Missing 'data' field in request body"); + return; + } + + EvalRequest.DataSpec dataSpec = request.getData(); + boolean hasInlineData = dataSpec.getData() != null && !dataSpec.getData().isEmpty(); + boolean hasByName = + dataSpec.getProjectName() != null && dataSpec.getDatasetName() != null; + boolean hasById = dataSpec.getDatasetId() != null; + + // Ensure exactly one dataset specification method is provided + int specCount = (hasInlineData ? 1 : 0) + (hasByName ? 1 : 0) + (hasById ? 1 : 0); + if (specCount == 0) { + sendResponse( + exchange, + 400, + "text/plain", + "Dataset must be specified using one of: inline data (data.data), by name" + + " (data.project_name + data.dataset_name), or by ID" + + " (data.dataset_id)"); + return; + } + if (specCount > 1) { + sendResponse( + exchange, + 400, + "text/plain", + "Only one dataset specification method should be provided"); + return; + } + + // TODO: Phase 2 - support remote scorers + + String datasetDescription = + hasInlineData + ? dataSpec.getData().size() + " inline cases" + : (hasByName + ? "dataset '" + + dataSpec.getProjectName() + + "/" + + dataSpec.getDatasetName() + + "'" + : "dataset ID '" + dataSpec.getDatasetId() + "'"); + log.info("Executing evaluator '{}' with {}", request.getName(), datasetDescription); + + // Check if streaming is requested + boolean isStreaming = request.getStream() != null && request.getStream(); + + if (isStreaming) { + // SSE streaming response - errors handled inside + log.info("Starting streaming evaluation for '{}'", request.getName()); + handleStreamingEval(exchange, eval, request); + } else { + // Regular JSON response + EvalResponse response = executeEval(eval, request); + String jsonResponse = JSON_MAPPER.writeValueAsString(response); + sendResponse(exchange, 200, "application/json", jsonResponse); + } + + } catch (Exception e) { + log.error("Error executing eval", e); + // Only send error response if we haven't started streaming + // (streaming errors are handled within handleStreamingEval) + sendResponse(exchange, 500, "text/plain", "Internal Server Error: " + e.getMessage()); + } + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private EvalResponse executeEval(RemoteEval eval, EvalRequest request) throws Exception { + // Get or create API client + BraintrustApiClient apiClient = BraintrustApiClient.of(eval.getConfig()); + + // Determine project name and ID + String projectName = eval.getProjectName(); + var orgAndProject = apiClient.getOrCreateProjectAndOrgInfo(eval.getConfig()); + String projectId = orgAndProject.project().id(); + + // Create experiment + String experimentName = + request.getExperimentName() != null + ? request.getExperimentName() + : eval.getName() + "-" + System.currentTimeMillis(); + + var experiment = + apiClient.getOrCreateExperiment( + new BraintrustApiClient.CreateExperimentRequest( + projectId, experimentName, Optional.empty(), Optional.empty())); + + log.info("Created experiment: {} ({})", experimentName, experiment.id()); + + // Load dataset using one of three methods + Dataset dataset; + EvalRequest.DataSpec dataSpec = request.getData(); + + if (dataSpec.getData() != null && !dataSpec.getData().isEmpty()) { + // Method 1: Inline data + List cases = new ArrayList<>(); + for (EvalRequest.EvalCaseData caseData : dataSpec.getData()) { + DatasetCase datasetCase = + new DatasetCase<>( + caseData.getInput(), + caseData.getExpected(), + caseData.getTags() != null ? caseData.getTags() : List.of(), + caseData.getMetadata() != null ? caseData.getMetadata() : Map.of()); + cases.add(datasetCase); + } + dataset = Dataset.of(cases.toArray(new DatasetCase[0])); + } else if (dataSpec.getProjectName() != null && dataSpec.getDatasetName() != null) { + // Method 2: Fetch by project name and dataset name + log.info( + "Fetching dataset from Braintrust: project={}, dataset={}", + dataSpec.getProjectName(), + dataSpec.getDatasetName()); + dataset = + Dataset.fetchFromBraintrust( + apiClient, + dataSpec.getProjectName(), + dataSpec.getDatasetName(), + null); // TODO: support dataset version from btql + } else if (dataSpec.getDatasetId() != null) { + // Method 3: Fetch by dataset ID + log.info("Fetching dataset from Braintrust by ID: {}", dataSpec.getDatasetId()); + // TODO: create a static method to fetch by ID in Dataset interface + var datasetMetadata = apiClient.getDataset(dataSpec.getDatasetId()); + if (datasetMetadata.isEmpty()) { + throw new IllegalArgumentException("Dataset not found: " + dataSpec.getDatasetId()); + } + + // Get project info to obtain project name + var project = apiClient.getProject(datasetMetadata.get().projectId()); + if (project.isEmpty()) { + throw new IllegalArgumentException( + "Project not found: " + datasetMetadata.get().projectId()); + } + + String fetchedProjectName = project.get().name(); + String fetchedDatasetName = datasetMetadata.get().name(); + log.info( + "Resolved dataset ID to project={}, dataset={}", + fetchedProjectName, + fetchedDatasetName); + + dataset = + Dataset.fetchFromBraintrust( + apiClient, fetchedProjectName, fetchedDatasetName, null); + } else { + throw new IllegalStateException("No dataset specification provided"); + } + // Execute task and scorers for each case + Map> scoresByName = new LinkedHashMap<>(); + dataset.forEach( + datasetCase -> { + // Run task + var task = eval.getTask(); + var taskResult = task.apply(datasetCase); + + // Run scorers + var scorers = eval.getScorers(); + for (Object scorerObj : scorers) { + dev.braintrust.eval.Scorer scorer = (dev.braintrust.eval.Scorer) scorerObj; + List scores = scorer.score(taskResult); + for (Score score : scores) { + scoresByName + .computeIfAbsent(score.name(), k -> new ArrayList<>()) + .add(score.value()); + } + } + }); + + // Aggregate scores + Map scoreSummaries = new LinkedHashMap<>(); + for (Map.Entry> entry : scoresByName.entrySet()) { + String scoreName = entry.getKey(); + List values = entry.getValue(); + + double avgScore = + values.stream().mapToDouble(Double::doubleValue).average().orElse(0.0); + + scoreSummaries.put( + scoreName, + EvalResponse.ScoreSummary.builder() + .name(scoreName) + .score(avgScore) + .improvements(0) // TODO: comparison with baseline + .regressions(0) // TODO: comparison with baseline + .build()); + } + + // Build response + String experimentUrl = + BraintrustUtils.createProjectURI(eval.getConfig().appUrl(), orgAndProject) + .toASCIIString() + + "/experiments/" + + experimentName; + String projectUrl = + BraintrustUtils.createProjectURI(eval.getConfig().appUrl(), orgAndProject) + .toASCIIString(); + + return EvalResponse.builder() + .experimentName(experimentName) + .projectName(projectName) + .projectId(projectId) + .experimentId(experiment.id()) + .experimentUrl(experimentUrl) + .projectUrl(projectUrl) + .comparisonExperimentName(null) + .scores(scoreSummaries) + .build(); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalRequest request) + throws Exception { + // Set SSE headers + log.debug("Setting SSE headers"); + exchange.getResponseHeaders().set("Content-Type", "text/event-stream"); + exchange.getResponseHeaders().set("Cache-Control", "no-cache"); + exchange.getResponseHeaders().set("Connection", "keep-alive"); + exchange.sendResponseHeaders(200, 0); // 0 = chunked encoding + log.debug("SSE headers sent, starting stream"); + + try (OutputStream os = exchange.getResponseBody()) { + try { + // Get or create API client + BraintrustApiClient apiClient = BraintrustApiClient.of(eval.getConfig()); + + // Determine project name and ID + String projectName = eval.getProjectName(); + var orgAndProject = apiClient.getOrCreateProjectAndOrgInfo(eval.getConfig()); + String projectId = orgAndProject.project().id(); + + // Parse parent from request for span linking + String parentSpec = null; + if (request.getParent() != null) { + if (request.getParent() instanceof Map) { + @SuppressWarnings("unchecked") + Map parentMap = (Map) request.getParent(); + String objectType = (String) parentMap.get("object_type"); + String objectId = (String) parentMap.get("object_id"); + if (objectType != null && objectId != null) { + parentSpec = objectType + ":" + objectId; + log.info("Using parent span: {}", parentSpec); + } + } + } + + // Send start event (no experiment for remote evals) + Map startData = new LinkedHashMap<>(); + startData.put("projectId", projectId); + startData.put("projectName", projectName); + String startJson = JSON_MAPPER.writeValueAsString(startData); + log.info("Sending start event: {}", startJson); + sendSSEEvent(os, "start", startJson); + os.flush(); + log.debug("Start event sent and flushed"); + + // Load dataset using one of three methods (same logic as executeEval) + Dataset dataset; + EvalRequest.DataSpec dataSpec = request.getData(); + + if (dataSpec.getData() != null && !dataSpec.getData().isEmpty()) { + // Method 1: Inline data + List cases = new ArrayList<>(); + for (EvalRequest.EvalCaseData caseData : dataSpec.getData()) { + DatasetCase datasetCase = + new DatasetCase<>( + caseData.getInput(), + caseData.getExpected(), + caseData.getTags() != null ? caseData.getTags() : List.of(), + caseData.getMetadata() != null + ? caseData.getMetadata() + : Map.of()); + cases.add(datasetCase); + } + dataset = Dataset.of(cases.toArray(new DatasetCase[0])); + } else if (dataSpec.getProjectName() != null && dataSpec.getDatasetName() != null) { + // Method 2: Fetch by project name and dataset name + log.info( + "Fetching dataset from Braintrust: project={}, dataset={}", + dataSpec.getProjectName(), + dataSpec.getDatasetName()); + dataset = + Dataset.fetchFromBraintrust( + apiClient, + dataSpec.getProjectName(), + dataSpec.getDatasetName(), + null); + } else if (dataSpec.getDatasetId() != null) { + // Method 3: Fetch by dataset ID + log.info("Fetching dataset from Braintrust by ID: {}", dataSpec.getDatasetId()); + var datasetMetadata = apiClient.getDataset(dataSpec.getDatasetId()); + if (datasetMetadata.isEmpty()) { + throw new IllegalArgumentException( + "Dataset not found: " + dataSpec.getDatasetId()); + } + + var project = apiClient.getProject(datasetMetadata.get().projectId()); + if (project.isEmpty()) { + throw new IllegalArgumentException( + "Project not found: " + datasetMetadata.get().projectId()); + } + + String fetchedProjectName = project.get().name(); + String fetchedDatasetName = datasetMetadata.get().name(); + log.info( + "Resolved dataset ID to project={}, dataset={}", + fetchedProjectName, + fetchedDatasetName); + + dataset = + Dataset.fetchFromBraintrust( + apiClient, fetchedProjectName, fetchedDatasetName, null); + } else { + throw new IllegalStateException("No dataset specification provided"); + } + + // Get tracer for creating spans + Tracer tracer = BraintrustTracing.getTracer(); + + // Execute task and scorers for each case, sending progress events and creating spans + Map> scoresByName = new LinkedHashMap<>(); + log.info("Starting dataset iteration"); + int[] caseCount = {0}; // Use array for mutability in lambda + final String finalParentSpec = parentSpec; // Make effectively final for lambda + if (finalParentSpec == null) { + throw new RuntimeException("parent required"); + } + + dataset.forEach( + datasetCase -> { + caseCount[0]++; + log.info("Processing dataset case #{}", caseCount[0]); + + // Create eval span for this dataset case (matches Eval.java pattern) + var evalSpan = + tracer.spanBuilder("eval") + .setNoParent() // each eval case is its own trace + .setSpanKind(SpanKind.CLIENT) + .setAttribute(PARENT, finalParentSpec) + .setAttribute("braintrust.span_attributes", json(Map.of("type", "eval"))) + .setAttribute("braintrust.input_json", json(Map.of("input", datasetCase.input()))) + .setAttribute("braintrust.expected", json(datasetCase.expected())) + .startSpan(); + + if (!datasetCase.tags().isEmpty()) { + evalSpan.setAttribute( + AttributeKey.stringArrayKey("braintrust.tags"), datasetCase.tags()); + } + if (!datasetCase.metadata().isEmpty()) { + evalSpan.setAttribute("braintrust.metadata", json(datasetCase.metadata())); + } + + try (var rootScope = Context.current().with(evalSpan).makeCurrent()) { + final dev.braintrust.eval.TaskResult taskResult; + { // run task + var taskSpan = + tracer.spanBuilder("task") + .setAttribute(PARENT, finalParentSpec) + .setAttribute("braintrust.span_attributes", json(Map.of("type", "task"))) + .startSpan(); + try (var unused = Context.current().with(taskSpan).makeCurrent()) { + var task = eval.getTask(); + taskResult = task.apply(datasetCase); + log.info("Task completed for case #{}, result type: {}", caseCount[0], taskResult.getClass().getSimpleName()); + } finally { + taskSpan.end(); + } + + // Set output on eval span (not task span) + evalSpan.setAttribute("braintrust.output_json", json(Map.of("output", taskResult.result()))); + + // Send progress event for task completion + Map progressData = new LinkedHashMap<>(); + progressData.put("format", "code"); + progressData.put("output_type", "completion"); + progressData.put("event", "json_delta"); + progressData.put("data", JSON_MAPPER.writeValueAsString(taskResult)); + String progressJson = JSON_MAPPER.writeValueAsString(progressData); + log.info("Sending progress event for completion, data length: {}", progressJson.length()); + sendSSEEvent(os, "progress", progressJson); + os.flush(); + log.info("Progress event sent and flushed"); + + } + { // run scorers - ONE score span for all scorers (matches Eval.java) + var scoreSpan = + tracer.spanBuilder("score") + .setAttribute(PARENT, finalParentSpec) + .setAttribute("braintrust.span_attributes", json(Map.of("type", "score"))) + .startSpan(); + try (var unused = Context.current().with(scoreSpan).makeCurrent()) { + Map nameToScore = new LinkedHashMap<>(); + var scorers = eval.getScorers(); + log.info("Running {} scorers", scorers.size()); + + for (Object scorerObj : scorers) { + dev.braintrust.eval.Scorer scorer = + (dev.braintrust.eval.Scorer) scorerObj; + List scores = scorer.score(taskResult); + log.info("Scorer '{}' produced {} scores", scorer.getName(), scores.size()); + + for (Score score : scores) { + scoresByName + .computeIfAbsent(score.name(), k -> new ArrayList<>()) + .add(score.value()); + nameToScore.put(score.name(), score.value()); + + // Send progress event for each score + Map scoreProgressData = new LinkedHashMap<>(); + scoreProgressData.put("format", "code"); + scoreProgressData.put("output_type", "score"); + scoreProgressData.put("event", "json_delta"); + Map scoreData = new LinkedHashMap<>(); + scoreData.put(score.name(), score.value()); + scoreProgressData.put( + "data", JSON_MAPPER.writeValueAsString(scoreData)); + String scoreProgressJson = JSON_MAPPER.writeValueAsString(scoreProgressData); + log.info("Sending progress event for score '{}': {}", score.name(), score.value()); + sendSSEEvent(os, "progress", scoreProgressJson); + os.flush(); + log.info("Score progress event sent and flushed"); + } + } + + // Set all scores on the single score span + scoreSpan.setAttribute("braintrust.scores", json(nameToScore)); + } finally { + scoreSpan.end(); + } + } + } catch (IOException e) { + throw new RuntimeException("Failed to send progress event", e); + } finally { + evalSpan.end(); + } + }); + log.info("Completed dataset iteration. Processed {} cases", caseCount[0]); + + // Aggregate scores + Map scoreSummaries = new LinkedHashMap<>(); + for (Map.Entry> entry : scoresByName.entrySet()) { + String scoreName = entry.getKey(); + List values = entry.getValue(); + + double avgScore = + values.stream().mapToDouble(Double::doubleValue).average().orElse(0.0); + + scoreSummaries.put( + scoreName, + EvalResponse.ScoreSummary.builder() + .name(scoreName) + .score(avgScore) + .improvements(0) + .regressions(0) + .build()); + } + + // Build final response (no experiment fields for remote evals) + String projectUrl = + BraintrustUtils.createProjectURI(eval.getConfig().appUrl(), orgAndProject) + .toASCIIString(); + + EvalResponse response = + EvalResponse.builder() + .projectName(projectName) + .projectId(projectId) + .projectUrl(projectUrl) + .scores(scoreSummaries) + .build(); + + // Send summary event + log.info("Sending summary event"); + sendSSEEvent(os, "summary", JSON_MAPPER.writeValueAsString(response)); + os.flush(); + log.debug("Summary event sent and flushed"); + + // Send done event + log.info("Sending done event"); + sendSSEEvent(os, "done", ""); + os.flush(); + log.debug("Done event sent and flushed"); + + } catch (Exception e) { + // Send error event via SSE + log.error("Error during streaming evaluation", e); + try { + sendSSEEvent(os, "error", e.getMessage() != null ? e.getMessage() : "Unknown error"); + os.flush(); + } catch (IOException ioException) { + log.error("Failed to send error event", ioException); + } + } + } + } + + private void sendSSEEvent(OutputStream os, String eventType, String data) + throws IOException { + String event = "event: " + eventType + "\n" + "data: " + data + "\n\n"; + log.debug("Writing SSE event: type={}, data length={}", eventType, data.length()); + os.write(event.getBytes(StandardCharsets.UTF_8)); + log.debug("SSE event written to output stream"); + } + + private String json(Object o) { + try { + return JSON_MAPPER.writeValueAsString(o); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize to JSON", e); + } + } + + private void sendResponse( + HttpExchange exchange, int statusCode, String contentType, String body) + throws IOException { + byte[] responseBytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", contentType); + exchange.sendResponseHeaders(statusCode, responseBytes.length); + try (OutputStream os = exchange.getResponseBody()) { + os.write(responseBytes); + } + } + + /** + * Check if the origin is whitelisted for CORS. + * + * @param origin The Origin header value + * @return true if the origin is allowed, false otherwise + */ + private static boolean isOriginAllowed(@Nullable String origin) { + if (origin == null || origin.isEmpty()) { + return true; // Allow requests without origin (e.g., same-origin) + } + + // Check against whitelisted origins + for (String allowedOrigin : WHITELISTED_ORIGINS) { + if (allowedOrigin != null && allowedOrigin.equals(origin)) { + return true; + } + } + + // Check against preview domain pattern + return PREVIEW_DOMAIN_PATTERN.matcher(origin).matches(); + } + + /** + * Apply CORS headers to the response. + * + * @param exchange The HTTP exchange + */ + private static void applyCorsHeaders(HttpExchange exchange) { + String origin = exchange.getRequestHeaders().getFirst("Origin"); + + if (isOriginAllowed(origin)) { + var headers = exchange.getResponseHeaders(); + if (origin != null && !origin.isEmpty()) { + headers.set("Access-Control-Allow-Origin", origin); + } + headers.set("Access-Control-Allow-Credentials", "true"); + headers.set("Access-Control-Expose-Headers", EXPOSED_HEADERS); + } + } + + /** + * Handle CORS preflight requests. + * + * @param exchange The HTTP exchange + */ + private static void handlePreflightRequest(HttpExchange exchange) throws IOException { + String origin = exchange.getRequestHeaders().getFirst("Origin"); + + if (!isOriginAllowed(origin)) { + exchange.sendResponseHeaders(403, -1); + return; + } + + var headers = exchange.getResponseHeaders(); + if (origin != null && !origin.isEmpty()) { + headers.set("Access-Control-Allow-Origin", origin); + } + headers.set("Access-Control-Allow-Methods", "GET, PATCH, POST, PUT, DELETE, OPTIONS"); + headers.set("Access-Control-Allow-Headers", ALLOWED_HEADERS); + headers.set("Access-Control-Allow-Credentials", "true"); + headers.set("Access-Control-Max-Age", "86400"); + + // Support for Chrome's Private Network Access + String requestPrivateNetwork = + exchange.getRequestHeaders().getFirst("Access-Control-Request-Private-Network"); + if ("true".equals(requestPrivateNetwork)) { + headers.set("Access-Control-Allow-Private-Network", "true"); + } + + exchange.sendResponseHeaders(204, -1); + } + + /** + * Wrap a handler with CORS support. + * + * @param handler The handler to wrap + * @return A handler that applies CORS headers + */ + private static HttpHandler withCors(HttpHandler handler) { + return exchange -> { + // Handle OPTIONS preflight requests + if ("OPTIONS".equals(exchange.getRequestMethod())) { + handlePreflightRequest(exchange); + return; + } + + // Apply CORS headers to all responses + applyCorsHeaders(exchange); + + // Delegate to the actual handler + handler.handle(exchange); + }; + } + + public static class Builder { + private String host = "localhost"; + private int port = 8300; + private @Nullable String orgName = null; + private List> evals = new ArrayList<>(); + + public Devserver build() { + if (evals.isEmpty()) { + throw new IllegalStateException("At least one evaluator must be registered"); + } + return new Devserver(this); + } + + public Builder registerEval(RemoteEval eval) { + this.evals.add(eval); + return this; + } + + public Builder host(String host) { + this.host = host; + return this; + } + + public Builder port(int port) { + this.port = port; + return this; + } + + public Builder orgName(String orgName) { + this.orgName = orgName; + return this; + } + } +} diff --git a/src/main/java/dev/braintrust/devserver/EvalRequest.java b/src/main/java/dev/braintrust/devserver/EvalRequest.java new file mode 100644 index 00000000..e3a71034 --- /dev/null +++ b/src/main/java/dev/braintrust/devserver/EvalRequest.java @@ -0,0 +1,116 @@ +package dev.braintrust.devserver; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import lombok.Data; + +/** Request body for POST /eval endpoint */ +@Data +public class EvalRequest { + /** Name of the evaluator to run */ + private String name; + + /** Optional parameter overrides */ + @Nullable private Map parameters; + + /** Dataset specification */ + private DataSpec data; + + /** Optional experiment name override */ + @JsonProperty("experiment_name") + @Nullable + private String experimentName; + + /** Optional project ID override */ + @JsonProperty("project_id") + @Nullable + private String projectId; + + /** Optional additional remote scorers */ + @Nullable private List scores; + + /** Optional parent span for tracing (can be string or object) */ + @Nullable private Object parent; + + /** Enable SSE streaming (default: false) */ + @Nullable private Boolean stream; + + /** Dataset specification - supports inline data, by name, or by ID */ + @Data + public static class DataSpec { + /** Inline data array */ + @Nullable private List data; + + /** Project name (for loading by name) */ + @JsonProperty("project_name") + @Nullable + private String projectName; + + /** Dataset name (for loading by name) */ + @JsonProperty("dataset_name") + @Nullable + private String datasetName; + + /** Dataset ID (for loading by ID) */ + @JsonProperty("dataset_id") + @Nullable + private String datasetId; + + /** Optional BTQL filter (can be string or structured query object) */ + @JsonProperty("_internal_btql") + @Nullable + private Object btql; + } + + /** Individual evaluation case data */ + @Data + public static class EvalCaseData { + /** Input for the task */ + private Object input; + + /** Expected output (optional) */ + @Nullable private Object expected; + + /** Metadata (optional) */ + @Nullable private Map metadata; + + /** Tags (optional) */ + @Nullable private List tags; + } + + /** Remote scorer specification */ + @Data + public static class RemoteScorer { + /** Scorer name */ + private String name; + + /** Function ID specification */ + @JsonProperty("function_id") + private FunctionId functionId; + } + + /** Function ID specification (multiple formats supported) */ + @Data + public static class FunctionId { + @JsonProperty("function_id") + @Nullable + private String functionId; + + @Nullable private String version; + @Nullable private String name; + + @JsonProperty("prompt_session_id") + @Nullable + private String promptSessionId; + + @JsonProperty("inline_code") + @Nullable + private String inlineCode; + + @JsonProperty("global_function") + @Nullable + private String globalFunction; + } +} diff --git a/src/main/java/dev/braintrust/devserver/EvalResponse.java b/src/main/java/dev/braintrust/devserver/EvalResponse.java new file mode 100644 index 00000000..f5303d21 --- /dev/null +++ b/src/main/java/dev/braintrust/devserver/EvalResponse.java @@ -0,0 +1,62 @@ +package dev.braintrust.devserver; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Data; + +/** Response body for POST /eval endpoint (non-streaming) */ +@Data +@Builder +public class EvalResponse { + /** Experiment name */ + @JsonProperty("experimentName") + private String experimentName; + + /** Project name */ + @JsonProperty("projectName") + private String projectName; + + /** Project ID */ + @JsonProperty("projectId") + private String projectId; + + /** Experiment ID */ + @JsonProperty("experimentId") + private String experimentId; + + /** Experiment URL */ + @JsonProperty("experimentUrl") + private String experimentUrl; + + /** Project URL */ + @JsonProperty("projectUrl") + private String projectUrl; + + /** Comparison experiment name (optional) */ + @JsonProperty("comparisonExperimentName") + @Nullable + private String comparisonExperimentName; + + /** Score summaries by scorer name */ + @JsonProperty("scores") + private Map scores; + + /** Summary statistics for a scorer */ + @Data + @Builder + public static class ScoreSummary { + /** Scorer name */ + private String name; + + /** Average score across all cases */ + private double score; + + /** Number of improvements vs baseline */ + private int improvements; + + /** Number of regressions vs baseline */ + private int regressions; + } +} diff --git a/src/main/java/dev/braintrust/devserver/RemoteEval.java b/src/main/java/dev/braintrust/devserver/RemoteEval.java new file mode 100644 index 00000000..46552c95 --- /dev/null +++ b/src/main/java/dev/braintrust/devserver/RemoteEval.java @@ -0,0 +1,152 @@ +package dev.braintrust.devserver; + +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import dev.braintrust.eval.Task; +import java.util.*; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Getter; +import lombok.Singular; + +/** + * Represents a remote evaluator that can be exposed via the dev server. + * + * @param The type of input data for the evaluation + * @param The type of output produced by the task + */ +@Getter +@Builder(builderClassName = "Builder", buildMethodName = "internalBuild") +public class RemoteEval { + /** The name of this evaluator (used as identifier) */ + @Nonnull private final String name; + + /** + * The Braintrust project name. If not specified, uses config.defaultProjectName() or + * config.defaultProjectId() + */ + @Nonnull private final String projectName; + + /** Braintrust configuration for API access */ + @Nonnull private final BraintrustConfig config; + + /** The task function that performs the evaluation */ + @Nonnull private final Task task; + + /** List of scorers for this evaluator */ + @Singular @Nonnull private final List> scorers; + + /** Optional parameters that can be configured from the UI */ + @Singular @Nonnull private final Map parameters; + + public static class Builder { + /** + * Convenience builder method to create a RemoteEval with a simple task function. + * + * @param taskFn Function that takes input and returns output + * @return this builder + */ + public Builder taskFunction(Function taskFn) { + return task( + datasetCase -> { + var result = taskFn.apply(datasetCase.input()); + return new dev.braintrust.eval.TaskResult<>(result, datasetCase); + }); + } + + /** Build the RemoteEval, using config's default project name if projectName is not set. */ + public RemoteEval build() { + // If projectName is not set, try to use the default from config + if (this.projectName == null && this.config != null) { + // Try defaultProjectName first, fall back to defaultProjectId + this.projectName = + this.config + .defaultProjectName() + .or(() -> this.config.defaultProjectId()) + .orElseThrow( + () -> + new IllegalArgumentException( + "projectName must be specified either" + + " explicitly or via" + + " config.defaultProjectName/defaultProjectId")); + } + + if (this.projectName == null) { + throw new IllegalArgumentException( + "projectName must be specified (config is not set yet)"); + } + + return internalBuild(); + } + } + + /** Represents a configurable parameter for the evaluator */ + @Getter + @lombok.Builder(builderClassName = "Builder") + public static class Parameter { + /** Type of parameter: "prompt" or "data" */ + @Nonnull private final ParameterType type; + + /** Optional description of the parameter */ + @Nullable private final String description; + + /** Optional default value for the parameter */ + @Nullable private final Object defaultValue; + + /** + * JSON Schema for data type parameters. Only applicable when type is DATA. Should be a Map + * representing a JSON Schema object. + */ + @Nullable private final Map schema; + + public static Parameter promptParameter(String description, Object defaultValue) { + return Parameter.builder() + .type(ParameterType.PROMPT) + .description(description) + .defaultValue(defaultValue) + .build(); + } + + public static Parameter promptParameter(Object defaultValue) { + return promptParameter(null, defaultValue); + } + + public static Parameter dataParameter( + String description, Map schema, Object defaultValue) { + return Parameter.builder() + .type(ParameterType.DATA) + .description(description) + .schema(schema) + .defaultValue(defaultValue) + .build(); + } + + public static Parameter dataParameter(Map schema, Object defaultValue) { + return dataParameter(null, schema, defaultValue); + } + + public static Parameter dataParameter(Map schema) { + return dataParameter(null, schema, null); + } + } + + /** Parameter type enumeration */ + public enum ParameterType { + /** Prompt parameter (for LLM prompts) */ + PROMPT("prompt"), + /** Data parameter (for other configuration data) */ + DATA("data"); + + private final String value; + + ParameterType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } +} diff --git a/src/main/java/dev/braintrust/eval/Scorer.java b/src/main/java/dev/braintrust/eval/Scorer.java index f28d6109..2ea8c9a6 100644 --- a/src/main/java/dev/braintrust/eval/Scorer.java +++ b/src/main/java/dev/braintrust/eval/Scorer.java @@ -17,7 +17,7 @@ public interface Scorer { List score(TaskResult taskResult); static Scorer of( - String scorerName, Function scorerFn) { + String scorerName, Function, Double> scorerFn) { return new Scorer<>() { @Override public String getName() { @@ -26,15 +26,13 @@ public String getName() { @Override public List score(TaskResult taskResult) { - return List.of(new Score(scorerName, scorerFn.apply(taskResult.result()))); + return List.of(new Score(scorerName, scorerFn.apply(taskResult))); } }; } - /** Deprecated. Use {@link #of(String, Function)} or implement the Scorer interface instead. */ - @Deprecated static Scorer of( - String scorerName, BiFunction, OUTPUT, Double> scorerFn) { + String scorerName, BiFunction scorerFn) { return new Scorer<>() { @Override public String getName() { @@ -47,8 +45,7 @@ public List score(TaskResult taskResult) { new Score( scorerName, scorerFn.apply( - EvalCase.from(taskResult.datasetCase()), - taskResult.result()))); + taskResult.datasetCase().expected(), taskResult.result()))); } }; } diff --git a/src/test/java/dev/braintrust/devserver/CorsTest.java b/src/test/java/dev/braintrust/devserver/CorsTest.java new file mode 100644 index 00000000..a3e7db5b --- /dev/null +++ b/src/test/java/dev/braintrust/devserver/CorsTest.java @@ -0,0 +1,165 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class CorsTest { + private static Devserver server; + private static Thread serverThread; + private static final int TEST_PORT = 18301; + + @BeforeAll + static void setup() throws Exception { + RemoteEval testEval = + RemoteEval.builder() + .name("test-eval") + .projectName("test-project") + .config(BraintrustConfig.of("BRAINTRUST_API_KEY", "bogus")) + .taskFunction(input -> input.toUpperCase()) + .scorer( + Scorer.of( + "length", + (expected, result) -> (double) result.length() / 10.0)) + .build(); + + server = + Devserver.builder() + .registerEval(testEval) + .host("localhost") + .port(TEST_PORT) + .build(); + + serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + serverThread.start(); + Thread.sleep(1000); // Give server time to start + } + + @AfterAll + static void teardown() { + server.stop(); + serverThread.interrupt(); + } + + @Test + void testCorsPreflightRequest() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/")) + .method("OPTIONS", HttpRequest.BodyPublishers.noBody()) + .header("Origin", "https://www.braintrust.dev") + .header("Access-Control-Request-Method", "POST") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(204, response.statusCode()); + assertEquals( + "https://www.braintrust.dev", + response.headers().firstValue("Access-Control-Allow-Origin").orElse(null)); + assertEquals( + "true", + response.headers().firstValue("Access-Control-Allow-Credentials").orElse(null)); + assertTrue( + response.headers() + .firstValue("Access-Control-Allow-Methods") + .orElse("") + .contains("POST")); + assertTrue( + response.headers() + .firstValue("Access-Control-Allow-Headers") + .orElse("") + .contains("x-bt-auth-token")); + } + + @Test + void testCorsActualRequest() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/")) + .GET() + .header("Origin", "https://www.braintrust.dev") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals("Hello, world!", response.body()); + assertEquals( + "https://www.braintrust.dev", + response.headers().firstValue("Access-Control-Allow-Origin").orElse(null)); + assertEquals( + "true", + response.headers().firstValue("Access-Control-Allow-Credentials").orElse(null)); + } + + @Test + void testCorsPreviewDomain() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/")) + .GET() + .header("Origin", "https://pr-123.preview.braintrust.dev") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals( + "https://pr-123.preview.braintrust.dev", + response.headers().firstValue("Access-Control-Allow-Origin").orElse(null)); + } + + @Test + void testCorsUnauthorizedOrigin() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/")) + .method("OPTIONS", HttpRequest.BodyPublishers.noBody()) + .header("Origin", "https://evil.com") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(403, response.statusCode()); + } + + @Test + void testPrivateNetworkAccess() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/")) + .method("OPTIONS", HttpRequest.BodyPublishers.noBody()) + .header("Origin", "https://www.braintrust.dev") + .header("Access-Control-Request-Private-Network", "true") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(204, response.statusCode()); + assertEquals( + "true", + response.headers().firstValue("Access-Control-Allow-Private-Network").orElse(null)); + } +} diff --git a/src/test/java/dev/braintrust/devserver/DevserverTest.java b/src/test/java/dev/braintrust/devserver/DevserverTest.java new file mode 100644 index 00000000..cf67ac09 --- /dev/null +++ b/src/test/java/dev/braintrust/devserver/DevserverTest.java @@ -0,0 +1,66 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import org.junit.jupiter.api.Test; + +class DevserverTest { + @Test + void testHealthCheck() throws Exception { + RemoteEval testEval = + RemoteEval.builder() + .name("test-eval") + .projectName("test-project") + .config(BraintrustConfig.of("BRAINTRUST_API_KEY", "bogus")) + .taskFunction(input -> input.toUpperCase()) + .scorer( + Scorer.of( + "length", + (expected, result) -> (double) result.length() / 10.0)) + .build(); + + Devserver server = + Devserver.builder().registerEval(testEval).host("localhost").port(18300).build(); + + // Start server in background thread + Thread serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + serverThread.start(); + + // Give server time to start + Thread.sleep(1000); + + try { + // Test health check endpoint + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:18300/")) + .GET() + .build(); + + HttpResponse response = + client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals("Hello, world!", response.body()); + assertEquals("text/plain", response.headers().firstValue("Content-Type").orElse("")); + } finally { + server.stop(); + serverThread.interrupt(); + } + } +} diff --git a/src/test/java/dev/braintrust/devserver/EvalEndpointTest.java b/src/test/java/dev/braintrust/devserver/EvalEndpointTest.java new file mode 100644 index 00000000..830391cb --- /dev/null +++ b/src/test/java/dev/braintrust/devserver/EvalEndpointTest.java @@ -0,0 +1,175 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.List; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class EvalEndpointTest { + private static Devserver server; + private static Thread serverThread; + private static final int TEST_PORT = 18303; + private static final ObjectMapper mapper = new ObjectMapper(); + + @BeforeAll + static void setup() throws Exception { + // Create a test eval + RemoteEval testEval = + RemoteEval.builder() + .name("uppercase-eval") + .projectName("test-project") + .config(BraintrustConfig.of("BRAINTRUST_API_KEY", "bogus")) + .taskFunction(String::toUpperCase) + .scorer( + Scorer.of( + "length", + (expected, result) -> + Math.min((double) result.length() / 10.0, 1.0))) + .scorer( + Scorer.of( + "has_hello", + (expected, result) -> result.contains("HELLO") ? 1.0 : 0.0)) + .build(); + + server = + Devserver.builder() + .registerEval(testEval) + .host("localhost") + .port(TEST_PORT) + .build(); + + serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + serverThread.start(); + Thread.sleep(1000); // Give server time to start + } + + @AfterAll + static void teardown() { + server.stop(); + serverThread.interrupt(); + } + + @Test + @Disabled + void testEvalEndpointWithInlineData() throws Exception { + // Build request + EvalRequest request = new EvalRequest(); + request.setName("uppercase-eval"); + + EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec(); + + EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData(); + case1.setInput("hello"); + case1.setExpected("HELLO"); + + EvalRequest.EvalCaseData case2 = new EvalRequest.EvalCaseData(); + case2.setInput("world"); + case2.setExpected("WORLD"); + + dataSpec.setData(List.of(case1, case2)); + request.setData(dataSpec); + + String requestJson = mapper.writeValueAsString(request); + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest httpRequest = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/eval")) + .POST(HttpRequest.BodyPublishers.ofString(requestJson)) + .header("Content-Type", "application/json") + .build(); + + HttpResponse response = + client.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals( + "application/json", response.headers().firstValue("Content-Type").orElse(null)); + + // Parse and validate JSON response + JsonNode root = mapper.readTree(response.body()); + + assertTrue(root.has("experimentName")); + assertTrue(root.has("projectName")); + assertTrue(root.has("projectId")); + assertTrue(root.has("experimentId")); + assertTrue(root.has("experimentUrl")); + assertTrue(root.has("projectUrl")); + assertTrue(root.has("scores")); + + JsonNode scores = root.get("scores"); + assertTrue(scores.has("length")); + assertTrue(scores.has("has_hello")); + + // Check length scorer (average of 5/10 and 5/10 = 0.5) + JsonNode lengthScore = scores.get("length"); + assertEquals("length", lengthScore.get("name").asText()); + assertEquals(0.5, lengthScore.get("score").asDouble(), 0.01); + + // Check has_hello scorer (1.0 for "hello", 0.0 for "world" = 0.5 average) + JsonNode helloScore = scores.get("has_hello"); + assertEquals("has_hello", helloScore.get("name").asText()); + assertEquals(0.5, helloScore.get("score").asDouble(), 0.01); + } + + @Test + void testEvalEndpointEvaluatorNotFound() throws Exception { + EvalRequest request = new EvalRequest(); + request.setName("non-existent-eval"); + + EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec(); + EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData(); + case1.setInput("test"); + dataSpec.setData(List.of(case1)); + request.setData(dataSpec); + + String requestJson = mapper.writeValueAsString(request); + + HttpClient client = HttpClient.newHttpClient(); + HttpRequest httpRequest = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/eval")) + .POST(HttpRequest.BodyPublishers.ofString(requestJson)) + .header("Content-Type", "application/json") + .build(); + + HttpResponse response = + client.send(httpRequest, HttpResponse.BodyHandlers.ofString()); + + assertEquals(404, response.statusCode()); + assertTrue(response.body().contains("Evaluator not found")); + } + + @Test + void testEvalEndpointMethodNotAllowed() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/eval")) + .GET() + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(405, response.statusCode()); + } +} diff --git a/src/test/java/dev/braintrust/devserver/ListEndpointTest.java b/src/test/java/dev/braintrust/devserver/ListEndpointTest.java new file mode 100644 index 00000000..651146c9 --- /dev/null +++ b/src/test/java/dev/braintrust/devserver/ListEndpointTest.java @@ -0,0 +1,161 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class ListEndpointTest { + private static Devserver server; + private static Thread serverThread; + private static final int TEST_PORT = 18302; + private static final ObjectMapper mapper = new ObjectMapper(); + + @BeforeAll + static void setup() throws Exception { + // Create a test eval with parameters + RemoteEval testEval = + RemoteEval.builder() + .name("test-classifier") + .projectName("test-project") + .config(BraintrustConfig.of("BRAINTRUST_API_KEY", "bogus")) + .taskFunction(input -> input.toUpperCase()) + .scorer(Scorer.of("accuracy", result -> 0.95)) + .scorer( + Scorer.of( + "length", + (expected, result) -> (double) result.length() / 10.0)) + .parameter( + "model", + RemoteEval.Parameter.dataParameter( + "The model to use", + Map.of( + "type", + "string", + "enum", + new String[] {"gpt-4", "gpt-3.5"}), + "gpt-4")) + .parameter( + "temperature", + RemoteEval.Parameter.dataParameter( + "Temperature for sampling", + Map.of("type", "number", "minimum", 0.0, "maximum", 2.0), + 0.7)) + .build(); + + server = + Devserver.builder() + .registerEval(testEval) + .host("localhost") + .port(TEST_PORT) + .build(); + + serverThread = + new Thread( + () -> { + try { + server.start(); + } catch (Exception e) { + e.printStackTrace(); + } + }); + serverThread.start(); + Thread.sleep(1000); // Give server time to start + } + + @AfterAll + static void teardown() { + server.stop(); + serverThread.interrupt(); + } + + @Test + void testListEndpoint() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/list")) + .GET() + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals( + "application/json", response.headers().firstValue("Content-Type").orElse(null)); + + // Parse and validate JSON response + JsonNode root = mapper.readTree(response.body()); + + // Should have one evaluator + assertTrue(root.has("test-classifier")); + + JsonNode eval = root.get("test-classifier"); + + // Check parameters + assertTrue(eval.has("parameters")); + JsonNode parameters = eval.get("parameters"); + + assertTrue(parameters.has("model")); + JsonNode modelParam = parameters.get("model"); + assertEquals("data", modelParam.get("type").asText()); + assertEquals("The model to use", modelParam.get("description").asText()); + assertEquals("gpt-4", modelParam.get("default").asText()); + assertTrue(modelParam.has("schema")); + + assertTrue(parameters.has("temperature")); + JsonNode tempParam = parameters.get("temperature"); + assertEquals("data", tempParam.get("type").asText()); + assertEquals(0.7, tempParam.get("default").asDouble()); + + // Check scores + assertTrue(eval.has("scores")); + JsonNode scores = eval.get("scores"); + assertEquals(2, scores.size()); + + assertEquals("accuracy", scores.get(0).get("name").asText()); + assertEquals("length", scores.get(1).get("name").asText()); + } + + @Test + void testListEndpointMethodNotAllowed() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/list")) + .POST(HttpRequest.BodyPublishers.noBody()) + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(405, response.statusCode()); + } + + @Test + void testListEndpointWithCors() throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create("http://localhost:" + TEST_PORT + "/list")) + .GET() + .header("Origin", "https://www.braintrust.dev") + .build(); + + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + assertEquals(200, response.statusCode()); + assertEquals( + "https://www.braintrust.dev", + response.headers().firstValue("Access-Control-Allow-Origin").orElse(null)); + } +} diff --git a/src/test/java/dev/braintrust/devserver/RemoteEvalTest.java b/src/test/java/dev/braintrust/devserver/RemoteEvalTest.java new file mode 100644 index 00000000..03c690ea --- /dev/null +++ b/src/test/java/dev/braintrust/devserver/RemoteEvalTest.java @@ -0,0 +1,104 @@ +package dev.braintrust.devserver; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Scorer; +import org.junit.jupiter.api.Test; + +class RemoteEvalTest { + + @Test + void testProjectNameFallbackToDefault() { + BraintrustConfig config = + BraintrustConfig.builder() + .apiKey("test-key") + .defaultProjectName("default-project") + .build(); + + RemoteEval eval = + RemoteEval.builder() + .name("test-eval") + .config(config) + .taskFunction(input -> input) + .scorer(Scorer.of("test", result -> 1.0)) + .build(); + + assertEquals("default-project", eval.getProjectName()); + } + + @Test + void testProjectNameExplicitOverridesDefault() { + BraintrustConfig config = + BraintrustConfig.builder() + .apiKey("test-key") + .defaultProjectName("default-project") + .build(); + + RemoteEval eval = + RemoteEval.builder() + .name("test-eval") + .projectName("explicit-project") + .config(config) + .taskFunction(input -> input) + .scorer(Scorer.of("test", result -> 1.0)) + .build(); + + assertEquals("explicit-project", eval.getProjectName()); + } + + @Test + void testProjectNameRequiredWhenNoDefault() { + // Create a minimal config without defaults + // Note: This test may pass if environment variables provide defaults + BraintrustConfig config = BraintrustConfig.builder().apiKey("test-key").build(); + + // Only throw if config truly has no defaults + if (config.defaultProjectName().isEmpty() && config.defaultProjectId().isEmpty()) { + assertThrows( + IllegalArgumentException.class, + () -> + RemoteEval.builder() + .name("test-eval") + .config(config) + .taskFunction(input -> input) + .scorer(Scorer.of("test", result -> 1.0)) + .build()); + } else { + // If environment provides defaults, just verify build succeeds + RemoteEval eval = + RemoteEval.builder() + .name("test-eval") + .config(config) + .taskFunction(input -> input) + .scorer(Scorer.of("test", result -> 1.0)) + .build(); + assertNotNull(eval.getProjectName()); + } + } + + @Test + void testProjectNameFallsBackToProjectId() { + BraintrustConfig config = + BraintrustConfig.builder() + .apiKey("test-key") + .defaultProjectId("project-id-123") + .build(); + + RemoteEval eval = + RemoteEval.builder() + .name("test-eval") + .config(config) + .taskFunction(input -> input) + .scorer(Scorer.of("test", result -> 1.0)) + .build(); + + // Project name should be either the explicitly set project ID or from environment + // If environment has a project name, it takes precedence + assertNotNull(eval.getProjectName()); + // Only assert exact match if environment doesn't override + if (config.defaultProjectName().isEmpty()) { + assertEquals("project-id-123", eval.getProjectName()); + } + } +} From 0f3178a89575c7d9ee57fbae97c2a16986354610 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 8 Dec 2025 15:18:02 -0500 Subject: [PATCH 2/3] wip -- DT --- .../java/dev/braintrust/devserver/Devserver.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/java/dev/braintrust/devserver/Devserver.java b/src/main/java/dev/braintrust/devserver/Devserver.java index 14d759b6..eb73e13e 100644 --- a/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/src/main/java/dev/braintrust/devserver/Devserver.java @@ -66,9 +66,6 @@ public class Devserver { private static final String EXPOSED_HEADERS = "x-bt-cursor, x-bt-found-existing-experiment, x-bt-span-id, x-bt-span-export"; - private static final AttributeKey PARENT = - AttributeKey.stringKey(BraintrustTracing.PARENT_KEY); - @Getter @Accessors(fluent = true) private final String host; @@ -573,12 +570,15 @@ private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalReq caseCount[0]++; log.info("Processing dataset case #{}", caseCount[0]); + // CLAUDE -- I need to set up the root span context with distributed trace info from the reqquest.getParent() + // the value will look like this: + // {"object_type":"playground_logs","object_id":"61738d9e-8c77-48db-bbb3-e69ef85a472f","propagated_event":{"span_attributes":{"generation":"470bb640-7dd7-4e5c-ba0f-7777365cbf46"}}} + // See BraintrustContext.java for distributed tracing examples + // Create eval span for this dataset case (matches Eval.java pattern) var evalSpan = tracer.spanBuilder("eval") .setNoParent() // each eval case is its own trace - .setSpanKind(SpanKind.CLIENT) - .setAttribute(PARENT, finalParentSpec) .setAttribute("braintrust.span_attributes", json(Map.of("type", "eval"))) .setAttribute("braintrust.input_json", json(Map.of("input", datasetCase.input()))) .setAttribute("braintrust.expected", json(datasetCase.expected())) @@ -597,7 +597,6 @@ private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalReq { // run task var taskSpan = tracer.spanBuilder("task") - .setAttribute(PARENT, finalParentSpec) .setAttribute("braintrust.span_attributes", json(Map.of("type", "task"))) .startSpan(); try (var unused = Context.current().with(taskSpan).makeCurrent()) { @@ -627,7 +626,6 @@ private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalReq { // run scorers - ONE score span for all scorers (matches Eval.java) var scoreSpan = tracer.spanBuilder("score") - .setAttribute(PARENT, finalParentSpec) .setAttribute("braintrust.span_attributes", json(Map.of("type", "score"))) .startSpan(); try (var unused = Context.current().with(scoreSpan).makeCurrent()) { From 55e5ef7cdcc13f749f3d61446b04482a8aafddfb Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 8 Dec 2025 15:57:53 -0500 Subject: [PATCH 3/3] wip: baggage --- .../dev/braintrust/devserver/Devserver.java | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/src/main/java/dev/braintrust/devserver/Devserver.java b/src/main/java/dev/braintrust/devserver/Devserver.java index eb73e13e..bbb7c397 100644 --- a/src/main/java/dev/braintrust/devserver/Devserver.java +++ b/src/main/java/dev/braintrust/devserver/Devserver.java @@ -9,7 +9,9 @@ import dev.braintrust.eval.Dataset; import dev.braintrust.eval.DatasetCase; import dev.braintrust.eval.Score; +import dev.braintrust.trace.BraintrustContext; import dev.braintrust.trace.BraintrustTracing; +import io.opentelemetry.api.baggage.Baggage; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; @@ -570,16 +572,46 @@ private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalReq caseCount[0]++; log.info("Processing dataset case #{}", caseCount[0]); - // CLAUDE -- I need to set up the root span context with distributed trace info from the reqquest.getParent() - // the value will look like this: - // {"object_type":"playground_logs","object_id":"61738d9e-8c77-48db-bbb3-e69ef85a472f","propagated_event":{"span_attributes":{"generation":"470bb640-7dd7-4e5c-ba0f-7777365cbf46"}}} - // See BraintrustContext.java for distributed tracing examples + // Set up the root span context with distributed trace info from request.getParent() + // Parse parent object to extract object_type:object_id and propagated_event + @SuppressWarnings("unchecked") + Map parentMap = (Map) request.getParent(); + String objectType = (String) parentMap.get("object_type"); + String objectId = (String) parentMap.get("object_id"); + + // Extract propagated span attributes if present + Map propagatedSpanAttributes = new java.util.HashMap<>(); + Object propagatedEvent = parentMap.get("propagated_event"); + if (propagatedEvent instanceof Map) { + @SuppressWarnings("unchecked") + Map propagatedEventMap = (Map) propagatedEvent; + Object spanAttrs = propagatedEventMap.get("span_attributes"); + if (spanAttrs instanceof Map) { + @SuppressWarnings("unchecked") + Map spanAttrsMap = (Map) spanAttrs; + propagatedSpanAttributes.putAll(spanAttrsMap); + log.info("Propagating span attributes: {}", propagatedSpanAttributes); + } + } + + // Create context with parent info in baggage for distributed tracing + String parentValue = (new BraintrustUtils.Parent("experiment_id", objectId)).toParentValue(); + Baggage baggage = Baggage.fromContext(Context.current()) + .toBuilder() + .put(BraintrustTracing.PARENT_KEY, parentValue) + .build(); + Context parentContext = Context.current().with(baggage); + + // Build span attributes, merging propagated attributes with eval type + Map spanAttributes = new java.util.HashMap<>(); + spanAttributes.put("type", "eval"); + spanAttributes.putAll(propagatedSpanAttributes); - // Create eval span for this dataset case (matches Eval.java pattern) + // Create eval span for this dataset case with parent context var evalSpan = tracer.spanBuilder("eval") - .setNoParent() // each eval case is its own trace - .setAttribute("braintrust.span_attributes", json(Map.of("type", "eval"))) + .setParent(parentContext) + .setAttribute("braintrust.span_attributes", json(spanAttributes)) .setAttribute("braintrust.input_json", json(Map.of("input", datasetCase.input()))) .setAttribute("braintrust.expected", json(datasetCase.expected())) .startSpan(); @@ -592,7 +624,8 @@ private void handleStreamingEval(HttpExchange exchange, RemoteEval eval, EvalReq evalSpan.setAttribute("braintrust.metadata", json(datasetCase.metadata())); } - try (var rootScope = Context.current().with(evalSpan).makeCurrent()) { + // Make the parent context (with baggage) and eval span current so child spans inherit both + try (var rootScope = parentContext.with(evalSpan).makeCurrent()) { final dev.braintrust.eval.TaskResult taskResult; { // run task var taskSpan =