Support Remote Evals

README.md

@@ -57,11 +57,8 @@ var eval = braintrust.<String, String>evalBuilder()
                 .taskFunction(getFoodType)
                 .scorers(
                         Scorer.of(
-                                "fruit_scorer",
-                                result -> "fruit".equals(result) ? 1.0 : 0.0),
-                        Scorer.of(
-                                "vegetable_scorer",
-                                result -> "vegetable".equals(result) ? 1.0 : 0.0))
+                                "exact_match",
+                                (expected, result) -> expected.equals(result) ? 1.0 : 0.0))
                 .build();
 var result = eval.run();
 System.out.println("\n\n" + result.createReportString());

examples/build.gradle

@@ -141,3 +141,18 @@ task runSpringAI(type: JavaExec) {
     suspend = false
   }
 }
+
+
+task runRemoteEval(type: JavaExec) {
+  group = 'Braintrust SDK Examples'
+  description = 'Run the remote eval example'
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass = 'dev.braintrust.examples.RemoteEvalExample'
+  systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel
+  debugOptions {
+    enabled = true
+    port = 5566
+    server = true
+    suspend = false
+  }
+}

examples/src/main/java/dev/braintrust/examples/ExperimentExample.java

@@ -34,10 +34,8 @@ public static void main(String[] args) throws Exception {
         var eval =
                 braintrust
                         .<String, String>evalBuilder()
-                        .name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a
-                        // constant, additional runs
-                        // will append new cases to
-                        // the same experiment
+                        // NOTE: pre-existing experiment names will append results
+                        .name("java-eval-x-" + System.currentTimeMillis())
                         .cases(
                                 new DatasetCase<>(
                                         "strawberry",
@@ -49,14 +47,13 @@ public static void main(String[] args) throws Exception {
                                 DatasetCase.of("asparagus", "vegetable"),
                                 DatasetCase.of("apple", "fruit"),
                                 DatasetCase.of("banana", "fruit"))
+                        // Or, to fetch a remote dataset:
+                        // .dataset(braintrust.fetchDataset("my-dataset-name"))
                         .taskFunction(getFoodType)
                         .scorers(
                                 Scorer.of(
-                                        "fruit_scorer",
-                                        result -> "fruit".equals(result) ? 1.0 : 0.0),
-                                Scorer.of(
-                                        "vegetable_scorer",
-                                        result -> "vegetable".equals(result) ? 1.0 : 0.0))
+                                        "exact_match",
+                                        (expected, result) -> expected.equals(result) ? 1.0 : 0.0))
                         .build();
         var result = eval.run();
         System.out.println("\n\n" + result.createReportString());

examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java

@@ -0,0 +1,74 @@
+package dev.braintrust.examples;
+
+import com.openai.client.okhttp.OpenAIOkHttpClient;
+import com.openai.models.ChatModel;
+import com.openai.models.chat.completions.ChatCompletionCreateParams;
+import dev.braintrust.Braintrust;
+import dev.braintrust.devserver.Devserver;
+import dev.braintrust.devserver.RemoteEval;
+import dev.braintrust.eval.Scorer;
+import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+/** Simple Dev Server for Remote Evals */
+public class RemoteEvalExample {
+    public static void main(String[] args) throws Exception {
+        var braintrust = Braintrust.get();
+        var openTelemetry = braintrust.openTelemetryCreate();
+        var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
+
+        RemoteEval<String, String> foodTypeEval =
+                RemoteEval.<String, String>builder()
+                        .name("food-type-classifier")
+                        .config(braintrust.config())
+                        .taskFunction(
+                                food -> {
+                                    var request =
+                                            ChatCompletionCreateParams.builder()
+                                                    .model(ChatModel.GPT_4O_MINI)
+                                                    .addSystemMessage("Return a one word answer")
+                                                    .addUserMessage(
+                                                            "What kind of food is " + food + "?")
+                                                    .maxTokens(50L)
+                                                    .temperature(0.0)
+                                                    .build();
+                                    var response =
+                                            openAIClient.chat().completions().create(request);
+                                    return response.choices()
+                                            .get(0)
+                                            .message()
+                                            .content()
+                                            .orElse("")
+                                            .toLowerCase();
+                                })
+                        .scorers(
+                                List.of(
+                                        Scorer.of(
+                                                "exact_match",
+                                                (expected, result) ->
+                                                        expected.equals(result) ? 1.0 : 0.0)))
+                        .build();
+
+        Devserver devserver =
+                Devserver.builder()
+                        .registerEval(foodTypeEval)
+                        .host("localhost") // set to 0.0.0.0 to bind all interfaces
+                        .port(8300)
+                        .build();
+
+        System.out.println("Starting Braintrust dev server");
+        devserver.start();
+        // await shutdown
+        CountDownLatch latch = new CountDownLatch(1);
+        Runtime.getRuntime()
+                .addShutdownHook(
+                        new Thread(
+                                () -> {
+                                    System.out.println("Shutting down...");
+                                    devserver.stop();
+                                    latch.countDown();
+                                }));
+        latch.await();
+    }
+}