Ark/unit tests

build.gradle

@@ -37,6 +37,7 @@ dependencies {
   api "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-exporter-otlp:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-exporter-logging:${otelVersion}"
+  implementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-semconv:1.25.0-alpha"
 
   // HTTP Client (Java 11+)
@@ -61,8 +62,6 @@ dependencies {
   testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"
   testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
   testImplementation "org.assertj:assertj-core:${assertjVersion}"
-  testImplementation "org.mockito:mockito-junit-jupiter:${mockitoVersion}"
-  testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}"
 }
 
 test {

examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java

@@ -6,7 +6,7 @@
 public class SimpleOpenTelemetryExample {
     public static void main(String[] args) throws Exception {
         var braintrustConfig = BraintrustConfig.fromEnvironment();
-        var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
+        var openTelemetry = BraintrustTracing.quickstart();
         var tracer = BraintrustTracing.getTracer(openTelemetry);
 
         var span = tracer.spanBuilder("hello-java").startSpan();

src/main/java/dev/braintrust/api/BraintrustApiClient.java

@@ -13,10 +13,10 @@
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.time.Duration;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 
 public interface BraintrustApiClient {
@@ -224,6 +224,71 @@ private static ObjectMapper createObjectMapper() {
         }
     }
 
+    /** Implementation for test doubling */
+    class InMemoryImpl implements BraintrustApiClient {
+        private final List<OrganizationAndProjectInfo> organizationAndProjectInfos;
+        private final Set<Experiment> experiments =
+                Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+        public InMemoryImpl(OrganizationAndProjectInfo... organizationAndProjectInfos) {
+            this.organizationAndProjectInfos = List.of(organizationAndProjectInfos);
+        }
+
+        @Override
+        public Project getOrCreateProject(String projectName) {
+            // Find existing project by name
+            for (var orgAndProject : organizationAndProjectInfos) {
+                if (orgAndProject.project().name().equals(projectName)) {
+                    return orgAndProject.project();
+                }
+            }
+            throw new RuntimeException(
+                    "Project '"
+                            + projectName
+                            + "' not found in test data. Please add it to the InMemoryImpl"
+                            + " constructor.");
+        }
+
+        @Override
+        public Optional<Project> getProject(String projectId) {
+            return organizationAndProjectInfos.stream()
+                    .map(OrganizationAndProjectInfo::project)
+                    .filter(project -> project.id().equals(projectId))
+                    .findFirst();
+        }
+
+        @Override
+        public Experiment getOrCreateExperiment(CreateExperimentRequest request) {
+            var existing =
+                    experiments.stream()
+                            .filter(exp -> exp.name().equals(request.name()))
+                            .findFirst();
+            return existing.orElseGet(
+                    () ->
+                            new Experiment(
+                                    request.name().hashCode() + "",
+                                    request.projectId(),
+                                    request.name(),
+                                    request.description(),
+                                    "notused",
+                                    "notused"));
+        }
+
+        @Override
+        public Optional<OrganizationAndProjectInfo> getProjectAndOrgInfo() {
+            return organizationAndProjectInfos.isEmpty()
+                    ? Optional.empty()
+                    : Optional.of(organizationAndProjectInfos.get(0));
+        }
+
+        @Override
+        public Optional<OrganizationAndProjectInfo> getProjectAndOrgInfo(String projectId) {
+            return organizationAndProjectInfos.stream()
+                    .filter(orgAndProject -> orgAndProject.project().id().equals(projectId))
+                    .findFirst();
+        }
+    }
+
     // Request/Response DTOs
 
     record CreateProjectRequest(String name) {}

src/main/java/dev/braintrust/config/BraintrustConfig.java

@@ -30,6 +30,10 @@ public final class BraintrustConfig extends BaseConfig {
     private final Duration requestTimeout =
             Duration.ofSeconds(getConfig("BRAINTRUST_REQUEST_TIMEOUT", 30));
 
+    /** Setting for unit testing. Do not use in production. */
+    private final boolean unitTetJavaExportSpansInMemory =
+            getConfig("BRAINTRUST_TEST_JAVA_EXPORT_SPANS_IN_MEMORY", false);
+
     public static BraintrustConfig fromEnvironment() {
         return of();
     }
@@ -79,15 +83,22 @@ public Optional<String> getBraintrustParentValue() {
 
     /** fetch all project info and IDs from the braintrust api */
     public URI fetchProjectURI() {
+        return fetchProjectURI(BraintrustApiClient.of(this));
+    }
+
+    URI fetchProjectURI(BraintrustApiClient client) {
         try {
-            var client = BraintrustApiClient.of(this);
             var orgAndProject = client.getProjectAndOrgInfo().orElseThrow();
+            var baseURI = new URI(appUrl());
             return new URI(
-                    appUrl()
+                    baseURI.getScheme(),
+                    baseURI.getHost(),
+                    baseURI.getPath()
                             + "/app/"
                             + orgAndProject.orgInfo().name()
                             + "/p/"
-                            + orgAndProject.project().name());
+                            + orgAndProject.project().name(),
+                    null);
         } catch (URISyntaxException e) {
             throw new RuntimeException(e);
         }

src/main/java/dev/braintrust/eval/Eval.java

@@ -4,15 +4,19 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import dev.braintrust.api.BraintrustApiClient;
 import dev.braintrust.config.BraintrustConfig;
+import dev.braintrust.spec.SdkSpec;
 import dev.braintrust.trace.BraintrustContext;
-import dev.braintrust.trace.BraintrustSpanProcessor;
 import dev.braintrust.trace.BraintrustTracing;
+import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.api.trace.Tracer;
+import java.net.URI;
 import java.util.*;
 import java.util.function.Function;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import lombok.Getter;
+import lombok.SneakyThrows;
 
 /**
  * An evaluation framework for testing AI models.
@@ -21,6 +25,8 @@
  * @param <OUTPUT> The type of output produced by the task
  */
 public final class Eval<INPUT, OUTPUT> {
+    private static final AttributeKey<String> PARENT =
+            AttributeKey.stringKey(SdkSpec.Attributes.PARENT);
     private static final ObjectMapper JSON_MAPPER =
             new com.fasterxml.jackson.databind.ObjectMapper();
     private final @Nonnull String experimentName;
@@ -35,7 +41,7 @@ public final class Eval<INPUT, OUTPUT> {
     private Eval(Builder<INPUT, OUTPUT> builder) {
         this.experimentName = builder.experimentName;
         this.config = Objects.requireNonNull(builder.config);
-        this.client = BraintrustApiClient.of(config);
+        this.client = Objects.requireNonNull(builder.apiClient);
         if (null == builder.projectId) {
             this.orgAndProject = client.getProjectAndOrgInfo().orElseThrow();
         } else {
@@ -73,31 +79,24 @@ private EvalCase.Result<INPUT, OUTPUT> evalOne(
                 tracer.spanBuilder("eval") // TODO: allow names for eval cases
                         .setNoParent() // each eval case is its own trace
                         .setSpanKind(SpanKind.CLIENT)
-                        .setAttribute(
-                                BraintrustSpanProcessor.PARENT, "experiment_id:" + experimentId)
+                        .setAttribute(PARENT, "experiment_id:" + experimentId)
                         .setAttribute("braintrust.span_attributes", "{\"type\":\"eval\"}")
                         // FIXME: use proper object mapper for json stuff
                         .setAttribute(
                                 "braintrust.input_json",
                                 "{ \"input\":\"" + evalCase.input() + "\"}")
                         .setAttribute("braintrust.expected", "\"" + evalCase.expected() + "\"")
-                        // TODO: these attributes are deprecated apparently? Do we need to set them?
-                        .setAttribute(BraintrustSpanProcessor.PARENT_EXPERIMENT_ID, experimentId)
-                        .setAttribute(BraintrustSpanProcessor.PARENT_TYPE, "experiment")
                         .startSpan();
-        try (var rootScope =
-                BraintrustContext.forExperiment(experimentId, rootSpan).makeCurrent()) {
+        try (var rootScope = BraintrustContext.ofExperiment(experimentId, rootSpan).makeCurrent()) {
             final OUTPUT result;
             { // run task
                 var taskSpan =
                         tracer.spanBuilder("task")
-                                .setAttribute(
-                                        BraintrustSpanProcessor.PARENT,
-                                        "experiment_id:" + experimentId)
+                                .setAttribute(PARENT, "experiment_id:" + experimentId)
                                 .setAttribute("braintrust.span_attributes", "{\"type\":\"task\"}")
                                 .startSpan();
                 try (var unused =
-                        BraintrustContext.forExperiment(experimentId, taskSpan).makeCurrent()) {
+                        BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
                     result = task.apply(evalCase);
                 } finally {
                     taskSpan.end();
@@ -114,13 +113,11 @@ private EvalCase.Result<INPUT, OUTPUT> evalOne(
             { // run scorers
                 var scoreSpan =
                         tracer.spanBuilder("score")
-                                .setAttribute(
-                                        BraintrustSpanProcessor.PARENT,
-                                        "experiment_id:" + experimentId)
+                                .setAttribute(PARENT, "experiment_id:" + experimentId)
                                 .setAttribute("braintrust.span_attributes", "{\"type\":\"score\"}")
                                 .startSpan();
                 try (var unused =
-                        BraintrustContext.forExperiment(experimentId, scoreSpan).makeCurrent()) {
+                        BraintrustContext.ofExperiment(experimentId, scoreSpan).makeCurrent()) {
                     // NOTE: linked hash map to preserve ordering. Not in the spec but nice user
                     // experience
                     final HashMap<String, Double> nameToScore = new LinkedHashMap<>();
@@ -158,17 +155,23 @@ private EvalCase.Result<INPUT, OUTPUT> evalOne(
 
     /** Results of all eval cases of an experiment. */
     public class Result {
-        private final String experimentUrl;
+        @Getter private final String experimentUrl;
 
+        @SneakyThrows
         private Result() {
+            var baseURI = new URI(config.appUrl());
             this.experimentUrl =
-                    config.appUrl()
-                            + "/app/"
-                            + orgAndProject.orgInfo().name()
-                            + "/p/"
-                            + orgAndProject.project().name()
-                            + "/experiments/"
-                            + experimentName;
+                    new URI(
+                                    baseURI.getScheme(),
+                                    baseURI.getHost(),
+                                    "/app/"
+                                            + orgAndProject.orgInfo().name()
+                                            + "/p/"
+                                            + orgAndProject.project().name()
+                                            + "/experiments/"
+                                            + experimentName,
+                                    null)
+                            .toASCIIString();
         }
 
         public String createReportString() {
@@ -189,6 +192,7 @@ public static <INPUT, OUTPUT> Builder<INPUT, OUTPUT> builder() {
     public static final class Builder<INPUT, OUTPUT> {
         private @Nonnull String experimentName = "unnamed-java-eval";
         private @Nullable BraintrustConfig config;
+        private @Nullable BraintrustApiClient apiClient;
         private @Nullable String projectId;
         private @Nullable Tracer tracer = null;
         private @Nonnull List<EvalCase<INPUT, OUTPUT>> evalCases = List.of();
@@ -211,6 +215,9 @@ public Eval<INPUT, OUTPUT> build() {
             if (scorers.isEmpty()) {
                 throw new RuntimeException("must provide at least one scorer");
             }
+            if (null == apiClient) {
+                apiClient = BraintrustApiClient.of(config);
+            }
             Objects.requireNonNull(task);
             return new Eval<>(this);
         }
@@ -230,6 +237,11 @@ public Builder<INPUT, OUTPUT> config(BraintrustConfig config) {
             return this;
         }
 
+        public Builder<INPUT, OUTPUT> apiClient(BraintrustApiClient apiClient) {
+            this.apiClient = apiClient;
+            return this;
+        }
+
         public Builder<INPUT, OUTPUT> tracer(Tracer tracer) {
             this.tracer = tracer;
             return this;

src/main/java/dev/braintrust/spec/SdkSpec.java

@@ -0,0 +1,12 @@
+package dev.braintrust.spec;
+
+public final class SdkSpec {
+
+    public static final class Attributes {
+        public static final String PARENT = "braintrust.parent";
+    }
+
+    public static final class Context {}
+
+    private SdkSpec() {}
+}