support devserver experiment snapshotting

braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java

@@ -377,10 +377,31 @@ private <I, O> void handleStreamingEval(
                         BraintrustUtils.createProjectURI(
                                         braintrust.config().appUrl(), orgName, projectName)
                                 .toASCIIString();
-                final var experimentUrl = projectUrl + "/experiments/" + experimentName;
 
                 var tracer = BraintrustTracing.getTracer();
 
+                // A remote eval can be triggered two ways:
+                //   1. Playground run: the request carries a `parent` object (playground_id). We
+                //      stream per-case progress under that parent (handled by the loop below).
+                //   2. Experiment "snapshot": no such parent (instead experiment_name + project).
+                //      We run a standard Eval, which creates a real experiment and emits standard
+                //      spans, then send a summary + done with the experiment link.
+                final var playgroundParent = extractPlaygroundParent(request);
+                if (playgroundParent.isEmpty()) {
+                    handleExperimentSnapshot(
+                            os,
+                            eval,
+                            request,
+                            braintrust,
+                            apiClient,
+                            projectId,
+                            projectName,
+                            projectUrl,
+                            experimentName,
+                            remoteScorers);
+                    return;
+                }
+
                 // Merge parameters: evaluator defaults + request overrides
                 final Parameters mergedParameters =
                         new Parameters(
@@ -391,7 +412,7 @@ private <I, O> void handleStreamingEval(
 
                 // Execute task and scorers for each case
                 final Map<String, List<Double>> scoresByName = new ConcurrentHashMap<>();
-                final var parentInfo = extractParentInfo(request);
+                final var parentInfo = playgroundParent.get();
                 final var braintrustParent = parentInfo.braintrustParent();
                 final var braintrustGeneration = parentInfo.generation();
 
@@ -547,13 +568,7 @@ private <I, O> void handleStreamingEval(
                 }
 
                 sendSummaryEvent(
-                        os,
-                        projectName,
-                        projectId,
-                        experimentName,
-                        projectUrl,
-                        experimentUrl,
-                        scoreSummaries);
+                        os, projectName, projectId, experimentName, projectUrl, scoreSummaries);
                 sendDoneEvent(os);
             } catch (Exception e) {
                 // Send error event via SSE
@@ -577,6 +592,73 @@ private <I, O> void handleStreamingEval(
         }
     }
 
+    /**
+     * Handles an experiment "snapshot" run: a remote eval triggered as an Experiment from the UI
+     * (no playground parent). Rather than re-implementing experiment creation and span emission, it
+     * builds a first-class {@link Eval} and runs it synchronously — so snapshots get the exact same
+     * behavior as a normal {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset
+     * id/version linkage for Braintrust-backed datasets, standard span shape). When the run
+     * completes it streams a single {@code summary} (with the created experiment's id/name/url) and
+     * a {@code done} event.
+     *
+     * <p>Unlike playground runs, snapshots do not stream per-case {@code progress} events: the user
+     * is handed the experiment link and views results in the experiment UI.
+     */
+    @SuppressWarnings({"unchecked", "rawtypes"})
+    private <I, O> void handleExperimentSnapshot(
+            OutputStream os,
+            RemoteEval<I, O> eval,
+            EvalRequest request,
+            Braintrust braintrust,
+            BraintrustOpenApiClient apiClient,
+            String projectId,
+            String projectName,
+            String projectUrl,
+            String experimentName,
+            List<Scorer<I, O>> remoteScorers)
+            throws IOException {
+        // Combine local scorers (from the RemoteEval) with remote scorers (from the request).
+        List<Scorer<I, O>> allScorers = new ArrayList<>(eval.getScorers());
+        allScorers.addAll(remoteScorers);
+
+        // TODO: when async evals are supported, simply begin the eval and hand back the link to the
+        // user and finish eval in the background
+
+        var evalResult =
+                Eval.<I, O>builder()
+                        .name(experimentName)
+                        .config(braintrust.config())
+                        .apiClient(apiClient)
+                        .projectId(projectId)
+                        .dataset((Dataset<I, O>) extractDataset(request, apiClient))
+                        .task(eval.getTask())
+                        .scorers(allScorers.toArray(new Scorer[0]))
+                        .parameters(eval.getParameters())
+                        .parameterValues(
+                                request.getParameters() == null
+                                        ? Map.of()
+                                        : request.getParameters())
+                        // Each snapshot run should produce a distinct experiment even if a prior
+                        // run used the same name (the backend dedupes the name on conflict).
+                        .ensureNew(true)
+                        .build()
+                        .run();
+
+        // Snapshots don't stream per-scorer progress. The scores are recorded on the experiment
+        // and visible via the experiment link.
+        sendExperimentSnapshotSummaryEvent(
+                os,
+                projectName,
+                projectId,
+                evalResult.getExperimentId(),
+                evalResult.getExperimentName() != null
+                        ? evalResult.getExperimentName()
+                        : experimentName,
+                projectUrl,
+                evalResult.getExperimentUrl());
+        sendExperimentSnapshotDoneEvent(os);
+    }
+
     private void setEvalSpanAttributes(
             Span evalSpan,
             BraintrustUtils.Parent braintrustParent,
@@ -802,7 +884,6 @@ private void sendSummaryEvent(
             String projectId,
             String experimentName,
             String projectUrl,
-            String experimentUrl,
             Map<String, EvalResponse.ScoreSummary> scoreSummaries)
             throws IOException {
         Map<String, Object> summary = new LinkedHashMap<>();
@@ -835,6 +916,40 @@ private void sendDoneEvent(OutputStream os) throws IOException {
         sendSSEEvent(os, "done", "");
     }
 
+    /**
+     * Sends the {@code summary} event for an experiment snapshot run. Unlike {@link
+     * #sendSummaryEvent} (playground), this references the created experiment via {@code
+     * experimentId}/{@code experimentUrl} so the UI can link straight to it, and carries no
+     * streamed per-scorer scores (results live on the experiment itself).
+     */
+    private void sendExperimentSnapshotSummaryEvent(
+            OutputStream os,
+            String projectName,
+            String projectId,
+            @Nullable String experimentId,
+            String experimentName,
+            String projectUrl,
+            @Nullable String experimentUrl)
+            throws IOException {
+        Map<String, Object> summary = new LinkedHashMap<>();
+        summary.put("projectName", projectName);
+        summary.put("projectId", projectId);
+        summary.put("experimentId", experimentId);
+        summary.put("experimentName", experimentName);
+        summary.put("projectUrl", projectUrl);
+        summary.put("experimentUrl", experimentUrl);
+        summary.put("comparisonExperimentName", null);
+        summary.put("scores", Map.of());
+        summary.put("metrics", Map.of());
+
+        sendSSEEvent(os, "summary", toJson(summary));
+    }
+
+    /** Sends the terminating {@code done} event for an experiment snapshot run. */
+    private void sendExperimentSnapshotDoneEvent(OutputStream os) throws IOException {
+        sendSSEEvent(os, "done", "");
+    }
+
     private void sendResponse(
             HttpExchange exchange, int statusCode, String contentType, String body)
             throws IOException {
@@ -1105,44 +1220,50 @@ private record ParentInfo(
             @Nonnull BraintrustUtils.Parent braintrustParent, @Nullable String generation) {}
 
     /**
-     * Extracts parent information from the eval request.
+     * Extracts the playground parent (and generation) from the eval request, if present.
+     *
+     * <p>Playground runs send a {@code parent} object carrying {@code object_type}/{@code
+     * object_id}; experiment ("snapshot") runs send no such parent (instead {@code experiment_name}
+     * + project via headers) and are handled by {@link #handleExperimentSnapshot}. Returns empty
+     * for the latter case.
      *
      * @param request The eval request
-     * @return ParentInfo containing braintrustParent and generation
+     * @return the playground {@link ParentInfo} if this is a playground run, else empty
      */
-    private static ParentInfo extractParentInfo(EvalRequest request) {
-        String parentSpec = null;
-        String generation = null;
+    private static Optional<ParentInfo> extractPlaygroundParent(EvalRequest request) {
+        if (!(request.getParent() instanceof Map)) {
+            return Optional.empty();
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, Object> parentMap = (Map<String, Object>) request.getParent();
+        String objectType = (String) parentMap.get("object_type");
+        String objectId = (String) parentMap.get("object_id");
+        if (objectType == null && objectId == null) {
+            return Optional.empty();
+        }
 
-        // Extract parent spec and generation from request
-        if (request.getParent() != null && request.getParent() instanceof Map) {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> parentMap = (Map<String, Object>) request.getParent();
-            String objectType = (String) parentMap.get("object_type");
-            String objectId = (String) parentMap.get("object_id");
+        if (objectType == null || objectId == null) {
+            throw new IllegalArgumentException(
+                    "malformed braintrust parent: %s, %s".formatted(objectType, objectId));
+        }
 
-            // Extract generation from propagated_event.span_attributes.generation
-            Object propEventObj = parentMap.get("propagated_event");
-            if (propEventObj instanceof Map) {
+        // Extract generation from propagated_event.span_attributes.generation
+        String generation = null;
+        Object propEventObj = parentMap.get("propagated_event");
+        if (propEventObj instanceof Map) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> propEvent = (Map<String, Object>) propEventObj;
+            Object spanAttrsObj = propEvent.get("span_attributes");
+            if (spanAttrsObj instanceof Map) {
                 @SuppressWarnings("unchecked")
-                Map<String, Object> propEvent = (Map<String, Object>) propEventObj;
-                Object spanAttrsObj = propEvent.get("span_attributes");
-                if (spanAttrsObj instanceof Map) {
-                    @SuppressWarnings("unchecked")
-                    Map<String, Object> spanAttrs = (Map<String, Object>) spanAttrsObj;
-                    generation = (String) spanAttrs.get("generation");
-                }
-            }
-
-            if (objectType != null && objectId != null) {
-                parentSpec = "playground_id:" + objectId;
+                Map<String, Object> spanAttrs = (Map<String, Object>) spanAttrsObj;
+                generation = (String) spanAttrs.get("generation");
             }
         }
 
-        if (parentSpec == null) {
-            throw new IllegalArgumentException("braintrust parent (playground_id) not found");
-        }
-        return new ParentInfo(BraintrustUtils.parseParent(parentSpec), generation);
+        return Optional.of(
+                new ParentInfo(
+                        BraintrustUtils.parseParent("playground_id:" + objectId), generation));
     }
 
     /**

braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java

@@ -46,6 +46,7 @@ public final class Eval<INPUT, OUTPUT> {
     private final @Nonnull List<String> tags;
     private final @Nonnull Map<String, Object> metadata;
     private final @Nonnull Parameters parameters;
+    private final boolean ensureNew;
 
     private Eval(Builder<INPUT, OUTPUT> builder) {
         this.experimentName = builder.experimentName;
@@ -63,6 +64,7 @@ private Eval(Builder<INPUT, OUTPUT> builder) {
         this.tags = List.copyOf(builder.tags);
         this.metadata = Map.copyOf(builder.metadata);
         this.parameters = builder.buildParameters();
+        this.ensureNew = builder.ensureNew;
     }
 
     /** Runs the evaluation and returns results. */
@@ -78,6 +80,9 @@ public EvalResult run() {
             var createExperiment =
                     new CreateExperiment().projectId(project.getId()).name(experimentName);
 
+            if (ensureNew) {
+                createExperiment.ensureNew(true);
+            }
             if (!tags.isEmpty()) {
                 createExperiment.tags(tags);
             }
@@ -90,16 +95,22 @@ public EvalResult run() {
             var experiment = new ExperimentsApi(client).postExperiment(createExperiment);
 
             cursor.forEach(datasetCase -> evalOne(experiment.getId().toString(), datasetCase));
-        }
 
-        var experimentUrl =
-                "%s/experiments/%s"
-                        .formatted(
-                                BraintrustUtils.createProjectURI(
-                                                config.appUrl(), orgInfo.name(), project.getName())
-                                        .toASCIIString(),
-                                experimentName);
-        return new EvalResult(experimentUrl);
+            // Use the experiment's actual name from the response: with ensure_new the backend may
+            // dedupe a conflicting name (e.g. "foo" -> "foo-2f8ca776"), and the URL must point at
+            // the real, created experiment.
+            var resolvedName = experiment.getName() != null ? experiment.getName() : experimentName;
+            var experimentUrl =
+                    "%s/experiments/%s"
+                            .formatted(
+                                    BraintrustUtils.createProjectURI(
+                                                    config.appUrl(),
+                                                    orgInfo.name(),
+                                                    project.getName())
+                                            .toASCIIString(),
+                                    resolvedName);
+            return new EvalResult(experiment.getId().toString(), resolvedName, experimentUrl);
+        }
     }
 
     private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase) {
@@ -397,6 +408,7 @@ public static final class Builder<INPUT, OUTPUT> {
         private @Nonnull Map<String, Object> parameterValues = Map.of();
         private @Nonnull List<String> tags = List.of();
         private @Nonnull Map<String, Object> metadata = Map.of();
+        private boolean ensureNew = false;
 
         public Eval<INPUT, OUTPUT> build() {
             if (config == null) {
@@ -521,6 +533,17 @@ public Builder<INPUT, OUTPUT> metadata(Map<String, Object> metadata) {
             return this;
         }
 
+        /**
+         * When {@code true}, sets {@code ensure_new} on the create-experiment request so a new
+         * experiment is always created even if one with the same name already exists (the backend
+         * dedupes the name on conflict). Useful for repeated runs (e.g. UI/remote snapshots) that
+         * should each produce a distinct experiment. Defaults to {@code false}.
+         */
+        public Builder<INPUT, OUTPUT> ensureNew(boolean ensureNew) {
+            this.ensureNew = ensureNew;
+            return this;
+        }
+
         /**
          * Sets parameter definitions for this eval. Default values from the definitions are used
          * unless overridden via {@link #parameterValues(Map)}.

braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java

@@ -1,14 +1,20 @@
 package dev.braintrust.eval;
 
+import javax.annotation.Nullable;
 import lombok.Getter;
 import lombok.SneakyThrows;
 
 /** Results of all eval cases of an experiment. */
 public class EvalResult {
+    @Getter private final @Nullable String experimentId;
+    @Getter private final @Nullable String experimentName;
     @Getter private final String experimentUrl;
 
     @SneakyThrows
-    EvalResult(String experimentUrl) {
+    EvalResult(
+            @Nullable String experimentId, @Nullable String experimentName, String experimentUrl) {
+        this.experimentId = experimentId;
+        this.experimentName = experimentName;
         this.experimentUrl = experimentUrl;
     }