Skip to content

Commit c22fcba

Browse files
realarkrealark
committed
wip -- impl
1 parent 5f85ecd commit c22fcba

10 files changed

Lines changed: 295 additions & 132 deletions

File tree

‎braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java‎

Lines changed: 112 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static dev.braintrust.json.BraintrustJsonMapper.fromJson;
44
import static dev.braintrust.json.BraintrustJsonMapper.toJson;
55

6+
import com.fasterxml.jackson.databind.node.NullNode;
67
import com.sun.net.httpserver.HttpExchange;
78
import com.sun.net.httpserver.HttpHandler;
89
import com.sun.net.httpserver.HttpServer;
@@ -180,32 +181,36 @@ private void handleList(HttpExchange exchange) throws IOException {
180181

181182
Map<String, Object> metadata = new LinkedHashMap<>();
182183

183-
Map<String, Map<String, Object>> parametersMap = new LinkedHashMap<>();
184-
for (Map.Entry<String, RemoteEval.Parameter> paramEntry :
185-
eval.getParameters().entrySet()) {
186-
String paramName = paramEntry.getKey();
187-
RemoteEval.Parameter param = paramEntry.getValue();
184+
// Serialize parameters in the container format
185+
if (eval.getParameters().isEmpty()) {
186+
metadata.put("parameters", NullNode.getInstance());
187+
} else {
188+
Map<String, Map<String, Object>> schemaMap = new LinkedHashMap<>();
189+
for (ParameterDef<?> param : eval.getParameters()) {
190+
Map<String, Object> paramMetadata = new LinkedHashMap<>();
191+
paramMetadata.put("type", param.type().toString().toLowerCase());
188192

189-
Map<String, Object> paramMetadata = new LinkedHashMap<>();
190-
paramMetadata.put("type", param.getType().getValue());
193+
if (param.schema() != null) {
194+
paramMetadata.put("schema", param.schema());
195+
}
191196

192-
if (param.getDescription() != null) {
193-
paramMetadata.put("description", param.getDescription());
194-
}
197+
if (param.defaultValue() != null) {
198+
paramMetadata.put("default", param.defaultValue());
199+
}
195200

196-
if (param.getDefaultValue() != null) {
197-
paramMetadata.put("default", param.getDefaultValue());
198-
}
201+
if (param.description() != null) {
202+
paramMetadata.put("description", param.description());
203+
}
199204

200-
// Only include schema for data type parameters
201-
if (param.getType() == RemoteEval.ParameterType.DATA
202-
&& param.getSchema() != null) {
203-
paramMetadata.put("schema", param.getSchema());
205+
schemaMap.put(param.name(), paramMetadata);
204206
}
205207

206-
parametersMap.put(paramName, paramMetadata);
208+
Map<String, Object> parametersContainer = new LinkedHashMap<>();
209+
parametersContainer.put("type", "braintrust.staticParameters");
210+
parametersContainer.put("schema", schemaMap);
211+
parametersContainer.put("source", NullNode.getInstance());
212+
metadata.put("parameters", parametersContainer);
207213
}
208-
metadata.put("parameters", parametersMap);
209214

210215
// Add scores (list of scorer names)
211216
List<Map<String, String>> scores = new ArrayList<>();
@@ -245,7 +250,14 @@ private void handleEval(HttpExchange exchange) throws IOException {
245250
try {
246251
InputStream requestBody = exchange.getRequestBody();
247252
var requestBodyString = new String(requestBody.readAllBytes(), StandardCharsets.UTF_8);
248-
EvalRequest request = fromJson(requestBodyString, EvalRequest.class);
253+
EvalRequest request;
254+
try {
255+
request = fromJson(requestBodyString, EvalRequest.class);
256+
} catch (Exception e) {
257+
sendResponse(
258+
exchange, 400, "text/plain", "Invalid request body: " + e.getMessage());
259+
return;
260+
}
249261

250262
// Validate evaluator exists
251263
RemoteEval eval = evals.get(request.getName());
@@ -376,6 +388,9 @@ private <I, O> void handleStreamingEval(
376388

377389
var tracer = BraintrustTracing.getTracer();
378390

391+
// Merge parameters: evaluator defaults + request overrides
392+
final Parameters mergedParameters = mergeParameters(eval, request);
393+
379394
// Execute task and scorers for each case
380395
final Map<String, List<Double>> scoresByName = new ConcurrentHashMap<>();
381396
final var parentInfo = extractParentInfo(request);
@@ -414,7 +429,9 @@ private <I, O> void handleStreamingEval(
414429
.makeCurrent()) {
415430
var task = eval.getTask();
416431
try {
417-
taskResult = task.apply(datasetCase);
432+
taskResult =
433+
task.apply(
434+
datasetCase, mergedParameters);
418435
} catch (Exception e) {
419436
taskSpan.setStatus(
420437
StatusCode.ERROR, e.getMessage());
@@ -431,6 +448,21 @@ private <I, O> void handleStreamingEval(
431448
"Task threw exception for input: "
432449
+ datasetCase.input(),
433450
e);
451+
// Set eval span attributes so Braintrust can
452+
// resolve the trace
453+
setEvalSpanAttributesForError(
454+
evalSpan,
455+
braintrustParent,
456+
braintrustGeneration,
457+
datasetCase);
458+
// Send progress event even on error so the
459+
// Playground can link to the trace
460+
sendProgressEvent(
461+
os,
462+
evalSpan.getSpanContext().getSpanId(),
463+
datasetCase.origin(),
464+
eval.getName(),
465+
null);
434466
// run scoreForTaskException on each scorer
435467
List<Scorer<I, O>> allScorersForError =
436468
new ArrayList<>(eval.getScorers());
@@ -578,6 +610,38 @@ private void setEvalSpanAttributes(
578610
"braintrust.output_json", toJson(Map.of("output", taskResult.result())));
579611
}
580612

613+
/**
614+
* Sets eval span attributes when the task threw an exception. Similar to {@link
615+
* #setEvalSpanAttributes} but does not require a TaskResult.
616+
*/
617+
private void setEvalSpanAttributesForError(
618+
Span evalSpan,
619+
BraintrustUtils.Parent braintrustParent,
620+
String braintrustGeneration,
621+
DatasetCase<?, ?> datasetCase) {
622+
var spanAttrs = new LinkedHashMap<>();
623+
spanAttrs.put("type", "eval");
624+
spanAttrs.put("name", "eval");
625+
if (braintrustGeneration != null) {
626+
spanAttrs.put("generation", braintrustGeneration);
627+
}
628+
evalSpan.setAttribute(PARENT, braintrustParent.toParentValue())
629+
.setAttribute("braintrust.span_attributes", toJson(spanAttrs))
630+
.setAttribute("braintrust.input_json", toJson(Map.of("input", datasetCase.input())))
631+
.setAttribute("braintrust.expected_json", toJson(datasetCase.expected()));
632+
633+
if (datasetCase.origin().isPresent()) {
634+
evalSpan.setAttribute("braintrust.origin", toJson(datasetCase.origin().get()));
635+
}
636+
if (!datasetCase.tags().isEmpty()) {
637+
evalSpan.setAttribute(
638+
AttributeKey.stringArrayKey("braintrust.tags"), datasetCase.tags());
639+
}
640+
if (!datasetCase.metadata().isEmpty()) {
641+
evalSpan.setAttribute("braintrust.metadata", toJson(datasetCase.metadata()));
642+
}
643+
}
644+
581645
private void setTaskSpanAttributes(
582646
Span taskSpan,
583647
BraintrustUtils.Parent braintrustParent,
@@ -1083,6 +1147,33 @@ private static ParentInfo extractParentInfo(EvalRequest request) {
10831147
return new ParentInfo(BraintrustUtils.parseParent(parentSpec), generation);
10841148
}
10851149

1150+
/**
1151+
* Merges evaluator default parameter values with request overrides.
1152+
*
1153+
* <p>Request values take precedence over evaluator defaults. Unknown parameters from the
1154+
* request are passed through.
1155+
*/
1156+
@SuppressWarnings("rawtypes")
1157+
private static Parameters mergeParameters(RemoteEval eval, EvalRequest request) {
1158+
List<ParameterDef<?>> definitions = eval.getParameters();
1159+
1160+
// Collect defaults from evaluator definitions
1161+
Map<String, Object> merged = new LinkedHashMap<>();
1162+
for (ParameterDef<?> def : definitions) {
1163+
if (def.defaultValue() != null) {
1164+
merged.put(def.name(), def.defaultValue());
1165+
}
1166+
}
1167+
1168+
// Overlay request parameters (request wins)
1169+
Map<String, Object> requestParams = request.getParameters();
1170+
if (requestParams != null) {
1171+
merged.putAll(requestParams);
1172+
}
1173+
1174+
return new Parameters(merged, definitions);
1175+
}
1176+
10861177
/**
10871178
* Extracts and loads the dataset from the eval request.
10881179
*

‎braintrust-sdk/src/main/java/dev/braintrust/devserver/RemoteEval.java‎

Lines changed: 13 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
package dev.braintrust.devserver;
22

33
import dev.braintrust.eval.DatasetCase;
4+
import dev.braintrust.eval.ParameterDef;
45
import dev.braintrust.eval.Parameters;
56
import dev.braintrust.eval.Scorer;
67
import dev.braintrust.eval.Task;
78
import dev.braintrust.eval.TaskResult;
89
import java.util.*;
910
import java.util.function.Function;
1011
import javax.annotation.Nonnull;
11-
import javax.annotation.Nullable;
1212
import lombok.Builder;
1313
import lombok.Getter;
1414
import lombok.Singular;
@@ -39,8 +39,8 @@ public class RemoteEval<INPUT, OUTPUT> {
3939
*/
4040
@Singular @Nonnull private final List<Scorer<INPUT, OUTPUT>> scorers;
4141

42-
/** Optional parameters that can be configured from the UI */
43-
@Singular @Nonnull private final Map<String, Parameter> parameters;
42+
/** Optional parameter definitions that can be configured from the UI */
43+
@Singular @Nonnull private final List<ParameterDef<?>> parameters;
4444

4545
public static class Builder<INPUT, OUTPUT> {
4646
/**
@@ -64,76 +64,16 @@ public TaskResult<INPUT, OUTPUT> apply(
6464

6565
/** Build the RemoteEval */
6666
public RemoteEval<INPUT, OUTPUT> build() {
67-
// can add build hooks here later if desired
68-
return internalBuild();
69-
}
70-
}
71-
72-
/** Represents a configurable parameter for the evaluator */
73-
@Getter
74-
@lombok.Builder(builderClassName = "Builder")
75-
public static class Parameter {
76-
/** Type of parameter: "prompt" or "data" */
77-
@Nonnull private final ParameterType type;
78-
79-
/** Optional description of the parameter */
80-
@Nullable private final String description;
81-
82-
/** Optional default value for the parameter */
83-
@Nullable private final Object defaultValue;
84-
85-
/**
86-
* JSON Schema for data type parameters. Only applicable when type is DATA. Should be a Map
87-
* representing a JSON Schema object.
88-
*/
89-
@Nullable private final Map<String, Object> schema;
90-
91-
public static Parameter promptParameter(String description, Object defaultValue) {
92-
return Parameter.builder()
93-
.type(ParameterType.PROMPT)
94-
.description(description)
95-
.defaultValue(defaultValue)
96-
.build();
97-
}
98-
99-
public static Parameter promptParameter(Object defaultValue) {
100-
return promptParameter(null, defaultValue);
101-
}
102-
103-
public static Parameter dataParameter(
104-
String description, Map<String, Object> schema, Object defaultValue) {
105-
return Parameter.builder()
106-
.type(ParameterType.DATA)
107-
.description(description)
108-
.schema(schema)
109-
.defaultValue(defaultValue)
110-
.build();
111-
}
112-
113-
public static Parameter dataParameter(Map<String, Object> schema, Object defaultValue) {
114-
return dataParameter(null, schema, defaultValue);
115-
}
116-
117-
public static Parameter dataParameter(Map<String, Object> schema) {
118-
return dataParameter(null, schema, null);
119-
}
120-
}
121-
122-
/** Parameter type enumeration */
123-
public enum ParameterType {
124-
/** Prompt parameter (for LLM prompts) */
125-
PROMPT("prompt"),
126-
/** Data parameter (for other configuration data) */
127-
DATA("data");
128-
129-
private final String value;
130-
131-
ParameterType(String value) {
132-
this.value = value;
133-
}
134-
135-
public String getValue() {
136-
return value;
67+
var result = internalBuild();
68+
// Validate parameter names are unique
69+
var seen = new HashSet<String>();
70+
for (var param : result.getParameters()) {
71+
if (!seen.add(param.name())) {
72+
throw new IllegalArgumentException(
73+
"Duplicate parameter name: '" + param.name() + "'");
74+
}
75+
}
76+
return result;
13777
}
13878
}
13979
}

‎braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public final class Eval<INPUT, OUTPUT> {
3838
private final @Nonnull List<Scorer<INPUT, OUTPUT>> scorers;
3939
private final @Nonnull List<String> tags;
4040
private final @Nonnull Map<String, Object> metadata;
41+
private final @Nonnull Parameters parameters;
4142

4243
private Eval(Builder<INPUT, OUTPUT> builder) {
4344
this.experimentName = builder.experimentName;
@@ -59,6 +60,7 @@ private Eval(Builder<INPUT, OUTPUT> builder) {
5960
this.scorers = List.copyOf(builder.scorers);
6061
this.tags = List.copyOf(builder.tags);
6162
this.metadata = Map.copyOf(builder.metadata);
63+
this.parameters = builder.buildParameters();
6264
}
6365

6466
/** Runs the evaluation and returns results. */
@@ -129,7 +131,7 @@ private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase
129131
.startSpan();
130132
try (var unused =
131133
BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
132-
taskResult = task.apply(datasetCase);
134+
taskResult = task.apply(datasetCase, parameters);
133135
rootSpan.setAttribute(
134136
"braintrust.output_json",
135137
toJson(Map.of("output", taskResult.result())));
@@ -252,6 +254,8 @@ public static final class Builder<INPUT, OUTPUT> {
252254
private @Nullable Tracer tracer = null;
253255
private @Nullable Task<INPUT, OUTPUT> task;
254256
private @Nonnull List<Scorer<INPUT, OUTPUT>> scorers = List.of();
257+
private @Nonnull List<ParameterDef<?>> parameterDefs = List.of();
258+
private @Nonnull Map<String, Object> parameterValues = Map.of();
255259
private @Nonnull List<String> tags = List.of();
256260
private @Nonnull Map<String, Object> metadata = Map.of();
257261

@@ -366,5 +370,46 @@ public Builder<INPUT, OUTPUT> metadata(Map<String, Object> metadata) {
366370
this.metadata = Map.copyOf(metadata);
367371
return this;
368372
}
373+
374+
/**
375+
* Sets parameter definitions for this eval. Default values from the definitions are used
376+
* unless overridden via {@link #parameterValues(Map)}.
377+
*/
378+
@SuppressWarnings("rawtypes")
379+
public Builder<INPUT, OUTPUT> parameters(ParameterDef<?>... parameterDefs) {
380+
this.parameterDefs = List.of(parameterDefs);
381+
return this;
382+
}
383+
384+
/** Sets parameter definitions for this eval. */
385+
public Builder<INPUT, OUTPUT> parameters(List<ParameterDef<?>> parameterDefs) {
386+
this.parameterDefs = List.copyOf(parameterDefs);
387+
return this;
388+
}
389+
390+
/**
391+
* Sets explicit parameter values, overriding any defaults from parameter definitions. Keys
392+
* not present here fall back to the default value from the corresponding {@link
393+
* ParameterDef}.
394+
*/
395+
public Builder<INPUT, OUTPUT> parameterValues(Map<String, Object> values) {
396+
this.parameterValues = Map.copyOf(values);
397+
return this;
398+
}
399+
400+
/** Builds the merged Parameters from definitions and explicit values. */
401+
private Parameters buildParameters() {
402+
if (parameterDefs.isEmpty() && parameterValues.isEmpty()) {
403+
return Parameters.empty();
404+
}
405+
Map<String, Object> merged = new LinkedHashMap<>();
406+
for (var def : parameterDefs) {
407+
if (def.defaultValue() != null) {
408+
merged.put(def.name(), def.defaultValue());
409+
}
410+
}
411+
merged.putAll(parameterValues);
412+
return new Parameters(merged, parameterDefs);
413+
}
369414
}
370415
}

0 commit comments

Comments
 (0)