From f370b5c80ccd7fec10157584f578128819811848 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 18 Sep 2025 22:22:36 -0600 Subject: [PATCH 1/4] remove org name config We can get this from the api key --- .../braintrust/api/BraintrustApiClient.java | 37 +++++++++++-------- .../braintrust/config/BraintrustConfig.java | 18 --------- src/main/java/dev/braintrust/eval/Eval.java | 12 ++++-- 3 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/src/main/java/dev/braintrust/api/BraintrustApiClient.java index 974f791..3dc81f6 100644 --- a/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -42,18 +42,22 @@ public CompletableFuture createProject(String name) { } /** Gets a project by ID. */ - public CompletableFuture> getProject(String projectId) { - return getAsync("/v1/project/" + projectId, Project.class) - .handle( - (project, error) -> { - if (error != null && isNotFound(error)) { - return Optional.empty(); - } - if (error != null) { - throw new CompletionException(error); - } - return Optional.of(project); - }); + public Optional getProject(String projectId) { + try { + return getAsync("/v1/project/" + projectId, Project.class) + .handle( + (project, error) -> { + if (error != null && isNotFound(error)) { + return Optional.empty(); + } + if (error != null) { + throw new CompletionException(error); + } + return Optional.of(project); + }).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } } /** Lists all projects. */ @@ -111,9 +115,12 @@ public CompletableFuture> listDatasets(String projectId) { } /** Login to get user information including organization details. */ - public CompletableFuture login() { - var request = new LoginRequest(config.apiKey()); - return postAsync("/api/apikey/login", request, LoginResponse.class); + public LoginResponse login() { + try { + return postAsync("/api/apikey/login", new LoginRequest(config.apiKey()), LoginResponse.class).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } } // Low-level HTTP methods diff --git a/src/main/java/dev/braintrust/config/BraintrustConfig.java b/src/main/java/dev/braintrust/config/BraintrustConfig.java index b5de08a..8d8433c 100644 --- a/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -21,7 +21,6 @@ public final class BraintrustConfig { private final String logsPath; private final URI appUrl; @Nullable private final String defaultProjectId; - @Nullable private final String orgName; private final boolean enableTraceConsoleLog; private final boolean debug; private final Duration requestTimeout; @@ -35,7 +34,6 @@ private BraintrustConfig(Builder builder) { this.appUrl = builder.appUrl; this.experimentalOtelLogs = builder.experimentalOtelLogs; this.defaultProjectId = builder.defaultProjectId; - this.orgName = builder.orgName; this.enableTraceConsoleLog = builder.enableTraceConsoleLog; this.debug = builder.debug; this.requestTimeout = builder.requestTimeout; @@ -65,10 +63,6 @@ public Optional defaultProjectId() { return Optional.ofNullable(defaultProjectId); } - public Optional orgName() { - return Optional.ofNullable(orgName); - } - public boolean enableTraceConsoleLog() { return enableTraceConsoleLog; } @@ -107,7 +101,6 @@ public static final class Builder { private String logsPath; private URI appUrl; private String defaultProjectId; - private String orgName; private boolean enableTraceConsoleLog; private boolean debug; private boolean experimentalOtelLogs; @@ -120,7 +113,6 @@ private Builder() { this.tracesPath = getEnv("BRAINTRUST_TRACES_PATH", "/otel/v1/traces"); this.logsPath = getEnv("BRAINTRUST_LOGS_PATH", "/otel/v1/logs"); this.appUrl = URI.create(getEnv("BRAINTRUST_APP_URL", DEFAULT_APP_URL)); - this.orgName = getEnv("BRAINTRUST_ORG_NAME", null); this.defaultProjectId = getEnv("BRAINTRUST_DEFAULT_PROJECT_ID", null); this.enableTraceConsoleLog = Boolean.parseBoolean(getEnv("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "false")); @@ -156,11 +148,6 @@ public Builder defaultProjectId(String projectId) { return this; } - public Builder orgName(String orgName) { - this.orgName = orgName; - return this; - } - public Builder enableTraceConsoleLog(boolean enable) { this.enableTraceConsoleLog = enable; return this; @@ -181,11 +168,6 @@ public BraintrustConfig build() { throw new IllegalStateException("API key is required. Set BRAINTRUST_API_KEY environment variable or use apiKey() method."); } - // If orgName is not already set, try to retrieve it from the API - if (orgName == null || orgName.isBlank()) { - throw new IllegalStateException("org name is required"); - } - return new BraintrustConfig(this); } diff --git a/src/main/java/dev/braintrust/eval/Eval.java b/src/main/java/dev/braintrust/eval/Eval.java index 4e5a5a8..88ca9b3 100644 --- a/src/main/java/dev/braintrust/eval/Eval.java +++ b/src/main/java/dev/braintrust/eval/Eval.java @@ -13,6 +13,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; /** @@ -122,11 +123,16 @@ private EvalCase.Result evalOne(String experimentId, EvalCase Date: Fri, 19 Sep 2025 10:00:54 -0600 Subject: [PATCH 2/4] make otel PARENT logic consistent with other SDKs --- .../braintrust/config/BraintrustConfig.java | 30 +++++++++++++++++-- .../trace/BraintrustLogExporter.java | 3 +- .../trace/BraintrustSpanExporter.java | 3 +- .../trace/BraintrustSpanProcessor.java | 11 +++---- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/main/java/dev/braintrust/config/BraintrustConfig.java b/src/main/java/dev/braintrust/config/BraintrustConfig.java index 8d8433c..88d6bfd 100644 --- a/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -11,7 +11,7 @@ * variable support. */ public final class BraintrustConfig { - public static final String FALLBACK_PROJECT_NAME = "default-java-project"; + private static final String DEFAULT_PROJECT_NAME = "default-java-project"; private static final String DEFAULT_API_URL = "https://api.braintrust.dev"; private static final String DEFAULT_APP_URL = "https://www.braintrust.dev"; @@ -21,6 +21,7 @@ public final class BraintrustConfig { private final String logsPath; private final URI appUrl; @Nullable private final String defaultProjectId; + @Nullable private final String defaultProjectName; private final boolean enableTraceConsoleLog; private final boolean debug; private final Duration requestTimeout; @@ -34,6 +35,7 @@ private BraintrustConfig(Builder builder) { this.appUrl = builder.appUrl; this.experimentalOtelLogs = builder.experimentalOtelLogs; this.defaultProjectId = builder.defaultProjectId; + this.defaultProjectName = builder.defaultProjectName; this.enableTraceConsoleLog = builder.enableTraceConsoleLog; this.debug = builder.debug; this.requestTimeout = builder.requestTimeout; @@ -63,6 +65,10 @@ public Optional defaultProjectId() { return Optional.ofNullable(defaultProjectId); } + public Optional defaultProjectName() { + return Optional.ofNullable(defaultProjectName); + } + public boolean enableTraceConsoleLog() { return enableTraceConsoleLog; } @@ -75,6 +81,25 @@ public Duration requestTimeout() { return requestTimeout; } + /** + * The parent attribute tells braintrust where to send otel data + *

+ * The otel ingestion endpoint looks for (a) braintrust.parent = project_id|project_name|experiment_id:value otel attribute and routes accordingly + *

+ * (b) if a span has no parent marked explicitly, it will look to see if there's an x-bt-parent http header (with the same format marked above e.g. project_name:andrew) that parent will apply to all spans in a request that don't have one + *

+ * If neither (a) nor (b) exists, the data is dropped + */ + public Optional getBraintrustParentValue() { + if (null != defaultProjectId) { + return Optional.of("project_id:" + defaultProjectId); + } else if (null != defaultProjectName) { + return Optional.of("project_name:" + defaultProjectName); + } else { + return Optional.empty(); + } + } + /** Creates a new builder initialized with environment variables. */ public static Builder builder() { return new Builder(); @@ -101,19 +126,20 @@ public static final class Builder { private String logsPath; private URI appUrl; private String defaultProjectId; + private String defaultProjectName; private boolean enableTraceConsoleLog; private boolean debug; private boolean experimentalOtelLogs; private Duration requestTimeout = Duration.ofSeconds(30); private Builder() { - // Initialize from environment this.apiKey = getEnv("BRAINTRUST_API_KEY", null); this.apiUrl = URI.create(getEnv("BRAINTRUST_API_URL", DEFAULT_API_URL)); this.tracesPath = getEnv("BRAINTRUST_TRACES_PATH", "/otel/v1/traces"); this.logsPath = getEnv("BRAINTRUST_LOGS_PATH", "/otel/v1/logs"); this.appUrl = URI.create(getEnv("BRAINTRUST_APP_URL", DEFAULT_APP_URL)); this.defaultProjectId = getEnv("BRAINTRUST_DEFAULT_PROJECT_ID", null); + this.defaultProjectName = getEnv("BRAINTRUST_DEFAULT_PROJECT", DEFAULT_PROJECT_NAME); this.enableTraceConsoleLog = Boolean.parseBoolean(getEnv("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "false")); this.debug = Boolean.parseBoolean(getEnv("BRAINTRUST_DEBUG", "false")); diff --git a/src/main/java/dev/braintrust/trace/BraintrustLogExporter.java b/src/main/java/dev/braintrust/trace/BraintrustLogExporter.java index e6348ec..66d49ef 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustLogExporter.java +++ b/src/main/java/dev/braintrust/trace/BraintrustLogExporter.java @@ -67,8 +67,7 @@ private String getParentFromLog(LogRecordData log) { return "project_id:" + projectId; } - // Use default project ID if configured - return config.defaultProjectId().map(id -> "project_id:" + id).orElse(""); + return config.getBraintrustParentValue().orElse(""); } private CompletableResultCode exportWithParent(String parent, List logs) { diff --git a/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java b/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java index 869caa0..4bc7564 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java +++ b/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java @@ -67,8 +67,7 @@ private String getParentFromSpan(SpanData span) { return "project_id:" + projectId; } - // Use default project ID if configured - return config.defaultProjectId().map(id -> "project_id:" + id).orElse(""); + return config.getBraintrustParentValue().orElse(""); } private CompletableResultCode exportWithParent(String parent, List spans) { diff --git a/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java b/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java index f609f19..0fb0b20 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java +++ b/src/main/java/dev/braintrust/trace/BraintrustSpanProcessor.java @@ -65,16 +65,17 @@ public void onStart(@NotNull Context parentContext, ReadWriteSpan span) { var btContext = BraintrustContext.fromContext(parentContext); if (btContext == null) { // Use default project ID if configured - config.defaultProjectId() + config.getBraintrustParentValue() .ifPresent( - projectId -> { - span.setAttribute(PARENT, "project_id:" + projectId); + parentValue -> { + span.setAttribute(PARENT, parentValue); BraintrustLogger.debug( - "OnStart: set default project {} for span {}", - projectId, + "OnStart: set parent {} for span {}", + parentValue, span.getName()); }); } else { + // Can braintrust context have project_name instead? btContext .projectId() .ifPresent( From da4143c6b0254331f816702b9e3d455a274ea5ad Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 19 Sep 2025 11:20:54 -0600 Subject: [PATCH 3/4] make Eval parent config consistent with other SDKs --- .../braintrust/api/BraintrustApiClient.java | 59 +++---------------- .../braintrust/config/BraintrustConfig.java | 8 ++- src/main/java/dev/braintrust/eval/Eval.java | 42 ++++++++----- 3 files changed, 43 insertions(+), 66 deletions(-) diff --git a/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/src/main/java/dev/braintrust/api/BraintrustApiClient.java index 3dc81f6..99a3564 100644 --- a/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -36,9 +36,13 @@ public BraintrustApiClient(BraintrustConfig config) { } /** Creates or gets a project by name. */ - public CompletableFuture createProject(String name) { - var request = new CreateProjectRequest(name); - return postAsync("/v1/project", request, Project.class); + public Project getOrCreateProjectByName(String name) { + try { + var request = new CreateProjectRequest(name); + return postAsync("/v1/project", request, Project.class).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } } /** Gets a project by ID. */ @@ -60,13 +64,8 @@ public Optional getProject(String projectId) { } } - /** Lists all projects. */ - public CompletableFuture> listProjects() { - return getAsync("/v1/project", ProjectList.class).thenApply(ProjectList::projects); - } - /** Creates an experiment. */ - public Experiment createExperiment(CreateExperimentRequest request) { + public Experiment getOrCreateExperiment(CreateExperimentRequest request) { try { return postAsync("/v1/experiment", request, Experiment.class).get(); } catch (InterruptedException | ExecutionException e) { @@ -74,46 +73,6 @@ public Experiment createExperiment(CreateExperimentRequest request) { } } - /** Gets an experiment by ID. */ - public CompletableFuture> getExperiment(String experimentId) { - return getAsync("/v1/experiment/" + experimentId, Experiment.class) - .handle( - (experiment, error) -> { - if (error != null && isNotFound(error)) { - return Optional.empty(); - } - if (error != null) { - throw new CompletionException(error); - } - return Optional.of(experiment); - }); - } - - /** Lists experiments for a project. */ - public CompletableFuture> listExperiments(String projectId) { - return getAsync("/v1/experiment?project_id=" + projectId, ExperimentList.class) - .thenApply(ExperimentList::experiments); - } - - /** Creates a dataset. */ - public CompletableFuture createDataset(CreateDatasetRequest request) { - return postAsync("/v1/dataset", request, Dataset.class); - } - - /** Inserts events into a dataset. */ - public CompletableFuture insertDatasetEvents( - String datasetId, List events) { - var request = new InsertEventsRequest(events); - return postAsync( - "/v1/dataset/" + datasetId + "/insert", request, InsertEventsResponse.class); - } - - /** Lists datasets for a project. */ - public CompletableFuture> listDatasets(String projectId) { - return getAsync("/v1/dataset?project_id=" + projectId, DatasetList.class) - .thenApply(DatasetList::datasets); - } - /** Login to get user information including organization details. */ public LoginResponse login() { try { @@ -123,8 +82,6 @@ public LoginResponse login() { } } - // Low-level HTTP methods - private CompletableFuture getAsync(String path, Class responseType) { var request = HttpRequest.newBuilder() diff --git a/src/main/java/dev/braintrust/config/BraintrustConfig.java b/src/main/java/dev/braintrust/config/BraintrustConfig.java index 88d6bfd..ac18f68 100644 --- a/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -139,7 +139,13 @@ private Builder() { this.logsPath = getEnv("BRAINTRUST_LOGS_PATH", "/otel/v1/logs"); this.appUrl = URI.create(getEnv("BRAINTRUST_APP_URL", DEFAULT_APP_URL)); this.defaultProjectId = getEnv("BRAINTRUST_DEFAULT_PROJECT_ID", null); - this.defaultProjectName = getEnv("BRAINTRUST_DEFAULT_PROJECT", DEFAULT_PROJECT_NAME); + this.defaultProjectName = getEnv("BRAINTRUST_DEFAULT_PROJECT", DEFAULT_PROJECT_NAME).trim(); + if ((null == defaultProjectId || "".equalsIgnoreCase(defaultProjectId.trim())) + && "".equalsIgnoreCase(defaultProjectName)) { + // NOTE: this should not happen, + // but if someone happens to export their default project to the empty string and does not set a default project ID we don't have a valid parent for otel data. + throw new RuntimeException("Missing required envars. Please export BRAINTRUST_DEFAULT_PROJECT_ID or BRAINTRUST_DEFAULT_PROJECT"); + } this.enableTraceConsoleLog = Boolean.parseBoolean(getEnv("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "false")); this.debug = Boolean.parseBoolean(getEnv("BRAINTRUST_DEBUG", "false")); diff --git a/src/main/java/dev/braintrust/eval/Eval.java b/src/main/java/dev/braintrust/eval/Eval.java index 88ca9b3..48b1a12 100644 --- a/src/main/java/dev/braintrust/eval/Eval.java +++ b/src/main/java/dev/braintrust/eval/Eval.java @@ -13,7 +13,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; /** @@ -25,9 +24,10 @@ public final class Eval { private static final ObjectMapper JSON_MAPPER = new com.fasterxml.jackson.databind.ObjectMapper(); private final @Nonnull String experimentName; - private final @Nonnull String projectName; private final @Nonnull BraintrustConfig config; private final @Nonnull BraintrustApiClient client; + private final @Nonnull BraintrustApiClient.Project project; + private final @Nonnull BraintrustApiClient.OrganizationInfo orgInfo; private final @Nonnull Tracer tracer; private final @Nonnull List> evalCases; private final @Nonnull Task task; @@ -35,9 +35,25 @@ public final class Eval { private Eval(Builder builder) { this.experimentName = builder.experimentName; - this.projectName = Objects.requireNonNull(builder.projectName); this.config = Objects.requireNonNull(builder.config); this.client = new BraintrustApiClient(config); + // TODO: come up with a more consistent way to fetch project and org info + if (null == builder.projectId) { + this.project = client.getOrCreateProjectByName(config.defaultProjectName().orElseThrow()); + } else { + this.project = client.getProject(builder.projectId).orElseThrow(() -> new RuntimeException("invalid project id: " + builder.projectId)); + } + BraintrustApiClient.OrganizationInfo orgInfo = null; + for (var org : client.login().orgInfo()) { + if (project.orgId().equalsIgnoreCase(org.id())) { + orgInfo = org; + break; + } + } + if (null == orgInfo) { + throw new RuntimeException("Should not happen. Unable to find project's org: " + project.orgId()); + } + this.orgInfo = orgInfo; this.tracer = Objects.requireNonNull(builder.tracer); this.evalCases = List.copyOf(builder.evalCases); this.task = Objects.requireNonNull(builder.task); @@ -46,7 +62,7 @@ private Eval(Builder builder) { /** Runs the evaluation and returns results. */ public Result run() { - var experiment = client.createExperiment(new BraintrustApiClient.CreateExperimentRequest(projectName, experimentName, Optional.empty(), Optional.empty())); + var experiment = client.getOrCreateExperiment(new BraintrustApiClient.CreateExperimentRequest(project.id(), experimentName, Optional.empty(), Optional.empty())); var experimentID = experiment.id(); var evalCaseResults = evalCases.stream() .map(evalCase -> evalOne(experimentID, evalCase)) @@ -126,9 +142,7 @@ public class Result { private final String experimentUrl; private Result() { - var loginResponse = client.login(); - var project = client.getProject(projectName).orElseThrow(); - this.experimentUrl = config.appUrl() + "/app/" + loginResponse.orgInfo() + "/p/" + project.name() + "/experiments/" + experimentName; + this.experimentUrl = config.appUrl() + "/app/" + orgInfo.name() + "/p/" + project.name() + "/experiments/" + experimentName; } public String createReportString() { @@ -149,7 +163,7 @@ public static Builder builder() { public static final class Builder { private @Nonnull String experimentName = "unnamed-java-eval"; private @Nullable BraintrustConfig config; - private @Nullable String projectName; + private @Nullable String projectId; private @Nullable Tracer tracer = null; private @Nonnull List> evalCases = List.of(); private @Nullable Task task; @@ -162,8 +176,8 @@ public Eval build() { if (tracer == null) { tracer = BraintrustTracing.getTracer(); } - if (projectName == null) { - projectName = config.defaultProjectId().orElse(BraintrustConfig.FALLBACK_PROJECT_NAME); + if (projectId == null) { + projectId = config.defaultProjectId().orElse(null); } if (evalCases.isEmpty()) { throw new RuntimeException("must provide at least one eval case"); @@ -175,13 +189,13 @@ public Eval build() { return new Eval<>(this); } - public Builder name(String name) { - this.experimentName = name; + public Builder name(@Nonnull String name) { + this.experimentName = Objects.requireNonNull(name); return this; } - public Builder projectName(@Nonnull String projectName) { - this.projectName = Objects.requireNonNull(projectName); + public Builder projectId(@Nonnull String projectId) { + this.projectId = Objects.requireNonNull(projectId); return this; } From 746a0a07ae65e50a28aa53bd3d1dd98414644f8e Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 19 Sep 2025 15:25:06 -0600 Subject: [PATCH 4/4] print URL to see data in Braintrust for all examples --- .../examples/CustomOpenTelemetryExample.java | 7 ++-- .../examples/ExperimentExample.java | 2 +- .../OpenAIInstrumentationExample.java | 7 ++-- .../examples/SimpleOpenTelemetryExample.java | 7 ++-- .../braintrust/api/BraintrustApiClient.java | 36 +++++++++++++++++-- .../braintrust/config/BraintrustConfig.java | 18 ++++++---- src/main/java/dev/braintrust/eval/Eval.java | 24 ++++--------- 7 files changed, 66 insertions(+), 35 deletions(-) diff --git a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java index 49aeb47..ef0aa81 100644 --- a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java @@ -4,7 +4,6 @@ import dev.braintrust.log.BraintrustLogger; import dev.braintrust.trace.BraintrustTracing; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter; import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; @@ -43,7 +42,8 @@ public static void main(String[] args) throws Exception { // NOTE: there are many ways to set up otel builders, etc. // The important line is here: call enable with your otel builders and braintrust will export open telemetry data in addition to your existing setup - BraintrustTracing.enable(BraintrustConfig.fromEnvironment(), tracerBuilder, loggerBuilder, meterBuilder); + var braintrustConfig = BraintrustConfig.fromEnvironment(); + BraintrustTracing.enable(braintrustConfig, tracerBuilder, loggerBuilder, meterBuilder); var openTelemetry = OpenTelemetrySdk.builder() .setTracerProvider(tracerBuilder.build()) @@ -63,7 +63,8 @@ public static void main(String[] args) throws Exception { } finally { span.end(); } - System.out.println("custom example completed!"); + var url = braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s".formatted(span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); + System.out.println("\n\nExample complete! View your data in Braintrust: " + url); } private static void registerShutdownHook(OpenTelemetrySdk otel) { diff --git a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java index 295d5c2..9fd2cda 100644 --- a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java +++ b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java @@ -44,6 +44,6 @@ public static void main(String[] args) throws Exception { Scorer.of("vegetable_scorer", result -> "vegetable".equals(result) ? 1.0 : 0.0)) .build(); var result = eval.run(); - System.out.println(result.createReportString()); + System.out.println("\n\n example complete! View your data in Braintrust: " + result.createReportString()); } } \ No newline at end of file diff --git a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java index 93a2efe..fcd97e4 100644 --- a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java +++ b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java @@ -4,6 +4,7 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import dev.braintrust.config.BraintrustConfig; import dev.braintrust.instrumentation.openai.BraintrustOpenAI; import dev.braintrust.trace.BraintrustTracing; @@ -15,7 +16,8 @@ public static void main(String[] args) throws Exception { if (null == System.getenv("OPENAI_API_KEY")) { System.err.println("\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n"); } - var openTelemetry = BraintrustTracing.quickstart(); + var braintrustConfig = BraintrustConfig.fromEnvironment(); + var openTelemetry = BraintrustTracing.quickstart(braintrustConfig, true); var tracer = BraintrustTracing.getTracer(openTelemetry); OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); var rootSpan = tracer.spanBuilder("java-braintrust-example").startSpan(); @@ -36,6 +38,7 @@ public static void main(String[] args) throws Exception { } finally { rootSpan.end(); } - System.out.println("example completed!"); + var url = braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s".formatted(rootSpan.getSpanContext().getTraceId(), rootSpan.getSpanContext().getSpanId()); + System.out.println("\n\nExample complete! View your data in Braintrust: " + url); } } diff --git a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java index c42c49c..63ce51f 100644 --- a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java @@ -1,10 +1,12 @@ package dev.braintrust.examples; +import dev.braintrust.config.BraintrustConfig; import dev.braintrust.trace.BraintrustTracing; public class SimpleOpenTelemetryExample { public static void main(String[] args) throws Exception { - var openTelemetry = BraintrustTracing.quickstart(); + var braintrustConfig = BraintrustConfig.fromEnvironment(); + var openTelemetry = BraintrustTracing.quickstart(braintrustConfig, true); var tracer = BraintrustTracing.getTracer(openTelemetry); var span = tracer.spanBuilder("hello-java").startSpan(); @@ -15,6 +17,7 @@ public static void main(String[] args) throws Exception { } finally { span.end(); } - System.out.println("simple example completed!"); + var url = braintrustConfig.fetchProjectURI() + "/logs?r=%s&s=%s".formatted(span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); + System.out.println("\n\nExample complete! View your data in Braintrust: " + url); } } diff --git a/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/src/main/java/dev/braintrust/api/BraintrustApiClient.java index 99a3564..d2ef611 100644 --- a/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -36,9 +36,9 @@ public BraintrustApiClient(BraintrustConfig config) { } /** Creates or gets a project by name. */ - public Project getOrCreateProjectByName(String name) { + public Project getOrCreateProject(String projectName) { try { - var request = new CreateProjectRequest(name); + var request = new CreateProjectRequest(projectName); return postAsync("/v1/project", request, Project.class).get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); @@ -82,6 +82,36 @@ public LoginResponse login() { } } + public Optional getProjectAndOrgInfo() { + var projectId = config.defaultProjectId().orElse(null); + if (null == projectId) { + projectId = getOrCreateProject(config.defaultProjectName().orElseThrow()).id(); + } + return getProjectAndOrgInfo(projectId); + } + + public Optional getOrCreateProjectAndOrgInfo(String projectName) { + return getProjectAndOrgInfo(getOrCreateProject(projectName).id()); + } + + public Optional getProjectAndOrgInfo(String projectId) { + var project = getProject(projectId).orElse(null); + if (null == project) { + return Optional.empty(); + } + OrganizationInfo orgInfo = null; + for (var org : login().orgInfo()) { + if (project.orgId().equalsIgnoreCase(org.id())) { + orgInfo = org; + break; + } + } + if (null == orgInfo) { + throw new ApiException("Should not happen. Unable to find project's org: " + project.orgId()); + } + return Optional.of(new OrganizationAndProjectInfo(orgInfo, project)); + } + private CompletableFuture getAsync(String path, Class responseType) { var request = HttpRequest.newBuilder() @@ -237,4 +267,6 @@ public record OrganizationInfo(String id, String name) {} private record LoginRequest(String token) {} public record LoginResponse(List orgInfo) {} + + public record OrganizationAndProjectInfo(OrganizationInfo orgInfo, Project project) {} } diff --git a/src/main/java/dev/braintrust/config/BraintrustConfig.java b/src/main/java/dev/braintrust/config/BraintrustConfig.java index ac18f68..858ca18 100644 --- a/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -1,6 +1,9 @@ package dev.braintrust.config; +import dev.braintrust.api.BraintrustApiClient; + import java.net.URI; +import java.net.URISyntaxException; import java.time.Duration; import java.util.Optional; import java.util.function.Consumer; @@ -110,13 +113,14 @@ public static BraintrustConfig fromEnvironment() { return builder().build(); } - /** - * Creates a config with a consumer for customization and retrieves organization name from API. - */ - public static BraintrustConfig create(Consumer customizer) { - var builder = builder(); - customizer.accept(builder); - return builder.build(); + public URI fetchProjectURI() { + try { + var client = new BraintrustApiClient(this); + var orgAndProject = client.getProjectAndOrgInfo().orElseThrow(); + return new URI(appUrl() + "/app/" + orgAndProject.orgInfo().name() + "/p/" + orgAndProject.project().name()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } } public static final class Builder { diff --git a/src/main/java/dev/braintrust/eval/Eval.java b/src/main/java/dev/braintrust/eval/Eval.java index 48b1a12..b937574 100644 --- a/src/main/java/dev/braintrust/eval/Eval.java +++ b/src/main/java/dev/braintrust/eval/Eval.java @@ -26,8 +26,7 @@ public final class Eval { private final @Nonnull String experimentName; private final @Nonnull BraintrustConfig config; private final @Nonnull BraintrustApiClient client; - private final @Nonnull BraintrustApiClient.Project project; - private final @Nonnull BraintrustApiClient.OrganizationInfo orgInfo; + private final @Nonnull BraintrustApiClient.OrganizationAndProjectInfo orgAndProject; private final @Nonnull Tracer tracer; private final @Nonnull List> evalCases; private final @Nonnull Task task; @@ -37,23 +36,12 @@ private Eval(Builder builder) { this.experimentName = builder.experimentName; this.config = Objects.requireNonNull(builder.config); this.client = new BraintrustApiClient(config); - // TODO: come up with a more consistent way to fetch project and org info if (null == builder.projectId) { - this.project = client.getOrCreateProjectByName(config.defaultProjectName().orElseThrow()); + this.orgAndProject = client.getProjectAndOrgInfo().orElseThrow(); } else { - this.project = client.getProject(builder.projectId).orElseThrow(() -> new RuntimeException("invalid project id: " + builder.projectId)); + this.orgAndProject = client.getProjectAndOrgInfo(builder.projectId) + .orElseThrow(() -> new RuntimeException("invalid project id: " + builder.projectId)); } - BraintrustApiClient.OrganizationInfo orgInfo = null; - for (var org : client.login().orgInfo()) { - if (project.orgId().equalsIgnoreCase(org.id())) { - orgInfo = org; - break; - } - } - if (null == orgInfo) { - throw new RuntimeException("Should not happen. Unable to find project's org: " + project.orgId()); - } - this.orgInfo = orgInfo; this.tracer = Objects.requireNonNull(builder.tracer); this.evalCases = List.copyOf(builder.evalCases); this.task = Objects.requireNonNull(builder.task); @@ -62,7 +50,7 @@ private Eval(Builder builder) { /** Runs the evaluation and returns results. */ public Result run() { - var experiment = client.getOrCreateExperiment(new BraintrustApiClient.CreateExperimentRequest(project.id(), experimentName, Optional.empty(), Optional.empty())); + var experiment = client.getOrCreateExperiment(new BraintrustApiClient.CreateExperimentRequest(orgAndProject.project().id(), experimentName, Optional.empty(), Optional.empty())); var experimentID = experiment.id(); var evalCaseResults = evalCases.stream() .map(evalCase -> evalOne(experimentID, evalCase)) @@ -142,7 +130,7 @@ public class Result { private final String experimentUrl; private Result() { - this.experimentUrl = config.appUrl() + "/app/" + orgInfo.name() + "/p/" + project.name() + "/experiments/" + experimentName; + this.experimentUrl = config.appUrl() + "/app/" + orgAndProject.orgInfo().name() + "/p/" + orgAndProject.project().name() + "/experiments/" + experimentName; } public String createReportString() {