diff --git a/README.md b/README.md index 85ba217..9ff76a5 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,10 @@ # Braintrust Java SDK -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) +An OpenTelemetry-based Braintrust SDK for Java 17 and up. Contains instrumentation for major AI vendors (OpenAI, Anthropic, etc) -An OpenTelemetry-based Braintrust SDK for Java that leverages modern Java features and idiomatic patterns. +# Basic Usage -## Features - -- **OpenTelemetry Integration**: Built on OTEL for standards-based distributed tracing -- **Modern Java**: Uses Java 17+ features including records, pattern matching, and streams -- **Type-Safe Evaluation Framework**: Generic evaluation system with compile-time type safety -- **Async API Client**: Non-blocking HTTP client using Java 11+ HTTP APIs -- **Library Instrumentation**: Automatic tracing for OpenAI and Anthropic SDKs -- **Functional Programming**: Extensive use of lambdas, method references, and functional interfaces - -## Installation +## Using the SDK in your code Add to your `build.gradle`: @@ -23,393 +14,36 @@ dependencies { } ``` -## Quick Start - -### Try It Out - -```bash -# 1. Set up environment -cp .env.example .env -# Edit .env with your BRAINTRUST_API_KEY and OPENAI_API_KEY - -# 2. Run the canonical evaluation example -./run-with-env.sh :examples:runTriviaEval - -# 3. View results in Braintrust UI (link printed in output) -``` - -### Basic Usage - -```java -import config.dev.braintrust.claude.BraintrustConfig; -import trace.dev.braintrust.claude.BraintrustTracing; - -// Initialize with environment variables -var openTelemetry = BraintrustTracing.quickstart(); - -// Or with custom configuration -var config = BraintrustConfig.create(builder -> builder - .apiKey("your-api-key") - .defaultProjectId("my-project") - .enableTraceConsoleLog(true) -); - -var openTelemetry = BraintrustTracing.quickstart(config); -``` - -## Evaluation Framework - -The evaluation framework uses Java generics and functional interfaces for type-safe, flexible testing. - -### Canonical Example: TriviaEvaluation - -See `examples/src/main/java/dev/braintrust/examples/TriviaEvaluation.java` for a complete real-world example that: -- Makes actual OpenAI API calls using the wrapped client -- Evaluates LLM responses against ground truth -- Scores based on accuracy and response time -- Tracks token usage and costs -- Demonstrates proper OpenTelemetry integration - -Run it with: `./run-with-env.sh :examples:runTriviaEval` - -### Basic Usage - -```java -// Define your test cases using records -record TestCase(String input, String expected) {} - -// Simple approach with automatic experiment registration (like Go SDK) -var results = Evaluation.builder() - .name("My Evaluation") - .data(testCases) - .task(testCase -> myModel.process(testCase.input)) - .scorer("accuracy", (test, output) -> output.equals(test.expected) ? 1.0 : 0.0) - .experiment("my-experiment", "my-project") // Automatically creates both! - .build() - .run(); - -// Or use the lower-level API matching Go's RegisterExperiment -var project = Project.registerProjectSync("my-project"); -var experiment = Experiment.registerExperimentSync("my-experiment", project.id()); - -var evaluation = Evaluation.builder() - .name("My Evaluation") - .data(testCases) - .task(testCase -> myModel.process(testCase.input)) - .scorer(Scorer.StringScorers.exactMatch(tc -> tc.expected)) - .experimentId(experiment.id()) - .parallel(true) - .build(); -``` - -## Idiomatic Java Features - -### Records for Data Classes -```java -record MathProblem(String question, int expected) {} -record ModelOutput(String answer, double confidence) {} -``` +## Running examples -### Stream Processing -```java -// Process evaluation results -var scoreDistribution = results.results().stream() - .collect(Collectors.groupingBy( - r -> (int)(r.averageScore() * 10) / 10.0, - Collectors.counting() - )); -``` - -### CompletableFuture for Async Operations -```java -// Async API calls with composition -apiClient.createProject("my-project") - .thenCompose(project -> apiClient.createExperiment( - new CreateExperimentRequest(project.id(), "experiment-1") - )) - .thenAccept(experiment -> - System.out.println("Created experiment: " + experiment.id()) - ); -``` - -### Functional Interfaces and Method References -```java -// Custom scorers using functional interfaces -.scorer("length_match", (input, output) -> - output.length() == input.expectedLength() ? 1.0 : 0.0) - -// Method references for clean code -.data(files.stream().map(File::toPath)) -.task(this::processFile) -``` - -### Try-with-Resources for Span Management -```java -try (var ignored = tracer.spanBuilder("operation") - .startSpan() - .makeCurrent()) { - - // Your traced code here - performOperation(); -} -``` - -### Optional for Null Safety -```java -config.defaultProjectId() - .ifPresent(id -> span.setAttribute("project_id", id)); -``` - -## Library Instrumentation - -### OpenAI Integration - -The SDK provides a wrapped OpenAI client that automatically adds Braintrust-specific tracing: - -```java -import openai.dev.braintrust.claude.BraintrustOpenAI; -import com.openai.client.okhttp.OpenAIOkHttpClient; - -// Create wrapped client -var openAI = OpenAIOkHttpClient.builder().apiKey(apiKey).build(); -var wrappedClient = BraintrustOpenAI.wrap(openAI, tracer); - -// Use it exactly like the regular OpenAI client -var response = wrappedClient.chat().completions().create( - ChatCompletionCreateParams.builder() - .model(ChatModel.GPT_4O_MINI) - .addSystemMessage("You are a helpful assistant.") - .addUserMessage("What is 2 + 2?") - .build(), - // Optionally provide message info for better tracing - List.of( - BraintrustOpenAI.MessageInfo.system("You are a helpful assistant."), - BraintrustOpenAI.MessageInfo.user("What is 2 + 2?") - ) -); -``` - -This automatically captures: -- Input messages as `braintrust.input_json` -- Output responses as `braintrust.output_json` -- Model metadata as `braintrust.metadata_json` -- Token usage as `braintrust.metrics_json` - -See the TriviaEvaluation example for a complete working implementation. - -### Anthropic Integration -```java -// Async Anthropic calls with tracing -AnthropicInterceptor.traceMessageAsync( - request, - requestExtractor, - responseExtractor, - () -> anthropicClient.sendMessageAsync(request) -); -``` - -## Advanced Usage - -### Custom Span Processors -```java -var spanProcessor = new BraintrustSpanProcessor.Builder(config) - .withExporter(customExporter) - .enableConsoleLog(true) - .build(); -``` - -### Weighted Scorers -```java -var weightedScorer = Scorer.weightedAverage(Map.of( - accuracyScorer, 0.7, - speedScorer, 0.3 -)); -``` +NOTE: if you wish to develop the SDK you can skip this section (developer setup includes a better way to set up the JDK). -### Parallel Evaluation with Custom Executor -```java -var customExecutor = new ForkJoinPool(16); - -var results = Evaluation.builder() - .executor(customExecutor) - .parallel(true) - .timeout(Duration.ofSeconds(30)) - .run(); -``` - -## Environment Variables - -- `BRAINTRUST_API_KEY`: API key for authentication -- `BRAINTRUST_API_URL`: API endpoint (default: https://api.braintrust.dev) -- `BRAINTRUST_DEFAULT_PROJECT_ID`: Default project for spans -- `BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG`: Enable console trace output -- `BRAINTRUST_DEBUG`: Enable debug logging - -## Requirements +First, install java 17 or greater - **Java 17** (LTS) - Required for building and running - macOS: `brew install openjdk@17` - Ubuntu/Debian: `sudo apt install openjdk-17-jdk` - Windows: Download from [Adoptium](https://adoptium.net/) - -- **Gradle** - Build tool (wrapper included) - -## Setup - -### 1. Install Java 17 - -```bash -# macOS with Homebrew -brew install openjdk@17 -export JAVA_HOME=$(/usr/libexec/java_home -v 17) - -# Or add to your shell profile for permanent setup -echo 'export JAVA_HOME=$(/usr/libexec/java_home -v 17)' >> ~/.zshrc -``` - -### 2. Verify Java Version - -```bash -java -version -# Should output: openjdk version "17.0.x" -``` - -### 3. Building - -The project includes Gradle Wrapper, so you don't need to install Gradle separately: - -```bash -# First time setup (downloads Gradle automatically) -./gradlew build - -# Run tests -./gradlew test - -# Clean build -./gradlew clean build -``` - -## Development - -### IDE Setup - -- **IntelliJ IDEA**: Import as Gradle project -- **VS Code**: Install "Extension Pack for Java" -- **Eclipse**: Import as Gradle project - -### Code Formatting - -This project uses [Google Java Format](https://github.com/google/google-java-format) for consistent code style. - -```bash -# Check code formatting -./gradlew spotlessCheck -# Fix formatting issues -./gradlew spotlessApply - -# Install git pre-commit hooks -./gradlew installGitHooks -``` - -The pre-commit hook will automatically check formatting before each commit. To skip it temporarily: -```bash -git commit --no-verify ``` - -### Common Tasks - -```bash -# Run tests with detailed output -./gradlew test --info - -# Build without tests -./gradlew build -x test - -# Generate Javadoc -./gradlew javadoc - -# Check dependencies -./gradlew dependencies - -# Update Gradle wrapper -./gradlew wrapper --gradle-version=8.10.2 -``` - -## Running Examples - -### Setup Environment - -```bash -# Copy the example environment file -cp .env.example .env - -# Edit .env and add your API keys: -# BRAINTRUST_API_KEY=your-actual-api-key -# OPENAI_API_KEY=your-openai-api-key # Required for TriviaEvaluation example +# assumes you have BRAINTRUST_API_KEY and OPENAI_API_KEY exported +./gradlew :examples:runOpenAIInstrumentation ``` -### Canonical Examples - -#### Logging and Tracing -```bash -# Basic OpenTelemetry example with manual spans -./run-with-env.sh :examples:runOpenTelemetry +# Developer Docs +## Setup -# Simple logging example -./run-with-env.sh :examples:runSimple +TODO -- document sdkman, gradle, etc -# Project management example -./run-with-env.sh :examples:runProject -``` +## Running a local OpenTelemetry collector -#### Evaluations -```bash -# Canonical evaluation example - runs a real LLM evaluation with OpenAI -# This example evaluates GPT-4o-mini on trivia questions with accuracy scoring -./run-with-env.sh :examples:runTriviaEval +OpenTelemetry provides a local collector with a debug exporter which logs all traces, logs, and metrics to stdout. -# Go-style experiment example (simple math evaluation) -./run-with-env.sh :examples:runGoStyleExperiment +To run a local collector: -# Other evaluation examples -./run-with-env.sh :examples:runSimpleExperiment -./run-with-env.sh :examples:runExperimentWithApi ``` - -### What Each Example Demonstrates - -- **`:examples:runOpenTelemetry`** - Manual span creation, nested spans, and OpenAI integration -- **`:examples:runTriviaEval`** - Real LLM evaluation with OpenAI, scoring, and metrics tracking -- **`:examples:runGoStyleExperiment`** - Mimics Go SDK patterns with automatic project/experiment registration -- **`:examples:runSimple`** - Basic tracing without evaluations -- **`:examples:runProject`** - Project and dataset management via API - -## Testing - -```bash -./gradlew test +# Assumes you're in the repo root +docker run --rm -p 4318:4318 -v "$PWD/localcollector/collector.yaml:/etc/otelcol/config.yaml" otel/opentelemetry-collector:latest ``` -## Project Structure - -### Files to Commit to Git - -The following Gradle files should be committed: -- `build.gradle` - Build configuration -- `settings.gradle` - Project settings -- `gradle.properties` - Gradle properties -- `gradlew` - Gradle wrapper script (Unix) -- `gradlew.bat` - Gradle wrapper script (Windows) -- `gradle/wrapper/gradle-wrapper.jar` - Gradle wrapper JAR -- `gradle/wrapper/gradle-wrapper.properties` - Wrapper configuration - -The `.gitignore` file is configured to exclude: -- `.gradle/` - Gradle cache directory -- `build/` - Build outputs -- IDE files (`.idea/`, `*.iml`, etc.) -- Compiled classes (`*.class`) - -## License - -MIT +## TODO: finish these docs diff --git a/build.gradle b/build.gradle index 7ae25ef..b96649a 100644 --- a/build.gradle +++ b/build.gradle @@ -33,9 +33,10 @@ dependencies { api "io.opentelemetry:opentelemetry-api:${otelVersion}" api "io.opentelemetry:opentelemetry-sdk:${otelVersion}" api "io.opentelemetry:opentelemetry-sdk-trace:${otelVersion}" + api "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}" implementation "io.opentelemetry:opentelemetry-exporter-otlp:${otelVersion}" implementation "io.opentelemetry:opentelemetry-semconv:1.25.0-alpha" - implementation "io.opentelemetry:opentelemetry-exporter-logging:1.45.0" + implementation "io.opentelemetry:opentelemetry-exporter-logging:${otelVersion}" // HTTP Client (Java 11+) implementation 'com.fasterxml.jackson.core:jackson-databind:' + jacksonVersion diff --git a/localcollector/collector.yaml b/localcollector/collector.yaml new file mode 100644 index 0000000..c09287f --- /dev/null +++ b/localcollector/collector.yaml @@ -0,0 +1,15 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: { receivers: [otlp], exporters: [debug] } + metrics: { receivers: [otlp], exporters: [debug] } + logs: { receivers: [otlp], exporters: [debug] } diff --git a/src/main/java/dev/braintrust/config/BraintrustConfig.java b/src/main/java/dev/braintrust/config/BraintrustConfig.java index 203e9e6..b5de08a 100644 --- a/src/main/java/dev/braintrust/config/BraintrustConfig.java +++ b/src/main/java/dev/braintrust/config/BraintrustConfig.java @@ -17,17 +17,23 @@ public final class BraintrustConfig { private final String apiKey; private final URI apiUrl; + private final String tracesPath; + private final String logsPath; private final URI appUrl; @Nullable private final String defaultProjectId; @Nullable private final String orgName; private final boolean enableTraceConsoleLog; private final boolean debug; private final Duration requestTimeout; + private final boolean experimentalOtelLogs; private BraintrustConfig(Builder builder) { this.apiKey = builder.apiKey; this.apiUrl = builder.apiUrl; + this.tracesPath = builder.tracesPath; + this.logsPath = builder.logsPath; this.appUrl = builder.appUrl; + this.experimentalOtelLogs = builder.experimentalOtelLogs; this.defaultProjectId = builder.defaultProjectId; this.orgName = builder.orgName; this.enableTraceConsoleLog = builder.enableTraceConsoleLog; @@ -43,6 +49,14 @@ public URI apiUrl() { return apiUrl; } + public String tracesPath() { + return tracesPath; + } + + public String logsPath() { + return logsPath; + } + public URI appUrl() { return appUrl; } @@ -89,23 +103,29 @@ public static BraintrustConfig create(Consumer customizer) { public static final class Builder { private String apiKey; private URI apiUrl; + private String tracesPath; + private String logsPath; private URI appUrl; private String defaultProjectId; private String orgName; private boolean enableTraceConsoleLog; private boolean debug; + private boolean experimentalOtelLogs; private Duration requestTimeout = Duration.ofSeconds(30); private Builder() { // Initialize from environment this.apiKey = getEnv("BRAINTRUST_API_KEY", null); this.apiUrl = URI.create(getEnv("BRAINTRUST_API_URL", DEFAULT_API_URL)); + this.tracesPath = getEnv("BRAINTRUST_TRACES_PATH", "/otel/v1/traces"); + this.logsPath = getEnv("BRAINTRUST_LOGS_PATH", "/otel/v1/logs"); this.appUrl = URI.create(getEnv("BRAINTRUST_APP_URL", DEFAULT_APP_URL)); this.orgName = getEnv("BRAINTRUST_ORG_NAME", null); this.defaultProjectId = getEnv("BRAINTRUST_DEFAULT_PROJECT_ID", null); this.enableTraceConsoleLog = Boolean.parseBoolean(getEnv("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "false")); this.debug = Boolean.parseBoolean(getEnv("BRAINTRUST_DEBUG", "false")); + this.experimentalOtelLogs = Boolean.parseBoolean(getEnv("BRAINTRUST_X_OTEL_LOGS", "false")); } public Builder apiKey(String apiKey) { diff --git a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java index 1cfd92b..ec088a3 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java @@ -6,6 +6,16 @@ public class BraintrustOpenAI { public static OpenAIClient wrapOpenAI(OpenTelemetry openTelemetry, OpenAIClient openAIClient) { - return OpenAITelemetry.builder(openTelemetry).setCaptureMessageContent(true).build().wrap(openAIClient); + if ("true".equalsIgnoreCase(System.getenv("BRAINTRUST_X_OTEL_LOGS"))) { + return io.opentelemetry.instrumentation.openai.v1_1.OpenAITelemetry.builder(openTelemetry) + .setCaptureMessageContent(true) + .build() + .wrap(openAIClient); + } else { + return OpenAITelemetry.builder(openTelemetry) + .setCaptureMessageContent(true) + .build() + .wrap(openAIClient); + } } } diff --git a/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java b/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java index fe2ca08..3fe79d3 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java +++ b/src/main/java/dev/braintrust/trace/BraintrustSpanExporter.java @@ -25,7 +25,7 @@ public class BraintrustSpanExporter implements SpanExporter { public BraintrustSpanExporter(BraintrustConfig config) { this.config = config; - this.tracesEndpoint = config.apiUrl() + "/otel/v1/traces"; + this.tracesEndpoint = config.apiUrl() + config.tracesPath(); } @Override @@ -97,7 +97,6 @@ private CompletableResultCode exportWithParent(String parent, List spa }); BraintrustLogger.debug("Exporting {} spans with x-bt-parent: {}", spans.size(), parent); - // Export the spans return exporter.export(spans); } catch (Exception e) { diff --git a/src/main/java/dev/braintrust/trace/BraintrustTracing.java b/src/main/java/dev/braintrust/trace/BraintrustTracing.java index 405689b..df54415 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustTracing.java +++ b/src/main/java/dev/braintrust/trace/BraintrustTracing.java @@ -5,8 +5,12 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter; +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; @@ -146,9 +150,18 @@ public OpenTelemetry build() { .addSpanProcessor(spanProcessor) .build(); + var logExporter = OtlpHttpLogRecordExporter.builder() + .setEndpoint(config.apiUrl() + config.logsPath()) + .build(); + var loggerProvider = SdkLoggerProvider.builder() + .addLogRecordProcessor(BatchLogRecordProcessor.builder(logExporter).build()) + .build(); + // Create OpenTelemetry instance - var openTelemetry = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + var openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setLoggerProvider(loggerProvider) + .build(); // Register globally if requested if (registerGlobal) {