diff --git a/README.md b/README.md
index 5540eab5..85d34301 100644
--- a/README.md
+++ b/README.md
@@ -52,9 +52,9 @@ Function<String, String> getFoodType =
 var eval = braintrust.<String, String>evalBuilder()
                 .name("java-eval-x-" + System.currentTimeMillis())
                 .cases(
-                        EvalCase.of("asparagus", "vegetable"),
-                        EvalCase.of("banana", "fruit"))
-                .task(getFoodType)
+                        DatasetCase.of("asparagus", "vegetable"),
+                        DatasetCase.of("banana", "fruit"))
+                .taskFunction(getFoodType)
                 .scorers(
                         Scorer.of(
                                 "fruit_scorer",
diff --git a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java
index 41eaf9d6..74007eb2 100644
--- a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java
+++ b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java
@@ -4,7 +4,7 @@
 import com.openai.models.ChatModel;
 import com.openai.models.chat.completions.ChatCompletionCreateParams;
 import dev.braintrust.Braintrust;
-import dev.braintrust.eval.EvalCase;
+import dev.braintrust.eval.DatasetCase;
 import dev.braintrust.eval.Scorer;
 import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
 import java.util.function.Function;
@@ -37,10 +37,10 @@ public static void main(String[] args) throws Exception {
                         // will append new cases to
                         // the same experiment
                         .cases(
-                                EvalCase.of("strawberry", "fruit"),
-                                EvalCase.of("asparagus", "vegetable"),
-                                EvalCase.of("apple", "fruit"),
-                                EvalCase.of("banana", "fruit"))
+                                DatasetCase.of("strawberry", "fruit"),
+                                DatasetCase.of("asparagus", "vegetable"),
+                                DatasetCase.of("apple", "fruit"),
+                                DatasetCase.of("banana", "fruit"))
                         .taskFunction(getFoodType)
                         .scorers(
                                 Scorer.of(
diff --git a/src/main/java/dev/braintrust/eval/Dataset.java b/src/main/java/dev/braintrust/eval/Dataset.java
new file mode 100644
index 00000000..f07b0d79
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/Dataset.java
@@ -0,0 +1,42 @@
+package dev.braintrust.eval;
+
+import java.util.List;
+import java.util.Optional;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Datasets define the cases for evals. This interface provides a means of iterating through all
+ * cases of a particular dataset.
+ *
+ * <p>The most common implementations are in-memory datasets, and datasets fetched from the
+ * Braintrust API.
+ */
+public interface Dataset<INPUT, OUTPUT> {
+    Cursor<DatasetCase<INPUT, OUTPUT>> openCursor();
+
+    String id();
+
+    String version();
+
+    @NotThreadSafe
+    interface Cursor<CASE> extends AutoCloseable {
+        /**
+         * Fetch the next case. Returns empty if there are no more cases to fetch.
+         *
+         * <p>Implementations may make external requests to fetch data.
+         *
+         * <p>If this method is invoked after {@link #close()} an IllegalStateException will be
+         * thrown
+         */
+        Optional<CASE> next();
+
+        /** close all cursor resources */
+        void close();
+    }
+
+    /** Create an in-memory Dataset containing the provided cases. */
+    @SafeVarargs
+    static <INPUT, OUTPUT> Dataset<INPUT, OUTPUT> of(DatasetCase<INPUT, OUTPUT>... cases) {
+        return new DatasetInMemoryImpl<>(List.of(cases));
+    }
+}
diff --git a/src/main/java/dev/braintrust/eval/DatasetCase.java b/src/main/java/dev/braintrust/eval/DatasetCase.java
new file mode 100644
index 00000000..5591ba52
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/DatasetCase.java
@@ -0,0 +1,25 @@
+package dev.braintrust.eval;
+
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nonnull;
+
+/** A single row in a dataset. */
+public record DatasetCase<INPUT, OUTPUT>(
+        INPUT input,
+        OUTPUT expected,
+        @Nonnull List<String> tags,
+        @Nonnull Map<String, Object> metadata) {
+    public DatasetCase {
+        if (!metadata.isEmpty()) {
+            throw new RuntimeException("TODO: metadata support not yet implemented");
+        }
+        if (!tags.isEmpty()) {
+            throw new RuntimeException("TODO: tags support not yet implemented");
+        }
+    }
+
+    public static <INPUT, OUTPUT> DatasetCase<INPUT, OUTPUT> of(INPUT input, OUTPUT expected) {
+        return new DatasetCase<>(input, expected, List.of(), Map.of());
+    }
+}
diff --git a/src/main/java/dev/braintrust/eval/DatasetInMemoryImpl.java b/src/main/java/dev/braintrust/eval/DatasetInMemoryImpl.java
new file mode 100644
index 00000000..554fdd89
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/DatasetInMemoryImpl.java
@@ -0,0 +1,49 @@
+package dev.braintrust.eval;
+
+import java.util.List;
+import java.util.Optional;
+
+/** A dataset held entirely in memory */
+class DatasetInMemoryImpl<INPUT, OUTPUT> implements Dataset<INPUT, OUTPUT> {
+    private final List<DatasetCase<INPUT, OUTPUT>> cases;
+    private final String id;
+
+    DatasetInMemoryImpl(List<DatasetCase<INPUT, OUTPUT>> cases) {
+        this.cases = List.copyOf(cases);
+        id = "in-memory-dataset<" + this.cases.hashCode() + ">";
+    }
+
+    @Override
+    public String id() {
+        return id;
+    }
+
+    @Override
+    public String version() {
+        return "0";
+    }
+
+    @Override
+    public Cursor<DatasetCase<INPUT, OUTPUT>> openCursor() {
+        return new Cursor<>() {
+            int nextIndex = 0;
+            boolean closed = false;
+
+            @Override
+            public Optional<DatasetCase<INPUT, OUTPUT>> next() {
+                if (closed) {
+                    throw new IllegalStateException("this method may not be invoked after close");
+                } else if (nextIndex < cases.size()) {
+                    return Optional.of(cases.get(nextIndex++));
+                } else {
+                    return Optional.empty();
+                }
+            }
+
+            @Override
+            public void close() {
+                closed = true;
+            }
+        };
+    }
+}
diff --git a/src/main/java/dev/braintrust/eval/Eval.java b/src/main/java/dev/braintrust/eval/Eval.java
index 01d089a2..14f9fc15 100644
--- a/src/main/java/dev/braintrust/eval/Eval.java
+++ b/src/main/java/dev/braintrust/eval/Eval.java
@@ -14,7 +14,6 @@
 import java.util.function.Function;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import lombok.Getter;
 import lombok.SneakyThrows;
 
 /**
@@ -26,14 +25,13 @@
 public final class Eval<INPUT, OUTPUT> {
     private static final AttributeKey<String> PARENT =
             AttributeKey.stringKey(BraintrustTracing.PARENT_KEY);
-    private static final ObjectMapper JSON_MAPPER =
-            new com.fasterxml.jackson.databind.ObjectMapper();
+    static final ObjectMapper JSON_MAPPER = new com.fasterxml.jackson.databind.ObjectMapper();
     private final @Nonnull String experimentName;
     private final @Nonnull BraintrustConfig config;
     private final @Nonnull BraintrustApiClient client;
     private final @Nonnull BraintrustApiClient.OrganizationAndProjectInfo orgAndProject;
     private final @Nonnull Tracer tracer;
-    private final @Nonnull List<EvalCase<INPUT, OUTPUT>> evalCases;
+    private final @Nonnull Dataset<INPUT, OUTPUT> dataset;
     private final @Nonnull Task<INPUT, OUTPUT> task;
     private final @Nonnull List<Scorer<INPUT, OUTPUT>> scorers;
 
@@ -52,13 +50,13 @@ private Eval(Builder<INPUT, OUTPUT> builder) {
                                                     "invalid project id: " + builder.projectId));
         }
         this.tracer = Objects.requireNonNull(builder.tracer);
-        this.evalCases = List.copyOf(builder.evalCases);
+        this.dataset = builder.dataset;
         this.task = Objects.requireNonNull(builder.task);
         this.scorers = List.copyOf(builder.scorers);
     }
 
     /** Runs the evaluation and returns results. */
-    public Result run() {
+    public EvalResult run() {
         var experiment =
                 client.getOrCreateExperiment(
                         new BraintrustApiClient.CreateExperimentRequest(
@@ -67,44 +65,55 @@ public Result run() {
                                 Optional.empty(),
                                 Optional.empty()));
         var experimentID = experiment.id();
-        var evalCaseResults =
-                evalCases.stream().map(evalCase -> evalOne(experimentID, evalCase)).toList();
-        return new Result();
+
+        try (var cursor = dataset.openCursor()) {
+            for (var datsetCase = cursor.next();
+                    datsetCase.isPresent();
+                    datsetCase = cursor.next()) {
+                evalOne(experimentID, datsetCase.get());
+            }
+        }
+        var experimentUrl =
+                "%s/experiments/%s"
+                        .formatted(
+                                BraintrustUtils.createProjectURI(config.appUrl(), orgAndProject)
+                                        .toASCIIString(),
+                                experimentName);
+        return new EvalResult(experimentUrl);
     }
 
-    private EvalCase.Result<INPUT, OUTPUT> evalOne(
-            String experimentId, EvalCase<INPUT, OUTPUT> evalCase) {
+    @SneakyThrows
+    private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase) {
+        JSON_MAPPER.writeValueAsString(Map.of("type", "eval"));
         var rootSpan =
                 tracer.spanBuilder("eval") // TODO: allow names for eval cases
                         .setNoParent() // each eval case is its own trace
                         .setSpanKind(SpanKind.CLIENT)
                         .setAttribute(PARENT, "experiment_id:" + experimentId)
-                        .setAttribute("braintrust.span_attributes", "{\"type\":\"eval\"}")
-                        // FIXME: use proper object mapper for json stuff
+                        .setAttribute("braintrust.span_attributes", json(Map.of("type", "eval")))
                         .setAttribute(
-                                "braintrust.input_json",
-                                "{ \"input\":\"" + evalCase.input() + "\"}")
-                        .setAttribute("braintrust.expected", "\"" + evalCase.expected() + "\"")
+                                "braintrust.input_json", json(Map.of("input", datasetCase.input())))
+                        .setAttribute("braintrust.expected", json(datasetCase.expected()))
                         .startSpan();
         try (var rootScope = BraintrustContext.ofExperiment(experimentId, rootSpan).makeCurrent()) {
-            final OUTPUT result;
+            final TaskResult<INPUT, OUTPUT> taskResult;
             { // run task
                 var taskSpan =
                         tracer.spanBuilder("task")
                                 .setAttribute(PARENT, "experiment_id:" + experimentId)
-                                .setAttribute("braintrust.span_attributes", "{\"type\":\"task\"}")
+                                .setAttribute(
+                                        "braintrust.span_attributes", json(Map.of("type", "task")))
                                 .startSpan();
                 try (var unused =
                         BraintrustContext.ofExperiment(experimentId, taskSpan).makeCurrent()) {
-                    result = task.apply(evalCase);
+                    taskResult = task.apply(datasetCase);
                 } finally {
                     taskSpan.end();
                 }
                 try {
                     rootSpan.setAttribute(
                             "braintrust.output_json",
-                            JSON_MAPPER.writeValueAsString(
-                                    Map.of("output", String.valueOf(result))));
+                            JSON_MAPPER.writeValueAsString(Map.of("output", taskResult.result())));
                 } catch (JsonProcessingException e) {
                     throw new RuntimeException(e);
                 }
@@ -113,66 +122,40 @@ private EvalCase.Result<INPUT, OUTPUT> evalOne(
                 var scoreSpan =
                         tracer.spanBuilder("score")
                                 .setAttribute(PARENT, "experiment_id:" + experimentId)
-                                .setAttribute("braintrust.span_attributes", "{\"type\":\"score\"}")
+                                .setAttribute(
+                                        "braintrust.span_attributes", json(Map.of("type", "score")))
                                 .startSpan();
                 try (var unused =
                         BraintrustContext.ofExperiment(experimentId, scoreSpan).makeCurrent()) {
-                    // NOTE: linked hash map to preserve ordering. Not in the spec but nice user
-                    // experience
-                    final HashMap<String, Double> nameToScore = new LinkedHashMap<>();
-                    var scores =
-                            scorers.stream()
-                                    .map(
-                                            scorer -> {
-                                                var score = scorer.score(evalCase, result);
-                                                if (score < 0.0 || score > 1.0) {
-                                                    throw new RuntimeException(
-                                                            "score must be between 0 and 1: "
-                                                                    + scorer.getName()
-                                                                    + " : "
-                                                                    + score);
-                                                }
-                                                nameToScore.put(scorer.getName(), score);
-                                                return score;
-                                            })
-                                    .toList();
-                    try {
-                        scoreSpan.setAttribute(
-                                "braintrust.scores", JSON_MAPPER.writeValueAsString(nameToScore));
-                    } catch (JsonProcessingException e) {
-                        throw new RuntimeException(e);
-                    }
+                    // linked map to preserve ordering. Not in the spec but nice user experience
+                    final Map<String, Double> nameToScore = new LinkedHashMap<>();
+                    scorers.forEach(
+                            scorer -> {
+                                var scores = scorer.score(taskResult);
+                                scores.forEach(
+                                        score -> {
+                                            if (score.value() < 0.0 || score.value() > 1.0) {
+                                                throw new RuntimeException(
+                                                        "score must be between 0 and 1: %s : %s"
+                                                                .formatted(
+                                                                        scorer.getName(), score));
+                                            }
+                                            nameToScore.put(score.name(), score.value());
+                                        });
+                            });
+                    scoreSpan.setAttribute("braintrust.scores", json(nameToScore));
                 } finally {
                     scoreSpan.end();
                 }
             }
-            return new EvalCase.Result<>(evalCase, result);
         } finally {
             rootSpan.end();
         }
     }
 
-    /** Results of all eval cases of an experiment. */
-    public class Result {
-        @Getter private final String experimentUrl;
-
-        @SneakyThrows
-        private Result() {
-            this.experimentUrl =
-                    "%s/experiments/%s"
-                            .formatted(
-                                    BraintrustUtils.createProjectURI(config.appUrl(), orgAndProject)
-                                            .toASCIIString(),
-                                    experimentName);
-        }
-
-        public String createReportString() {
-            try {
-                return "Experiment complete. View results in braintrust: " + experimentUrl;
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
+    @SneakyThrows
+    private String json(Object o) {
+        return JSON_MAPPER.writeValueAsString(o);
     }
 
     /** Creates a new eval builder. */
@@ -182,12 +165,12 @@ public static <INPUT, OUTPUT> Builder<INPUT, OUTPUT> builder() {
 
     /** Builder for creating evaluations with fluent API. */
     public static final class Builder<INPUT, OUTPUT> {
+        public @Nonnull Dataset<INPUT, OUTPUT> dataset;
         private @Nonnull String experimentName = "unnamed-java-eval";
         private @Nullable BraintrustConfig config;
         private @Nullable BraintrustApiClient apiClient;
         private @Nullable String projectId;
         private @Nullable Tracer tracer = null;
-        private @Nonnull List<EvalCase<INPUT, OUTPUT>> evalCases = List.of();
         private @Nullable Task<INPUT, OUTPUT> task;
         private @Nonnull List<Scorer<INPUT, OUTPUT>> scorers = List.of();
 
@@ -201,15 +184,13 @@ public Eval<INPUT, OUTPUT> build() {
             if (projectId == null) {
                 projectId = config.defaultProjectId().orElse(null);
             }
-            if (evalCases.isEmpty()) {
-                throw new RuntimeException("must provide at least one eval case");
-            }
             if (scorers.isEmpty()) {
                 throw new RuntimeException("must provide at least one scorer");
             }
             if (null == apiClient) {
                 apiClient = BraintrustApiClient.of(config);
             }
+            Objects.requireNonNull(dataset);
             Objects.requireNonNull(task);
             return new Eval<>(this);
         }
@@ -239,10 +220,28 @@ public Builder<INPUT, OUTPUT> tracer(Tracer tracer) {
             return this;
         }
 
+        public Builder<INPUT, OUTPUT> dataset(Dataset<INPUT, OUTPUT> dataset) {
+            this.dataset = dataset;
+            return this;
+        }
+
+        /** Deprecated. Use {@link #cases(DatasetCase[])} or {@link #dataset(Dataset)} instead */
+        @Deprecated
         @SafeVarargs
         public final Builder<INPUT, OUTPUT> cases(EvalCase<INPUT, OUTPUT>... cases) {
-            this.evalCases = List.of(cases);
-            return this;
+            return cases(
+                    Arrays.stream(cases)
+                            .map(evalCase -> DatasetCase.of(evalCase.input(), evalCase.expected()))
+                            .toList()
+                            .toArray(new DatasetCase[0]));
+        }
+
+        @SafeVarargs
+        public final Builder<INPUT, OUTPUT> cases(DatasetCase<INPUT, OUTPUT>... cases) {
+            if (cases.length == 0) {
+                throw new RuntimeException("must provide at least one case");
+            }
+            return dataset(Dataset.of(cases));
         }
 
         public Builder<INPUT, OUTPUT> task(Task<INPUT, OUTPUT> task) {
@@ -251,7 +250,15 @@ public Builder<INPUT, OUTPUT> task(Task<INPUT, OUTPUT> task) {
         }
 
         public Builder<INPUT, OUTPUT> taskFunction(Function<INPUT, OUTPUT> taskFn) {
-            return task(evalCase -> taskFn.apply(evalCase.input()));
+            return task(
+                    new Task<>() {
+                        @Override
+                        public TaskResult<INPUT, OUTPUT> apply(
+                                DatasetCase<INPUT, OUTPUT> datasetCase) {
+                            var result = taskFn.apply(datasetCase.input());
+                            return new TaskResult<>(result, datasetCase);
+                        }
+                    });
         }
 
         @SafeVarargs
diff --git a/src/main/java/dev/braintrust/eval/EvalCase.java b/src/main/java/dev/braintrust/eval/EvalCase.java
index ae7c8fff..27a29a43 100644
--- a/src/main/java/dev/braintrust/eval/EvalCase.java
+++ b/src/main/java/dev/braintrust/eval/EvalCase.java
@@ -4,19 +4,15 @@
 import java.util.Map;
 import javax.annotation.Nonnull;
 
-/**
- * A single test case in an LLM eval.
- *
- * @param input test input
- * @param expected expected value
- * @param tags additional tags to apply in the braintrust UI
- * @param metadata additional key-value data to apply in the braintrust UI
- */
+/** Deprecated. Use {@link DatasetCase} instead */
+@Deprecated
 public record EvalCase<INPUT, OUTPUT>(
         INPUT input,
         OUTPUT expected,
         @Nonnull List<String> tags,
         @Nonnull Map<String, Object> metadata) {
+    /** Deprecated. Use {@link DatasetCase} instead */
+    @Deprecated
     public EvalCase {
         if (!metadata.isEmpty()) {
             throw new RuntimeException("TODO: metadata support not yet implemented");
@@ -26,10 +22,14 @@ public record EvalCase<INPUT, OUTPUT>(
         }
     }
 
+    /** Deprecated. Use {@link DatasetCase} instead */
+    @Deprecated
     public static <INPUT, OUTPUT> EvalCase<INPUT, OUTPUT> of(INPUT input, OUTPUT expected) {
         return of(input, expected, List.of(), Map.of());
     }
 
+    /** Deprecated. Use {@link DatasetCase} instead */
+    @Deprecated
     public static <INPUT, OUTPUT> EvalCase<INPUT, OUTPUT> of(
             INPUT input,
             OUTPUT expected,
@@ -38,5 +38,11 @@ public static <INPUT, OUTPUT> EvalCase<INPUT, OUTPUT> of(
         return new EvalCase<>(input, expected, tags, metadata);
     }
 
-    public record Result<INPUT, OUTPUT>(EvalCase<INPUT, OUTPUT> evalCase, OUTPUT result) {}
+    static <INPUT, OUTPUT> EvalCase<INPUT, OUTPUT> from(DatasetCase<INPUT, OUTPUT> datasetCase) {
+        return new EvalCase<>(
+                datasetCase.input(),
+                datasetCase.expected(),
+                datasetCase.tags(),
+                datasetCase.metadata());
+    }
 }
diff --git a/src/main/java/dev/braintrust/eval/EvalResult.java b/src/main/java/dev/braintrust/eval/EvalResult.java
new file mode 100644
index 00000000..70297d4e
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/EvalResult.java
@@ -0,0 +1,18 @@
+package dev.braintrust.eval;
+
+import lombok.Getter;
+import lombok.SneakyThrows;
+
+/** Results of all eval cases of an experiment. */
+public class EvalResult {
+    @Getter private final String experimentUrl;
+
+    @SneakyThrows
+    EvalResult(String experimentUrl) {
+        this.experimentUrl = experimentUrl;
+    }
+
+    public String createReportString() {
+        return "Experiment complete. View results in braintrust: %s".formatted(experimentUrl);
+    }
+}
diff --git a/src/main/java/dev/braintrust/eval/Score.java b/src/main/java/dev/braintrust/eval/Score.java
new file mode 100644
index 00000000..c79ee133
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/Score.java
@@ -0,0 +1,18 @@
+package dev.braintrust.eval;
+
+/** Individual metric value assigned by a scorer. */
+public record Score(
+        /**
+         * Name of the metric being scored.
+         *
+         * <p>This does not have to be the same as the scorer name, but it often will be.
+         */
+        String name,
+        /**
+         * Numeric representation of how well the task performed.
+         *
+         * <p>Must be between 0.0 (inclusive) and 1.0 (inclusive)
+         *
+         * <p>0 is completely incorrect. 1 is completely correct.
+         */
+        double value) {}
diff --git a/src/main/java/dev/braintrust/eval/Scorer.java b/src/main/java/dev/braintrust/eval/Scorer.java
index 189d5ed8..f28d6109 100644
--- a/src/main/java/dev/braintrust/eval/Scorer.java
+++ b/src/main/java/dev/braintrust/eval/Scorer.java
@@ -1,5 +1,6 @@
 package dev.braintrust.eval;
 
+import java.util.List;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
@@ -13,10 +14,10 @@
 public interface Scorer<INPUT, OUTPUT> {
     String getName();
 
-    double score(EvalCase<INPUT, OUTPUT> evalCase, OUTPUT result);
+    List<Score> score(TaskResult<INPUT, OUTPUT> taskResult);
 
     static <INPUT, OUTPUT> Scorer<INPUT, OUTPUT> of(
-            String scorerName, BiFunction<EvalCase<INPUT, OUTPUT>, OUTPUT, Double> scorerFn) {
+            String scorerName, Function<OUTPUT, Double> scorerFn) {
         return new Scorer<>() {
             @Override
             public String getName() {
@@ -24,14 +25,31 @@ public String getName() {
             }
 
             @Override
-            public double score(EvalCase<INPUT, OUTPUT> evalCase, OUTPUT result) {
-                return scorerFn.apply(evalCase, result);
+            public List<Score> score(TaskResult<INPUT, OUTPUT> taskResult) {
+                return List.of(new Score(scorerName, scorerFn.apply(taskResult.result())));
             }
         };
     }
 
-    static <INPUT, OUTPUT, RESULT> Scorer<INPUT, OUTPUT> of(
-            String scorerName, Function<OUTPUT, Double> scorerFn) {
-        return of(scorerName, (evalCase, result) -> scorerFn.apply(result));
+    /** Deprecated. Use {@link #of(String, Function)} or implement the Scorer interface instead. */
+    @Deprecated
+    static <INPUT, OUTPUT> Scorer<INPUT, OUTPUT> of(
+            String scorerName, BiFunction<EvalCase<INPUT, OUTPUT>, OUTPUT, Double> scorerFn) {
+        return new Scorer<>() {
+            @Override
+            public String getName() {
+                return scorerName;
+            }
+
+            @Override
+            public List<Score> score(TaskResult<INPUT, OUTPUT> taskResult) {
+                return List.of(
+                        new Score(
+                                scorerName,
+                                scorerFn.apply(
+                                        EvalCase.from(taskResult.datasetCase()),
+                                        taskResult.result())));
+            }
+        };
     }
 }
diff --git a/src/main/java/dev/braintrust/eval/Task.java b/src/main/java/dev/braintrust/eval/Task.java
index 0807eaeb..1611f0fd 100644
--- a/src/main/java/dev/braintrust/eval/Task.java
+++ b/src/main/java/dev/braintrust/eval/Task.java
@@ -7,5 +7,5 @@
  * @param <OUTPUT> type of the output data
  */
 public interface Task<INPUT, OUTPUT> {
-    OUTPUT apply(EvalCase<INPUT, OUTPUT> evalCase);
+    TaskResult<INPUT, OUTPUT> apply(DatasetCase<INPUT, OUTPUT> datasetCase);
 }
diff --git a/src/main/java/dev/braintrust/eval/TaskResult.java b/src/main/java/dev/braintrust/eval/TaskResult.java
new file mode 100644
index 00000000..55fe3e49
--- /dev/null
+++ b/src/main/java/dev/braintrust/eval/TaskResult.java
@@ -0,0 +1,8 @@
+package dev.braintrust.eval;
+
+/** Result from a single task run. */
+public record TaskResult<INPUT, OUTPUT>(
+        /** task output */
+        OUTPUT result,
+        /** The dataset case the task ran against to produce the result */
+        DatasetCase<INPUT, OUTPUT> datasetCase) {}
diff --git a/src/test/java/dev/braintrust/eval/EvalTest.java b/src/test/java/dev/braintrust/eval/EvalTest.java
index ca974162..6c7af7f9 100644
--- a/src/test/java/dev/braintrust/eval/EvalTest.java
+++ b/src/test/java/dev/braintrust/eval/EvalTest.java
@@ -6,6 +6,11 @@
 import dev.braintrust.api.BraintrustApiClient;
 import dev.braintrust.trace.BraintrustTracing;
 import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.SpanId;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import lombok.SneakyThrows;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -18,6 +23,7 @@ void beforeEach() {
     }
 
     @Test
+    @SneakyThrows
     public void evalOtelTraceWithProperAttributes() {
         var experimentName = "unit-test-eval";
 
@@ -27,9 +33,9 @@ public void evalOtelTraceWithProperAttributes() {
                         .<String, String>evalBuilder()
                         .name(experimentName)
                         .cases(
-                                EvalCase.of("strawberry", "fruit"),
-                                EvalCase.of("asparagus", "vegetable"))
-                        .task(food -> "fruit")
+                                DatasetCase.of("strawberry", "fruit"),
+                                DatasetCase.of("asparagus", "vegetable"))
+                        .taskFunction(food -> "fruit")
                         .scorers(
                                 Scorer.of(
                                         "fruit_scorer",
@@ -44,8 +50,6 @@ public void evalOtelTraceWithProperAttributes() {
                         .formatted(testHarness.braintrust().projectUri(), experimentName),
                 result.getExperimentUrl());
         var spans = testHarness.awaitExportedSpans();
-        assertEquals(6, spans.size(), "each eval case should make three spans");
-        // TODO: assert each case makes the expected spans
         var experiment =
                 testHarness
                         .braintrust()
@@ -53,15 +57,46 @@ public void evalOtelTraceWithProperAttributes() {
                         .getOrCreateExperiment(
                                 new BraintrustApiClient.CreateExperimentRequest(
                                         TestHarness.defaultProjectId(), experimentName));
-        spans.forEach(
-                span -> {
-                    var parent =
-                            span.getAttributes()
-                                    .get(AttributeKey.stringKey(BraintrustTracing.PARENT_KEY));
-                    assertEquals(
-                            "experiment_id:" + experiment.id(),
-                            parent,
-                            "all eval spans must set the parent to the experiment id");
-                });
+        final AtomicInteger numRootSpans = new AtomicInteger(0);
+        for (SpanData span : spans) {
+            var parent =
+                    span.getAttributes().get(AttributeKey.stringKey(BraintrustTracing.PARENT_KEY));
+            assertEquals(
+                    "experiment_id:" + experiment.id(),
+                    parent,
+                    "all eval spans must set the parent to the experiment id");
+            if (span.getParentSpanId().equals(SpanId.getInvalid())) {
+                numRootSpans.incrementAndGet();
+                var inputJson =
+                        Eval.JSON_MAPPER.readValue(
+                                span.getAttributes()
+                                        .get(AttributeKey.stringKey("braintrust.input_json")),
+                                Map.class);
+                assertNotNull(inputJson.get("input"), "invlaid input: " + inputJson);
+
+                var expected =
+                        Eval.JSON_MAPPER.readValue(
+                                span.getAttributes()
+                                        .get(AttributeKey.stringKey("braintrust.expected")),
+                                String.class);
+                assertTrue(isFruitOrVegetable(expected), "invalid expected: " + expected);
+
+                var outputJson =
+                        Eval.JSON_MAPPER.readValue(
+                                span.getAttributes()
+                                        .get(AttributeKey.stringKey("braintrust.output_json")),
+                                Map.class);
+                var output = outputJson.get("output");
+                assertNotNull(output, "invlaid output: " + outputJson);
+                assertTrue(isFruitOrVegetable(String.valueOf(output)), "invalid output: " + output);
+            }
+        }
+        assertEquals(2, numRootSpans.get(), "each case should make a root span");
+        assertEquals(
+                numRootSpans.get() * 3, spans.size(), "each eval case should make three spans");
+    }
+
+    boolean isFruitOrVegetable(String str) {
+        return "vegetable".equals(str) || "fruit".equals(str);
     }
 }