feat(eval): add go action runtime

README.md

@@ -18,12 +18,12 @@ You can configure the following variables:
 - `api_key`: Your
   [Braintrust API key](https://www.braintrust.dev/app/settings/api-keys).
 - `root`: The root directory containing your evals (defaults to `'.'`). The root
-  directory must either have `node` or `python` configured.
+  directory must have `node`, `python`, or `go` configured.
 - `paths`: Specific paths, relative to the root, containing evals you'd like to
   run.
-- `runtime`: Either `node` or `python`
-- `package_manager`: Either `npm`, `pnpm`, or `yarn` for a `node` runtime, or
-  `pip` or `uv` for a `python` runtime.
+- `runtime`: Either `node`, `python`, or `go`
+- `package_manager`: Either `npm` or `pnpm` for a `node` runtime, `pip` or `uv`
+  for a `python` runtime, or `go` for a `go` runtime. You can omit this for Go.
 - `use_proxy`: Either `true` or `false`. If set, `OPENAI_BASE_URL` will be set
   to `https://braintrustproxy.com/v1`, which will automatically cache repetitive
   LLM calls and run your evals faster. Defaults to `true`.
@@ -88,11 +88,32 @@ To see examples of fully configured templates, see the `examples` directory:
 - [`node with pnpm`](examples/node/pnpm.yml)
 - [`python with pip`](examples/python/pip.yml)
 - [`python with uv`](examples/python/uv.yml)
+- [`go`](examples/go/go.yml)
+
+## Go evals
+
+The Go runtime executes `go run ${paths}`. To include Go eval results in the PR
+comment, print each `ExperimentSummary` as one JSON line after calling
+`result.Summarize(ctx)`:
+
+```go
+summary, err := result.Summarize(ctx)
+if err != nil {
+    log.Fatal(err)
+}
+b, err := json.Marshal(summary)
+if err != nil {
+    log.Fatal(err)
+}
+fmt.Println(string(b))
+```
 
 ## How it works
 
-The action runs `braintrust eval` and collects experiment results, which are
-posted as a comment in the PR alongside a link to Braintrust. For example:
+For Node and Python, the action runs `braintrust eval`. For Go, the action runs
+`go run` on `paths` from `root`. It collects experiment results emitted as JSONL
+and posts them as a comment in the PR alongside a link to Braintrust. For
+example:
 
 ### Example braintrust eval report
 

action.yml

@@ -21,12 +21,12 @@ inputs:
     required: false
     default: "."
   runtime:
-    description: "The runtime to use for evals. Valid values: node, python."
+    description: "The runtime to use for evals. Valid values: node, python, go."
     required: true
   package_manager:
     description:
-      "The package manager to use for evals. Valid values: npm, pnpm, yarn, pip,
-      or uv depending on the runtime."
+      "The package manager to use for evals. Valid values: npm or pnpm for node,
+      pip or uv for python, or go for Go."
     required: false
     default: ""
   use_proxy:

eval/src/braintrust.ts

@@ -16,42 +16,103 @@ function snakeToCamelCase(str: string) {
   return str.replace(/([-_][a-z])/g, group => group.charAt(1).toUpperCase());
 }
 
+const summaryKeyMap: Record<string, string> = {
+  ProjectName: "projectName",
+  ExperimentName: "experimentName",
+  ProjectID: "projectId",
+  projectID: "projectId",
+  ExperimentID: "experimentId",
+  experimentID: "experimentId",
+  ProjectURL: "projectUrl",
+  projectURL: "projectUrl",
+  ExperimentURL: "experimentUrl",
+  experimentURL: "experimentUrl",
+  ComparisonExperimentName: "comparisonExperimentName",
+  Scores: "scores",
+  Metrics: "metrics",
+  EvaluatorName: "evaluatorName",
+  Errors: "errors",
+};
+
+function normalizeSummaryKeys(value: Record<string, unknown>) {
+  return Object.fromEntries(
+    Object.entries(value).map(([key, entry]) => [
+      summaryKeyMap[key] ?? snakeToCamelCase(key),
+      entry,
+    ]),
+  );
+}
+
+function parseSummaryLine(line: string) {
+  try {
+    const parsedLine = JSON.parse(line) as unknown;
+    if (
+      parsedLine === null ||
+      typeof parsedLine !== "object" ||
+      Array.isArray(parsedLine)
+    ) {
+      core.info(line);
+      return [];
+    }
+
+    // TODO: This is hacky and we should be parsing what comes off the wire.
+    // The JS/Python CLI emits snake_case JSONL while the Go SDK's
+    // ExperimentSummary marshals top-level fields as PascalCase.
+    const summary = normalizeSummaryKeys(parsedLine as Record<string, unknown>);
+    if (
+      ("errors" in summary && "evaluatorName" in summary) ||
+      ("experimentName" in summary &&
+        ("scores" in summary || "metrics" in summary))
+    ) {
+      return [summary as unknown as ExperimentSummary];
+    }
+
+    core.info(line);
+    return [];
+  } catch (e) {
+    if (line.startsWith("{") || line.startsWith("[")) {
+      core.error(`Failed to parse jsonl data: ${e}`);
+    } else {
+      core.info(line);
+    }
+    return [];
+  }
+}
+
 async function runCommand(command: string, onSummary: OnSummaryFn) {
   core.info(`> $ ${command}`);
   return new Promise((resolve, reject) => {
     const process = spawn(command, { shell: true });
+    let stdoutBuffer = "";
+
+    const handleStdoutLine = (line: string) => {
+      const trimmedLine = line.trim();
+      if (trimmedLine.length === 0) {
+        return;
+      }
+      const summaries = parseSummaryLine(trimmedLine);
+      if (summaries.length > 0) {
+        onSummary(summaries);
+      }
+    };
 
     process.stdout?.on("data", (data: Buffer) => {
-      onSummary(
-        data
-          .toString()
-          .split("\n")
-          .map(line => line.trim())
-          .filter(line => line.length > 0)
-          .flatMap(line => {
-            try {
-              const parsedLine = JSON.parse(line);
-              const camelCaseLine = Object.fromEntries(
-                Object.entries(parsedLine).map(([key, value]) => [
-                  snakeToCamelCase(key),
-                  value,
-                ]),
-              );
-              // TODO: This is hacky and we should be parsing what comes off the wire
-              return [camelCaseLine as unknown as ExperimentSummary];
-            } catch (e) {
-              core.error(`Failed to parse jsonl data: ${e}`);
-              return [];
-            }
-          }),
-      );
+      stdoutBuffer += data.toString();
+      const lines = stdoutBuffer.split("\n");
+      stdoutBuffer = lines.pop() ?? "";
+      lines.forEach(handleStdoutLine);
     });
 
     process.stderr?.on("data", (data: Buffer) => {
       core.info(data.toString()); // Outputs the stderr of the command
     });
 
     process.on("close", code => {
+      if (stdoutBuffer.length > 0) {
+        handleStdoutLine(stdoutBuffer);
+        stdoutBuffer = "";
+      }
+
       if (code === 0) {
         resolve(null);
       } else {
@@ -80,27 +141,48 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
 
   const terminateFlag = terminate_on_failure ? "--terminate-on-failure" : "";
 
-  const baseCommand = (() => {
+  const command = (() => {
     switch (args.runtime.toLowerCase().trim()) {
-      case "node":
-        switch (args.package_manager) {
-          case "":
-          case "npm":
-            return "npx braintrust";
-          case "pnpm":
-            return "pnpm dlx braintrust";
-          default:
-            throw new Error(
-              `Unsupported package manager: ${args.package_manager}`,
-            );
-        }
-      case "python":
+      case "node": {
+        const baseCommand = (() => {
+          switch (args.package_manager) {
+            case "":
+            case "npm":
+              return "npx braintrust";
+            case "pnpm":
+              return "pnpm dlx braintrust";
+            default:
+              throw new Error(
+                `Unsupported package manager: ${args.package_manager}`,
+              );
+          }
+        })();
+        return `${baseCommand} eval --jsonl ${terminateFlag} ${paths}`;
+      }
+      case "python": {
+        const baseCommand = (() => {
+          switch ((args.package_manager || "").toLowerCase().trim()) {
+            case "":
+            case "pip":
+              return `braintrust`;
+            case "uv":
+              return `uv run braintrust`;
+            default:
+              throw new Error(
+                `Unsupported package manager: ${args.package_manager}`,
+              );
+          }
+        })();
+        return `${baseCommand} eval --jsonl ${terminateFlag} ${paths}`;
+      }
+      case "go":
         switch ((args.package_manager || "").toLowerCase().trim()) {
           case "":
-          case "pip":
-            return `braintrust`;
-          case "uv":
-            return `uv run braintrust`;
+          case "go":
+            if (terminate_on_failure) {
+              core.info("Ignoring terminate_on_failure for Go evals");
+            }
+            return `go run ${paths}`;
           default:
             throw new Error(
               `Unsupported package manager: ${args.package_manager}`,
@@ -111,7 +193,5 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
     }
   })();
 
-  const command = `${baseCommand} eval --jsonl ${terminateFlag} ${paths}`;
-
   await runCommand(command, onSummary);
 }

eval/src/main.ts

@@ -7,6 +7,7 @@ import { z } from "zod";
 
 const nodeManagers = ["npm", "pnpm"];
 const pythonManagers = ["pip", "uv"];
+const goManagers = ["go"];
 const booleanInput = z.stringbool({ truthy: ["true"], falsy: ["false"] });
 
 function capitalize(value: string) {
@@ -18,9 +19,9 @@ const paramsSchema = z
     api_key: z.string(),
     root: z.string(),
     paths: z.string(),
-    runtime: z.enum(["node", "python"]),
+    runtime: z.enum(["node", "python", "go"]),
     package_manager: z
-      .enum(["", ...nodeManagers, ...pythonManagers])
+      .enum(["", ...nodeManagers, ...pythonManagers, ...goManagers])
       .describe("The preferred package manager for the runtime selected")
       .default(""),
     use_proxy: booleanInput,
@@ -37,6 +38,9 @@ const paramsSchema = z
       if (data.runtime === "python") {
         return pythonManagers.includes(data.package_manager as any);
       }
+      if (data.runtime === "go") {
+        return goManagers.includes(data.package_manager as any);
+      }
       return false;
     },
     {
@@ -157,7 +161,7 @@ function formatSummary(summary: ExperimentSummary) {
     .map((_, idx) => (idx > 1 ? "---:" : ":---"))
     .join(" | ");
 
-  const rowData = Object.entries(summary.scores)
+  const rowData = Object.entries(summary.scores ?? {})
     .map(([name, scoreSummary]) => {
       let diffText = "";
       if (scoreSummary.diff !== undefined) {

examples/go/go.yml

@@ -0,0 +1,42 @@
+name: Run Go evals
+
+on:
+  push:
+    # Uncomment to run only when files in the 'evals' directory change
+    # - paths:
+    #     - "evals/**"
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  eval:
+    name: Run evals
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        id: setup-go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      - name: Download Dependencies
+        id: install
+        working-directory: my_eval_dir
+        run: go mod download
+
+      - name: Run Evals
+        uses: braintrustdata/eval-action@v1
+        with:
+          api_key: ${{ secrets.BRAINTRUST_API_KEY }}
+          runtime: go
+          root: my_eval_dir
+          paths: .