fix(dashboard-agent): stop a retrieved run's own error counting as a failed tool call in evals

Author: kathiekiwi

internal-packages/dashboard-agent/src/eval-error-category.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
 import { extractToolActivity, unfoldEvalToolOutput } from "./dashboard-agent";
 import { classifyEvalError, redactEvalToolValue } from "./eval-policy";
 import { toolResultErrored } from "./eval-turn";
+import { curateRun } from "./tool-curation";
 
 type Messages = Parameters<typeof extractToolActivity>[0];
 
@@ -135,6 +136,32 @@ describe("the derived error category", () => {
     expect(redacted).not.toHaveProperty("errorCategory");
   });
 
+  it("does not label a run that simply carries an error field", () => {
+    // `curateRun` always emits the key, undefined when the run succeeded, and a run that
+    // failed is still a tool call that worked.
+    const succeeded = redactEvalToolValue(curateRun({ id: "run_1", status: "COMPLETED" })) as Record<
+      string,
+      unknown
+    >;
+    expect(succeeded).not.toHaveProperty("errorCategory");
+    expect(toolResultErrored(succeeded)).toBe(false);
+
+    const failed = redactEvalToolValue(
+      curateRun({
+        id: "run_2",
+        status: "FAILED",
+        error: { name: "TimeoutError", message: "the task timed out" },
+      })
+    ) as Record<string, unknown>;
+    expect(failed).not.toHaveProperty("errorCategory");
+    expect(toolResultErrored(failed)).toBe(false);
+  });
+
+  it("still reports a returned tool failure after redaction", () => {
+    const redacted = redactEvalToolValue({ error: "Couldn't get run run_1 (status 500)." });
+    expect(toolResultErrored(redacted)).toBe(true);
+  });
+
   it("ignores a tool's own errorCategory field", () => {
     const redacted = redactEvalToolValue({
       isError: true,

internal-packages/dashboard-agent/src/eval-policy.ts

@@ -253,10 +253,34 @@ export const EVAL_ERROR_CATEGORIES = [
 
 export type EvalErrorCategory = (typeof EVAL_ERROR_CATEGORIES)[number];
 
-/** True when an output is a tool failure: unfolded (`isError`) or a plain `error` field. */
+/**
+ * True when an output is the tool's own failure: unfolded (`isError`), or the `error`
+ * message string every tool returns when it gives up.
+ *
+ * The key alone is not the signal. `curateRun` and `curateDeployment` always emit
+ * `error`, holding the subject's own failure or nothing at all, and a run that failed is
+ * still a tool call that worked.
+ */
 export function evalOutputErrored(output: unknown): boolean {
   if (output === null || typeof output !== "object" || Array.isArray(output)) return false;
-  return (output as { isError?: unknown }).isError === true || "error" in output;
+  const fields = output as { isError?: unknown; error?: unknown };
+  return fields.isError === true || typeof fields.error === "string";
+}
+
+/**
+ * The same question asked of an output that has already been through redaction, where the
+ * message is gone and {@link annotateEvalErrorCategory} has left the derived label behind.
+ * A tool's own `errorCategory` cannot be mistaken for it: redaction turns any field we
+ * don't write ourselves into a shape descriptor.
+ */
+export function redactedEvalOutputErrored(output: unknown): boolean {
+  if (output === null || typeof output !== "object" || Array.isArray(output)) return false;
+  const fields = output as { isError?: unknown; errorCategory?: unknown };
+  if (fields.isError === true) return true;
+  return (
+    typeof fields.errorCategory === "string" &&
+    (EVAL_ERROR_CATEGORIES as readonly string[]).includes(fields.errorCategory)
+  );
 }
 
 /**

internal-packages/dashboard-agent/src/eval-redaction.test.ts

@@ -7,7 +7,7 @@ import {
   truncateEvalToolValue,
   unfoldEvalToolOutput,
 } from "./dashboard-agent";
-import { allowedEvalKeys, redactEvalToolValue } from "./eval-policy";
+import { allowedEvalKeys, evalOutputErrored, redactEvalToolValue } from "./eval-policy";
 import { toolResultErrored } from "./eval-turn";
 
 type Messages = Parameters<typeof extractToolActivity>[0];
@@ -185,8 +185,17 @@ describe("an errored tool result", () => {
     expect(toolResultErrored(activity[0]!.output)).toBe(true);
   });
 
-  it("reads a plain error field as an error too", () => {
-    expect(toolResultErrored({ error: { name: "TimeoutError" } })).toBe(true);
+  it("reads a tool's own error message as an error too", () => {
+    expect(evalOutputErrored({ error: "Couldn't get run run_1 (status 500)." })).toBe(true);
+    // The subject's failure, not the call's: a run that failed was still retrieved.
+    expect(evalOutputErrored({ error: { name: "TimeoutError" } })).toBe(false);
+    expect(evalOutputErrored({ id: "run_1" })).toBe(false);
+    expect(evalOutputErrored("boom")).toBe(false);
+  });
+
+  it("reads the derived category once the message is gone", () => {
+    expect(toolResultErrored({ errorCategory: "timeout", error: { redacted: "error" } })).toBe(true);
+    expect(toolResultErrored({ error: { name: "TimeoutError" } })).toBe(false);
     expect(toolResultErrored({ id: "run_1" })).toBe(false);
     expect(toolResultErrored("boom")).toBe(false);
   });

internal-packages/dashboard-agent/src/eval-turn.ts

@@ -5,7 +5,7 @@ import {
   type DashboardAgentDbClient,
 } from "@internal/dashboard-agent-db";
 import { logger, task } from "@trigger.dev/sdk";
-import { EVAL_ERROR_CATEGORIES, evalOutputErrored } from "./eval-policy";
+import { EVAL_ERROR_CATEGORIES, redactedEvalOutputErrored } from "./eval-policy";
 import { generateObject } from "ai";
 import { z } from "zod";
 
@@ -131,9 +131,9 @@ const TurnEval = z.object({
   summary: z.string().describe("One line: what the user asked and how it went."),
 });
 
-/** An errored result reaches here unfolded (`isError`) or as a plain `error` field. */
+/** A payload's outputs are redacted, so the derived category is what marks a failure. */
 export function toolResultErrored(output: unknown): boolean {
-  return evalOutputErrored(output);
+  return redactedEvalOutputErrored(output);
 }
 
 const JUDGE_SYSTEM = [