feat(scoring): configurable judge timeout and retries before withholding

AGENTS.md

@@ -144,6 +144,8 @@ Two silent-failure modes used to leak a fabricated composite (goal 0 + default c
 - **Empty-work signature**: a run that produced no transcript AND no result. `hasEmptyOutput()` in `types/output.ts` defines it; `isFailedRun()` treats it as failed even on a clean exit; the base adapter (`agent-adapter.ts`) stamps `"Agent produced no output"` so the error column is descriptive.
 - **Judge death / unparseable judge response**: `callJudge()` throws `ScoringError` when the judge invocation failed or returned nothing; `goal-achievement.ts` and `deep-eval.ts` throw `ScoringError` when a judge response can't be parsed at all (a parsed-but-incomplete response still gets per-item defaults). `scoreRunResult()` catches `ScoringError`, stamps `"Score withheld: …"` on the run, and returns a zero/failed result. Non-`ScoringError` throws propagate (real bugs should fail loud).
 
+Before withholding, `callJudgeAndParse()` (`scoring/judge-retry.ts`) retries the judge call + parse up to `judging.retries` times (default 0) on `ScoringError` only; `judging.timeout_minutes` becomes `AgentInput.timeoutMs` for every judge call (default: the adapter's 10 min). It lives in its own module so tests that mock `judge.js`'s `callJudge` still drive it.
+
 `buildScoredOutput()` recomputes `completed`/`failed` from the scored results, so a run that scoring flips to failed is reflected in the summary and exit code.
 
 The withheld path also sets `score.withheld = true`. Multi-run aggregation depends on that flag to keep a judge outage out of the agent's reliability denominator, so a new withhold site must set it rather than relying on the stamped error message.

src/cli.ts

@@ -10,6 +10,7 @@ import { run, buildJobEnv } from "./runner/runner.js";
 import { runLifecyclePhase } from "./runner/lifecycle.js";
 import { loadConfig } from "./config/loader.js";
 import { scoreRunResult, buildScoredOutput } from "./scoring/index.js";
+import { judgeLimitsFromConfig } from "./scoring/judge.js";
 import { initReport, finalizeReport } from "./reports/writer.js";
 import { listReports, readReport, readScenarioResults } from "./reports/reader.js";
 import { setBaseline, readBaseline, listBaselines, deleteBaseline, DEFAULT_BASELINE_NAME } from "./baselines/store.js";
@@ -410,6 +411,7 @@ async function runPipelineBody(
   // AgentConfig. The list is preserved so the scoring layer can pick the
   // best-matching judge per run (skip the run's own agent when possible).
   const judging = config.judging?.agents.filter((a): a is AgentConfig => typeof a === "object");
+  const judgeLimits = judgeLimitsFromConfig(config.judging);
 
   let runOutput: RunOutput | undefined;
   let output: ScoredOutput | RunOutput | undefined;
@@ -441,6 +443,7 @@ async function runPipelineBody(
               logger,
               reportDir,
               judging,
+              judgeLimits,
               onProgress: (event) => {
                 if (event.phase === "start") onScoringStart?.(event);
               },
@@ -476,6 +479,7 @@ async function runPipelineBody(
               logger,
               reportDir,
               judging,
+              judgeLimits,
             }).catch((err) => {
               logger.error(`Scoring fallback failed for ${r.scenarioKey} (${r.agentName}): ${formatError(err)}`);
               return undefined;

src/config/validator.ts

@@ -140,6 +140,16 @@ function validateJudging(data: unknown, filePath: string): void {
     }
     throw new Error(`Invalid config at ${filePath}: "judging.agents[${i}]" must be a string or an AgentConfig object`);
   }
+  if (obj.timeout_minutes !== undefined) {
+    if (typeof obj.timeout_minutes !== "number" || !(obj.timeout_minutes > 0)) {
+      throw new Error(`Invalid config at ${filePath}: "judging.timeout_minutes" must be a positive number`);
+    }
+  }
+  if (obj.retries !== undefined) {
+    if (typeof obj.retries !== "number" || !Number.isInteger(obj.retries) || obj.retries < 0 || obj.retries > 5) {
+      throw new Error(`Invalid config at ${filePath}: "judging.retries" must be a whole number from 0 to 5`);
+    }
+  }
 }
 
 function validateScenariosField(data: unknown, filePath: string): void {

src/docs-site/src/pages/configuration.astro

@@ -264,7 +264,9 @@ export default async () => {
         <td><span class="req-no">No</span></td>
         <td>
           Precedence-ordered list of judge agents for scoring. See <a href="#judging">Judging
-          Agents</a>. When omitted, each run is judged by its own agent.
+          Agents</a>. When omitted, each run is judged by its own agent. Optional
+          <code>timeout_minutes</code> and <code>retries</code> (see
+          <a href="#judging-limits">Judge timeout and retries</a>).
         </td>
       </tr>
       <tr>
@@ -868,6 +870,47 @@ curl -X POST "$SLACK_WEBHOOK" \\
     ]
   }
 }`}</code></pre>
+  <h3 id="judging-limits">Judge timeout and retries</h3>
+  <p>
+    A judge that dies or replies with something AXIS can't parse
+    <a href="/scoring#withheld">withholds the run's score</a>. Two optional fields make that rarer:
+  </p>
+  <table class="table-fields">
+    <thead>
+      <tr>
+        <th>Field</th>
+        <th>Default</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td><code>timeout_minutes</code></td>
+        <td>adapter default (10)</td>
+        <td>
+          Time limit for each judge invocation. Raise it when judges verify large workspaces and
+          time out.
+        </td>
+      </tr>
+      <tr>
+        <td><code>retries</code></td>
+        <td><code>0</code></td>
+        <td>
+          Extra attempts (0–5) when a judge invocation fails (crash, timeout, rate limit) or its
+          reply can't be parsed. Each retry is a full judge call and costs as much as the first.
+          Retries are logged with <code>--verbose</code>; a score withheld after retries says how
+          many attempts were made.
+        </td>
+      </tr>
+    </tbody>
+  </table>
+  <pre><code>{`{
+  "judging": {
+    "agents": ["claude-code"],
+    "timeout_minutes": 15,
+    "retries": 1
+  }
+}`}</code></pre>
 
   <h2>Skills</h2>
   <p>

src/docs-site/src/pages/scoring.astro

@@ -518,6 +518,12 @@ import DocsLayout from "../layouts/DocsLayout.astro";
     Withholding keeps a transient infrastructure failure from polluting your gate: the run fails
     loudly and can be retried, instead of quietly dragging an average down with a fabricated number.
   </p>
+  <p>
+    To retry a dying or unparseable judge before withholding, and to give judges longer than the
+    adapter's 10-minute default, set <code>judging.retries</code> and
+    <code>judging.timeout_minutes</code> (see
+    <a href="/configuration#judging-limits">Judge timeout and retries</a>).
+  </p>
 
   <script is:inline>
     const wheel = document.querySelector(".dimension-wheel");

src/scoring/deep-eval.ts

@@ -1,7 +1,8 @@
 import type { NormalizedEntry, NormalizedTranscript } from "../transcript/types.js";
-import type { RunResult } from "../types/output.js";
+import type { Logger, RunResult } from "../types/output.js";
 import type { AgentConfig, ScoringWeights } from "../types/config.js";
 import type {
+  JudgeLimits,
   CategoryEvalResult,
   DeepEvalResult,
   EvalPattern,
@@ -12,7 +13,7 @@ import type {
   SparseIndex,
 } from "../types/scoring.js";
 import { DEFAULT_AUDIT_SCORES } from "./category-score.js";
-import { callJudge } from "./judge.js";
+import { callJudgeAndParse } from "./judge-retry.js";
 import { parseJsonFromText } from "./parse-json.js";
 import { ScoringError } from "./errors.js";
 import { CATEGORY_GUIDANCE, getPromptTemplates, interpolate } from "./prompt-templates.js";
@@ -37,6 +38,10 @@ export interface DeepEvalOptions {
    * name differs from the run's own agent (or falls back to the first entry).
    */
   judging?: AgentConfig[];
+  /** Timeout and retry limits for each judge call. */
+  judgeLimits?: JudgeLimits;
+  /** Receives a verbose line per retried judge attempt. */
+  logger?: Logger;
 }
 
 /**
@@ -72,7 +77,7 @@ export async function runDeepEval(
         return buildDefaultCategoryResult(category);
       }
 
-      return runCategoryEval(result, sparseIndex, normalized, category, options?.reportDir, options?.judging);
+      return runCategoryEval(result, sparseIndex, normalized, category, options);
     }),
   );
 
@@ -138,25 +143,30 @@ async function runCategoryEval(
   sparseIndex: SparseIndex,
   normalized: NormalizedTranscript,
   category: InteractionCategory,
-  reportDir?: string,
-  judging?: AgentConfig[],
+  options?: DeepEvalOptions,
 ): Promise<CategoryEvalResult> {
-  const prompt = buildCategoryEvalPrompt(result, sparseIndex, normalized, category, reportDir);
-  const responseText = await callJudge(result, prompt, {
-    scenarioKey: `__${category}_eval__`,
-    scenarioName: `AXIS ${category} Evaluation`,
-    judging,
-  });
-
-  // A response with no parseable JSON means the judge didn't grade anything.
-  // Withhold rather than silently defaulting every interaction to a high score
-  // (which would inflate the category and hide the failure). A parsed response
-  // that omits some interactions is fine; those get per-interaction defaults.
-  if (!parseJsonFromText(responseText, isEvalVerdict)) {
-    throw new ScoringError(`Could not parse ${category} evaluation judge response`);
-  }
-
-  return parseCategoryEvalResponse(responseText, category, sparseIndex);
+  const prompt = buildCategoryEvalPrompt(result, sparseIndex, normalized, category, options?.reportDir);
+  return callJudgeAndParse(
+    result,
+    prompt,
+    {
+      scenarioKey: `__${category}_eval__`,
+      scenarioName: `AXIS ${category} Evaluation`,
+      judging: options?.judging,
+      limits: options?.judgeLimits,
+      logger: options?.logger,
+    },
+    (responseText) => {
+      // A response with no parseable JSON means the judge didn't grade anything.
+      // Withhold rather than silently defaulting every interaction to a high score
+      // (which would inflate the category and hide the failure). A parsed response
+      // that omits some interactions is fine; those get per-interaction defaults.
+      if (!parseJsonFromText(responseText, isEvalVerdict)) {
+        throw new ScoringError(`Could not parse ${category} evaluation judge response`);
+      }
+      return parseCategoryEvalResponse(responseText, category, sparseIndex);
+    },
+  );
 }
 
 function buildCategoryEvalPrompt(

src/scoring/goal-achievement.ts

@@ -1,10 +1,10 @@
 import type { NormalizedEntry } from "../transcript/types.js";
 import type { JudgeCriterion } from "../types/scenario.js";
-import type { RunResult } from "../types/output.js";
+import type { Logger, RunResult } from "../types/output.js";
 import type { AgentMetadata } from "../types/agent.js";
 import type { AgentConfig } from "../types/config.js";
-import type { GoalAchievementScore, CriterionGrade } from "../types/scoring.js";
-import { callJudge } from "./judge.js";
+import type { GoalAchievementScore, CriterionGrade, JudgeLimits } from "../types/scoring.js";
+import { callJudgeAndParse } from "./judge-retry.js";
 import { parseJsonFromText } from "./parse-json.js";
 import { ScoringError } from "./errors.js";
 import { getPromptTemplates, interpolate } from "./prompt-templates.js";
@@ -13,19 +13,20 @@ export async function scoreGoalAchievement(
   result: RunResult,
   normalizedEntries: NormalizedEntry[],
   judging?: AgentConfig[],
+  judgeOptions: { limits?: JudgeLimits; logger?: Logger } = {},
 ): Promise<GoalAchievementScore> {
   const { judge } = result;
   const { result: finalResult } = result.output;
 
   if (typeof judge === "string") {
-    return scoreStringJudge(result, judge, normalizedEntries, finalResult, judging);
+    return scoreStringJudge(result, judge, normalizedEntries, finalResult, judging, judgeOptions);
   }
 
   if (!judge || judge.length === 0) {
     return { score: 0, criteria: [] };
   }
 
-  return scoreArrayJudge(result, judge, normalizedEntries, finalResult, judging);
+  return scoreArrayJudge(result, judge, normalizedEntries, finalResult, judging, judgeOptions);
 }
 
 async function scoreStringJudge(
@@ -34,20 +35,23 @@ async function scoreStringJudge(
   entries: NormalizedEntry[],
   finalResult: string | null,
   judging: AgentConfig[] | undefined,
+  judgeOptions: { limits?: JudgeLimits; logger?: Logger },
 ): Promise<GoalAchievementScore> {
   const prompt = buildStringJudgePrompt(runResult, entries, finalResult, judge);
-  const responseText = await callJudge(runResult, prompt, {
-    scenarioKey: "__judge__",
-    scenarioName: "AXIS Judge",
-    judging,
-  });
-
-  const parsed = parseJsonFromText(responseText, (c) => typeof c.score === "number");
-  if (!parsed || typeof parsed.score !== "number") {
-    // The judge produced something we can't grade against. Withhold rather than
-    // fabricate a zero that would look like a genuine failure to meet the goal.
-    throw new ScoringError("Could not parse goal-achievement judge response");
-  }
+  const parsed = await callJudgeAndParse(
+    runResult,
+    prompt,
+    { scenarioKey: "__judge__", scenarioName: "AXIS Judge", judging, ...judgeOptions },
+    (responseText) => {
+      const reply = parseJsonFromText(responseText, (c) => typeof c.score === "number");
+      if (!reply || typeof reply.score !== "number") {
+        // The judge produced something we can't grade against. Withhold rather than
+        // fabricate a zero that would look like a genuine failure to meet the goal.
+        throw new ScoringError("Could not parse goal-achievement judge response");
+      }
+      return reply as typeof reply & { score: number };
+    },
+  );
 
   const score = Math.max(0, Math.min(10, Math.round(parsed.score)));
   return {
@@ -69,15 +73,15 @@ async function scoreArrayJudge(
   entries: NormalizedEntry[],
   finalResult: string | null,
   judging: AgentConfig[] | undefined,
+  judgeOptions: { limits?: JudgeLimits; logger?: Logger },
 ): Promise<GoalAchievementScore> {
   const prompt = buildArrayJudgePrompt(runResult, entries, finalResult, judge);
-  const responseText = await callJudge(runResult, prompt, {
-    scenarioKey: "__judge__",
-    scenarioName: "AXIS Judge",
-    judging,
-  });
-
-  const criteria = parseArrayJudgeResponse(responseText, judge);
+  const criteria = await callJudgeAndParse(
+    runResult,
+    prompt,
+    { scenarioKey: "__judge__", scenarioName: "AXIS Judge", judging, ...judgeOptions },
+    (responseText) => parseArrayJudgeResponse(responseText, judge),
+  );
   const score = computeWeightedScore(criteria);
 
   return { score, criteria };

src/scoring/index.ts

@@ -107,8 +107,10 @@ export async function scoreRunResult(result: RunResult, options?: ScoringOptions
         weights,
         reportDir: options?.reportDir,
         judging: resolvedJudging,
+        judgeLimits: options?.judgeLimits,
+        logger,
       }),
-      scoreGoalAchievement(result, normalized.entries, resolvedJudging),
+      scoreGoalAchievement(result, normalized.entries, resolvedJudging, { limits: options?.judgeLimits, logger }),
     ]);
 
     // Step 5: Compute category scores

src/scoring/judge-retry.ts

@@ -0,0 +1,33 @@
+import { callJudge, type JudgeCallOptions } from "./judge.js";
+import { ScoringError } from "./errors.js";
+import type { RunResult } from "../types/output.js";
+
+/**
+ * Call the judge and parse its reply, retrying up to `options.limits.retries`
+ * times when the score would otherwise be withheld: the invocation failed or
+ * `parse` rejected the reply (both surface as {@link ScoringError}). Any other
+ * error is a bug and propagates immediately. The final failure keeps its
+ * message, with the attempt count appended when retries were made.
+ */
+export async function callJudgeAndParse<T>(
+  runResult: RunResult,
+  prompt: string,
+  options: JudgeCallOptions,
+  parse: (responseText: string) => T,
+): Promise<T> {
+  const attempts = 1 + Math.max(0, Math.floor(options.limits?.retries ?? 0));
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return parse(await callJudge(runResult, prompt, options));
+    } catch (err) {
+      if (!(err instanceof ScoringError)) throw err;
+      if (attempt >= attempts) {
+        throw attempts > 1 ? new ScoringError(`${err.message} (after ${attempts} attempts)`) : err;
+      }
+      options.logger?.verbose?.(
+        `Retrying ${options.scenarioName} for ${runResult.scenarioKey} (${runResult.agentName}), ` +
+          `attempt ${attempt + 1}/${attempts}: ${err.message}`,
+      );
+    }
+  }
+}