Bound SDK event logs while preserving screenshot evidence

packages/evals/framework/harnesses/piAdapter.ts

@@ -133,12 +133,14 @@ function normalizeResult(value: unknown): {
     for (const block of value.content) {
       if (!isRecord(block)) continue;
       if (block.type === "text" && typeof block.text === "string") text.push(block.text);
-      if (
-        block.type === "image" &&
-        typeof block.data === "string" &&
-        typeof block.mimeType === "string"
-      ) {
-        images.push({ bytes: Buffer.from(block.data, "base64"), mediaType: block.mimeType });
+      if (block.type === "image" && typeof block.mimeType === "string") {
+        // The pi session decodes screenshots to a single Buffer when it retains
+        // the event; raw base64 only arrives from callers that bypass it.
+        if (Buffer.isBuffer(block.bytes)) {
+          images.push({ bytes: block.bytes, mediaType: block.mimeType });
+        } else if (typeof block.data === "string") {
+          images.push({ bytes: Buffer.from(block.data, "base64"), mediaType: block.mimeType });
+        }
       }
     }
   }

packages/evals/tests/framework/piScreenshotPipeline.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "vitest";
+import { compactPiEvent } from "@browserbasehq/stagehand-integrations-pi-sdk";
+import { piAdapter } from "../../framework/harnesses/piAdapter.js";
+
+describe("Pi screenshot evidence pipeline", () => {
+  it("keeps explicit omission evidence without re-decoding an over-budget screenshot", () => {
+    const event = compactPiEvent(
+      {
+        type: "tool_execution_end",
+        toolCallId: "oversize",
+        toolName: "screenshot",
+        result: { content: [{ type: "image", data: "AAAA", mimeType: "image/png" }] },
+      },
+      { remainingBytes: 0 },
+    );
+    const trajectory = piAdapter.fromHarnessResult(
+      { events: [event], finalAnswer: "done" },
+      { id: "omitted-image", instruction: "Capture a screenshot." },
+    );
+    const modalities = trajectory.steps[0].agentEvidence.modalities;
+    expect(modalities.filter((modality) => modality.type === "image")).toEqual([]);
+    expect(JSON.stringify(modalities)).toContain("Screenshot omitted");
+    expect(JSON.stringify(event)).not.toContain("AAAA");
+  });
+  it.each([false, true])("retains the screenshot after SDK compaction=%s", (compact) => {
+    const png = Buffer.from(
+      "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aZu0AAAAASUVORK5CYII=",
+      "base64",
+    );
+    const event = {
+      type: "tool_execution_end",
+      toolCallId: "screenshot-1",
+      toolName: "mcp__stagehand__screenshot",
+      result: {
+        content: [
+          { type: "text", text: "Screenshot captured." },
+          { type: "image", data: png.toString("base64"), mimeType: "image/png" },
+        ],
+      },
+      isError: false,
+    };
+
+    const trajectory = piAdapter.fromHarnessResult(
+      { events: [compact ? compactPiEvent(event) : event], finalAnswer: "done" },
+      {
+        id: "pi-screenshot-pipeline",
+        instruction: "Capture a screenshot.",
+        initUrl: "https://example.invalid",
+      },
+    );
+
+    expect(trajectory.steps).toHaveLength(1);
+    const images = trajectory.steps[0].agentEvidence.modalities.filter(
+      (modality) => modality.type === "image",
+    );
+    expect(images).toHaveLength(1);
+    expect(images[0]).toMatchObject({ bytes: png, mediaType: "image/png" });
+    expect(event.result.content[1].data).toBe(png.toString("base64"));
+  });
+});

packages/integrations/claude-agent-sdk/src/session.ts

@@ -1,5 +1,6 @@
 import {
   HarnessAdapterError,
+  harnessEventLogLevel,
   sanitizeErrorMessage,
   type HarnessLogger,
 } from "@browserbasehq/stagehand-integrations/harness";
@@ -115,7 +116,11 @@ export async function runClaudeAgentSession(input: {
         permissionMode: input.session.permissionMode ?? "default",
         settingSources: input.session.settingSources ?? [],
         stderr: (data: string) => {
-          input.logger.log({ category: "claude_code", message: data, level: 1 });
+          input.logger.log({
+            category: "claude_code",
+            message: data,
+            level: /\b(?:error|fatal|failed)\b/iu.test(data) ? 1 : 2,
+          });
         },
         ...(systemPrompt !== undefined && { systemPrompt }),
       },
@@ -277,11 +282,25 @@ export function buildClaudeCodeTranscript(messages: ClaudeSdkMessage[]): string
 }
 
 export function logClaudeCodeMessage(logger: HarnessLogger, message: ClaudeSdkMessage): void {
+  const type = String(message.type ?? "unknown");
+  const level = harnessEventLogLevel(type, {
+    isError:
+      (type === "result" && message.subtype !== undefined && message.subtype !== "success") ||
+      message.is_error === true ||
+      (type === "user" &&
+        isRecord(message.message) &&
+        Array.isArray(message.message.content) &&
+        message.message.content.some(
+          (block) => isRecord(block) && block.type === "tool_result" && block.is_error === true,
+        )),
+    hasContent: type === "assistant" || type === "user" || type === "result",
+  });
+  if (level === undefined) return;
   const summary = summarizeClaudeCodeMessage(message);
   logger.log({
     category: "claude_code",
     message: summary.message,
-    level: 1,
+    level,
     auxiliary: {
       type: { value: String(message.type ?? "unknown"), type: "string" },
       ...(summary.detail && { detail: { value: summary.detail, type: "string" } }),

packages/integrations/claude-agent-sdk/tests/eventLog.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+import { logClaudeCodeMessage } from "../src/index.js";
+
+function recordingLogger() {
+  const lines: Array<{ level?: number; message: string }> = [];
+  const push = (line: { level?: number; message: string }) => void lines.push(line);
+  return { lines, logger: { log: push, warn: push, error: push } };
+}
+
+describe("claude_code event log levels", () => {
+  it("drops partial stream events and demotes routine messages to debug", () => {
+    const { lines, logger } = recordingLogger();
+    logClaudeCodeMessage(logger, { type: "stream_event", event: { type: "content_block_delta" } });
+    expect(lines).toEqual([]);
+
+    logClaudeCodeMessage(logger, { type: "system", subtype: "init" });
+    logClaudeCodeMessage(logger, {
+      type: "assistant",
+      message: { content: [{ type: "text", text: "hello" }] },
+    });
+    logClaudeCodeMessage(logger, { type: "result", subtype: "success", result: "done" });
+    expect(lines.map((line) => [line.level, line.message])).toEqual([
+      [2, "system message"],
+      [2, "assistant: hello"],
+      [2, "result: success"],
+    ]);
+  });
+
+  it("keeps failures visible", () => {
+    const { lines, logger } = recordingLogger();
+    logClaudeCodeMessage(logger, { type: "result", subtype: "error_max_turns", is_error: true });
+    expect(lines).toEqual([
+      expect.objectContaining({ level: 1, message: "result: error_max_turns" }),
+    ]);
+  });
+  it.each([false, true])("classifies nested tool_result is_error=%s", (isError) => {
+    const { lines, logger } = recordingLogger();
+    logClaudeCodeMessage(logger, {
+      type: "user",
+      message: {
+        content: [
+          { type: "tool_result", tool_use_id: "fixture", is_error: isError, content: "result" },
+        ],
+      },
+    });
+    expect(lines[0].level).toBe(isError ? 1 : 2);
+  });
+});

packages/integrations/codex-sdk/src/session.ts

@@ -1,5 +1,6 @@
 import {
   HarnessAdapterError,
+  harnessEventLogLevel,
   sanitizeErrorMessage,
   type HarnessLogger,
 } from "@browserbasehq/stagehand-integrations/harness";
@@ -256,11 +257,26 @@ export function buildCodexTranscript(events: CodexEvent[]): string {
 }
 
 export function logCodexEvent(logger: HarnessLogger, event: CodexEvent): void {
+  const type = String(event.type ?? "unknown");
+  const item = isRecord(event.item) ? event.item : undefined;
+  const level = harnessEventLogLevel(type, {
+    isError:
+      type === "turn.failed" ||
+      type === "error" ||
+      item?.type === "error" ||
+      (type === "item.completed" &&
+        (item?.status === "failed" ||
+          (item?.type === "command_execution" &&
+            typeof item.exit_code === "number" &&
+            item.exit_code !== 0))),
+    hasContent: type === "item.completed" || type === "turn.completed",
+  });
+  if (level === undefined) return;
   const summary = summarizeCodexEvent(event);
   logger.log({
     category: "codex",
     message: summary.message,
-    level: 1,
+    level,
     auxiliary: {
       type: { value: String(event.type ?? "unknown"), type: "string" },
       ...(summary.detail && { detail: { value: summary.detail, type: "string" } }),

packages/integrations/codex-sdk/tests/eventLog.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { logCodexEvent } from "../src/index.js";
+
+function recordingLogger() {
+  const lines: Array<{ level?: number; message: string }> = [];
+  const push = (line: { level?: number; message: string }) => void lines.push(line);
+  return { lines, logger: { log: push, warn: push, error: push } };
+}
+
+describe("codex event log levels", () => {
+  it("drops item updates and bare lifecycle starts, demotes completed items to debug", () => {
+    const { lines, logger } = recordingLogger();
+    logCodexEvent(logger, { type: "item.updated", item: { type: "agent_message", text: "par" } });
+    logCodexEvent(logger, { type: "thread.started", thread_id: "t1" });
+    logCodexEvent(logger, { type: "turn.started" });
+    logCodexEvent(logger, {
+      type: "item.started",
+      item: { type: "mcp_tool_call", server: "stagehand", tool: "run" },
+    });
+    expect(lines).toEqual([]);
+
+    logCodexEvent(logger, {
+      type: "item.completed",
+      item: { type: "mcp_tool_call", server: "stagehand", tool: "run", status: "completed" },
+    });
+    logCodexEvent(logger, { type: "turn.completed", usage: { input_tokens: 1 } });
+    expect(lines.map((line) => [line.level, line.message])).toEqual([
+      [2, "mcp: stagehand.run completed"],
+      [2, "turn completed"],
+    ]);
+  });
+
+  it("keeps failures visible", () => {
+    const { lines, logger } = recordingLogger();
+    logCodexEvent(logger, { type: "turn.failed", error: { message: "boom" } });
+    logCodexEvent(logger, {
+      type: "item.completed",
+      item: { type: "mcp_tool_call", server: "stagehand", tool: "run", status: "failed" },
+    });
+    logCodexEvent(logger, { type: "item.completed", item: { type: "error", message: "bad" } });
+    expect(lines.map((line) => line.level)).toEqual([1, 1, 1]);
+  });
+  it.each([0, 1, -1])("classifies completed command exit_code=%s", (exitCode) => {
+    const { lines, logger } = recordingLogger();
+    logCodexEvent(logger, {
+      type: "item.completed",
+      item: {
+        type: "command_execution",
+        status: "completed",
+        command: "fixture",
+        exit_code: exitCode,
+      },
+    });
+    expect(lines[0].level).toBe(exitCode === 0 ? 2 : 1);
+  });
+});

packages/integrations/core/src/harness/eventLog.ts

@@ -0,0 +1,43 @@
+/** Log level a raw SDK event should be recorded at. */
+export type HarnessEventLogLevel = 0 | 1 | 2;
+
+export interface HarnessEventClassification {
+  /** The event reports a failure the operator should see without debug output. */
+  isError?: boolean;
+  /**
+   * The event carries complete content of its own (a finished message, a tool
+   * result, usage). Lifecycle boundary events without content are dropped.
+   */
+  hasContent?: boolean;
+}
+
+/**
+ * Stream fragments — deltas, partial updates, chunks — that only accumulate
+ * into a later completed event and never carry standalone content.
+ */
+export function isStreamDeltaEventType(type: string): boolean {
+  return (
+    type === "stream_event" || /(?:^|[-_.:])(?:delta|update|updated|partial|chunk)$/iu.test(type)
+  );
+}
+
+/** Lifecycle markers (`*-start`, `*.started`, `*_end`, ...) that bracket other events. */
+export function isLifecycleBoundaryEventType(type: string): boolean {
+  return /(?:^|[-_.:])(?:start|started|begin|end|ended|stop|stopped)$/iu.test(type);
+}
+
+/**
+ * Decide how a raw SDK event lands in the harness log. The readable per-step
+ * trace is emitted from the normalized trajectory after the run, so raw events
+ * are debug material: errors stay visible at level 1, pure stream noise is
+ * dropped (`undefined`), and everything else is kept at level 2.
+ */
+export function harnessEventLogLevel(
+  type: string,
+  classification: HarnessEventClassification = {},
+): HarnessEventLogLevel | undefined {
+  if (classification.isError) return 1;
+  if (isStreamDeltaEventType(type)) return undefined;
+  if (isLifecycleBoundaryEventType(type) && !classification.hasContent) return undefined;
+  return 2;
+}

packages/integrations/core/src/harness/index.ts

@@ -1,3 +1,4 @@
 export * from "./contract.js";
 export * from "./env.js";
+export * from "./eventLog.js";
 export * from "./redact.js";

packages/integrations/core/tests/event-log.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vitest";
+import {
+  harnessEventLogLevel,
+  isLifecycleBoundaryEventType,
+  isStreamDeltaEventType,
+} from "../src/harness/eventLog.js";
+
+describe("harness event log levels", () => {
+  it("recognizes stream fragments across SDK vocabularies", () => {
+    for (const type of [
+      "text-delta",
+      "reasoning-delta",
+      "tool-call-delta",
+      "message_update",
+      "tool_execution_update",
+      "item.updated",
+      "message.delta",
+      "stream_event",
+      "content_block_delta",
+    ]) {
+      expect(isStreamDeltaEventType(type), type).toBe(true);
+    }
+    expect(isStreamDeltaEventType("tool-call")).toBe(false);
+    expect(isStreamDeltaEventType("message_end")).toBe(false);
+  });
+
+  it("recognizes lifecycle boundaries", () => {
+    for (const type of ["reasoning-start", "turn.started", "message_end", "step-start"]) {
+      expect(isLifecycleBoundaryEventType(type), type).toBe(true);
+    }
+    expect(isLifecycleBoundaryEventType("step-finish")).toBe(false);
+    expect(isLifecycleBoundaryEventType("turn.completed")).toBe(false);
+  });
+
+  it("drops noise, demotes the rest, and keeps errors at level 1", () => {
+    expect(harnessEventLogLevel("text-delta")).toBeUndefined();
+    expect(harnessEventLogLevel("reasoning-start")).toBeUndefined();
+    expect(harnessEventLogLevel("message_end", { hasContent: true })).toBe(2);
+    expect(harnessEventLogLevel("tool-call")).toBe(2);
+    expect(harnessEventLogLevel("text-delta", { isError: true })).toBe(1);
+    expect(harnessEventLogLevel("turn.failed", { isError: true })).toBe(1);
+  });
+});

packages/integrations/deepagents-sdk/src/session.ts

@@ -3,6 +3,7 @@ import { fileURLToPath } from "node:url";
 import path from "node:path";
 import { createInterface } from "node:readline";
 import {
+  harnessEventLogLevel,
   sanitizeErrorMessage,
   type HarnessLogger,
 } from "@browserbasehq/stagehand-integrations/harness";
@@ -366,11 +367,17 @@ export function buildDeepagentsTranscript(events: DeepagentsEvent[]): string {
 }
 
 export function logDeepagentsEvent(logger: HarnessLogger, event: DeepagentsEvent): void {
+  const type = String(event.type ?? "unknown");
+  const level = harnessEventLogLevel(type, {
+    isError: type === "error" || (type === "tool_result" && event.ok === false),
+    hasContent: type === "assistant" || type === "tool_result" || type === "final",
+  });
+  if (level === undefined) return;
   const summary = summarizeDeepagentsEvent(event);
   logger.log({
     category: "deepagents",
     message: sanitizeErrorMessage(summary.message),
-    level: 1,
+    level,
     auxiliary: {
       type: { value: String(event.type ?? "unknown"), type: "string" },
       ...(summary.detail && {