diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts index 02fdc5be8..d0390ddb5 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts @@ -4,7 +4,7 @@ import { createUnsupportedIssue, mswServer, } from "@sentry/mcp-server-mocks"; -import { http, HttpResponse } from "msw"; +import { http, HttpResponse, type JsonBodyType } from "msw"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { formatErrorForUser } from "../../internal/error-handling.js"; import analyzeIssueWithSeer from "./analyze-issue-with-seer.js"; @@ -548,27 +548,59 @@ describe("analyze_issue_with_seer", () => { ); }); - it("explains a run that stopped because the org ran out of credits", async () => { + const RESTART_URL = + "https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-RESTART/autofix/"; + + function failedRun( + runId: number, + failureReason?: string, + blocks: JsonBodyType[] = [], + ) { + return { + autofix: { + run_id: runId, + status: "error", + failure_reason: failureReason, + blocks, + }, + }; + } + + function completedRun(runId: number) { + return { + ...autofixStateFixture, + autofix: { ...autofixStateFixture.autofix, run_id: runId }, + }; + } + + /** + * Serves `states` to successive GETs (repeating the last one), answers the + * POST with `postResponse`, and counts POSTs. + */ + async function analyzeWithStates( + states: JsonBodyType[], + postResponse: () => Response = () => HttpResponse.json({ run_id: 789 }), + ) { + let getCallCount = 0; + let postCallCount = 0; + mswServer.use( - http.get( - "https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-CREDITS/autofix/", - () => - HttpResponse.json({ - autofix: { - run_id: 456, - status: "error", - failure_reason: "out_of_credits", - blocks: [], - }, - }), - ), + http.get(RESTART_URL, () => { + const state = states[Math.min(getCallCount, states.length - 1)]; + getCallCount++; + return HttpResponse.json(state); + }), + http.post(RESTART_URL, () => { + postCallCount++; + return postResponse(); + }), ); - const result = await analyzeIssueWithSeer.handler( + const promise = analyzeIssueWithSeer.handler( { organizationSlug: "sentry-mcp-evals", regionUrl: null, - issueId: "CLOUDFLARE-MCP-CREDITS", + issueId: "CLOUDFLARE-MCP-RESTART", instruction: undefined, }, { @@ -578,6 +610,81 @@ describe("analyze_issue_with_seer", () => { }, ); + await vi.runAllTimersAsync(); + + return { result: await promise, postCallCount }; + } + + it.each([ + ["out of credits", "out_of_credits"], + ["without a reason", undefined], + ])( + "starts a new run when the last one failed (%s)", + async (_label, failureReason) => { + const { result, postCallCount } = await analyzeWithStates([ + failedRun(456, failureReason), + completedRun(789), + ]); + + expect(postCallCount).toBe(1); + expect(result).toContain( + "Previous analysis (Run ID: 456) failed, starting a new one...", + ); + expect(result).toContain("Analysis started with Run ID: 789"); + expect(result).toContain("## Analysis Complete"); + expect(result).not.toContain("## Analysis Failed"); + }, + ); + + it("keeps the failed run's partial analysis when a restart gets a 402", async () => { + const { result, postCallCount } = await analyzeWithStates( + [ + failedRun(456, "out_of_credits", [ + { + id: "block-1", + artifacts: [ + { + key: "root_cause", + reason: "Root cause analysis completed", + data: { + one_line_description: "The request used the wrong bottle ID.", + five_whys: [], + reproduction_steps: [], + }, + }, + ], + }, + ]), + ], + () => + HttpResponse.json( + { + detail: + "Your organization does not have enough credits to start an Autofix run.", + code: "seer_out_of_credits", + }, + { status: 402 }, + ), + ); + + expect(postCallCount).toBe(1); + expect(result).toContain("## Analysis Not Started"); + expect(result).toContain( + "Your organization does not have enough credits to start an Autofix run.", + ); + expect(result).toContain( + "The previous analysis (Run ID: 456) failed before it finished. Its partial results:", + ); + expect(result).toContain(''); + expect(result).toContain("The request used the wrong bottle ID."); + }); + + it("explains a run that stopped because the org ran out of credits", async () => { + const { result } = await analyzeWithStates( + [{ autofix: null }, failedRun(456, "out_of_credits")], + () => HttpResponse.json({ run_id: 456 }), + ); + expect(result).toContain("## Analysis Failed"); expect(result).toContain( "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry.", diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts index 69ba1d20e..456047875 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts @@ -141,8 +141,10 @@ export default defineTool({ }, ); - // Step 2: Start analysis if none exists - if (!autofixState.autofix) { + // Step 2: Start analysis if none exists or the last one failed, so a run + // that stopped (e.g. out of credits) can be retried. + const previousRun = autofixState.autofix; + if (!previousRun || previousRun.status === "error") { let startResult: Awaited>; try { startResult = await apiService.startAutofix({ @@ -155,11 +157,21 @@ export default defineTool({ // Seer credits or quota. if (error instanceof ApiError && error.status === 402) { output += `## Analysis Not Started\n\n${getSeerPaymentRequiredMessage(error.responseBody)}\n`; + // The failed run may already hold paid-for analysis, e.g. a root + // cause found before the org ran out of credits. + const previousOutput = previousRun + ? getOutputForAutofixRun(previousRun) + : ""; + if (previousRun && previousOutput) { + output += `\nThe previous analysis (Run ID: ${previousRun.run_id}) failed before it finished. Its partial results:\n\n${previousOutput}`; + } return output; } throw error; } - output += `Starting new analysis...\n\n`; + output += previousRun + ? `Previous analysis (Run ID: ${previousRun.run_id}) failed, starting a new one...\n\n` + : `Starting new analysis...\n\n`; output += `Analysis started with Run ID: ${startResult.run_id}\n\n`; // Give it a moment to initialize @@ -184,20 +196,16 @@ export default defineTool({ }, ); } else { - output += `Found existing analysis (Run ID: ${autofixState.autofix.run_id})\n\n`; + output += `Found existing analysis (Run ID: ${previousRun.run_id})\n\n`; - // Check if existing analysis is already complete - const existingStatus = autofixState.autofix.status; + // Completed and waiting-for-input runs are returned as they are + const existingStatus = previousRun.status; if (isTerminalStatus(existingStatus)) { - // Return results immediately, no polling needed output += `## Analysis ${getStatusDisplayName(existingStatus)}\n\n`; - output += getOutputForAutofixRun(autofixState.autofix); + output += getOutputForAutofixRun(previousRun); if (existingStatus !== "completed") { output += `\n**Status**: ${existingStatus}\n`; - output += getFailureReasonGuidance( - autofixState.autofix.failure_reason, - ); output += getHumanInterventionGuidance(existingStatus); output += "\n"; }