From b1ccc6a997be2cca8920198e7b900d7e2a1b4bd3 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Nov 2026 13:34:27 +0100 Subject: [PATCH 1/5] fix(mcp): Start a new Seer run when the last one failed analyze_issue_with_seer returned any existing finished run as is, so a run that stopped because the org was out of credits could never be retried after a top-up. Like the CLI, start a new run whenever the last one ended in error, and keep polling until Sentry returns the new run so the old failure isn't reported as the result. Fixes CW-3094 Co-Authored-By: Claude Opus 5.5 --- .../src/internal/tool-helpers/seer.ts | 2 +- packages/mcp-core/src/skillDefinitions.json | 2 +- packages/mcp-core/src/toolDefinitions.json | 2 +- .../catalog/analyze-issue-with-seer.test.ts | 135 +++++++++++++++--- .../tools/catalog/analyze-issue-with-seer.ts | 42 +++--- 5 files changed, 144 insertions(+), 39 deletions(-) diff --git a/packages/mcp-core/src/internal/tool-helpers/seer.ts b/packages/mcp-core/src/internal/tool-helpers/seer.ts index 608eab2a8..0941ea013 100644 --- a/packages/mcp-core/src/internal/tool-helpers/seer.ts +++ b/packages/mcp-core/src/internal/tool-helpers/seer.ts @@ -90,7 +90,7 @@ export const SEER_START_OUT_OF_CREDITS_MESSAGE = "Your organization does not have enough credits to start an Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry."; export const SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE = - "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry."; + "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry, then run the analysis again to start a new run."; const SEER_NO_QUOTA_MESSAGE = "Your organization does not have enough Seer quota to start an Autofix run."; diff --git a/packages/mcp-core/src/skillDefinitions.json b/packages/mcp-core/src/skillDefinitions.json index 8c9257566..a2017902e 100644 --- a/packages/mcp-core/src/skillDefinitions.json +++ b/packages/mcp-core/src/skillDefinitions.json @@ -259,7 +259,7 @@ "tools": [ { "name": "analyze_issue_with_seer", - "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if needed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly\n", + "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly, except that a failed analysis is retried\n", "requiredScopes": [] }, { diff --git a/packages/mcp-core/src/toolDefinitions.json b/packages/mcp-core/src/toolDefinitions.json index 85faf0a6a..18466c058 100644 --- a/packages/mcp-core/src/toolDefinitions.json +++ b/packages/mcp-core/src/toolDefinitions.json @@ -152,7 +152,7 @@ { "name": "analyze_issue_with_seer", "title": "Analyze Issue with Seer", - "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if needed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly\n", + "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly, except that a failed analysis is retried\n", "inputSchema": { "type": "object", "properties": { diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts index 006d26040..90fe7d2a2 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts @@ -4,7 +4,7 @@ import { createUnsupportedIssue, mswServer, } from "@sentry/mcp-server-mocks"; -import { http, HttpResponse } from "msw"; +import { http, HttpResponse, type JsonBodyType } from "msw"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import analyzeIssueWithSeer from "./analyze-issue-with-seer.js"; @@ -390,7 +390,7 @@ describe("analyze_issue_with_seer", () => { async ({ request }) => { requestBody = await request.json(); return HttpResponse.json({ - run_id: 123, + run_id: autofixStateFixture.autofix.run_id, }); }, ), @@ -426,7 +426,9 @@ describe("analyze_issue_with_seer", () => { expect(requestBody).toEqual({ user_context: "Focus on memory leaks" }); expect(result).toContain("Starting new analysis..."); - expect(result).toContain("Analysis started with Run ID: 123"); + expect(result).toContain( + `Analysis started with Run ID: ${autofixStateFixture.autofix.run_id}`, + ); expect(result).toContain("## Analysis Complete"); }); @@ -500,27 +502,55 @@ describe("analyze_issue_with_seer", () => { }, ); - it("explains a run that stopped because the org ran out of credits", async () => { + const RESTART_URL = + "https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-RESTART/autofix/"; + + function failedRun(runId: number, failureReason?: string) { + return { + autofix: { + run_id: runId, + status: "error", + failure_reason: failureReason, + blocks: [], + }, + }; + } + + function completedRun(runId: number) { + return { + ...autofixStateFixture, + autofix: { ...autofixStateFixture.autofix, run_id: runId }, + }; + } + + /** + * Serves `states` to successive GETs (repeating the last one), answers the + * POST with `postResponse`, and counts POSTs. + */ + async function analyzeWithStates( + states: JsonBodyType[], + postResponse: () => Response = () => HttpResponse.json({ run_id: 789 }), + ) { + let getCallCount = 0; + let postCallCount = 0; + mswServer.use( - http.get( - "https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-CREDITS/autofix/", - () => - HttpResponse.json({ - autofix: { - run_id: 456, - status: "error", - failure_reason: "out_of_credits", - blocks: [], - }, - }), - ), + http.get(RESTART_URL, () => { + const state = states[Math.min(getCallCount, states.length - 1)]; + getCallCount++; + return HttpResponse.json(state); + }), + http.post(RESTART_URL, () => { + postCallCount++; + return postResponse(); + }), ); - const result = await analyzeIssueWithSeer.handler( + const promise = analyzeIssueWithSeer.handler( { organizationSlug: "sentry-mcp-evals", regionUrl: null, - issueId: "CLOUDFLARE-MCP-CREDITS", + issueId: "CLOUDFLARE-MCP-RESTART", instruction: undefined, }, { @@ -530,9 +560,76 @@ describe("analyze_issue_with_seer", () => { }, ); + await vi.runAllTimersAsync(); + + return { result: await promise, postCallCount }; + } + + it.each([ + ["out of credits", "out_of_credits"], + ["without a reason", undefined], + ])( + "starts a new run when the last one failed (%s)", + async (_label, failureReason) => { + const { result, postCallCount } = await analyzeWithStates([ + failedRun(456, failureReason), + completedRun(789), + ]); + + expect(postCallCount).toBe(1); + expect(result).toContain( + "Previous analysis (Run ID: 456) failed, starting a new one...", + ); + expect(result).toContain("Analysis started with Run ID: 789"); + expect(result).toContain("## Analysis Complete"); + expect(result).not.toContain("## Analysis Failed"); + }, + ); + + it("keeps polling until the new run replaces the failed one", async () => { + const { result } = await analyzeWithStates([ + failedRun(456, "out_of_credits"), + // Sentry can still return the old run right after the POST. + failedRun(456, "out_of_credits"), + failedRun(456, "out_of_credits"), + completedRun(789), + ]); + + expect(result).toContain("## Analysis Complete"); + expect(result).toContain(''); + expect(result).not.toContain("## Analysis Failed"); + }); + + it("explains why a restart did not start when Sentry answers 402", async () => { + const { result, postCallCount } = await analyzeWithStates( + [failedRun(456, "out_of_credits")], + () => + HttpResponse.json( + { + detail: + "Your organization does not have enough credits to start an Autofix run.", + code: "seer_out_of_credits", + }, + { status: 402 }, + ), + ); + + expect(postCallCount).toBe(1); + expect(result).toContain("## Analysis Not Started"); + expect(result).toContain( + "Your organization does not have enough credits to start an Autofix run.", + ); + }); + + it("explains a run that stopped because the org ran out of credits", async () => { + const { result } = await analyzeWithStates( + [{ autofix: null }, failedRun(456, "out_of_credits")], + () => HttpResponse.json({ run_id: 456 }), + ); + expect(result).toContain("## Analysis Failed"); expect(result).toContain( - "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry.", + "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry, then run the analysis again to start a new run.", ); }); diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts index 69ba1d20e..780c7bbe4 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts @@ -53,7 +53,7 @@ export default defineTool({ "", "This tool automatically:", "1. Checks if analysis already exists (instant results)", - "2. Starts new AI analysis if needed (~2-5 minutes)", + "2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)", "3. Returns complete fix recommendations", "", "", @@ -75,7 +75,7 @@ export default defineTool({ "- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead", "- If the user provides an issueUrl, extract it and use that parameter alone", "- The analysis includes actual code snippets and fixes, not just error descriptions", - "- Results are cached - subsequent calls return instantly", + "- Results are cached - subsequent calls return instantly, except that a failed analysis is retried", "", ].join("\n"), inputSchema: { @@ -141,8 +141,11 @@ export default defineTool({ }, ); - // Step 2: Start analysis if none exists - if (!autofixState.autofix) { + // Step 2: Start analysis if none exists or the last one failed, so a run + // that stopped (e.g. out of credits) can be retried after a top-up. + const previousRun = autofixState.autofix; + let newRunId: number | undefined; + if (!previousRun || previousRun.status === "error") { let startResult: Awaited>; try { startResult = await apiService.startAutofix({ @@ -159,8 +162,11 @@ export default defineTool({ } throw error; } - output += `Starting new analysis...\n\n`; - output += `Analysis started with Run ID: ${startResult.run_id}\n\n`; + newRunId = startResult.run_id; + output += previousRun + ? `Previous analysis (Run ID: ${previousRun.run_id}) failed, starting a new one...\n\n` + : `Starting new analysis...\n\n`; + output += `Analysis started with Run ID: ${newRunId}\n\n`; // Give it a moment to initialize await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -184,20 +190,16 @@ export default defineTool({ }, ); } else { - output += `Found existing analysis (Run ID: ${autofixState.autofix.run_id})\n\n`; + output += `Found existing analysis (Run ID: ${previousRun.run_id})\n\n`; - // Check if existing analysis is already complete - const existingStatus = autofixState.autofix.status; + // Completed and waiting-for-input runs are returned as they are + const existingStatus = previousRun.status; if (isTerminalStatus(existingStatus)) { - // Return results immediately, no polling needed output += `## Analysis ${getStatusDisplayName(existingStatus)}\n\n`; - output += getOutputForAutofixRun(autofixState.autofix); + output += getOutputForAutofixRun(previousRun); if (existingStatus !== "completed") { output += `\n**Status**: ${existingStatus}\n`; - output += getFailureReasonGuidance( - autofixState.autofix.failure_reason, - ); output += getHumanInterventionGuidance(existingStatus); output += "\n"; } @@ -223,9 +225,12 @@ export default defineTool({ } const status = autofixState.autofix.status; + // Right after a restart, Sentry can still return the failed run. + const isStaleRun = + newRunId !== undefined && autofixState.autofix.run_id !== newRunId; // Check if completed (terminal state) - if (isTerminalStatus(status)) { + if (!isStaleRun && isTerminalStatus(status)) { output += `## Analysis ${getStatusDisplayName(status)}\n\n`; output += getOutputForAutofixRun(autofixState.autofix); @@ -241,7 +246,7 @@ export default defineTool({ } // Update status if changed - if (status !== lastStatus) { + if (!isStaleRun && status !== lastStatus) { const activeTodo = getActiveAutofixTodo(autofixState.autofix); if (activeTodo) { output += `Processing: ${activeTodo}...\n`; @@ -297,7 +302,10 @@ export default defineTool({ } // Show current progress - if (autofixState.autofix) { + if ( + autofixState.autofix && + (newRunId === undefined || autofixState.autofix.run_id === newRunId) + ) { output += `**Current Status**: ${getStatusDisplayName(autofixState.autofix.status)}\n\n`; output += getOutputForAutofixRun(autofixState.autofix); } From c9a87eb0bf6b1c0eea9674324c336d486ef449d4 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Nov 2026 13:38:57 +0100 Subject: [PATCH 2/5] fix(mcp): Keep the original out-of-credits wording The added "then run the analysis again" read as if the billing admin should rerun the analysis. Co-Authored-By: Claude Opus 5.5 --- packages/mcp-core/src/internal/tool-helpers/seer.ts | 2 +- .../mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mcp-core/src/internal/tool-helpers/seer.ts b/packages/mcp-core/src/internal/tool-helpers/seer.ts index 0941ea013..608eab2a8 100644 --- a/packages/mcp-core/src/internal/tool-helpers/seer.ts +++ b/packages/mcp-core/src/internal/tool-helpers/seer.ts @@ -90,7 +90,7 @@ export const SEER_START_OUT_OF_CREDITS_MESSAGE = "Your organization does not have enough credits to start an Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry."; export const SEER_CONTINUE_OUT_OF_CREDITS_MESSAGE = - "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry, then run the analysis again to start a new run."; + "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry."; const SEER_NO_QUOTA_MESSAGE = "Your organization does not have enough Seer quota to start an Autofix run."; diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts index 90fe7d2a2..be39ea0e6 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts @@ -629,7 +629,7 @@ describe("analyze_issue_with_seer", () => { expect(result).toContain("## Analysis Failed"); expect(result).toContain( - "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry, then run the analysis again to start a new run.", + "Your organization does not have enough credits to continue your Autofix run. Someone with billing access can adjust Seer spend and auto-reload in Sentry.", ); }); From edf9e1217d1d8b4031889bc368831a07d44d680e Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Nov 2026 13:41:16 +0100 Subject: [PATCH 3/5] fix(mcp): Keep the original analyze_issue_with_seer description Retrying a failed run is an implementation detail the agent doesn't need in the tool description. Co-Authored-By: Claude Opus 5.5 --- packages/mcp-core/src/skillDefinitions.json | 2 +- packages/mcp-core/src/toolDefinitions.json | 2 +- .../mcp-core/src/tools/catalog/analyze-issue-with-seer.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mcp-core/src/skillDefinitions.json b/packages/mcp-core/src/skillDefinitions.json index a2017902e..8c9257566 100644 --- a/packages/mcp-core/src/skillDefinitions.json +++ b/packages/mcp-core/src/skillDefinitions.json @@ -259,7 +259,7 @@ "tools": [ { "name": "analyze_issue_with_seer", - "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly, except that a failed analysis is retried\n", + "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if needed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly\n", "requiredScopes": [] }, { diff --git a/packages/mcp-core/src/toolDefinitions.json b/packages/mcp-core/src/toolDefinitions.json index 18466c058..85faf0a6a 100644 --- a/packages/mcp-core/src/toolDefinitions.json +++ b/packages/mcp-core/src/toolDefinitions.json @@ -152,7 +152,7 @@ { "name": "analyze_issue_with_seer", "title": "Analyze Issue with Seer", - "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly, except that a failed analysis is retried\n", + "description": "Use Seer to analyze production errors and get detailed root cause analysis with specific code fixes.\n\nUse this tool when:\n- The user explicitly asks for root cause analysis, Seer analysis, or help fixing/debugging an issue\n- You are unable to accurately determine the root cause from the issue details alone\n\nDo NOT call this tool as an automatic follow-up to get_sentry_resource.\n\nWhat this tool provides:\n- Root cause analysis with code-level explanations\n- Specific file locations and line numbers where errors occur\n- Concrete code fixes you can apply\n- Step-by-step implementation guidance\n\nThis tool automatically:\n1. Checks if analysis already exists (instant results)\n2. Starts new AI analysis if needed (~2-5 minutes)\n3. Returns complete fix recommendations\n\n\n### User: \"Run Seer on this issue\"\n\n```\nanalyze_issue_with_seer(issueUrl='https://my-org.sentry.io/issues/PROJECT-1Z43')\n```\n\n### User: \"Analyze this issue and suggest a fix\"\n\n```\nanalyze_issue_with_seer(organizationSlug='my-organization', issueId='ERROR-456')\n```\n\n\n\n- Only use when the user explicitly requests analysis or you cannot determine the root cause from issue details alone\n- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead\n- If the user provides an issueUrl, extract it and use that parameter alone\n- The analysis includes actual code snippets and fixes, not just error descriptions\n- Results are cached - subsequent calls return instantly\n", "inputSchema": { "type": "object", "properties": { diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts index 780c7bbe4..da4b9d289 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts @@ -53,7 +53,7 @@ export default defineTool({ "", "This tool automatically:", "1. Checks if analysis already exists (instant results)", - "2. Starts new AI analysis if none exists or the last one failed (~2-5 minutes)", + "2. Starts new AI analysis if needed (~2-5 minutes)", "3. Returns complete fix recommendations", "", "", @@ -75,7 +75,7 @@ export default defineTool({ "- Seer Autofix does not support metric alert issues (issueCategory: metric); use get_issue_details and search_metrics or search_traces instead", "- If the user provides an issueUrl, extract it and use that parameter alone", "- The analysis includes actual code snippets and fixes, not just error descriptions", - "- Results are cached - subsequent calls return instantly, except that a failed analysis is retried", + "- Results are cached - subsequent calls return instantly", "", ].join("\n"), inputSchema: { From ef32976f43184d31fd26eb5ff94c344920cb45f2 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Nov 2026 13:45:46 +0100 Subject: [PATCH 4/5] fix(mcp): Keep partial Seer results when a restart gets a 402 A run that ran out of credits partway through can already hold a root cause. If the restart is refused with a 402, show that partial analysis instead of only "Analysis Not Started". Drop the stale-run guard while polling. Sentry creates the run before the POST returns and the GET returns the most recently triggered run, so the old failed run never comes back. When the IDs do differ, another caller started a run after ours, and the guard made us wait out the full timeout. Co-Authored-By: Claude Opus 5.5 --- .../catalog/analyze-issue-with-seer.test.ts | 54 +++++++++++-------- .../tools/catalog/analyze-issue-with-seer.ts | 24 ++++----- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts index be39ea0e6..4e8685f24 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.test.ts @@ -390,7 +390,7 @@ describe("analyze_issue_with_seer", () => { async ({ request }) => { requestBody = await request.json(); return HttpResponse.json({ - run_id: autofixStateFixture.autofix.run_id, + run_id: 123, }); }, ), @@ -426,9 +426,7 @@ describe("analyze_issue_with_seer", () => { expect(requestBody).toEqual({ user_context: "Focus on memory leaks" }); expect(result).toContain("Starting new analysis..."); - expect(result).toContain( - `Analysis started with Run ID: ${autofixStateFixture.autofix.run_id}`, - ); + expect(result).toContain("Analysis started with Run ID: 123"); expect(result).toContain("## Analysis Complete"); }); @@ -505,13 +503,17 @@ describe("analyze_issue_with_seer", () => { const RESTART_URL = "https://sentry.io/api/0/organizations/sentry-mcp-evals/issues/CLOUDFLARE-MCP-RESTART/autofix/"; - function failedRun(runId: number, failureReason?: string) { + function failedRun( + runId: number, + failureReason?: string, + blocks: JsonBodyType[] = [], + ) { return { autofix: { run_id: runId, status: "error", failure_reason: failureReason, - blocks: [], + blocks, }, }; } @@ -586,23 +588,26 @@ describe("analyze_issue_with_seer", () => { }, ); - it("keeps polling until the new run replaces the failed one", async () => { - const { result } = await analyzeWithStates([ - failedRun(456, "out_of_credits"), - // Sentry can still return the old run right after the POST. - failedRun(456, "out_of_credits"), - failedRun(456, "out_of_credits"), - completedRun(789), - ]); - - expect(result).toContain("## Analysis Complete"); - expect(result).toContain(''); - expect(result).not.toContain("## Analysis Failed"); - }); - - it("explains why a restart did not start when Sentry answers 402", async () => { + it("keeps the failed run's partial analysis when a restart gets a 402", async () => { const { result, postCallCount } = await analyzeWithStates( - [failedRun(456, "out_of_credits")], + [ + failedRun(456, "out_of_credits", [ + { + id: "block-1", + artifacts: [ + { + key: "root_cause", + reason: "Root cause analysis completed", + data: { + one_line_description: "The request used the wrong bottle ID.", + five_whys: [], + reproduction_steps: [], + }, + }, + ], + }, + ]), + ], () => HttpResponse.json( { @@ -619,6 +624,11 @@ describe("analyze_issue_with_seer", () => { expect(result).toContain( "Your organization does not have enough credits to start an Autofix run.", ); + expect(result).toContain( + "The previous analysis (Run ID: 456) failed before it finished. Its partial results:", + ); + expect(result).toContain(''); + expect(result).toContain("The request used the wrong bottle ID."); }); it("explains a run that stopped because the org ran out of credits", async () => { diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts index da4b9d289..dff574dd1 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts @@ -144,7 +144,6 @@ export default defineTool({ // Step 2: Start analysis if none exists or the last one failed, so a run // that stopped (e.g. out of credits) can be retried after a top-up. const previousRun = autofixState.autofix; - let newRunId: number | undefined; if (!previousRun || previousRun.status === "error") { let startResult: Awaited>; try { @@ -158,15 +157,22 @@ export default defineTool({ // Seer credits or quota. if (error instanceof ApiError && error.status === 402) { output += `## Analysis Not Started\n\n${getSeerPaymentRequiredMessage(error.responseBody)}\n`; + // The failed run may already hold paid-for analysis, e.g. a root + // cause found before the org ran out of credits. + const previousOutput = previousRun + ? getOutputForAutofixRun(previousRun) + : ""; + if (previousRun && previousOutput) { + output += `\nThe previous analysis (Run ID: ${previousRun.run_id}) failed before it finished. Its partial results:\n\n${previousOutput}`; + } return output; } throw error; } - newRunId = startResult.run_id; output += previousRun ? `Previous analysis (Run ID: ${previousRun.run_id}) failed, starting a new one...\n\n` : `Starting new analysis...\n\n`; - output += `Analysis started with Run ID: ${newRunId}\n\n`; + output += `Analysis started with Run ID: ${startResult.run_id}\n\n`; // Give it a moment to initialize await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -225,12 +231,9 @@ export default defineTool({ } const status = autofixState.autofix.status; - // Right after a restart, Sentry can still return the failed run. - const isStaleRun = - newRunId !== undefined && autofixState.autofix.run_id !== newRunId; // Check if completed (terminal state) - if (!isStaleRun && isTerminalStatus(status)) { + if (isTerminalStatus(status)) { output += `## Analysis ${getStatusDisplayName(status)}\n\n`; output += getOutputForAutofixRun(autofixState.autofix); @@ -246,7 +249,7 @@ export default defineTool({ } // Update status if changed - if (!isStaleRun && status !== lastStatus) { + if (status !== lastStatus) { const activeTodo = getActiveAutofixTodo(autofixState.autofix); if (activeTodo) { output += `Processing: ${activeTodo}...\n`; @@ -302,10 +305,7 @@ export default defineTool({ } // Show current progress - if ( - autofixState.autofix && - (newRunId === undefined || autofixState.autofix.run_id === newRunId) - ) { + if (autofixState.autofix) { output += `**Current Status**: ${getStatusDisplayName(autofixState.autofix.status)}\n\n`; output += getOutputForAutofixRun(autofixState.autofix); } From a43fd209acaadc05bbfe6cc70d954176232cd719 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Nov 2026 20:27:29 +0100 Subject: [PATCH 5/5] fix(mcp): Drop the top-up wording from the Seer restart comment Co-Authored-By: Claude Opus 5.5 --- packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts index dff574dd1..456047875 100644 --- a/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts +++ b/packages/mcp-core/src/tools/catalog/analyze-issue-with-seer.ts @@ -142,7 +142,7 @@ export default defineTool({ ); // Step 2: Start analysis if none exists or the last one failed, so a run - // that stopped (e.g. out of credits) can be retried after a top-up. + // that stopped (e.g. out of credits) can be retried. const previousRun = autofixState.autofix; if (!previousRun || previousRun.status === "error") { let startResult: Awaited>;