diff --git a/static/app/components/events/autofix/autofixCredits.tsx b/static/app/components/events/autofix/autofixCredits.tsx index 1fe60f123b1e..9bbd18a014ca 100644 --- a/static/app/components/events/autofix/autofixCredits.tsx +++ b/static/app/components/events/autofix/autofixCredits.tsx @@ -17,7 +17,10 @@ import { type ExplorerAutofixState, } from './useExplorerAutofix'; -type AutofixAction = AutofixNextStep['action']; +/** + * A billable autofix action: a step, opening a PR, or iterating on one. + */ +export type AutofixAction = AutofixNextStep['action']; const AUTOFIX_CREDIT_OPERATIONS: Record = { root_cause: 'autofix_root_cause', diff --git a/static/app/components/events/autofix/useExplorerAutofix.spec.tsx b/static/app/components/events/autofix/useExplorerAutofix.spec.tsx index 580aa4e81656..55945ea23914 100644 --- a/static/app/components/events/autofix/useExplorerAutofix.spec.tsx +++ b/static/app/components/events/autofix/useExplorerAutofix.spec.tsx @@ -5,6 +5,7 @@ import {act, renderHookWithProviders, waitFor} from 'sentry-test/reactTestingLib import {toast} from '@sentry/scraps/toast'; +import {useSeerCreditCheck} from 'sentry/utils/seer/useSeerCreditCheck'; import type {Artifact, Block, ExplorerFilePatch} from 'sentry/views/seerExplorer/types'; import {DiffFileType, DiffLineType} from './types'; @@ -18,6 +19,7 @@ import { isCodeChangesArtifact, isCodingAgentsArtifact, isLastStepPrIteration, + isOutOfCreditsSection, isPrIterationBlock, isPullRequestsArtifact, isRootCauseArtifact, @@ -640,6 +642,34 @@ describe('getOrderedAutofixSections', () => { expect(prSection.artifacts).toEqual([[prState]]); }); + it('marks only the section the run stopped in with its failure reason', () => { + const sections = getOrderedAutofixSections({ + ...makeState([ + makeBlock({message: {metadata: {step: 'root_cause'}}}), + makeBlock({message: {metadata: {step: 'solution'}}}), + ]), + status: 'error', + failure_reason: 'out_of_credits', + }); + + expect(sections.map(s => [s.status, s.failureReason])).toEqual([ + ['completed', undefined], + ['error', 'out_of_credits'], + ]); + expect(sections.map(isOutOfCreditsSection)).toEqual([false, true]); + }); + + it('does not treat other failures as out of credits', () => { + const [section] = getOrderedAutofixSections({ + ...makeState([makeBlock({message: {metadata: {step: 'root_cause'}}})]), + status: 'error', + failure_reason: 'timeout', + }); + + expect(section!.status).toBe('error'); + expect(isOutOfCreditsSection(section!)).toBe(false); + }); + it('marks the synthetic pull_request section as processing while a PR is creating', () => { const sections = getOrderedAutofixSections({ ...makeState([makeBlock({message: {metadata: {step: 'code_changes'}}})]), @@ -1254,6 +1284,71 @@ describe('useExplorerAutofix - createPR', () => { ); }); + it('says a push was refused for credits and rechecks credits', async () => { + const errorSpy = jest.spyOn(toast, 'error'); + const organization = OrganizationFixture({ + features: ['seer-credit-checks', 'token-billed-seer-enabled'], + }); + const creditCheck = MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + body: {hasEnoughCredits: true}, + }); + MockApiClient.addMockResponse({ + url: AUTOFIX_URL, + method: 'GET', + body: { + autofix: { + run_id: 42, + blocks: [], + status: 'completed', + updated_at: '2026-01-01T00:00:00Z', + repo_pr_states: { + 'org/repo': { + repo_name: 'org/repo', + branch_name: 'seer/fix', + commit_sha: 'abc1234', + pr_creation_status: 'error', + pr_creation_error: null, + pr_id: 7, + pr_number: 7, + pr_url: 'https://github.com/org/repo/pull/7', + title: 'Fix', + }, + }, + }, + }, + }); + MockApiClient.addMockResponse({ + url: AUTOFIX_URL, + method: 'POST', + statusCode: 402, + body: { + detail: + 'Your organization does not have enough credits to continue your Autofix run.', + code: 'seer_out_of_credits', + }, + }); + + const {result} = renderHookWithProviders( + () => ({ + autofix: useExplorerAutofix(MOCK_GROUP), + credits: useSeerCreditCheck('autofix_open_pr'), + }), + {organization} + ); + await waitFor(() => expect(result.current.autofix.runState?.run_id).toBe(42)); + await waitFor(() => expect(creditCheck).toHaveBeenCalledTimes(1)); + + await expect( + act(() => result.current.autofix.createPR(42, 'org/repo')) + ).rejects.toThrow(); + + expect(errorSpy).toHaveBeenCalledWith( + "Seer couldn't push changes to org/repo#7 because your organization is out of credits." + ); + await waitFor(() => expect(creditCheck).toHaveBeenCalledTimes(2)); + }); + it('shows an error toast and throws on API error', async () => { MockApiClient.addMockResponse({ url: AUTOFIX_URL, @@ -1719,6 +1814,51 @@ describe('useExplorerAutofix - startStep errors', () => { ); }); + it('says a kickoff was refused for credits, rechecks credits, and paints no failed run', async () => { + const errorSpy = jest.spyOn(toast, 'error'); + const organization = OrganizationFixture({ + features: ['seer-credit-checks', 'token-billed-seer-enabled'], + }); + const creditCheck = MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + body: {hasEnoughCredits: true}, + }); + MockApiClient.addMockResponse({ + url: AUTOFIX_URL, + method: 'GET', + body: {autofix: null}, + }); + MockApiClient.addMockResponse({ + url: AUTOFIX_URL, + method: 'POST', + statusCode: 402, + body: { + detail: 'Your organization does not have enough credits to start an Autofix run.', + code: 'seer_out_of_credits', + }, + }); + + const {result} = renderHookWithProviders( + () => ({ + autofix: useExplorerAutofix(MOCK_GROUP), + credits: useSeerCreditCheck('autofix_root_cause'), + }), + {organization} + ); + await waitFor(() => expect(result.current.autofix.isLoading).toBe(false)); + await waitFor(() => expect(creditCheck).toHaveBeenCalledTimes(1)); + + await expect( + act(() => result.current.autofix.startStep('root_cause')) + ).rejects.toThrow(); + + expect(errorSpy).toHaveBeenCalledWith( + "Seer couldn't start on the root cause because your organization is out of credits." + ); + expect(result.current.autofix.runState).toBeNull(); + await waitFor(() => expect(creditCheck).toHaveBeenCalledTimes(2)); + }); + it('falls back to the error state when a kickoff fails with no existing run', async () => { MockApiClient.addMockResponse({ url: AUTOFIX_URL, diff --git a/static/app/components/events/autofix/useExplorerAutofix.tsx b/static/app/components/events/autofix/useExplorerAutofix.tsx index 1dccd53b5109..b6d1f26932c3 100644 --- a/static/app/components/events/autofix/useExplorerAutofix.tsx +++ b/static/app/components/events/autofix/useExplorerAutofix.tsx @@ -4,6 +4,11 @@ import {useQuery, useQueryClient} from '@tanstack/react-query'; import {useModal} from '@sentry/scraps/modal'; import {toast} from '@sentry/scraps/toast'; +import { + getOutOfCreditsPRMessage, + OUT_OF_CREDITS_OPEN_PRS_MESSAGE, + OUT_OF_CREDITS_START_STEP_MESSAGE, +} from 'sentry/components/seer/outOfCredits/messages'; import {useServiceWorker} from 'sentry/serviceWorker/client/serviceWorkerContext'; import type {Group} from 'sentry/types/group'; import type {User} from 'sentry/types/user'; @@ -12,6 +17,7 @@ import {trackAnalytics} from 'sentry/utils/analytics'; import {apiOptions} from 'sentry/utils/api/apiOptions'; import {getApiUrl} from 'sentry/utils/api/getApiUrl'; import {defined} from 'sentry/utils/defined'; +import {seerCreditChecksQueryKey} from 'sentry/utils/seer/useSeerCreditCheck'; import {useApi} from 'sentry/utils/useApi'; import {useOrganization} from 'sentry/utils/useOrganization'; import {useUser} from 'sentry/utils/useUser'; @@ -25,6 +31,7 @@ import { type Artifact, type Block, type ExplorerCodingAgentState, + type ExplorerFailureReason, type ExplorerFilePatch, type RepoPRState, type SeerExplorerRunId, @@ -216,6 +223,7 @@ export interface ExplorerAutofixState { status: 'processing' | 'completed' | 'error' | 'awaiting_user_input'; updated_at: string; coding_agents?: Record; + failure_reason?: ExplorerFailureReason | null; feature_id?: string | null; pending_user_input?: { data: Record; @@ -326,6 +334,19 @@ function isAutofixSetupRequiredError( ); } +/** + * A step or PR the backend refused because the organization is out of Seer + * credits. The triggers' credit checks fail open and can be out of date, so + * the backend has the final say. + */ +function isOutOfCreditsError(e: unknown): boolean { + const error = e as + | {responseJSON?: {code?: unknown}; status?: number} + | null + | undefined; + return error?.status === 402 && error.responseJSON?.code === 'seer_out_of_credits'; +} + const makeErrorExplorerAutofixData = (errorMessage: string): ExplorerAutofixResponse => ({ autofix: { run_id: 0, @@ -415,6 +436,10 @@ export interface AutofixSection { blocks: Block[]; status: 'processing' | 'completed' | 'error'; step: string; + /** + * Why the run stopped, on the section it stopped in. + */ + failureReason?: ExplorerFailureReason | null; index?: number; } @@ -460,6 +485,7 @@ function buildSection( if (currentStep && runState?.status === 'error') { section.status = 'error'; + section.failureReason = runState.failure_reason; } else if ( (!currentStep && runState?.status !== 'processing') || isLastBlockOfSection(blocks[blocks.length - 1]) || @@ -672,6 +698,15 @@ export function isCodingAgentsSection(section: AutofixSection): boolean { return section.step === 'coding_agents'; } +/** + * The step stopped because the organization ran out of Seer credits. Running it + * again fails the same way until someone adds credits, and the blocks it wrote + * before stopping are still worth showing. + */ +export function isOutOfCreditsSection(section: AutofixSection): boolean { + return section.status === 'error' && section.failureReason === 'out_of_credits'; +} + export function isPrIterationPaused(runState: ExplorerAutofixState | null): boolean { return runState?.pr_iteration_paused === true; } @@ -998,6 +1033,18 @@ export function useExplorerAutofix( return {runId: getAutofixRunId(e.responseJSON)!, isSingleRun: false}; } + // Recheck every credit check, so the credit gates bring up the + // out-of-credits CTA and hold their buttons back, and say why the step + // didn't start. A refused kickoff paints no failed run: the start card + // offers the CTA instead. + if (isOutOfCreditsError(e)) { + queryClient.invalidateQueries({ + queryKey: seerCreditChecksQueryKey(organization), + }); + toast.error(OUT_OF_CREDITS_START_STEP_MESSAGE[step]); + throw e; + } + // The kickoff was refused because Seer setup is incomplete. There is no // failed run to paint, and unlike the in-flight race there is nothing to // recover by refetching — surface the backend's setup copy (it names the @@ -1063,11 +1110,30 @@ export function useExplorerAutofix( queryKey: explorerAutofixApiOptions(orgSlug, groupId).queryKey, }); } catch (e: any) { - toast.error(getApiErrorMessage(e, 'Failed to create PR')); + if (!isOutOfCreditsError(e)) { + toast.error(getApiErrorMessage(e, 'Failed to create PR')); + throw e; + } + + // Recheck every credit check, so the credit gates bring up the + // out-of-credits CTA and hold their buttons back. + queryClient.invalidateQueries({queryKey: seerCreditChecksQueryKey(organization)}); + // A retry for a repo whose PR exists pushes to it rather than opening one. + const repoState = repoName + ? queryClient.getQueryData(explorerAutofixApiOptions(orgSlug, groupId).queryKey) + ?.json?.autofix?.repo_pr_states?.[repoName] + : undefined; + toast.error( + repoName + ? getOutOfCreditsPRMessage( + repoState ?? {repo_name: repoName, pr_number: null} + ) + : OUT_OF_CREDITS_OPEN_PRS_MESSAGE + ); throw e; } }, - [api, orgSlug, groupId, queryClient] + [api, orgSlug, groupId, organization, queryClient] ); /** diff --git a/static/app/components/events/autofix/v3/artifactLoadingDetails.tsx b/static/app/components/events/autofix/v3/artifactLoadingDetails.tsx index 21f2a3ae946c..546040e1b760 100644 --- a/static/app/components/events/autofix/v3/artifactLoadingDetails.tsx +++ b/static/app/components/events/autofix/v3/artifactLoadingDetails.tsx @@ -8,6 +8,7 @@ import { ResponseGroup, } from 'sentry/views/seerExplorer/components/chat/responseGroup'; import {findLatestTodos} from 'sentry/views/seerExplorer/components/chat/toolUse'; +import {isSeerErrorBlock} from 'sentry/views/seerExplorer/types'; import {ArtifactDetails} from './artifactDetails'; @@ -21,28 +22,12 @@ export function ArtifactLoadingDetails({ blocks, }: ArtifactLoadingDetailsProps) { const [appearedAt] = useState(() => new Date()); - const responses = useMemo( - () => - groupTranscript(blocks).flatMap(segment => - segment.kind === 'response' ? [segment] : [] - ), - [blocks] - ); - const latestTodos = useMemo(() => findLatestTodos(blocks), [blocks]); + const transcript = useStepTranscript(blocks); return ( - {responses.length ? ( - responses.map(segment => ( - - )) + {transcript.responses.length ? ( + ) : ( @@ -51,3 +36,54 @@ export function ArtifactLoadingDetails({ ); } + +/** + * What a step wrote before it stopped, drawn as it was while the step ran. + * Seer's own error block is left out, since the caller explains why the step + * stopped. + */ +export function ArtifactPartialDetails({blocks}: {blocks: AutofixSection['blocks']}) { + const partialBlocks = useMemo( + () => blocks.filter(block => !isSeerErrorBlock(block)), + [blocks] + ); + const transcript = useStepTranscript(partialBlocks); + + if (!transcript.responses.length) { + return null; + } + + return ( + + + + ); +} + +function useStepTranscript(blocks: AutofixSection['blocks']) { + const responses = useMemo( + () => + groupTranscript(blocks).flatMap(segment => + segment.kind === 'response' ? [segment] : [] + ), + [blocks] + ); + const latestTodos = useMemo(() => findLatestTodos(blocks), [blocks]); + return {responses, latestTodos}; +} + +/** + * A step's responses, the way the Seer Agent chat draws them. + */ +function StepResponses({responses, latestTodos}: ReturnType) { + return responses.map(segment => ( + + )); +} diff --git a/static/app/components/events/autofix/v3/autofixCards.spec.tsx b/static/app/components/events/autofix/v3/autofixCards.spec.tsx index e96e8d078dad..79f24c38b313 100644 --- a/static/app/components/events/autofix/v3/autofixCards.spec.tsx +++ b/static/app/components/events/autofix/v3/autofixCards.spec.tsx @@ -2702,3 +2702,352 @@ describe('ArtifactCard', () => { }); }); }); + +describe('out of credits', () => { + const organization = OrganizationFixture({ + features: ['seer-credit-checks', 'token-billed-seer-enabled'], + }); + const manualPrIterationCreditsOrganization = OrganizationFixture({ + features: [ + 'seer-credit-checks', + 'token-billed-seer-enabled', + 'autofix-pr-iteration-manual', + ], + }); + const retryTooltip = 'Your organization does not have enough Seer credits for this.'; + + function mockCreditCheck(operation: string, hasEnoughCredits: boolean) { + return MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + match: [MockApiClient.matchQuery({operation})], + body: {hasEnoughCredits}, + }); + } + + afterEach(() => { + MockApiClient.clearMockResponses(); + }); + + // While the check is in flight the button is natively disabled. Once it + // settles it is only aria-disabled, so it stays focusable and its tooltip can + // say why. + async function expectDisabledForCredits(button: HTMLElement) { + await waitFor(() => expect(button).toBeEnabled()); + expect(button).toHaveAttribute('aria-disabled', 'true'); + await userEvent.hover(button); + expect(await screen.findByText(retryTooltip)).toBeInTheDocument(); + } + + function makeOutOfCreditsSection( + step: string, + blocks: AutofixSection['blocks'], + artifacts: AutofixArtifact[] = [] + ): AutofixSection { + return { + ...makeSection(step, 'error', artifacts, blocks), + failureReason: 'out_of_credits', + }; + } + + function makeBlock( + id: string, + content: string, + metadata?: Record + ): AutofixSection['blocks'][number] { + return { + id, + timestamp: '2026-01-01T00:00:00Z', + message: {role: 'assistant', content, metadata}, + }; + } + + // Seer appends this when it stops a run for credits. + const seerErrorBlock = makeBlock( + 'error-1', + "Your organization doesn't have enough Seer credits for this, so I stopped here." + ); + + const stoppedRunState = { + run_id: 123, + blocks: [], + status: 'error' as const, + failure_reason: 'out_of_credits' as const, + updated_at: '2026-01-01T00:00:00Z', + }; + + const rootCauseSection = makeOutOfCreditsSection('root_cause', [ + makeBlock('block-1', 'The cache key omits the org slug.', {step: 'root_cause'}), + seerErrorBlock, + ]); + + const codeChangesSection = makeOutOfCreditsSection( + 'code_changes', + [ + makeBlock('block-1', 'Editing the cart service.', {step: 'code_changes'}), + seerErrorBlock, + ], + [[makePatch('org/repo', 'src/cart.py')]] + ); + + it('keeps what the root cause step wrote and disables Re-run while out of credits', async () => { + mockCreditCheck('autofix_root_cause', false); + render( + , + {organization} + ); + + expect(screen.getByText('The cache key omits the org slug.')).toBeInTheDocument(); + expect( + screen.getByText( + 'Seer stopped before finishing the root cause because your organization is out of credits.' + ) + ).toBeInTheDocument(); + expect(screen.queryByText(/This one is on us/)).not.toBeInTheDocument(); + expect(screen.queryByText(/so I stopped here/)).not.toBeInTheDocument(); + + await expectDisabledForCredits(screen.getByRole('button', {name: 'Re-run'})); + expect(screen.getByRole('button', {name: 'Re-run step'})).toBeDisabled(); + }); + + it('re-runs a root cause that stopped for credits once the organization can afford it', async () => { + mockCreditCheck('autofix_root_cause', true); + const startStep = jest.fn(); + render( + , + {organization} + ); + + await waitFor(() => + expect(screen.getByRole('button', {name: 'Re-run'})).toHaveAttribute( + 'aria-disabled', + 'false' + ) + ); + expect(screen.getByRole('button', {name: 'Re-run step'})).toBeEnabled(); + + await userEvent.click(screen.getByRole('button', {name: 'Re-run'})); + expect(startStep).toHaveBeenCalledWith('root_cause', { + runId: 123, + userContext: undefined, + insertIndex: undefined, + }); + }); + + it('keeps what the plan step wrote and disables Re-run while out of credits', async () => { + mockCreditCheck('autofix_solution', false); + render( + , + {organization} + ); + + expect(screen.getByText('Seed the reduction with zero.')).toBeInTheDocument(); + expect( + screen.getByText( + 'Seer stopped before finishing the plan because your organization is out of credits.' + ) + ).toBeInTheDocument(); + + await expectDisabledForCredits(screen.getByRole('button', {name: 'Re-run'})); + }); + + it('keeps the partial diff of code changes that stopped for credits and disables Re-run', async () => { + mockCreditCheck('autofix_code_changes', false); + render( + , + {organization} + ); + + expect(screen.getByText('Editing the cart service.')).toBeInTheDocument(); + expect(screen.getByTestId('file-diff-viewer')).toBeInTheDocument(); + expect( + screen.getByText( + 'Seer stopped before finishing the code changes because your organization is out of credits.' + ) + ).toBeInTheDocument(); + expect( + screen.queryByText("Seer proposed a fix but couldn't apply it automatically") + ).not.toBeInTheDocument(); + // Partial changes are not the finished artifact. + expect(screen.getByRole('button', {name: 'Copy as Markdown'})).toHaveAttribute( + 'aria-disabled', + 'true' + ); + + await expectDisabledForCredits(screen.getByRole('button', {name: 'Re-run'})); + expect(screen.getByRole('button', {name: 'Re-run step'})).toBeDisabled(); + }); + + it('re-runs code changes that stopped for credits once the organization can afford it', async () => { + mockCreditCheck('autofix_code_changes', true); + const startStep = jest.fn(); + render( + , + {organization} + ); + + await waitFor(() => + expect(screen.getByRole('button', {name: 'Re-run'})).toHaveAttribute( + 'aria-disabled', + 'false' + ) + ); + + await userEvent.click(screen.getByRole('button', {name: 'Re-run'})); + expect(startStep).toHaveBeenCalledWith('code_changes', { + runId: 123, + userContext: undefined, + insertIndex: undefined, + }); + }); + + it('leaves the retry to the backend without seer-credit-checks', () => { + const creditCheck = mockCreditCheck('autofix_code_changes', false); + render( + , + {organization: OrganizationFixture({features: []})} + ); + + expect(screen.getByRole('button', {name: 'Re-run'})).toHaveAttribute( + 'aria-disabled', + 'false' + ); + expect(creditCheck).not.toHaveBeenCalled(); + }); + + it.each([ + [false, 'true'], + [true, 'false'], + ])( + 'says a PR iteration stopped for credits and gates its retry (enough credits: %s)', + async (hasEnoughCredits, ariaDisabled) => { + const creditCheck = mockCreditCheck('autofix_pr_iteration', hasEnoughCredits); + render( + , + {organization: manualPrIterationCreditsOrganization} + ); + + expect( + screen.getByText( + 'Seer stopped before updating the pull request because your organization is out of credits.' + ) + ).toBeInTheDocument(); + // Only the iteration that stopped is replayed, not the original step. + expect(screen.queryByText('Applied the fix.')).not.toBeInTheDocument(); + + await waitFor(() => expect(creditCheck).toHaveBeenCalled()); + await waitFor(() => + expect(screen.getByRole('button', {name: 'Add context & retry'})).toHaveAttribute( + 'aria-disabled', + ariaDisabled + ) + ); + } + ); + + describe('a PR that failed for credits', () => { + const section = makeSection('pull_request', 'completed', [ + [ + makePR({ + repo_name: 'org/api', + pr_creation_status: 'error', + pr_creation_error: "Your organization doesn't have enough Seer credits.", + pr_creation_error_code: 'out_of_credits', + pr_number: null, + pr_url: null, + }), + makePR({ + repo_name: 'org/web', + pr_creation_status: 'error', + pr_creation_error: 'Push rejected', + pr_number: null, + pr_url: null, + }), + ], + ]); + + it('says why and disables its retry while out of credits', async () => { + mockCreditCheck('autofix_open_pr', false); + render(, { + organization, + }); + + expect( + screen.getByText( + "Seer couldn't open a pull request in org/api because your organization is out of credits." + ) + ).toBeInTheDocument(); + + await expectDisabledForCredits( + screen.getByRole('button', {name: 'Retry PR in org/api'}) + ); + // Other failures stay retryable. + expect(screen.getByRole('button', {name: 'Retry PR in org/web'})).toHaveAttribute( + 'aria-disabled', + 'false' + ); + }); + + it('retries it once the organization can afford it', async () => { + mockCreditCheck('autofix_open_pr', true); + const createPR = jest.fn(); + render( + , + {organization} + ); + + await waitFor(() => + expect(screen.getByRole('button', {name: 'Retry PR in org/api'})).toHaveAttribute( + 'aria-disabled', + 'false' + ) + ); + + await userEvent.click(screen.getByRole('button', {name: 'Retry PR in org/api'})); + expect(createPR).toHaveBeenCalledWith(123, 'org/api'); + }); + }); +}); diff --git a/static/app/components/events/autofix/v3/autofixPreviews.spec.tsx b/static/app/components/events/autofix/v3/autofixPreviews.spec.tsx index 1555f379c7ca..02b87dea1bd6 100644 --- a/static/app/components/events/autofix/v3/autofixPreviews.spec.tsx +++ b/static/app/components/events/autofix/v3/autofixPreviews.spec.tsx @@ -213,6 +213,33 @@ describe('CodeChangesPreview', () => { }); }); +describe('a step that stopped for credits', () => { + function makeStoppedSection(step: string): AutofixSection { + return { + step, + artifacts: [], + blocks: [], + status: 'error', + failureReason: 'out_of_credits', + }; + } + + it.each([ + [RootCausePreview, 'root_cause', 'Seer stopped before finishing the root cause'], + [SolutionPreview, 'solution', 'Seer stopped before finishing the plan'], + [ + CodeChangesPreview, + 'code_changes', + 'Seer stopped before finishing the code changes', + ], + ])('says so instead of blaming Seer (%#)', (Preview, step, message) => { + render(); + + expect(screen.getByText(new RegExp(message))).toBeInTheDocument(); + expect(screen.queryByText(/This one is on us/)).not.toBeInTheDocument(); + }); +}); + describe('PullRequestsPreview', () => { function makePR(overrides: Partial = {}): RepoPRState { return { diff --git a/static/app/components/events/autofix/v3/autofixPreviews.tsx b/static/app/components/events/autofix/v3/autofixPreviews.tsx index 4f9e019f4db8..72e198b1ead7 100644 --- a/static/app/components/events/autofix/v3/autofixPreviews.tsx +++ b/static/app/components/events/autofix/v3/autofixPreviews.tsx @@ -24,6 +24,7 @@ import { getSingleRunStop, isCodeChangesArtifact, isCodingAgentsArtifact, + isOutOfCreditsSection, isPullRequestsArtifact, isRootCauseArtifact, isSolutionArtifact, @@ -32,6 +33,7 @@ import { import {LoadingIndicator} from 'sentry/components/loadingIndicator'; import {Placeholder} from 'sentry/components/placeholder'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import {OUT_OF_CREDITS_STEP_MESSAGE} from 'sentry/components/seer/outOfCredits/messages'; import {t, tn} from 'sentry/locale'; interface ArtifactPreviewProps { @@ -56,6 +58,8 @@ export function RootCausePreview({section}: ArtifactPreviewProps) { ) : stop ? ( + ) : isOutOfCreditsSection(section) ? ( + {OUT_OF_CREDITS_STEP_MESSAGE.root_cause} ) : ( {t( @@ -82,6 +86,8 @@ export function SolutionPreview({section}: ArtifactPreviewProps) { ) : artifact?.data ? ( + ) : isOutOfCreditsSection(section) ? ( + {OUT_OF_CREDITS_STEP_MESSAGE.solution} ) : ( {t('Seer failed to generate a plan. This one is on us. Try running it again.')} @@ -110,6 +116,10 @@ export function CodeChangesPreview({section}: ArtifactPreviewProps) { } } + if (isOutOfCreditsSection(section)) { + return {OUT_OF_CREDITS_STEP_MESSAGE.code_changes}; + } + if (reposChanged === 0) { return ( @@ -135,7 +145,7 @@ export function CodeChangesPreview({section}: ArtifactPreviewProps) { return ( {t('%s files changed in %s repos', filesChanged.size, reposChanged)} ); - }, [patchesByRepo]); + }, [patchesByRepo, section]); return ( } title={t('Code Changes')}> diff --git a/static/app/components/events/autofix/v3/codeChangesCard.stories.tsx b/static/app/components/events/autofix/v3/codeChangesCard.stories.tsx index 44265a57c1c7..307292753760 100644 --- a/static/app/components/events/autofix/v3/codeChangesCard.stories.tsx +++ b/static/app/components/events/autofix/v3/codeChangesCard.stories.tsx @@ -9,6 +9,7 @@ import type { RawFeedback, useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; +import {SeerCreditCheckStub} from 'sentry/components/seer/outOfCredits/__stories__/seerCreditCheckStub'; import * as Storybook from 'sentry/stories'; import type {User} from 'sentry/types/user'; import {useUser} from 'sentry/utils/useUser'; @@ -110,6 +111,29 @@ function makeAssistantBlock( }; } +function makeStepBlock( + step: string | undefined, + content: string +): AutofixSection['blocks'][number] { + return { + id: `block-${step ?? 'continued'}`, + timestamp: '2026-07-20T00:00:00Z', + message: {role: 'assistant', content, metadata: step ? {step} : undefined}, + }; +} + +// The block Seer appends when a run stops for credits. The card leaves it out +// in favor of its own line saying why the step stopped. +const SEER_OUT_OF_CREDITS_BLOCK: AutofixSection['blocks'][number] = { + id: 'error-1', + timestamp: '2026-07-20T00:00:00Z', + message: { + role: 'assistant', + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here. Add credits to continue.", + }, +}; + function makeSection( status: AutofixSection['status'], blocks: AutofixSection['blocks'], @@ -436,6 +460,85 @@ export default Storybook.story('CodeChangesCard Feedback', story => { ); }); + story('Stopped for credits', () => ( + + + The organization ran out of Seer credits while Seer was writing the code changes. + The run ends with failure_reason: out_of_credits. The card keeps what + Seer wrote and the partial diff, then says why the step stopped. The credit check + for autofix_code_changes says the organization still can't afford it, + so Re-run and the header's Re-run step are disabled (hover Re-run for why). Copy + as Markdown stays disabled for partial changes. The out-of-credits CTA is not + here: the alert at the top of the Autofix tab and drawer covers it. + + + + + + )); + + story('Stopped for credits, after a top-up', () => ( + + + The same stopped run once the organization has added credits: the credit check now + says it can afford autofix_code_changes, so Re-run and Re-run step + are enabled again without reloading. + + + + + + )); + + story('PR iteration stopped for credits', () => { + const user = useUser(); + return ( + + + The organization ran out of credits while Seer was working on feedback for an + open PR. The feedback item is marked failed (hover the red icon), the code + already pushed stays on screen, and a line says the PR update stopped. The + credit check for autofix_pr_iteration says the organization still + can't afford it, so “Add context & retry” and Re-run step are disabled. + + + + + + ); + }); + story('PR reviews', () => { return ( @@ -548,6 +651,29 @@ export default Storybook.story('CodeChangesCard Feedback', story => { }); }); +/** + * Code changes that stopped part way, one file in, because the organization ran + * out of credits. + */ +function CodeChangesStoppedForCredits() { + return ( + + ); +} + function StatusExample({children, label}: {children: ReactNode; label: string}) { return ( diff --git a/static/app/components/events/autofix/v3/codeChangesCard.tsx b/static/app/components/events/autofix/v3/codeChangesCard.tsx index 63d521a07992..b226189aedb0 100644 --- a/static/app/components/events/autofix/v3/codeChangesCard.tsx +++ b/static/app/components/events/autofix/v3/codeChangesCard.tsx @@ -14,6 +14,7 @@ import { collectPatches, getAutofixArtifactFromSection, isCodeChangesArtifact, + isOutOfCreditsSection, isPrIterationBlock, isPrIterationPaused, isSingleRunAutofix, @@ -21,6 +22,8 @@ import { type useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import {OUT_OF_CREDITS_STEP_MESSAGE} from 'sentry/components/seer/outOfCredits/messages'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {t, tn} from 'sentry/locale'; import {defined} from 'sentry/utils/defined'; import {useCopyToClipboard} from 'sentry/utils/useCopyToClipboard'; @@ -30,9 +33,10 @@ import {FileDiffViewer} from 'sentry/views/seerExplorer/components/fileDiffViewe import {ArtifactCard} from './artifactCard'; import {ArtifactDetails} from './artifactDetails'; -import {ArtifactLoadingDetails} from './artifactLoadingDetails'; +import {ArtifactLoadingDetails, ArtifactPartialDetails} from './artifactLoadingDetails'; import {AutofixResetPrompt} from './autofixResetPrompt'; import {FeedbackList, usePrIterationFeedback} from './feedbackList'; +import {OutOfCreditsStatus} from './outOfCreditsStatus'; import { PR_ITERATION_PAUSED_TOOLTIP, PrIterationFeedbackForm, @@ -85,21 +89,36 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp const {feedback, latestIterationIndex} = usePrIterationFeedback(section, autofix); - const loadingBlocks = useMemo(() => { - if (section.status !== 'processing') { + const isOutOfCredits = isOutOfCreditsSection(section); + const stoppedStep = isLatestStepPrIteration ? 'pr_iteration' : 'code_changes'; + const outOfCreditsRetry = useOutOfCreditsRetry(stoppedStep, { + enabled: isOutOfCredits, + }); + + // The blocks of the step that is running, or that stopped for credits. + const currentStepBlocks = useMemo(() => { + if (section.status !== 'processing' && !isOutOfCredits) { return []; } if (currentStepStart === -1) { return section.blocks; } return section.blocks.slice(currentStepStart); - }, [section.status, section.blocks, currentStepStart]); + }, [section.status, section.blocks, currentStepStart, isOutOfCredits]); const artifact = useMemo(() => { const sectionArtifact = getAutofixArtifactFromSection(section); return isCodeChangesArtifact(sectionArtifact) ? sectionArtifact : null; }, [section]); + // A step that stopped for credits keeps the diff it got to, which is not + // the finished artifact, so it can't be copied or turned into a PR. + const partialArtifact = useMemo( + () => + isOutOfCredits ? (section.artifacts.findLast(isCodeChangesArtifact) ?? null) : null, + [isOutOfCredits, section.artifacts] + ); + const {copy} = useCopyToClipboard(); const isSingleRun = isSingleRunAutofix(autofix.runState); const singleRunRerun = useRerunSingleRun(autofix, { @@ -120,11 +139,14 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp // Once a PR exists, reset opens the PR iteration form, so it stays available // while an iteration is processing. - const isResetEligible = + const isResetAvailable = !isPaused && !hasFailedOnlyPRs && noCodingAgents && (hasPRs || autofix.runState?.status !== 'processing'); + // A step that stopped for credits fails the same way until the organization + // can afford it again. + const isResetEligible = isResetAvailable && !outOfCreditsRetry.disabled; const {canReset, shouldShowReset, setShouldShowReset, handleReset} = useResetAutofixStep({ @@ -149,7 +171,10 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp ); }, [location, navigate, shouldShowReset]); - const patchesByRepo = useMemo(() => collectPatches(artifact ?? []), [artifact]); + const patchesByRepo = useMemo( + () => collectPatches(artifact ?? partialArtifact ?? []), + [artifact, partialArtifact] + ); const shouldExpandDiffs = useMemo(() => { let lineCount = 0; @@ -244,6 +269,32 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp ); } + const diffDetails = ( + + + {summary} + + {Array.from(patchesByRepo.entries(), ([repo, patches]) => ( + + + {t('Repository:')} + {repo} + + {patches.map((patch, index) => ( + + ))} + + ))} + + ); + let content: React.ReactNode; if (section.status === 'processing') { content = ( @@ -252,7 +303,7 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp the form available even mid-run. */} {shouldShowReset && showPrIterationForm && prIterationForm} - {summary} - - {Array.from(patchesByRepo.entries(), ([repo, patches]) => ( - - - {t('Repository:')} - {repo} - - {patches.map((patch, index) => ( - - ))} - - ))} + {diffDetails} + + ); + } else if (isOutOfCredits) { + let retry: React.ReactNode = null; + if (shouldShowReset) { + retry = resetPrompt( + t('What additional context should Seer use?'), + t('Add context that could unblock the change, e.g. the repo or files to edit.') + ); + } else if (isResetAvailable) { + // An iteration is retried with feedback, through the PR iteration form. + retry = ( + + + + ); + } + content = ( + + + {patchesByRepo.size > 0 && diffDetails} + + {retry} + ); } else if (explanation && isLatestStepPrIteration) { @@ -373,7 +438,7 @@ export function CodeChangesCard({autofix, groupId, section}: CodeChangesCardProp allowReset={!isSingleRun} onReset={canReset ? (resetInChat ?? (() => setShouldShowReset(true))) : undefined} resetInChat={defined(resetInChat)} - resetTooltip={isPaused ? PR_ITERATION_PAUSED_TOOLTIP : undefined} + resetTooltip={isPaused ? PR_ITERATION_PAUSED_TOOLTIP : outOfCreditsRetry.tooltip} > {content} diff --git a/static/app/components/events/autofix/v3/feedbackList.spec.tsx b/static/app/components/events/autofix/v3/feedbackList.spec.tsx index 7c043f93dd4b..1786d0c889ec 100644 --- a/static/app/components/events/autofix/v3/feedbackList.spec.tsx +++ b/static/app/components/events/autofix/v3/feedbackList.spec.tsx @@ -138,4 +138,82 @@ describe('FeedbackList', () => { expect(screen.getByText('rename it')).toBeInTheDocument(); expect(screen.getByTestId('feedback-processed')).toBeInTheDocument(); }); + + function iterationBlocks() { + return [ + ExplorerAutofixBlockFixture({ + id: 'iteration-0', + artifacts: [], + message: { + role: 'user', + content: 'Please add a null check', + metadata: { + step: 'pr_iteration', + iteration_index: '0', + feedback: JSON.stringify({ + text: 'Please add a null check', + source: {type: 'user-ui'}, + }), + }, + }, + }), + ]; + } + + it('says the iteration stopped for credits', async () => { + const state = ExplorerAutofixStateFixture({ + status: 'error', + failure_reason: 'out_of_credits', + blocks: [ + ...iterationBlocks(), + ExplorerAutofixBlockFixture({ + id: 'error-1', + artifacts: [], + message: { + role: 'assistant', + content: "Your organization doesn't have enough Seer credits for this.", + metadata: null, + }, + }), + ], + }); + + render(); + + await userEvent.hover(screen.getByRole('img', {name: 'Feedback failed'})); + expect( + await screen.findByText( + "Seer couldn't finish this feedback because your organization is out of credits." + ) + ).toBeInTheDocument(); + }); + + it('says the iteration push failed for credits', async () => { + const state = ExplorerAutofixStateFixture({ + status: 'completed', + blocks: [ + ...iterationBlocks(), + ExplorerAutofixBlockFixture({ + id: 'iteration-completed', + artifacts: [], + message: {role: 'assistant', content: 'Done', metadata: null}, + }), + ], + repo_pr_states: { + 'org/repo': AutofixRepoPRStateFixture({ + pr_creation_status: 'error', + pr_creation_error_code: 'out_of_credits', + }), + }, + }); + + render(); + + await userEvent.hover(screen.getByRole('img', {name: 'Feedback failed'})); + expect( + await screen.findByText( + "Seer couldn't finish this feedback because your organization is out of credits." + ) + ).toBeInTheDocument(); + }); }); diff --git a/static/app/components/events/autofix/v3/feedbackList.tsx b/static/app/components/events/autofix/v3/feedbackList.tsx index a45ef1050bb3..6e825225e390 100644 --- a/static/app/components/events/autofix/v3/feedbackList.tsx +++ b/static/app/components/events/autofix/v3/feedbackList.tsx @@ -18,6 +18,7 @@ import {Tooltip} from '@sentry/scraps/tooltip'; import {CollapsibleContent} from 'sentry/components/ai/chat/collapsibleContent'; import { + isOutOfCreditsSection, isPrIterationBlock, type AutofixSection, type RawFeedback, @@ -29,6 +30,7 @@ import {t} from 'sentry/locale'; import type {AvatarUser, User} from 'sentry/types/user'; import {defined} from 'sentry/utils/defined'; import {userDisplayName} from 'sentry/utils/formatters'; +import {isOutOfCreditsPRState} from 'sentry/views/seerExplorer/types'; import {ArtifactDetails} from './artifactDetails'; @@ -198,9 +200,12 @@ export function usePrIterationFeedback( const blockFeedback = useMemo(() => { const repoStates = Object.values(autofix.runState?.repo_pr_states ?? {}); const pushErrors = repoStates.filter(repo => repo.pr_creation_status === 'error'); - const errorCode = pushErrors.find( - repo => repo.pr_creation_error_code === 'branch_moved' - )?.pr_creation_error_code; + // Running out of credits stops either the iteration itself or its push. + const errorCode = + isOutOfCreditsSection(section) || pushErrors.some(isOutOfCreditsPRState) + ? 'out_of_credits' + : pushErrors.find(repo => repo.pr_creation_error_code === 'branch_moved') + ?.pr_creation_error_code; const inProgress = section.status === 'processing' || autofix.runState?.status === 'processing' || @@ -251,7 +256,7 @@ export function usePrIterationFeedback( errorCode: status === 'error' ? errorCode : undefined, })); }); - }, [section.blocks, section.status, autofix.runState, currentStepStart]); + }, [section, autofix.runState, currentStepStart]); const latestIterationIndex = useMemo( () => @@ -797,6 +802,21 @@ function PrimaryIconAvatar({Icon, tooltip}: {Icon: typeof IconSeer; tooltip: str ); } +function getFeedbackErrorMessage(errorCode: string | null | undefined): string { + switch (errorCode) { + case 'branch_moved': + return t( + 'Seer did not push these changes because it would have had to force-push over newer commits on the PR branch.' + ); + case 'out_of_credits': + return t( + "Seer couldn't finish this feedback because your organization is out of credits." + ); + default: + return t('Changes from this feedback could not be pushed'); + } +} + // Source-agnostic: the same status glyphs regardless of where the iteration came from. function FeedbackStatusIcon({ status, @@ -808,16 +828,7 @@ function FeedbackStatusIcon({ switch (status) { case 'error': return ( - + + {message} + {children} + + ); +} diff --git a/static/app/components/events/autofix/v3/pullRequestsCard.stories.tsx b/static/app/components/events/autofix/v3/pullRequestsCard.stories.tsx index b4163edf4411..eafaf1610ea4 100644 --- a/static/app/components/events/autofix/v3/pullRequestsCard.stories.tsx +++ b/static/app/components/events/autofix/v3/pullRequestsCard.stories.tsx @@ -8,6 +8,7 @@ import type { ExplorerAutofixState, useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; +import {SeerCreditCheckStub} from 'sentry/components/seer/outOfCredits/__stories__/seerCreditCheckStub'; import * as Storybook from 'sentry/stories'; import type {RepoPRState} from 'sentry/views/seerExplorer/types'; @@ -63,6 +64,17 @@ function failedPR(repoName = REPO): RepoPRState { }; } +// What the backend records when the LLM calls behind a PR (description, commit +// message) run out of credits. +function outOfCreditsPR(repoName = REPO): RepoPRState { + return { + ...creatingPR(repoName), + pr_creation_status: 'error', + pr_creation_error: "Your organization doesn't have enough Seer credits for this.", + pr_creation_error_code: 'out_of_credits', + }; +} + function makeSection(pullRequests: RepoPRState[]): AutofixSection { return { step: 'pull_request', @@ -143,6 +155,44 @@ export default Storybook.story('PullRequestsCard', story => { ); }); + story('Failed for credits', () => { + return ( + + + Creating the PR failed because the organization ran out of Seer credits ( + pr_creation_error_code: out_of_credits). A line under the PRs says + so, and the credit check for autofix_open_pr says the organization + still can't afford it, so its “Retry PR” is disabled (hover it for why). The + out-of-credits CTA is not here: the alert at the top of the Autofix tab and + drawer covers it. + + + + + + + + + + + + + + + ); + }); + story('Every state at once', () => { return ( diff --git a/static/app/components/events/autofix/v3/pullRequestsCard.tsx b/static/app/components/events/autofix/v3/pullRequestsCard.tsx index 0bcac8621499..5644346ec2a6 100644 --- a/static/app/components/events/autofix/v3/pullRequestsCard.tsx +++ b/static/app/components/events/autofix/v3/pullRequestsCard.tsx @@ -6,6 +6,7 @@ import {IconRefresh} from '@sentry/icons/refresh'; import {Button, LinkButton} from '@sentry/scraps/button'; import {Flex} from '@sentry/scraps/layout'; +import {Text} from '@sentry/scraps/text'; import {getAutofixRunId} from 'sentry/components/events/autofix/autofixRunId'; import {getRepoPullRequestLink} from 'sentry/components/events/autofix/pullRequests'; @@ -15,9 +16,15 @@ import { type AutofixSection, type useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; +import { + getOutOfCreditsPRMessage, + getOutOfCreditsPRsMessage, +} from 'sentry/components/seer/outOfCredits/messages'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {t} from 'sentry/locale'; import {defined} from 'sentry/utils/defined'; import {useCopyToClipboard} from 'sentry/utils/useCopyToClipboard'; +import {isOutOfCreditsPRState} from 'sentry/views/seerExplorer/types'; import {ArtifactCard} from './artifactCard'; import {artifactToMarkdown} from './utils'; @@ -40,6 +47,12 @@ export function PullRequestsCard({autofix, section}: PullRequestsCardProps) { [artifact] ); + const outOfCreditsPullRequests = artifact?.filter(isOutOfCreditsPRState) ?? []; + // Their "Retry PR" fails the same way until the organization can afford it. + const outOfCreditsRetry = useOutOfCreditsRetry('create_pr', { + enabled: defined(runId) && outOfCreditsPullRequests.length > 0, + }); + if (!runId) { return null; } @@ -83,19 +96,28 @@ export function PullRequestsCard({autofix, section}: PullRequestsCardProps) { ); } + const isOutOfCredits = isOutOfCreditsPRState(pullRequest); return ( ); })} + {outOfCreditsPullRequests.length > 0 && ( + {getOutOfCreditsPRsMessage(outOfCreditsPullRequests)} + )} ); } diff --git a/static/app/components/events/autofix/v3/rootCauseCard.stories.tsx b/static/app/components/events/autofix/v3/rootCauseCard.stories.tsx new file mode 100644 index 000000000000..d03bef5c6cb1 --- /dev/null +++ b/static/app/components/events/autofix/v3/rootCauseCard.stories.tsx @@ -0,0 +1,126 @@ +import {Stack} from '@sentry/scraps/layout'; +import {Text} from '@sentry/scraps/text'; + +import type { + AutofixSection, + useExplorerAutofix, +} from 'sentry/components/events/autofix/useExplorerAutofix'; +import {SeerCreditCheckStub} from 'sentry/components/seer/outOfCredits/__stories__/seerCreditCheckStub'; +import * as Storybook from 'sentry/stories'; + +import {RootCauseCard} from './rootCauseCard'; + +const noop = () => {}; + +const OUT_OF_CREDITS_AUTOFIX: ReturnType = { + runState: { + run_id: 123, + blocks: [], + status: 'error', + failure_reason: 'out_of_credits', + updated_at: '2026-07-20T00:00:00Z', + }, + autofixFormatted: null, + isLoading: false, + isWaitingForRun: false, + isPolling: false, + isProcessing: false, + // None of these are invoked by the static examples below. + startStep: () => Promise.resolve({runId: 0, isSingleRun: false}), + createPR: () => Promise.resolve(), + reset: noop, + triggerCodingAgentHandoff: () => Promise.resolve(), + codingAgentErrors: [], + dismissCodingAgentError: noop, + warnings: [], +}; + +// The block Seer appends when a run stops for credits. The card leaves it out +// and says why the step stopped instead. +const SEER_OUT_OF_CREDITS_BLOCK: AutofixSection['blocks'][number] = { + id: 'error-1', + timestamp: '2026-07-20T00:00:00Z', + message: { + role: 'assistant', + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here. Add credits to continue.", + }, +}; + +const PARTIAL_BLOCK: AutofixSection['blocks'][number] = { + id: 'block-root-cause', + timestamp: '2026-07-20T00:00:00Z', + message: { + role: 'assistant', + content: + '`CartService.total()` calls `reduce` without an initial value, so an empty cart throws `TypeError`. Next I was going to check which callers can pass an empty cart.', + metadata: {step: 'root_cause'}, + }, +}; + +function makeOutOfCreditsSection(blocks: AutofixSection['blocks']): AutofixSection { + return { + step: 'root_cause', + status: 'error', + failureReason: 'out_of_credits', + blocks: [...blocks, SEER_OUT_OF_CREDITS_BLOCK], + artifacts: [], + }; +} + +function StoppedForCredits({ + blocks = [PARTIAL_BLOCK], +}: { + blocks?: AutofixSection['blocks']; +}) { + return ( + + ); +} + +export default Storybook.story('RootCauseCard', story => { + story('Stopped for credits', () => ( + + + The organization ran out of Seer credits while Seer was finding the root cause. + The run ends with failure_reason: out_of_credits. The card keeps what + Seer wrote so far, then says why the step stopped. The credit check for{' '} + autofix_root_cause says the organization still can't afford it, so + Re-run and the header's Re-run step are disabled (hover Re-run for why). The + out-of-credits CTA is not here: the alert at the top of the Autofix tab and drawer + covers it. + + + + + + )); + + story('Stopped for credits, after a top-up', () => ( + + + The same stopped run once the organization has added credits: the credit check now + says it can afford autofix_root_cause, so Re-run and Re-run step are + enabled again without reloading. + + + + + + )); + + story('Stopped before writing anything', () => ( + + + When the run stops before Seer writes anything, the card only says why it stopped. + + + + + + )); +}); diff --git a/static/app/components/events/autofix/v3/rootCauseCard.tsx b/static/app/components/events/autofix/v3/rootCauseCard.tsx index ae70a19de78b..0f5b2394b863 100644 --- a/static/app/components/events/autofix/v3/rootCauseCard.tsx +++ b/static/app/components/events/autofix/v3/rootCauseCard.tsx @@ -9,21 +9,25 @@ import {Text} from '@sentry/scraps/text'; import { getAutofixArtifactFromSection, getSingleRunStop, + isOutOfCreditsSection, isRootCauseArtifact, isSingleRunAutofix, type AutofixSection, type useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import {OUT_OF_CREDITS_STEP_MESSAGE} from 'sentry/components/seer/outOfCredits/messages'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {t} from 'sentry/locale'; import {defined} from 'sentry/utils/defined'; import {useCopyToClipboard} from 'sentry/utils/useCopyToClipboard'; import {ArtifactCard} from './artifactCard'; import {ArtifactDetails} from './artifactDetails'; -import {ArtifactLoadingDetails} from './artifactLoadingDetails'; +import {ArtifactLoadingDetails, ArtifactPartialDetails} from './artifactLoadingDetails'; import {AutofixEvidence} from './autofixEvidence'; import {AutofixResetPrompt} from './autofixResetPrompt'; +import {OutOfCreditsStatus} from './outOfCreditsStatus'; import {useAutofixSectionEvidence} from './useAutofixSectionEvidence'; import {useResetAutofixStep} from './useResetAutofixStep'; import {useRethinkInChat} from './useRethinkInChat'; @@ -61,20 +65,52 @@ export function RootCauseCard({autofix, groupId, section}: RootCauseCardProps) { [artifact] ); + const isOutOfCredits = isOutOfCreditsSection(section); + const outOfCreditsRetry = useOutOfCreditsRetry('root_cause', { + enabled: isOutOfCredits, + }); + const {canReset, shouldShowReset, setShouldShowReset, handleReset} = useResetAutofixStep({ autofix, + // A step that stopped for credits fails the same way until the + // organization can afford it again. + canReset: outOfCreditsRetry.disabled ? false : undefined, section, step: 'root_cause', }); const rethinkPrompt = t('How can this root cause be improved?'); + const resetPrompt = shouldShowReset ? ( + setShouldShowReset(false)} + onReset={handleReset} + placeholder={t('Give seer additional context to improve this root cause.')} + prompt={rethinkPrompt} + /> + ) : null; const rethinkInChat = useRethinkInChat({ prompt: rethinkPrompt, step: 'root_cause', }); const evidence = useAutofixSectionEvidence({section}); + const evidenceDetails = + evidence.length > 0 ? ( + + {t('Evidence')} + + {evidence.map(e => ( + + ))} + + + ) : null; return ( setShouldShowReset(true))) : undefined} resetInChat={defined(rethinkInChat)} + resetTooltip={outOfCreditsRetry.tooltip} > {section.status === 'processing' ? ( ) : artifact?.data ? ( - {shouldShowReset && ( - setShouldShowReset(false)} - onReset={handleReset} - placeholder={t('Give seer additional context to improve this root cause.')} - prompt={rethinkPrompt} - /> - )} + {resetPrompt} {artifact.data.cause ? : null} @@ -135,23 +165,29 @@ export function RootCauseCard({autofix, groupId, section}: RootCauseCardProps) { ) : null} ) : null} - {evidence.length > 0 && ( - - {t('Evidence')} - - {evidence.map(e => ( - - ))} - - - )} + {evidenceDetails} {stopDetails} + ) : isOutOfCredits ? ( + + + {evidenceDetails} + + {resetPrompt ?? ( + + + + )} + + ) : stopDetails ? ( stopDetails ) : ( diff --git a/static/app/components/events/autofix/v3/solutionCard.stories.tsx b/static/app/components/events/autofix/v3/solutionCard.stories.tsx new file mode 100644 index 000000000000..1968387fa6c8 --- /dev/null +++ b/static/app/components/events/autofix/v3/solutionCard.stories.tsx @@ -0,0 +1,125 @@ +import {Stack} from '@sentry/scraps/layout'; +import {Text} from '@sentry/scraps/text'; + +import type { + AutofixSection, + useExplorerAutofix, +} from 'sentry/components/events/autofix/useExplorerAutofix'; +import {SeerCreditCheckStub} from 'sentry/components/seer/outOfCredits/__stories__/seerCreditCheckStub'; +import * as Storybook from 'sentry/stories'; + +import {SolutionCard} from './solutionCard'; + +const noop = () => {}; + +const OUT_OF_CREDITS_AUTOFIX: ReturnType = { + runState: { + run_id: 123, + blocks: [], + status: 'error', + failure_reason: 'out_of_credits', + updated_at: '2026-07-20T00:00:00Z', + }, + autofixFormatted: null, + isLoading: false, + isWaitingForRun: false, + isPolling: false, + isProcessing: false, + // None of these are invoked by the static examples below. + startStep: () => Promise.resolve({runId: 0, isSingleRun: false}), + createPR: () => Promise.resolve(), + reset: noop, + triggerCodingAgentHandoff: () => Promise.resolve(), + codingAgentErrors: [], + dismissCodingAgentError: noop, + warnings: [], +}; + +// The block Seer appends when a run stops for credits. The card leaves it out +// and says why the step stopped instead. +const SEER_OUT_OF_CREDITS_BLOCK: AutofixSection['blocks'][number] = { + id: 'error-1', + timestamp: '2026-07-20T00:00:00Z', + message: { + role: 'assistant', + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here. Add credits to continue.", + }, +}; + +const PARTIAL_BLOCK: AutofixSection['blocks'][number] = { + id: 'block-solution', + timestamp: '2026-07-20T00:00:00Z', + message: { + role: 'assistant', + content: + 'Pass `0` as the initial value to `reduce` in `CartService.total()` so an empty cart totals to zero. Next I was going to plan a regression test for the empty cart.', + metadata: {step: 'solution'}, + }, +}; + +function makeOutOfCreditsSection(blocks: AutofixSection['blocks']): AutofixSection { + return { + step: 'solution', + status: 'error', + failureReason: 'out_of_credits', + blocks: [...blocks, SEER_OUT_OF_CREDITS_BLOCK], + artifacts: [], + }; +} + +function StoppedForCredits({ + blocks = [PARTIAL_BLOCK], +}: { + blocks?: AutofixSection['blocks']; +}) { + return ( + + ); +} + +export default Storybook.story('SolutionCard', story => { + story('Stopped for credits', () => ( + + + The organization ran out of Seer credits while Seer was writing the plan. The run + ends with failure_reason: out_of_credits. The card keeps what Seer + wrote so far, then says why the step stopped. The credit check for{' '} + autofix_solution says the organization still can't afford it, so + Re-run and the header's Re-run step are disabled (hover Re-run for why). The + out-of-credits CTA is not here: the alert at the top of the Autofix tab and drawer + covers it. + + + + + + )); + + story('Stopped for credits, after a top-up', () => ( + + + The same stopped run once the organization has added credits: the credit check now + says it can afford autofix_solution, so Re-run and Re-run step are + enabled again without reloading. + + + + + + )); + + story('Stopped before writing anything', () => ( + + + When the run stops before Seer writes anything, the card only says why it stopped. + + + + + + )); +}); diff --git a/static/app/components/events/autofix/v3/solutionCard.tsx b/static/app/components/events/autofix/v3/solutionCard.tsx index 1a74fce7ce32..af5c5e1458d5 100644 --- a/static/app/components/events/autofix/v3/solutionCard.tsx +++ b/static/app/components/events/autofix/v3/solutionCard.tsx @@ -3,24 +3,28 @@ import {IconList} from '@sentry/icons/list'; import {IconRefresh} from '@sentry/icons/refresh'; import {Button} from '@sentry/scraps/button'; -import {Container, Stack} from '@sentry/scraps/layout'; +import {Container, Flex, Stack} from '@sentry/scraps/layout'; import {Text} from '@sentry/scraps/text'; import { getAutofixArtifactFromSection, + isOutOfCreditsSection, isSolutionArtifact, type AutofixSection, type useExplorerAutofix, } from 'sentry/components/events/autofix/useExplorerAutofix'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import {OUT_OF_CREDITS_STEP_MESSAGE} from 'sentry/components/seer/outOfCredits/messages'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {t} from 'sentry/locale'; import {defined} from 'sentry/utils/defined'; import {useCopyToClipboard} from 'sentry/utils/useCopyToClipboard'; import {ArtifactCard} from './artifactCard'; import {ArtifactDetails} from './artifactDetails'; -import {ArtifactLoadingDetails} from './artifactLoadingDetails'; +import {ArtifactLoadingDetails, ArtifactPartialDetails} from './artifactLoadingDetails'; import {AutofixResetPrompt} from './autofixResetPrompt'; +import {OutOfCreditsStatus} from './outOfCreditsStatus'; import {useResetAutofixStep} from './useResetAutofixStep'; import {useRethinkInChat} from './useRethinkInChat'; import {artifactToMarkdown} from './utils'; @@ -42,14 +46,30 @@ export function SolutionCard({autofix, section}: SolutionCardProps) { [artifact] ); + const isOutOfCredits = isOutOfCreditsSection(section); + const outOfCreditsRetry = useOutOfCreditsRetry('solution', { + enabled: isOutOfCredits, + }); + const {canReset, shouldShowReset, setShouldShowReset, handleReset} = useResetAutofixStep({ autofix, + // A step that stopped for credits fails the same way until the + // organization can afford it again. + canReset: outOfCreditsRetry.disabled ? false : undefined, section, step: 'solution', }); const rethinkPrompt = t('How can this plan be improved?'); + const resetPrompt = shouldShowReset ? ( + setShouldShowReset(false)} + onReset={handleReset} + placeholder={t('Give seer additional context to improve this plan.')} + prompt={rethinkPrompt} + /> + ) : null; const rethinkInChat = useRethinkInChat({ prompt: rethinkPrompt, step: 'solution', @@ -67,6 +87,7 @@ export function SolutionCard({autofix, section}: SolutionCardProps) { allowReset onReset={canReset ? (rethinkInChat ?? (() => setShouldShowReset(true))) : undefined} resetInChat={defined(rethinkInChat)} + resetTooltip={outOfCreditsRetry.tooltip} > {section.status === 'processing' ? ( ) : artifact?.data ? ( - {shouldShowReset && ( - setShouldShowReset(false)} - onReset={handleReset} - placeholder={t('Give seer additional context to improve this plan.')} - prompt={rethinkPrompt} - /> - )} + {resetPrompt} @@ -104,6 +118,25 @@ export function SolutionCard({autofix, section}: SolutionCardProps) { ) : null} + ) : isOutOfCredits ? ( + + + + {resetPrompt ?? ( + + + + )} + + ) : ( diff --git a/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx b/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx index 0094c22754ed..e2daa5256d00 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.spec.tsx @@ -493,6 +493,171 @@ describe('AutofixRef embed', () => { } ); + describe('out of credits', () => { + const creditsOrganization = OrganizationFixture({ + slug: 'org-slug', + features: ['seer-credit-checks', 'token-billed-seer-enabled'], + }); + const retryTooltip = 'Your organization does not have enough Seer credits for this.'; + + function mockCreditCheck(operation: string, hasEnoughCredits: boolean) { + return MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + match: [MockApiClient.matchQuery({operation})], + body: {hasEnoughCredits}, + }); + } + + // While the check is in flight the button is natively disabled. Once it + // settles it is only aria-disabled, so it stays focusable and its tooltip + // can say why. + async function expectDisabledForCredits(button: HTMLElement) { + await waitFor(() => expect(button).toBeEnabled()); + expect(button).toHaveAttribute('aria-disabled', 'true'); + await userEvent.hover(button); + expect(await screen.findByText(retryTooltip)).toBeInTheDocument(); + } + + function makeStoppedRun(step: AutofixExplorerStep): ExplorerAutofixState { + const run = makeRunThrough(step, {status: 'error'}); + return { + ...run, + failure_reason: 'out_of_credits', + blocks: [ + ...run.blocks, + { + id: 'error-1', + timestamp: '2026-01-01T00:00:00Z', + message: { + role: 'assistant', + content: "Your organization doesn't have enough Seer credits for this.", + }, + }, + ], + }; + } + + function withOutOfCreditsPR(run: ExplorerAutofixState): ExplorerAutofixState { + const prState = run.repo_pr_states!['getsentry/sentry']!; + return { + ...run, + repo_pr_states: { + 'getsentry/sentry': {...prState, pr_creation_error_code: 'out_of_credits'}, + }, + }; + } + + it.each([ + [ + 'root_cause', + 'autofix_root_cause', + 'Seer stopped before finishing the root cause', + ], + ['solution', 'autofix_solution', 'Seer stopped before finishing the plan'], + [ + 'code_changes', + 'autofix_code_changes', + 'Seer stopped before finishing the code changes', + ], + ] as const)( + 'keeps what the %s step wrote and disables Try again while out of credits', + async (step, operation, message) => { + mockRun(makeStoppedRun(step)); + mockCreditCheck(operation, false); + render( + + + , + {organization: creditsOrganization} + ); + await expandAll(1); + + expect(screen.getByText(`Finished ${step}`)).toBeInTheDocument(); + expect(screen.getByText(new RegExp(message))).toBeInTheDocument(); + expect(screen.queryByText(/This one is on us/)).not.toBeInTheDocument(); + expect( + screen.queryByText(/doesn't have enough Seer credits/) + ).not.toBeInTheDocument(); + expect(getButtons('Draft a pull request')).toHaveLength(0); + await expectDisabledForCredits(screen.getByRole('button', {name: 'Try again'})); + } + ); + + it('retries a step that stopped for credits once the organization can afford it', async () => { + mockRun(makeStoppedRun('code_changes')); + mockCreditCheck('autofix_code_changes', true); + const sendMessage = jest.fn(); + render( + + + , + {organization: creditsOrganization} + ); + await expandAll(1); + + await waitFor(() => + expect(screen.getByRole('button', {name: 'Try again'})).toHaveAttribute( + 'aria-disabled', + 'false' + ) + ); + await userEvent.click(screen.getByRole('button', {name: 'Try again'})); + expect(sendMessage).toHaveBeenCalledWith( + 'Retry the Code Changes step for JAVASCRIPT-1.' + ); + }); + + it('disables drafting the pull request again while out of credits', async () => { + mockRun(withOutOfCreditsPR(makeRunThrough('code_changes', {pullRequest: 'error'}))); + mockCreditCheck('autofix_open_pr', false); + render( + + + , + {organization: creditsOrganization} + ); + await expandAll(1); + + await expectDisabledForCredits( + screen.getByRole('button', {name: 'Draft a pull request'}) + ); + }); + + it('drafts the pull request again once the organization can afford it', async () => { + mockRun(withOutOfCreditsPR(makeRunThrough('code_changes', {pullRequest: 'error'}))); + mockCreditCheck('autofix_open_pr', true); + const sendMessage = jest.fn(); + render( + + + , + {organization: creditsOrganization} + ); + await expandAll(1); + + await waitFor(() => + expect( + screen.getByRole('button', {name: 'Draft a pull request'}) + ).toHaveAttribute('aria-disabled', 'false') + ); + await userEvent.click(screen.getByRole('button', {name: 'Draft a pull request'})); + expect(sendMessage).toHaveBeenCalledWith('Draft a pull request for JAVASCRIPT-1.'); + }); + + it('says the pull request step failed for credits', async () => { + mockRun(withOutOfCreditsPR(makeRunThrough('code_changes', {pullRequest: 'error'}))); + render(, {organization: creditsOrganization}); + await expandAll(1); + + expect( + screen.getByText( + "Seer couldn't open a pull request in getsentry/sentry because your organization is out of credits." + ) + ).toBeInTheDocument(); + expect(screen.queryByText(/This one is on us/)).not.toBeInTheDocument(); + }); + }); + // One embed per step, as a conversation that walked the whole run holds. // Only the action the run has not taken yet is on offer, from one place. it('offers only the untaken action across a conversation spanning every step', async () => { diff --git a/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx b/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx index 60acfdca6c11..11c96a41b3d6 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.stories.tsx @@ -1,4 +1,4 @@ -import {useMemo, useState, type ReactNode} from 'react'; +import {Fragment, useEffect, useMemo, useRef, useState, type ReactNode} from 'react'; import {IconArrow} from '@sentry/icons/arrow'; import { QueryClient, @@ -512,6 +512,58 @@ function creditsRun(blocks: Block[]): ExplorerAutofixState { }; } +// The block Seer appends when a run stops for credits. +const SEER_OUT_OF_CREDITS_BLOCK: Block = { + id: 'error-1', + timestamp: '2026-10-07T12:00:00Z', + loading: false, + message: { + role: 'assistant', + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here. Add credits to continue.", + }, +}; + +// Root cause and plan finished; code changes stopped part way, one file in. +const CODE_CHANGES_STOPPED_RUN: ExplorerAutofixState = { + run_id: CREDITS_RUN_ID, + status: 'error', + failure_reason: 'out_of_credits', + updated_at: '2026-10-07T12:00:00Z', + blocks: [ + ROOT_CAUSE_BLOCK, + SOLUTION_BLOCK, + { + ...CODE_CHANGES_BLOCK, + message: { + ...CODE_CHANGES_BLOCK.message, + content: + 'Passed `0` as the initial accumulator in `cartService.ts`. Next up was the empty-cart test in `cartService.test.ts`.', + }, + }, + SEER_OUT_OF_CREDITS_BLOCK, + ], +}; + +// Code changes finished, then opening the PR failed for credits. +const PR_STOPPED_RUN: ExplorerAutofixState = { + ...creditsRun([ROOT_CAUSE_BLOCK, SOLUTION_BLOCK, CODE_CHANGES_BLOCK]), + repo_pr_states: { + 'acme/checkout': { + repo_name: 'acme/checkout', + branch_name: null, + commit_sha: null, + pr_creation_status: 'error', + pr_creation_error: "Your organization doesn't have enough Seer credits for this.", + pr_creation_error_code: 'out_of_credits', + pr_id: null, + pr_number: null, + pr_url: null, + title: null, + }, + }, +}; + /** * Answers the embeds' run request with the given run, the credit check with the * given balance, and every other request with an empty list. @@ -545,6 +597,25 @@ class CreditsQueryClient extends QueryClient { }; } +/** + * Autofix embeds ship collapsed. Open them so the state is visible without a + * click. + */ +function ExpandedEmbeds({children}: {children: ReactNode}) { + const ref = useRef(null); + const opened = useRef(false); + useEffect(() => { + if (opened.current) { + return; + } + opened.current = true; + ref.current + ?.querySelectorAll('button[aria-expanded="false"]') + .forEach(button => button.click()); + }, []); + return {children}; +} + /** * A conversation in an org with credit checks on, out of credits unless * `hasEnoughCredits` is set, where the chat can still send, so each embed @@ -555,12 +626,17 @@ function CreditsChat({ children, description, run, + expanded = false, hasEnoughCredits = false, isBillingMember = false, }: { children: ReactNode; description: ReactNode; run: ExplorerAutofixState; + /** + * Open the embeds on load, so the story needs no click. + */ + expanded?: boolean; hasEnoughCredits?: boolean; /** * Billing members configure auto-reload; everyone else contacts their admins. @@ -571,6 +647,21 @@ function CreditsChat({ const [queryClient] = useState(() => new CreditsQueryClient(run, hasEnoughCredits)); const {isAutofixOutOfCredits, autofixDenialReason, reportOutOfCredits} = useAutofixOutOfCreditsEmbeds(); + const chat = ( + + ) : null + } + > + {children} + + ); return ( @@ -595,19 +686,7 @@ function CreditsChat({ sendMessage={() => {}} reportOutOfCredits={reportOutOfCredits} > - - ) : null - } - > - {children} - + {expanded ? {chat} : chat} @@ -664,6 +743,50 @@ export default Storybook.story('Autofix', story => { )); + story('Stopped for credits', () => ( + + The organization ran out of Seer credits while Autofix was writing code changes, + so the run ended with failure_reason: out_of_credits. The card + keeps what Seer wrote and the partial diff, then says why it stopped. The credit + check for autofix_code_changes says the organization still can't + afford it, so “Try again” is disabled (hover it for why). No “Draft a pull + request” is offered for the unfinished changes. + + } + run={CODE_CHANGES_STOPPED_RUN} + expanded + > + Fix {ISSUE.shortId} all the way. + + + )); + + story('Pull request failed for credits', () => ( + + The code changes finished, then opening the PR failed because the organization + ran out of credits (pr_creation_error_code: out_of_credits). The + Pull Request card says so in place of Seer's generic failure. The credit check + for autofix_open_pr says the organization still can't afford it, so + the Code Changes card's “Draft a pull request” is disabled (hover it for why), + and the notice above the composer offers more credits. + + } + run={PR_STOPPED_RUN} + expanded + > + Open a pull request for {ISSUE.shortId}. + + + )); + story('Live Autofix run (autofixRef)', () => ); story('Out of credits: after the root cause', () => ( diff --git a/static/app/components/seer/markdown/embeds/components/autofix.tsx b/static/app/components/seer/markdown/embeds/components/autofix.tsx index 8a0cfe226257..566459bf305a 100644 --- a/static/app/components/seer/markdown/embeds/components/autofix.tsx +++ b/static/app/components/seer/markdown/embeds/components/autofix.tsx @@ -23,6 +23,9 @@ import { getAutofixArtifactFromSection, getOrderedAutofixSections, isCodeChangesArtifact, + isLastStepPrIteration, + isOutOfCreditsSection, + isPrIterationBlock, isPullRequestsArtifact, isPullRequestsSection, isCodeChangesSection, @@ -42,11 +45,20 @@ import { useReportAutofixOutOfCredits, } from 'sentry/components/seer/autofixChatContext'; import {defineSeerEmbed} from 'sentry/components/seer/markdown/embeds/utils'; +import { + getOutOfCreditsPRsMessage, + OUT_OF_CREDITS_RETRY_TOOLTIP, + OUT_OF_CREDITS_STEP_MESSAGE, +} from 'sentry/components/seer/outOfCredits/messages'; import {SeerOutOfCreditsCta} from 'sentry/components/seer/outOfCredits/seerOutOfCreditsCta'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {t, tn} from 'sentry/locale'; import type {Group} from 'sentry/types/group'; +import {defined} from 'sentry/utils/defined'; +import {useAutoScroll} from 'sentry/utils/useAutoScroll'; import {useOrganization} from 'sentry/utils/useOrganization'; import {FileDiffViewer} from 'sentry/views/seerExplorer/components/fileDiffViewer'; +import {isOutOfCreditsPRState, isSeerErrorBlock} from 'sentry/views/seerExplorer/types'; import {resourceLinkMarkdown} from './resourceLink'; import {SeerEmbedBlock} from './seerEmbedBlock'; @@ -297,6 +309,20 @@ function AutofixRefContent({id, shortId, step}: AutofixRefContentProps) { const canCreatePR = step === 'code_changes' && !hasCreatedPullRequests(runState?.repo_pr_states); + // A step or PR that stopped for credits fails the same way until the + // organization can afford it again, so its retry waits for the credit check. + const outOfCreditsSection = + section && isOutOfCreditsSection(section) ? section : undefined; + const stepRetry = useOutOfCreditsRetry( + outOfCreditsSection ? getStoppedStep(outOfCreditsSection, step) : step, + {enabled: defined(outOfCreditsSection)} + ); + // Draft a pull request retries a create that failed for credits. The credit + // check below holds it back like any other create; this only explains why. + const prFailedForCredits = Object.values(runState?.repo_pr_states ?? {}).some( + isOutOfCreditsPRState + ); + const showsNextAction = section?.status === 'completed' && (canCreatePR || !!pendingNextStep); const creditCheck = useAutofixCreditCheck(canCreatePR ? 'create_pr' : pendingNextStep, { @@ -323,13 +349,32 @@ function AutofixRefContent({id, shortId, step}: AutofixRefContentProps) { creditCheck.denialReason ); + // Once a PR exists this embed shows the PR section, so an iteration that ran + // out of credits is only visible on the code_changes section it folds into. + const codeChangesSection = sections.find(isCodeChangesSection); + const iterationOutOfCredits = + step === 'pr_iteration' && + !!codeChangesSection && + isOutOfCreditsSection(codeChangesSection) && + isLastStepPrIteration(runState); + return ( - + {section?.status === 'error' && ( - @@ -350,6 +395,12 @@ function AutofixRefContent({id, shortId, step}: AutofixRefContentProps) { variant="primary" onClick={handleCreatePR} disabled={!canContinue} + tooltipProps={{ + title: + prFailedForCredits && !creditCheck.hasEnoughCredits + ? OUT_OF_CREDITS_RETRY_TOOLTIP + : undefined, + }} > {t('Draft a pull request')} @@ -374,10 +425,16 @@ function AutofixRefContent({id, shortId, step}: AutofixRefContentProps) { interface AutofixRefBodyProps { isLoading: boolean; step: AutofixExplorerStep; + iterationOutOfCredits?: boolean; section?: AutofixSection; } -function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { +function AutofixRefBody({ + isLoading, + iterationOutOfCredits, + section, + step, +}: AutofixRefBodyProps) { if (isLoading || !section || section.status === 'processing') { return ( @@ -387,6 +444,10 @@ function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { ); } + if (isOutOfCreditsSection(section)) { + return ; + } + if (section.status === 'error') { return {ERROR_TEXT[step]}; } @@ -395,25 +456,43 @@ function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { const artifact = getAutofixArtifactFromSection(section); const pullRequests = isPullRequestsArtifact(artifact) ? artifact : []; const links = pullRequests.map(getRepoPullRequestLink).filter(link => link !== null); + // The PR section never reads as errored, so a create or push that ran out + // of credits only shows on its repo's state. + const outOfCreditsPullRequests = pullRequests.filter(isOutOfCreditsPRState); + let outOfCreditsStatus: ReactNode = null; + if (outOfCreditsPullRequests.length > 0) { + outOfCreditsStatus = ( + + {getOutOfCreditsPRsMessage(outOfCreditsPullRequests)} + + ); + } else if (iterationOutOfCredits) { + outOfCreditsStatus = ( + {OUT_OF_CREDITS_STEP_MESSAGE.pr_iteration} + ); + } if (!links.length) { - return {ERROR_TEXT.pr_iteration}; + return outOfCreditsStatus ?? {ERROR_TEXT.pr_iteration}; } return ( - - {links.map(link => ( - } - href={link.url} - external - > - {link.label} - - ))} - + + + {links.map(link => ( + } + href={link.url} + external + > + {link.label} + + ))} + + {outOfCreditsStatus} + ); } @@ -448,6 +527,103 @@ function AutofixRefBody({isLoading, section, step}: AutofixRefBodyProps) { return {ERROR_TEXT[step]}; } +/** + * The blocks of the step a section stopped in. A code_changes section also + * holds every PR iteration, and only the one that stopped is relevant. + */ +function getStoppedStepBlocks(section: AutofixSection): AutofixSection['blocks'] { + const stepStart = section.blocks.findLastIndex(block => + defined(block.message.metadata?.step) + ); + return section.blocks.slice(Math.max(stepStart, 0)); +} + +/** + * The step a section stopped in: a code_changes section can stop in a PR + * iteration. + */ +function getStoppedStep( + section: AutofixSection, + step: AutofixExplorerStep +): AutofixExplorerStep { + if (!isCodeChangesSection(section)) { + return step; + } + const firstBlock = getStoppedStepBlocks(section)[0]; + return firstBlock && isPrIterationBlock(firstBlock) ? 'pr_iteration' : 'code_changes'; +} + +function hasText(content: string | null | undefined): content is string { + const trimmed = content?.trim(); + return !!trimmed && trimmed !== '.'; +} + +/** + * What a stopped step wrote, without Seer's own error block, since the line + * below says why it stopped. It renders with this embed's markdown rather than + * `ArtifactPartialDetails`, whose `SeerMarkdown` renders embeds and so would + * import this file back. + */ +function StoppedStepText({blocks}: {blocks: AutofixSection['blocks']}) { + const {containerRef, onScrollHandler} = useAutoScroll({key: blocks}); + const texts = blocks.flatMap(block => { + if (block.message.role === 'user' || isSeerErrorBlock(block)) { + return []; + } + const {content, thinking_content} = block.message; + if (hasText(content) && content !== 'Thinking...') { + return [content]; + } + return hasText(thinking_content) ? [thinking_content] : []; + }); + + if (texts.length === 0) { + return null; + } + + return ( + + + {texts.map((text, index) => ( + + ))} + + + ); +} + +/** + * A step that stopped for credits: what it wrote and, for code changes, the + * diff it got to, then a line saying why it stopped. The credit gates show the + * CTA to add credits, so this has none of its own. + */ +function OutOfCreditsStepBody({ + section, + step, +}: { + section: AutofixSection; + step: AutofixExplorerStep; +}) { + const partialPatches = section.artifacts.findLast(isCodeChangesArtifact); + const patchesByRepo = collectPatches(partialPatches ?? []); + + return ( + + + {patchesByRepo.size > 0 && } + + {OUT_OF_CREDITS_STEP_MESSAGE[getStoppedStep(section, step)]} + + + ); +} + interface RootCauseBodyProps { description: string; fiveWhys: string[]; diff --git a/static/app/components/seer/outOfCredits/__stories__/seerCreditCheckStub.tsx b/static/app/components/seer/outOfCredits/__stories__/seerCreditCheckStub.tsx new file mode 100644 index 000000000000..9ede540e6cfd --- /dev/null +++ b/static/app/components/seer/outOfCredits/__stories__/seerCreditCheckStub.tsx @@ -0,0 +1,62 @@ +import {useState, type ReactNode} from 'react'; +import {QueryClient, QueryClientProvider, skipToken} from '@tanstack/react-query'; + +import {OrganizationContext} from 'sentry/utils/organizationContext'; +import {useOrganization} from 'sentry/utils/useOrganization'; + +/** + * Answers every Seer credit check with a fixed result. Other queries go to the + * API as usual. + */ +class CreditCheckQueryClient extends QueryClient { + hasEnoughCredits: boolean; + + constructor(hasEnoughCredits: boolean) { + super({defaultOptions: {queries: {retry: false}}}); + this.hasEnoughCredits = hasEnoughCredits; + } + + override defaultQueryOptions: QueryClient['defaultQueryOptions'] = options => { + const defaulted = super.defaultQueryOptions(options); + const url = String(defaulted.queryKey[0]); + if (defaulted.queryFn === skipToken || !url.includes('/seer/credit-check/')) { + return defaulted; + } + const json = {hasEnoughCredits: this.hasEnoughCredits}; + const queryFn = () => Promise.resolve({json, headers: {}}); + return {...defaulted, queryFn: queryFn as never}; + }; +} + +interface SeerCreditCheckStubProps { + children: ReactNode; + hasEnoughCredits: boolean; +} + +/** + * Turns on `seer-credit-checks` and `token-billed-seer-enabled`, and answers + * every credit check with `hasEnoughCredits`, so a story can show what a + * credit-gated button does before and after the organization tops up. + */ +export function SeerCreditCheckStub({ + children, + hasEnoughCredits, +}: SeerCreditCheckStubProps) { + const organization = useOrganization(); + const [queryClient] = useState(() => new CreditCheckQueryClient(hasEnoughCredits)); + + return ( + + {children} + + ); +} diff --git a/static/app/components/seer/outOfCredits/messages.tsx b/static/app/components/seer/outOfCredits/messages.tsx new file mode 100644 index 000000000000..9b948b1f0a19 --- /dev/null +++ b/static/app/components/seer/outOfCredits/messages.tsx @@ -0,0 +1,94 @@ +import type {AutofixExplorerStep} from 'sentry/components/events/autofix/useExplorerAutofix'; +import {t} from 'sentry/locale'; +import type {RepoPRState} from 'sentry/views/seerExplorer/types'; + +/** + * Inline status for an autofix step that stopped part way for credits. The + * out-of-credits CTA is shown by the credit gates, so this only says what + * happened. + */ +export const OUT_OF_CREDITS_STEP_MESSAGE: Record = { + root_cause: t( + 'Seer stopped before finishing the root cause because your organization is out of credits.' + ), + solution: t( + 'Seer stopped before finishing the plan because your organization is out of credits.' + ), + code_changes: t( + 'Seer stopped before finishing the code changes because your organization is out of credits.' + ), + pr_iteration: t( + 'Seer stopped before updating the pull request because your organization is out of credits.' + ), +}; + +/** + * Toast for an autofix step the backend refused to start because the + * organization is out of credits. + */ +export const OUT_OF_CREDITS_START_STEP_MESSAGE: Record = { + root_cause: t( + "Seer couldn't start on the root cause because your organization is out of credits." + ), + solution: t( + "Seer couldn't start on the plan because your organization is out of credits." + ), + code_changes: t( + "Seer couldn't start on the code changes because your organization is out of credits." + ), + pr_iteration: t( + "Seer couldn't update the pull request because your organization is out of credits." + ), +}; + +/** + * Toast for pull requests the backend refused to open because the + * organization is out of credits, when the request named no repo. + */ +export const OUT_OF_CREDITS_OPEN_PRS_MESSAGE = t( + "Seer couldn't open a pull request because your organization is out of credits." +); + +export const OUT_OF_CREDITS_CHAT_MESSAGE = t( + 'Seer stopped because your organization is out of credits.' +); + +/** + * Tooltip for a retry that stays disabled until the organization can afford it. + */ +export const OUT_OF_CREDITS_RETRY_TOOLTIP = t( + 'Your organization does not have enough Seer credits for this.' +); + +/** + * Stands in for the backend's `pr_creation_error` on a PR that failed for + * credits. A PR number means the PR already exists, so the push failed. + */ +export function getOutOfCreditsPRMessage( + state: Pick +): string { + return state.pr_number + ? t( + "Seer couldn't push changes to %s#%s because your organization is out of credits.", + state.repo_name, + state.pr_number + ) + : t( + "Seer couldn't open a pull request in %s because your organization is out of credits.", + state.repo_name + ); +} + +/** + * One status for every PR in a run that failed for credits, however many repos + * the run touched. + */ +export function getOutOfCreditsPRsMessage(states: RepoPRState[]): string { + if (states.length === 1) { + return getOutOfCreditsPRMessage(states[0]!); + } + return t( + "Seer couldn't finish %s pull requests because your organization is out of credits.", + states.length + ); +} diff --git a/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.spec.tsx b/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.spec.tsx new file mode 100644 index 000000000000..7bcb73106760 --- /dev/null +++ b/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.spec.tsx @@ -0,0 +1,111 @@ +import {QueryClientProvider} from '@tanstack/react-query'; +import {OrganizationFixture} from 'sentry-fixture/organization'; + +import {makeTestQueryClient} from 'sentry-test/queryClient'; +import { + render, + renderHookWithProviders, + screen, + waitFor, +} from 'sentry-test/reactTestingLibrary'; + +import {useOutOfCreditsRetry} from './useOutOfCreditsRetry'; + +describe('useOutOfCreditsRetry', () => { + const organization = OrganizationFixture({ + features: ['seer-credit-checks', 'token-billed-seer-enabled'], + }); + + function mockCreditCheck(hasEnoughCredits: boolean) { + return MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + match: [MockApiClient.matchQuery({operation: 'autofix_code_changes'})], + body: {hasEnoughCredits}, + }); + } + + afterEach(() => { + MockApiClient.clearMockResponses(); + }); + + it('holds the retry back while the check is in flight, then disables it with a reason', async () => { + mockCreditCheck(false); + + const {result} = renderHookWithProviders( + () => useOutOfCreditsRetry('code_changes', {enabled: true}), + {organization} + ); + + expect(result.current).toEqual({disabled: true, tooltip: undefined}); + await waitFor(() => + expect(result.current.tooltip).toBe( + 'Your organization does not have enough Seer credits for this.' + ) + ); + expect(result.current.disabled).toBe(true); + }); + + it('enables the retry once the organization can afford it', async () => { + mockCreditCheck(true); + + const {result} = renderHookWithProviders( + () => useOutOfCreditsRetry('code_changes', {enabled: true}), + {organization} + ); + + await waitFor(() => + expect(result.current).toEqual({disabled: false, tooltip: undefined}) + ); + }); + + it('holds the retry back until its own check answers, even with an answer cached', async () => { + function RetryButton() { + const {disabled} = useOutOfCreditsRetry('code_changes', {enabled: true}); + return ; + } + // One cache across both mounts, as the app keeps one. + const queryClient = makeTestQueryClient(); + const tree = ( + + + + ); + + mockCreditCheck(true); + const {unmount} = render(tree, {organization}); + await waitFor(() => expect(screen.getByRole('button')).toBeEnabled()); + unmount(); + + // Credits run out before the retry is back on screen, and the recheck + // hasn't answered yet. The cached answer from before must not enable it. + let answerRecheck!: () => void; + MockApiClient.clearMockResponses(); + MockApiClient.addMockResponse({ + url: `/organizations/${organization.slug}/seer/credit-check/`, + match: [MockApiClient.matchQuery({operation: 'autofix_code_changes'})], + body: {hasEnoughCredits: false}, + asyncDelay: new Promise(resolve => { + answerRecheck = resolve; + }), + }); + render(tree, {organization}); + + expect(screen.getByRole('button')).toBeDisabled(); + + answerRecheck(); + await waitFor(() => expect(queryClient.isFetching()).toBe(0)); + expect(screen.getByRole('button')).toBeDisabled(); + }); + + it('does not check when the retry is not on screen', () => { + const request = mockCreditCheck(false); + + const {result} = renderHookWithProviders( + () => useOutOfCreditsRetry('code_changes', {enabled: false}), + {organization} + ); + + expect(result.current).toEqual({disabled: false, tooltip: undefined}); + expect(request).not.toHaveBeenCalled(); + }); +}); diff --git a/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.tsx b/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.tsx new file mode 100644 index 000000000000..20ffd19a694a --- /dev/null +++ b/static/app/components/seer/outOfCredits/useOutOfCreditsRetry.tsx @@ -0,0 +1,38 @@ +import { + useAutofixCreditCheck, + type AutofixAction, +} from 'sentry/components/events/autofix/autofixCredits'; + +import {OUT_OF_CREDITS_RETRY_TOOLTIP} from './messages'; + +interface UseOutOfCreditsRetryOptions { + /** + * The retry is on screen for something that stopped for credits. Nothing is + * checked otherwise. + */ + enabled: boolean; +} + +/** + * Gates the retry of a step or PR that stopped because the organization ran + * out of Seer credits. The retry stays on screen, disabled while the + * organization can't afford `action`, and comes back once it can. + * + * Fails open like `useSeerCreditCheck`: without `seer-credit-checks` the retry + * is enabled and the backend decides. + */ +export function useOutOfCreditsRetry( + action: AutofixAction, + {enabled}: UseOutOfCreditsRetryOptions +) { + const {hasCheckedSinceMount, hasEnoughCredits} = useAutofixCreditCheck(action, { + enabled, + }); + + return { + // Held back until this mount's check answers, so it doesn't flash enabled + // on an earlier mount's cached answer. + disabled: !hasCheckedSinceMount || !hasEnoughCredits, + tooltip: hasEnoughCredits ? undefined : OUT_OF_CREDITS_RETRY_TOOLTIP, + }; +} diff --git a/static/app/views/issueList/pages/inbox/index.spec.tsx b/static/app/views/issueList/pages/inbox/index.spec.tsx index b79378d0640b..a399a3095516 100644 --- a/static/app/views/issueList/pages/inbox/index.spec.tsx +++ b/static/app/views/issueList/pages/inbox/index.spec.tsx @@ -1436,6 +1436,94 @@ describe('InboxPage', () => { ).toHaveAttribute('href', 'https://github.com/org/repository/pull/649'); }); + describe('an Autofix pull request that failed for credits', () => { + const creditsOrganization = OrganizationFixture({ + features: [ + ...organization.features, + 'seer-credit-checks', + 'token-billed-seer-enabled', + ], + }); + + function mockOutOfCreditsPullRequest() { + mockSuccessfulSections(); + mockIssuePreview(); + mockAutofixResponse( + ExplorerAutofixResponseFixture({ + autofix: ExplorerAutofixStateFixture({ + repo_pr_states: { + 'org/repository': AutofixRepoPRStateFixture({ + pr_creation_error: + "Your organization doesn't have enough Seer credits for this.", + pr_creation_error_code: 'out_of_credits', + pr_creation_status: 'error', + pr_id: null, + pr_number: null, + pr_url: null, + }), + }, + }), + }) + ); + } + + function mockCreditCheck(hasEnoughCredits: boolean) { + MockApiClient.addMockResponse({ + url: `/organizations/${creditsOrganization.slug}/seer/credit-check/`, + match: [MockApiClient.matchQuery({operation: 'autofix_open_pr'})], + body: {hasEnoughCredits}, + }); + } + + it('keeps its retry, disabled, while out of credits', async () => { + mockOutOfCreditsPullRequest(); + mockCreditCheck(false); + + render(, { + organization: creditsOrganization, + initialRouterConfig, + }); + + const preview = await openFixProposedPreview(); + const retryButton = await within(preview).findByRole('button', { + name: 'Retry PR in org/repository', + }); + expect(retryButton).toHaveAttribute('aria-disabled', 'true'); + + await userEvent.hover(retryButton); + expect( + await screen.findByText( + 'Your organization does not have enough Seer credits for this.' + ) + ).toBeInTheDocument(); + }); + + it('enables its retry once the organization can afford it', async () => { + mockOutOfCreditsPullRequest(); + mockCreditCheck(true); + + render(, { + organization: creditsOrganization, + initialRouterConfig, + }); + + const preview = await openFixProposedPreview(); + const retryButton = await within(preview).findByRole('button', { + name: 'Retry PR in org/repository', + }); + await waitFor(() => + expect(retryButton).not.toHaveAttribute('aria-disabled', 'true') + ); + + await userEvent.hover(retryButton); + expect( + await screen.findByText( + "Seer couldn't open a pull request in org/repository because your organization is out of credits." + ) + ).toBeInTheDocument(); + }); + }); + it('retries a failed Autofix pull request', async () => { mockSuccessfulSections(); mockIssuePreview(); diff --git a/static/app/views/issueList/pages/inbox/issuePreview/issuePreviewSeerActions.tsx b/static/app/views/issueList/pages/inbox/issuePreview/issuePreviewSeerActions.tsx index 787d995a9c2c..2b8022c67ed5 100644 --- a/static/app/views/issueList/pages/inbox/issuePreview/issuePreviewSeerActions.tsx +++ b/static/app/views/issueList/pages/inbox/issuePreview/issuePreviewSeerActions.tsx @@ -30,11 +30,14 @@ import { } from 'sentry/components/events/autofix/useExplorerAutofix'; import {useCodingAgents} from 'sentry/components/events/autofix/v3/useCodingAgents'; import {Placeholder} from 'sentry/components/placeholder'; +import {getOutOfCreditsPRMessage} from 'sentry/components/seer/outOfCredits/messages'; +import {useOutOfCreditsRetry} from 'sentry/components/seer/outOfCredits/useOutOfCreditsRetry'; import {PluginIcon} from 'sentry/icons/pluginIcon'; import {t} from 'sentry/locale'; import type {Group} from 'sentry/types/group'; import {defined} from 'sentry/utils/defined'; import {useOrganization} from 'sentry/utils/useOrganization'; +import {isOutOfCreditsPRState, type RepoPRState} from 'sentry/views/seerExplorer/types'; import { hidesCodingAgentResultLink, @@ -235,6 +238,56 @@ function RunSeerActionButton({ ); } +interface RetryPullRequestButtonProps extends Pick< + SeerActionButtonProps, + 'autofix' | 'disabled' | 'group' | 'onContinueInSeer' | 'variant' +> { + failedPullRequest: RepoPRState; + runId: number; +} + +/** + * A PR that failed for credits fails the same way until the organization can + * afford it, so its retry stays on screen, disabled, rather than letting + * another action step in as the next one. + */ +function RetryPullRequestButton({ + autofix, + disabled, + failedPullRequest, + group, + onContinueInSeer, + runId, + variant, +}: RetryPullRequestButtonProps) { + const isOutOfCredits = isOutOfCreditsPRState(failedPullRequest); + const outOfCreditsRetry = useOutOfCreditsRetry('create_pr', { + enabled: isOutOfCredits, + }); + + return ( + autofix.createPR(runId, failedPullRequest.repo_name)} + analyticsAction="create_pr" + analyticsEventKey="issue_inbox.create_pr_clicked" + analyticsEventName="Issue Inbox: Create PR Setup Clicked" + analyticsParams={{is_retry: true}} + autofix={autofix} + disabled={disabled || outOfCreditsRetry.disabled} + group={group} + icon={} + label={t('Retry PR in %s', failedPullRequest.repo_name)} + onContinueInSeer={onContinueInSeer} + tooltip={ + isOutOfCredits + ? (outOfCreditsRetry.tooltip ?? getOutOfCreditsPRMessage(failedPullRequest)) + : failedPullRequest.pr_creation_error + } + variant={variant} + /> + ); +} + function SeerActionButton({ autofix, creditsDisabled, @@ -300,19 +353,13 @@ function SeerActionButton({ if (failedPullRequest) { return ( - autofix.createPR(runState.run_id, failedPullRequest.repo_name)} - analyticsAction="create_pr" - analyticsEventKey="issue_inbox.create_pr_clicked" - analyticsEventName="Issue Inbox: Create PR Setup Clicked" - analyticsParams={{is_retry: true}} + } - label={t('Retry PR in %s', failedPullRequest.repo_name)} onContinueInSeer={onContinueInSeer} - tooltip={failedPullRequest.pr_creation_error} + runId={runState.run_id} variant={variant} /> ); diff --git a/static/app/views/seerExplorer/components/chat/chatWindow.stories.tsx b/static/app/views/seerExplorer/components/chat/chatWindow.stories.tsx index 29eb622f621c..88f91414e608 100644 --- a/static/app/views/seerExplorer/components/chat/chatWindow.stories.tsx +++ b/static/app/views/seerExplorer/components/chat/chatWindow.stories.tsx @@ -42,6 +42,7 @@ interface Wallet { /** * Answers every query the chat window makes from fixtures, so the story never reaches the API. + * Without a wallet, the wallet's requests fail as they do for an org without token-billed Seer. * Per-query options outrank client defaults, which is why this replaces `queryFn` after * defaulting rather than setting a default. Sending a message is a mutation and is not stubbed. */ @@ -72,7 +73,10 @@ class FixtureQueryClient extends QueryClient { dailySpendCapCents: 5000, }; } - if (this.wallet && url.includes('/seer-wallet/')) { + if (url.includes('/seer-wallet/')) { + if (!this.wallet) { + throw new Error('This story has no wallet'); + } const {remaining, total} = this.wallet; return { balance: remaining, @@ -88,7 +92,9 @@ class FixtureQueryClient extends QueryClient { override defaultQueryOptions: QueryClient['defaultQueryOptions'] = options => { const defaulted = super.defaultQueryOptions(options); const url = String(defaulted.queryKey[0]); - const queryFn = () => Promise.resolve({json: this.respond(url), headers: {}}); + // `then` turns a fixture that throws into a failed query. + const queryFn = () => + Promise.resolve().then(() => ({json: this.respond(url), headers: {}})); // Each fixture matches the response type of the query that asked for it. return {...defaulted, queryFn: queryFn as never}; }; @@ -123,8 +129,11 @@ function ChatWindow({session, hasEnoughCredits, initialQuery, wallet}: ChatWindo scope !== 'org:billing'), openMembership: true, hideAiFeatures: false, features: [ @@ -416,6 +425,29 @@ const AWAITING_ANSWER_SESSION: Session = { blocks: [USER_QUESTION, SEARCH_SPANS], }; +// Seer ran out of credits after two tool calls. It keeps the blocks it wrote and appends a +// plain-text error block, which the chat swaps for a line saying why it stopped. +const OUT_OF_CREDITS_SESSION: Session = { + status: 'error', + failure_reason: 'out_of_credits', + updated_at: at(14), + blocks: [ + USER_QUESTION, + SEARCH_SPANS, + INSPECT_ISSUE, + { + id: 'error-1', + message: { + role: 'assistant', + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here. Add credits to continue.", + }, + timestamp: at(14), + loading: false, + }, + ], +}; + export default Storybook.story('ChatWindow', story => { story('Completed conversation', () => ( @@ -542,6 +574,21 @@ export default Storybook.story('ChatWindow', story => { )); + story('Stopped for credits', () => ( + +

+ The organization ran out of Seer credits mid-response. The run ends with{' '} + failure_reason: out_of_credits. The reasoning and tool calls Seer + made before it stopped stay in the thinking block, and a muted line saying Seer + stopped for credits replaces Seer's plain-text error message. There is no CTA in + the transcript: the chat rechecks credits when a run stops for them, and the + out-of-credits alert above the composer covers it. Open source renders no CTA; + getsentry fills it in. +

+ + + )); + story('Awaiting an answer', () => (

diff --git a/static/app/views/seerExplorer/components/chat/responseGroup.tsx b/static/app/views/seerExplorer/components/chat/responseGroup.tsx index a37e12b278f5..8d2e1a8d821b 100644 --- a/static/app/views/seerExplorer/components/chat/responseGroup.tsx +++ b/static/app/views/seerExplorer/components/chat/responseGroup.tsx @@ -2,16 +2,19 @@ import {Fragment, memo, useState} from 'react'; import styled from '@emotion/styled'; import {motion} from 'framer-motion'; -import {MessageRow, ThinkingBlock} from '@sentry/scraps/chat'; +import {AssistantMessage, MessageRow, ThinkingBlock} from '@sentry/scraps/chat'; import {Container} from '@sentry/scraps/layout'; +import {Text} from '@sentry/scraps/text'; import {SeerMarkdown} from 'sentry/components/seer/markdown'; +import {OUT_OF_CREDITS_CHAT_MESSAGE} from 'sentry/components/seer/outOfCredits/messages'; import {t} from 'sentry/locale'; import {callRecordLabel, visibleCallRecords} from 'sentry/views/seerExplorer/callRecords'; -import type { - Block, - PendingUserInput, - SeerExplorerRunId, +import { + isSeerErrorBlock, + type Block, + type PendingUserInput, + type SeerExplorerRunId, } from 'sentry/views/seerExplorer/types'; import {getToolsStringFromBlock} from 'sentry/views/seerExplorer/utils'; @@ -141,6 +144,11 @@ interface ResponseGroupProps { interactionPending?: boolean; /** The conversation's newest todo snapshot, from `findLatestTodos`. */ latestTodos?: LatestTodos | null; + /** + * The run stopped here because the organization ran out of Seer credits. + * Only set on the conversation's last response. + */ + outOfCredits?: boolean; pendingInput?: PendingUserInput | null; readOnly?: boolean; runId?: SeerExplorerRunId; @@ -177,6 +185,7 @@ export const ResponseGroup = memo(function ResponseGroup({ latestTodos, getPageReferrer, interactionPending, + outOfCredits, pendingInput, readOnly, runId, @@ -189,6 +198,7 @@ export const ResponseGroup = memo(function ResponseGroup({ const answer = finalAnswer(group); const settledAnswer = answer && !answer.loading ? answer : null; const active = group.some(block => block.loading); + const showOutOfCredits = outOfCredits && !active; // The reasoning trace is everything except the answer's content: thinking prose (gated on the // `showThinking` toggle), any intermediate narration, and the tool calls. @@ -265,7 +275,8 @@ export const ResponseGroup = memo(function ResponseGroup({ ) : null} - {settledAnswer ? ( + {/* Seer's plain-text error block gives way to a line saying why it stopped. */} + {settledAnswer && !(showOutOfCredits && isSeerErrorBlock(settledAnswer)) ? ( ) : null} + {showOutOfCredits ? : null} ); }, areResponseGroupPropsEqual); +/** + * Why the run stopped. The out-of-credits alert above the composer offers the + * way out, so this has no CTA of its own. + */ +function OutOfCreditsMessage({compact}: {compact: boolean}) { + return ( + + + {OUT_OF_CREDITS_CHAT_MESSAGE} + + + ); +} + // The response's raw reasoning. When it sits between tool calls it is set apart with extra vertical // space (`data-spaced`); leading or trailing reasoning gets none so it stays tight against the // answer or the block edge. Shared with the investigation status block, which draws its agent's diff --git a/static/app/views/seerExplorer/components/prLinksBar.spec.tsx b/static/app/views/seerExplorer/components/prLinksBar.spec.tsx index b688f4b4c401..89dbed46f3c1 100644 --- a/static/app/views/seerExplorer/components/prLinksBar.spec.tsx +++ b/static/app/views/seerExplorer/components/prLinksBar.spec.tsx @@ -129,4 +129,30 @@ describe('PRLinksBar', () => { expect(screen.queryByRole('button')).not.toBeInTheDocument(); }); + + it('explains a PR that failed for credits in its tooltip', async () => { + render( + + ); + + await userEvent.hover(screen.getByRole('button', {name: REPO})); + expect( + await screen.findByText( + "Seer couldn't open a pull request in acme/web because your organization is out of credits." + ) + ).toBeInTheDocument(); + expect( + screen.queryByText("Your organization doesn't have enough Seer credits.") + ).not.toBeInTheDocument(); + }); }); diff --git a/static/app/views/seerExplorer/components/prLinksBar.stories.tsx b/static/app/views/seerExplorer/components/prLinksBar.stories.tsx index aa6344d56da1..77c90388d696 100644 --- a/static/app/views/seerExplorer/components/prLinksBar.stories.tsx +++ b/static/app/views/seerExplorer/components/prLinksBar.stories.tsx @@ -64,6 +64,16 @@ const PUSH_FAILED = prState({ pr_creation_status: 'error', }); +const OUT_OF_CREDITS = prState({ + repo_name: 'acme/billing', + pr_creation_error: "Your organization doesn't have enough Seer credits for this.", + pr_creation_error_code: 'out_of_credits', + pr_creation_status: 'error', + pr_number: null, + pr_url: null, + title: 'fix: Retry failed invoice webhooks', +}); + const ALL_STATES = [COMPLETED, CREATING, UPDATING, ERRORED, PUSH_FAILED]; function byRepo(...states: RepoPRState[]): Record { @@ -109,6 +119,20 @@ export default Storybook.story('PRLinksBar', story => { )); + story('Failed for credits', () => ( + + + The PR could not be opened because the organization ran out of Seer credits. The + button has no retry; hover it and its tooltip says the PR failed for credits + instead of the raw backend error. There is no CTA under the bar: the + out-of-credits alert above the chat composer covers it. + + + + + + )); + story('Several repos', () => ( diff --git a/static/app/views/seerExplorer/components/prLinksBar.tsx b/static/app/views/seerExplorer/components/prLinksBar.tsx index fd4a798ae3fe..e1472a43d6bc 100644 --- a/static/app/views/seerExplorer/components/prLinksBar.tsx +++ b/static/app/views/seerExplorer/components/prLinksBar.tsx @@ -5,9 +5,10 @@ import {Button, LinkButton} from '@sentry/scraps/button'; import {Flex} from '@sentry/scraps/layout'; import {LoadingIndicator} from 'sentry/components/loadingIndicator'; +import {getOutOfCreditsPRMessage} from 'sentry/components/seer/outOfCredits/messages'; import {t} from 'sentry/locale'; import {isValidUrl} from 'sentry/utils/string/isValidUrl'; -import type {RepoPRState} from 'sentry/views/seerExplorer/types'; +import {isOutOfCreditsPRState, type RepoPRState} from 'sentry/views/seerExplorer/types'; interface PRLinksBarProps { repoPRStates: Record; @@ -31,6 +32,17 @@ export function PRLinksBar({repoPRStates}: PRLinksBarProps) { ); } +function getErrorTooltip(state: RepoPRState): string { + if (isOutOfCreditsPRState(state)) { + return getOutOfCreditsPRMessage(state); + } + // A PR number means the PR already exists, so the failure was a push to it. + return ( + state.pr_creation_error ?? + (state.pr_number ? t('Could not push changes') : t('Could not open PR')) + ); +} + /** Styled like the autofix evidence buttons: status icon, then owner/repo#number. */ function PRButton({state}: {state: RepoPRState}) { // The URL comes from session data, so only link http(s) URLs and never e.g. `javascript:`. @@ -56,14 +68,7 @@ function PRButton({state}: {state: RepoPRState}) { const isError = state.pr_creation_status === 'error'; const icon = isError ? : ; - const tooltipProps = isError - ? { - // A PR number means the PR already exists, so the failure was a push to it. - title: - state.pr_creation_error ?? - (state.pr_number ? t('Could not push changes') : t('Could not open PR')), - } - : undefined; + const tooltipProps = isError ? {title: getErrorTooltip(state)} : undefined; // A failed push to an existing PR still links to it. if (prUrl) { diff --git a/static/app/views/seerExplorer/components/seerExplorerContent.spec.tsx b/static/app/views/seerExplorer/components/seerExplorerContent.spec.tsx index 58a467019eb5..7c0c805b999f 100644 --- a/static/app/views/seerExplorer/components/seerExplorerContent.spec.tsx +++ b/static/app/views/seerExplorer/components/seerExplorerContent.spec.tsx @@ -1282,6 +1282,95 @@ describe('SeerExplorerContent', () => { }); }); + describe('Run stopped for credits', () => { + const stoppedBlocks = [ + { + id: 'user-1', + message: {role: 'user' as const, content: 'Why is checkout slow?'}, + timestamp: '2024-01-01T00:00:00Z', + }, + { + id: 'tool-1', + message: { + role: 'tool_use' as const, + content: 'Looking at the slowest checkout spans.', + }, + timestamp: '2024-01-01T00:00:10Z', + }, + { + id: 'error-1', + message: { + role: 'assistant' as const, + content: + "Your organization doesn't have enough Seer credits for this, so I stopped here.", + }, + timestamp: '2024-01-01T00:00:20Z', + }, + ]; + + function renderContent() { + render( + + + {}} + /> + + , + {organization} + ); + } + + it("replaces Seer's error block with why it stopped and keeps the partial response", async () => { + jest.spyOn(useSeerExplorerModule, 'useSeerExplorer').mockReturnValue({ + ...defaultHookReturn, + runId: 123, + sessionData: { + blocks: stoppedBlocks, + status: 'error', + updated_at: '2024-01-01T00:00:20Z', + failure_reason: 'out_of_credits', + }, + }); + + renderContent(); + + expect( + await screen.findByText( + 'Seer stopped because your organization is out of credits.' + ) + ).toBeInTheDocument(); + expect(screen.queryByText(/so I stopped here/)).not.toBeInTheDocument(); + expect(screen.queryByText('Response timed out.')).not.toBeInTheDocument(); + // The reasoning before it stopped stays in the thinking block. + await userEvent.click( + screen.getByRole('button', {name: /See thinking and tool calls/}) + ); + expect(screen.getByText('Looking at the slowest checkout spans.')).toBeVisible(); + }); + + it("leaves Seer's error block alone for other failures", async () => { + jest.spyOn(useSeerExplorerModule, 'useSeerExplorer').mockReturnValue({ + ...defaultHookReturn, + runId: 123, + sessionData: { + blocks: stoppedBlocks, + status: 'error', + updated_at: '2024-01-01T00:00:20Z', + failure_reason: null, + }, + }); + + renderContent(); + + expect(await screen.findByText(/so I stopped here/)).toBeInTheDocument(); + expect( + screen.queryByText('Seer stopped because your organization is out of credits.') + ).not.toBeInTheDocument(); + }); + }); + describe('Input Persistence', () => { it.each([ ['legacy string', 'Ask @Alice #team'], diff --git a/static/app/views/seerExplorer/components/seerExplorerContent.tsx b/static/app/views/seerExplorer/components/seerExplorerContent.tsx index c73cb632c532..622beefd3657 100644 --- a/static/app/views/seerExplorer/components/seerExplorerContent.tsx +++ b/static/app/views/seerExplorer/components/seerExplorerContent.tsx @@ -941,6 +941,7 @@ export function SeerExplorerContent({ isQuestionPending || showReauth } + outOfCredits={isRunOutOfCredits} pendingInput={pendingInput} readOnly={readOnly} showThinking={showThinking} @@ -1036,6 +1037,8 @@ interface SeerExplorerTranscriptProps { blocks: Block[]; getPageReferrer: () => string; interactionPending: boolean; + /** The run stopped because the organization ran out of Seer credits. */ + outOfCredits: boolean; pendingInput: PendingUserInput | null; readOnly: boolean; runId: SeerExplorerRunId | null; @@ -1051,6 +1054,7 @@ const SeerExplorerTranscript = memo(function SeerExplorerTranscript({ blocks, getPageReferrer, interactionPending, + outOfCredits, pendingInput, readOnly, runId, @@ -1060,7 +1064,7 @@ const SeerExplorerTranscript = memo(function SeerExplorerTranscript({ // Walk the conversation once here rather than once per tool block. const latestTodos = useMemo(() => findLatestTodos(blocks), [blocks]); - return segments.map(segment => { + return segments.map((segment, segmentIndex) => { if (segment.kind === 'user') { // For slide-in animation that runs on mount. Avoid running this twice on user // blocks when blocks are hydrated. @@ -1083,6 +1087,8 @@ const SeerExplorerTranscript = memo(function SeerExplorerTranscript({ runId={runId ?? undefined} getPageReferrer={getPageReferrer} interactionPending={interactionPending} + // The run stopped at its end, so only the last response explains why. + outOfCredits={outOfCredits && segmentIndex === segments.length - 1} pendingInput={pendingInput} readOnly={readOnly} showThinking={showThinking} diff --git a/static/app/views/seerExplorer/hooks/useSeerExplorer.spec.tsx b/static/app/views/seerExplorer/hooks/useSeerExplorer.spec.tsx index 6c48c678a30d..394ac92a1d54 100644 --- a/static/app/views/seerExplorer/hooks/useSeerExplorer.spec.tsx +++ b/static/app/views/seerExplorer/hooks/useSeerExplorer.spec.tsx @@ -4,6 +4,8 @@ import {OrganizationFixture} from 'sentry-fixture/organization'; import {act, renderHookWithProviders, waitFor} from 'sentry-test/reactTestingLibrary'; +import {toast} from '@sentry/scraps/toast'; + import * as llmContextModule from 'sentry/views/seerExplorer/contexts/llmContext'; import { SeerExplorerChatStateProvider, @@ -1192,6 +1194,68 @@ describe('useSeerExplorer', () => { }); }); + describe('PR creation errors', () => { + const chatUrl = `/organizations/${organization.slug}/seer/explorer-chat/`; + const runId = 789; + + function mockPRState(prState: Record) { + MockApiClient.addMockResponse({ + url: `${chatUrl}${runId}/`, + method: 'GET', + body: { + session: { + status: 'completed', + blocks: [], + updated_at: new Date().toISOString(), + repo_pr_states: { + 'acme/web': {repo_name: 'acme/web', pr_number: null, ...prState}, + }, + }, + }, + }); + } + + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(async () => { + try { + await act(() => jest.runOnlyPendingTimersAsync()); + } finally { + jest.useRealTimers(); + } + }); + + it('toasts out-of-credits copy instead of the raw error', async () => { + const errorSpy = jest.spyOn(toast, 'error'); + mockPRState({pr_creation_status: 'creating'}); + + const {result} = renderHookWithProviders(() => useSeerExplorer(), { + organization, + additionalWrapper: SeerExplorerChatStateProvider, + }); + act(() => { + result.current.switchToRun(runId); + }); + await waitFor(() => expect(result.current.isPolling).toBe(true)); + + mockPRState({ + pr_creation_status: 'error', + pr_creation_error: "Your organization doesn't have enough Seer credits for this.", + pr_creation_error_code: 'out_of_credits', + }); + + // The next poll picks up the failure. + await act(() => jest.advanceTimersByTimeAsync(500)); + await waitFor(() => + expect(errorSpy).toHaveBeenCalledWith( + "Seer couldn't open a pull request in acme/web because your organization is out of credits." + ) + ); + }); + }); + describe('Optimistic Thinking Block', () => { it('sets optimistic blocks when session is processing with no user block in DB', async () => { const chatUrl = `/organizations/${organization.slug}/seer/explorer-chat/`; diff --git a/static/app/views/seerExplorer/hooks/useSeerExplorer.tsx b/static/app/views/seerExplorer/hooks/useSeerExplorer.tsx index 0be7985d2024..4aa6e5d03658 100644 --- a/static/app/views/seerExplorer/hooks/useSeerExplorer.tsx +++ b/static/app/views/seerExplorer/hooks/useSeerExplorer.tsx @@ -5,6 +5,7 @@ import moment from 'moment-timezone'; import {useTimezone} from '@sentry/scraps/datetime'; import {toast} from '@sentry/scraps/toast'; +import {getOutOfCreditsPRMessage} from 'sentry/components/seer/outOfCredits/messages'; import {t} from 'sentry/locale'; import {trackAnalytics} from 'sentry/utils/analytics'; import {parseQueryKey} from 'sentry/utils/api/apiQueryKey'; @@ -24,12 +25,13 @@ import { useSeerExplorerChatDispatch, useSeerExplorerChatState, } from 'sentry/views/seerExplorer/seerExplorerChatStateContext'; -import type { - Block, - RepoPRState, - RespondToUserInputOptions, - SeerExplorerResponse, - SeerExplorerRunId, +import { + isOutOfCreditsPRState, + type Block, + type RepoPRState, + type RespondToUserInputOptions, + type SeerExplorerResponse, + type SeerExplorerRunId, } from 'sentry/views/seerExplorer/types'; import { makeSeerExplorerQueryKey, @@ -640,7 +642,11 @@ export const useSeerExplorer = () => { currentState.pr_creation_status === 'error' && currentState.pr_creation_error ) { - toast.error(currentState.pr_creation_error ?? 'Failed to create PR'); + toast.error( + isOutOfCreditsPRState(currentState) + ? getOutOfCreditsPRMessage(currentState) + : (currentState.pr_creation_error ?? 'Failed to create PR') + ); } } diff --git a/static/app/views/seerExplorer/hooks/useSeerExplorerPolling.spec.tsx b/static/app/views/seerExplorer/hooks/useSeerExplorerPolling.spec.tsx index 4f76907864b1..9ea52970672c 100644 --- a/static/app/views/seerExplorer/hooks/useSeerExplorerPolling.spec.tsx +++ b/static/app/views/seerExplorer/hooks/useSeerExplorerPolling.spec.tsx @@ -52,6 +52,8 @@ describe('useSeerExplorerPolling', () => { it.each([ ['timeout', 'timed-out', true], ['stalled', 'not-polling', false], + // A run that stopped for credits is finished, not timed out, so no Retry shows. + ['out_of_credits', 'not-polling', false], ] as const)( 'maps a backend %s reason to %s', async (failureReason, expectedPollingState, expectedTimedOut) => { diff --git a/static/app/views/seerExplorer/types.tsx b/static/app/views/seerExplorer/types.tsx index 9732ab0db687..4938df6af1ef 100644 --- a/static/app/views/seerExplorer/types.tsx +++ b/static/app/views/seerExplorer/types.tsx @@ -258,12 +258,41 @@ export interface ReauthMonitoringProviderData { export type SeerExplorerRunId = number | string; +/** + * Why a run ended with `status: 'error'`, when Seer knows. `out_of_credits` + * runs keep the blocks they produced before stopping. + */ +export type ExplorerFailureReason = 'timeout' | 'stalled' | 'out_of_credits'; + +/** + * Seer's id prefix for the assistant block it appends when a run fails, whose + * content is its plain-text error message. + */ +const SEER_ERROR_BLOCK_ID_PREFIX = 'error-'; + +export function isSeerErrorBlock(block: Block): boolean { + return ( + block.message.role === 'assistant' && block.id.startsWith(SEER_ERROR_BLOCK_ID_PREFIX) + ); +} + +/** + * A PR create or push that failed because the organization ran out of Seer + * credits. + */ +export function isOutOfCreditsPRState(state: RepoPRState): boolean { + return ( + state.pr_creation_status === 'error' && + state.pr_creation_error_code === 'out_of_credits' + ); +} + export type SeerExplorerResponse = { session: { status: 'processing' | 'completed' | 'error' | 'awaiting_user_input'; updated_at: string; blocks?: Block[]; - failure_reason?: 'timeout' | 'stalled' | 'out_of_credits' | null; + failure_reason?: ExplorerFailureReason | null; owner_user_id?: number | null; pending_user_input?: PendingUserInput | null; repo_pr_states?: Record;