diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py index a4086f19bebc..33c2119afaf0 100644 --- a/src/sentry/seer/autofix/issue_summary.py +++ b/src/sentry/seer/autofix/issue_summary.py @@ -34,6 +34,7 @@ IssueSummaryHidden, IssueSummarySelfHosted, NoSeerQuotaException, + SeerOutOfCreditsException, ) from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import ( @@ -204,7 +205,7 @@ def _trigger_autofix_task( data={"referrer": referrer.value}, send_notification=False, ) - except NoSeerQuotaException: + except (NoSeerQuotaException, SeerOutOfCreditsException): pass if run and SeerAutofixOperator.has_access(organization=group.project.organization): diff --git a/src/sentry/seer/autofix/on_completion_hook.py b/src/sentry/seer/autofix/on_completion_hook.py index 5383a1919fd1..194e93b412eb 100644 --- a/src/sentry/seer/autofix/on_completion_hook.py +++ b/src/sentry/seer/autofix/on_completion_hook.py @@ -28,6 +28,7 @@ ) from sentry.seer.autofix.coding_agent import IntegrationNotFound from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import LEGACY_FEATURE_ID as LEGACY_AUTOFIX_FEATURE_ID from sentry.seer.autofix.pr_iteration.completion import ( @@ -786,12 +787,22 @@ def _maybe_continue_pipeline( "iteration_index": get_latest_iteration_index(state), }, ) - trigger_autofix_agent( - group=group, - step=next_step, - referrer=referrer, - run_id=run_id, - ) + try: + trigger_autofix_agent( + group=group, + step=next_step, + referrer=referrer, + run_id=run_id, + ) + except SeerOutOfCreditsException: + logger.info( + "autofix.on_completion_hook.next_step_out_of_credits", + extra={ + "run_id": run_id, + "organization_id": organization.id, + "next_step": next_step, + }, + ) @classmethod def _push_changes( @@ -841,6 +852,12 @@ def _push_changes( state=state, verify_content=True, ) + except SeerOutOfCreditsException: + logger.info( + "autofix.on_completion_hook.push_changes_out_of_credits", + extra={"run_id": run_id, "organization_id": group.organization.id}, + ) + return False except Exception: logger.exception( "autofix.on_completion_hook.push_changes_failed", diff --git a/src/sentry/seer/autofix/pr_iteration/completion.py b/src/sentry/seer/autofix/pr_iteration/completion.py index dbf1a07e83b9..3a9f1120b206 100644 --- a/src/sentry/seer/autofix/pr_iteration/completion.py +++ b/src/sentry/seer/autofix/pr_iteration/completion.py @@ -15,6 +15,7 @@ from sentry.seer.autofix.autofix_agent import trigger_push_changes from sentry.seer.autofix.commit_author import SeerCommitAuthor, parse_commit_author from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature from sentry.seer.autofix.feature.models import PrIterationStepArgs from sentry.seer.autofix.pr_iteration.current_iteration import triggered_iteration_id @@ -489,6 +490,16 @@ def _push_iteration_changes( # Never overwrite commits on the PR that Seer didn't push. allow_force_push=False, ) + except SeerOutOfCreditsException: + log_ctx.info("autofix.pr_iteration.push", outcome="failed", reason="out_of_credits") + # TODO(CW-3066): Resume after a top-up. Until then nothing retries the push, so pause + # the run so that new feedback is refused. + pause_pr_iteration( + run_id=run_id, + organization_id=group.organization.id, + reason=PauseReason.RUN_ERRORED, + ) + return False except Exception as e: log_ctx.error( "autofix.pr_iteration.push", @@ -572,6 +583,11 @@ def _retry_push_on_moved_branch( ), ), ) + except SeerOutOfCreditsException: + log_ctx.info("autofix.pr_iteration.push", outcome="not_pushed", reason="out_of_credits") + # TODO(CW-3066): Resume after a top-up. Until then nothing retries this, so end the + # iteration as errored, and the caller pauses the run and records the outcome. + return PrIterationOutcome.PR_CREATION_ERRORED except Exception as e: # End the iteration as errored, so the caller pauses the run and records the outcome. log_ctx.error( diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index 8eab5630069d..c53e909e070e 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -25,7 +25,12 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource from sentry.seer.autofix.pr_iteration.pause import PauseReason, get_pause_reason from sentry.seer.autofix.pr_iteration.queue import enqueue_autofix_feedback -from sentry.seer.endpoints.utils import SEER_PERMISSION_DENIED +from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.endpoints.utils import ( + SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, + SEER_OUT_OF_CREDITS_CODE, + SEER_PERMISSION_DENIED, +) from sentry.seer.models import SeerPermissionError from sentry.tasks.seer.pr_iteration import trigger_consume_pr_iteration_feedback from sentry.users.services.user.service import user_service @@ -52,8 +57,9 @@ def handle_ui_feedback( Returns None once the feedback has been handed off, or the error response the endpoint should send back when the request cannot be acted on: 400 when - the feedback is empty or the run has no pull request yet, and 409 when - iteration on the run is paused. Raises ``PermissionDenied`` when Seer + the feedback is empty or the run has no pull request yet, 409 when + iteration on the run is paused, and 402 when the organization doesn't have + enough Seer credits. Raises ``PermissionDenied`` when Seer refuses to show us the run. """ if not user_feedback: @@ -80,6 +86,18 @@ def handle_ui_feedback( status=status.HTTP_409_CONFLICT, ) + # Check credits here, since the queued feedback is consumed after this request returns. + if not can_run_seer_operation( + group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + ): + return Response( + { + "detail": SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, + "code": SEER_OUT_OF_CREDITS_CODE, + }, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) + serialized_users = user_service.serialize_many( filter={"user_ids": [user_id]}, ) diff --git a/src/sentry/seer/endpoints/admin_autofix_retry.py b/src/sentry/seer/endpoints/admin_autofix_retry.py index 1329ce811b2a..7a45ad6108d9 100644 --- a/src/sentry/seer/endpoints/admin_autofix_retry.py +++ b/src/sentry/seer/endpoints/admin_autofix_retry.py @@ -18,6 +18,7 @@ trigger_autofix_agent, ) from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID, LEGACY_FEATURE_ID from sentry.seer.autofix.on_completion_hook import PIPELINE_ORDER from sentry.seer.models import SeerRun @@ -120,6 +121,8 @@ def _retry_run(organization: Organization, run_id: int) -> dict[str, Any]: run_id=run_id, insert_index=insert_index, ) + except SeerOutOfCreditsException: + return _skipped(run_id, "Organization does not have enough Seer credits") except Exception: logger.exception( "autofix.admin_retry.trigger_failed", diff --git a/src/sentry/seer/endpoints/admin_autofix_single_run.py b/src/sentry/seer/endpoints/admin_autofix_single_run.py index 5772b42f6e98..0ae26e851eb6 100644 --- a/src/sentry/seer/endpoints/admin_autofix_single_run.py +++ b/src/sentry/seer/endpoints/admin_autofix_single_run.py @@ -11,7 +11,7 @@ from sentry.models.organization import Organization from sentry.seer.autofix.autofix_agent import trigger_autofix_agent from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import get_automation_handoff from sentry.seer.models import SeerPermissionError @@ -75,6 +75,10 @@ def post(self, request: Request) -> Response: ) except NoSeerQuotaException: return Response({"detail": "Organization has no Seer quota"}, status=400) + except SeerOutOfCreditsException: + return Response( + {"detail": "Organization does not have enough Seer credits"}, status=400 + ) except SeerPermissionError as e: return Response({"detail": e.message}, status=400) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4767dde92f13..1f35185cc203 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -59,7 +59,7 @@ ) from sentry.seer.autofix.commit_author import commit_author_for_user from sentry.seer.autofix.constants import PUBLISH_SCOPES, AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import project_stopping_point, should_use_single_run from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID @@ -81,7 +81,15 @@ from sentry.seer.endpoints.organization_seer_onboarding_check import ( has_supported_scm_integration, ) -from sentry.seer.endpoints.utils import SEER_PERMISSION_DENIED, get_seer_run, resolve_seer_run +from sentry.seer.endpoints.utils import ( + SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, + SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL, + SEER_OUT_OF_CREDITS_CODE, + SEER_OUT_OF_QUOTA_CODE, + SEER_PERMISSION_DENIED, + get_seer_run, + resolve_seer_run, +) from sentry.seer.models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.types.activity import ActivityType from sentry.types.ratelimit import RateLimit, RateLimitCategory @@ -455,6 +463,14 @@ def post( ), user=acting_user(request.user), ) + except SeerOutOfCreditsException: + return Response( + { + "detail": SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, + "code": SEER_OUT_OF_CREDITS_CODE, + }, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) except SeerPermissionError: return Response(status=status.HTTP_404_NOT_FOUND) @@ -583,7 +599,20 @@ def post( ) except NoSeerQuotaException: return Response( - "No budget for Seer Autofix.", status=status.HTTP_402_PAYMENT_REQUIRED + {"detail": "No budget for Seer Autofix.", "code": SEER_OUT_OF_QUOTA_CODE}, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) + except SeerOutOfCreditsException: + return Response( + { + "detail": ( + SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL + if resolved_run_id is None + else SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL + ), + "code": SEER_OUT_OF_CREDITS_CODE, + }, + status=status.HTTP_402_PAYMENT_REQUIRED, ) except SeerPermissionError as e: if _is_unknown_run_id_error(e): diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 02e9b3619d8a..12416ecab212 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -34,12 +34,18 @@ PUBLISH_SCOPES, AutofixReferrer, ) +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import continue_single_run from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation from sentry.seer.endpoints.group_ai_autofix import RUN_IN_FLIGHT_CODE -from sentry.seer.endpoints.utils import ResolvedSeerRun, resolve_seer_run +from sentry.seer.endpoints.utils import ( + SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, + SEER_OUT_OF_CREDITS_CODE, + ResolvedSeerRun, + resolve_seer_run, +) from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun from sentry.seer.seer_setup import has_seer_access_with_detail @@ -268,6 +274,11 @@ def _maybe_reply_to_single_run( can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), referrer=AutofixReferrer.SEER_EXPLORER, ) + except SeerOutOfCreditsException: + return Response( + {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, + status=402, + ) except SeerApiError as e: if e.status == 404: return Response({"session": None}, status=404) @@ -442,9 +453,7 @@ def post( organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE ): return Response( - { - "detail": "Your organization does not have enough credits left for you to message Seer." - }, + {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, status=402, ) diff --git a/src/sentry/seer/endpoints/utils.py b/src/sentry/seer/endpoints/utils.py index 734cb7f6efdf..3414c3bd2aba 100644 --- a/src/sentry/seer/endpoints/utils.py +++ b/src/sentry/seer/endpoints/utils.py @@ -17,6 +17,20 @@ SEER_PERMISSION_DENIED = "You are not authorized to perform this action" +# Let API clients like the CLI and MCP tell billing 402s apart without parsing detail text. +SEER_OUT_OF_CREDITS_CODE = "seer_out_of_credits" +SEER_OUT_OF_QUOTA_CODE = "seer_out_of_quota" + +SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL = ( + "Your organization does not have enough credits to start an Autofix run." +) +SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL = ( + "Your organization does not have enough credits to continue your Autofix run." +) +SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL = ( + "Your organization does not have enough credits left for you to message Seer." +) + class OrganizationTraceExplorerAIPermission(OrganizationPermission): scope_map = { diff --git a/src/sentry/seer/entrypoints/operator.py b/src/sentry/seer/entrypoints/operator.py index 8f3a1eb7f314..b2d9ef8e074c 100644 --- a/src/sentry/seer/entrypoints/operator.py +++ b/src/sentry/seer/entrypoints/operator.py @@ -209,7 +209,7 @@ def trigger_autofix_agent( trigger_autofix_agent, trigger_push_changes, ) - from sentry.seer.autofix.exceptions import NoSeerQuotaException + from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.steps import AutofixStep @@ -313,6 +313,16 @@ def trigger_autofix_agent( self.entrypoint.on_trigger_autofix_error(error=error) lifecycle.record_failure(failure_reason=error) return + except SeerOutOfCreditsException: + with SeerOperatorEventLifecycleMetric( + interaction_type=SeerOperatorInteractionType.ENTRYPOINT_ON_TRIGGER_AUTOFIX_ERROR, + entrypoint_key=self.entrypoint.key, + ).capture(): + self.entrypoint.on_trigger_autofix_error( + error="Your organization does not have enough Seer credits" + ) + lifecycle.record_halt(halt_reason="not_enough_seer_credits") + return except Exception as e: with SeerOperatorEventLifecycleMetric( interaction_type=SeerOperatorInteractionType.ENTRYPOINT_ON_TRIGGER_AUTOFIX_ERROR, diff --git a/src/sentry/tasks/seer/pr_iteration.py b/src/sentry/tasks/seer/pr_iteration.py index 7462b2d65887..c95fce7757b6 100644 --- a/src/sentry/tasks/seer/pr_iteration.py +++ b/src/sentry/tasks/seer/pr_iteration.py @@ -52,6 +52,7 @@ from sentry.seer.autofix.autofix_agent import trigger_autofix_agent from sentry.seer.autofix.commit_author import commit_author_for_feedback from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.pr_iteration.bot_identity import bot_logins_for_feedback from sentry.seer.autofix.pr_iteration.cap_exhausted import assign_user_for_exhausted_cap from sentry.seer.autofix.pr_iteration.constants import PR_ITERATION_PROVIDER @@ -840,7 +841,11 @@ def _drain_queued_autofix_feedback( commit_author=commit_author_for_feedback(feedback_items, organization_id), iteration_id=iteration_id, ) - except (PrIterationNoPullRequestException, SeerPermissionError) as error: + except ( + PrIterationNoPullRequestException, + SeerPermissionError, + SeerOutOfCreditsException, + ) as error: log_ctx.info( "autofix.pr_iteration.consume_feedback.trigger_agent", outcome="skipped", @@ -849,10 +854,13 @@ def _drain_queued_autofix_feedback( trigger_id=trigger_id, trigger_source=trigger_source, ) - _record_drain_outcome( - "skipped_permission" if isinstance(error, SeerPermissionError) else "skipped_no_pr", - trigger_source, - ) + if isinstance(error, SeerPermissionError): + drain_outcome = "skipped_permission" + elif isinstance(error, SeerOutOfCreditsException): + drain_outcome = "skipped_out_of_credits" + else: + drain_outcome = "skipped_no_pr" + _record_drain_outcome(drain_outcome, trigger_source) # The drain popped the queue, so this iteration will never run. _discard_iteration(log_ctx, run_id, organization_id, iteration_id) return diff --git a/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py b/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py index 7e131d10b7e7..351e7554a0c5 100644 --- a/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py +++ b/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py @@ -1,6 +1,10 @@ from typing import Any, Literal, TypedDict from unittest.mock import MagicMock, patch +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) + from sentry.analytics.events.pr_iteration_events import ( AiAutofixPrIterationFeedbackBatchCompletedEvent, ) @@ -18,6 +22,7 @@ from sentry.seer.autofix.autofix_agent import _group_and_referrer_from_run, get_current_step from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID, LEGACY_FEATURE_ID, SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.on_completion_hook import ( PIPELINE_ORDER, @@ -1316,6 +1321,41 @@ def test_a_failed_push_reaches_completion_details_as_that_outcome( assert mock_complete.call_args.kwargs["outcome"] == PrIterationOutcome.PUSH_FAILED.value + @patch(f"{COMPLETION_PATH}.complete_pr_iteration_details") + @patch(f"{COMPLETION_PATH}.consume_queued_feedback") + @patch(f"{COMPLETION_PATH}.trigger_push_changes", side_effect=SeerOutOfCreditsException()) + def test_a_push_refused_for_credits_pauses_as_an_error( + self, mock_push: MagicMock, mock_consume: MagicMock, mock_complete: MagicMock + ) -> None: + self.create_seer_run(organization=self.organization, seer_run_state_id=123) + + continue_pr_iteration( + self.organization, self.group, 123, self._unsynced(), AutofixReferrer.ON_COMPLETION_HOOK + ) + + assert ( + get_pause_reason(run_id=123, organization_id=self.organization.id) + == PauseReason.RUN_ERRORED + ) + assert mock_complete.call_args.kwargs["outcome"] == PrIterationOutcome.PUSH_FAILED.value + mock_consume.assert_not_called() + + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_a_branch_moved_retry_refused_for_credits_ends_the_iteration_as_errored( + self, mock_check_seer_credits: MagicMock + ) -> None: + with self.feature( + ["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"] + ): + outcome, retried = self._branch_moved_push(self._branch_moved()) + + assert outcome == PrIterationOutcome.PR_CREATION_ERRORED + assert retried is False + mock_check_seer_credits.assert_called_once() + class TestFailedRunCompletionHook(TestCase): """What the hook does with a run that Seer marked ``error``.""" diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py index 5d3fc7abc0b6..e95ecb37cb35 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py @@ -6,6 +6,7 @@ from sentry.models.organization import Organization from sentry.seer.agent.client_models import MemoryBlock, Message, RepoPRState, SeerRunState from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID from sentry.seer.autofix.steps import AutofixStep from sentry.testutils.cases import APITestCase @@ -156,6 +157,21 @@ def test_trigger_failure(self, mock_fetch: MagicMock, mock_trigger: MagicMock) - {"run_id": 1, "retried": False, "reason": "Failed to trigger the step"} ] + def test_trigger_out_of_credits(self, mock_fetch: MagicMock, mock_trigger: MagicMock) -> None: + self.create_autofix_run(1) + mock_fetch.return_value = run_state(1, [block("root_cause")], group_id=self.group.id) + mock_trigger.side_effect = SeerOutOfCreditsException() + + response = self.get_success_response(run_ids=[1]) + + assert response.data["results"] == [ + { + "run_id": 1, + "retried": False, + "reason": "Organization does not have enough Seer credits", + } + ] + def test_invalid_payload(self, mock_fetch: MagicMock, mock_trigger: MagicMock) -> None: payloads: list[dict[str, Any]] = [ {}, diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py b/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py index 17a30811c3f3..a27667cdaa6b 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py @@ -11,7 +11,7 @@ from sentry.models.groupopenperiod import get_latest_open_period from sentry.notifications.utils.issue_notification_context import IssueNotificationContext from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.seer.models.workflow import SeerWorkflowRun, SeerWorkflowStrategy from sentry.testutils.cases import APITestCase @@ -107,6 +107,15 @@ def test_out_of_quota_returns_reason(self, mock_trigger: MagicMock) -> None: assert response.data["detail"] + def test_out_of_credits_returns_reason(self, mock_trigger: MagicMock) -> None: + mock_trigger.side_effect = SeerOutOfCreditsException() + + response = self.get_error_response( + organization_slug="sentry", short_id=self.group.qualified_short_id, status_code=400 + ) + + assert response.data["detail"] + def test_unknown_issue_returns_404(self, mock_trigger: MagicMock) -> None: self.get_error_response(organization_slug="sentry", short_id="NOPE-1", status_code=404) diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 10c804a02408..d574dce8cb74 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -23,7 +23,7 @@ SeerRunState, ) from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.github_perms import MissingGithubPermissions @@ -38,6 +38,7 @@ from sentry.seer.autofix.pr_iteration.queue import QueuedAutofixFeedback from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerFeature, SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -1665,6 +1666,41 @@ def test_pr_iteration_rejected_when_paused( assert response.data["detail"] == "Seer can no longer iterate on this pull request" mock_enqueue.assert_not_called() + @patch( + "sentry.seer.autofix.pr_iteration.manual_trigger.can_run_seer_operation", + return_value=False, + ) + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") + def test_pr_iteration_rejected_when_out_of_credits( + self, mock_run_state: MagicMock, mock_enqueue: MagicMock, mock_can_run: MagicMock + ) -> None: + group = self.create_group() + mock_run_state.return_value = SeerRunState( + run_id=123, + blocks=[], + status="completed", + updated_at="2024-01-01T00:00:00Z", + repo_pr_states={"owner/repo": RepoPRState(repo_name="owner/repo", pr_number=7)}, + ) + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "pr_iteration", "run_id": 123, "user_context": "please fix this"}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run.", + "code": "seer_out_of_credits", + } + mock_can_run.assert_called_once_with( + group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + ) + mock_enqueue.assert_not_called() + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") @@ -1739,7 +1775,60 @@ def test_post_returns_402_when_no_seer_quota(self, mock_trigger_explorer: MagicM ) assert response.status_code == 402, response.data - assert response.data == "No budget for Seer Autofix." + assert response.data == { + "detail": "No budget for Seer Autofix.", + "code": "seer_out_of_quota", + } + + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_returns_402_when_out_of_seer_credits( + self, mock_trigger_explorer: MagicMock + ) -> None: + """POST returns 402 Payment Required when the credit check fails.""" + mock_trigger_explorer.side_effect = SeerOutOfCreditsException() + group = self.create_group() + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause"}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to start an Autofix run.", + "code": "seer_out_of_credits", + } + + @patch("sentry.seer.endpoints.group_ai_autofix.get_autofix_run_state") + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_returns_402_when_out_of_seer_credits_continuing_a_run( + self, mock_trigger_explorer: MagicMock, mock_run_state: MagicMock + ) -> None: + """POST says the run can't continue when a later step fails the credit check.""" + mock_trigger_explorer.side_effect = SeerOutOfCreditsException() + mock_run_state.return_value = SeerRunState( + run_id=123, + blocks=[], + status="completed", + updated_at="2024-01-01T00:00:00Z", + repo_pr_states={}, + ) + group = self.create_group() + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "solution", "run_id": 123}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run.", + "code": "seer_out_of_credits", + } def test_post_coding_agent_handoff_errors_with_both_provider_and_integration_id(self) -> None: """POST returns 400 when both provider and integration_id are specified for coding_agent_handoff.""" @@ -1951,6 +2040,24 @@ def test_open_pr_permission_error(self, mock_explorer_state_request: MagicMock) assert response.status_code == 404, response.data + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_push_changes") + def test_open_pr_out_of_credits(self, mock_push_changes: MagicMock) -> None: + mock_push_changes.side_effect = SeerOutOfCreditsException() + self.login_as(user=self.user) + group = self.create_group() + + response = self.client.post( + self._get_url(group.id), + data={"step": "open_pr", "run_id": 123}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run.", + "code": "seer_out_of_credits", + } + def test_open_pr_coding_disabled(self) -> None: self.login_as(user=self.user) group = self.create_group() diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index fb8363e49947..bda832ba562d 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -16,6 +16,7 @@ UsageAccumulator, ) from sentry.seer.agent.client_utils import snapshot_to_markdown +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.endpoints.organization_seer_agent_chat import SeerAgentChatSerializer from sentry.seer.models import SeerApiError @@ -514,6 +515,26 @@ def test_reply_to_single_run_while_it_is_processing_is_409( assert response.data["code"] == "run_in_flight" mock_continue.assert_not_called() + @patch( + "sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run", + side_effect=SeerOutOfCreditsException(), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_reply_to_single_run_without_credits_is_402( + self, mock_client_class: MagicMock, mock_continue: MagicMock + ) -> None: + run = self._single_run(user_id=self.user.id) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "make a code fix"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer.", + "code": "seer_out_of_credits", + } + @patch("sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run") @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_reply_to_single_run_with_no_seer_session_is_404( @@ -953,7 +974,8 @@ def test_post_new_conversation_without_credits_returns_402( assert response.status_code == 402 assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." + "detail": "Your organization does not have enough credits left for you to message Seer.", + "code": "seer_out_of_credits", } mock_client_class.assert_not_called() @@ -994,7 +1016,8 @@ def test_post_continue_without_credits_returns_402( assert response.status_code == 402 assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." + "detail": "Your organization does not have enough credits left for you to message Seer.", + "code": "seer_out_of_credits", } mock_client_class.assert_not_called()