From c51801b736344632e06ee64257849347bf758e74 Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Fri, 13 Nov 2026 17:08:55 -0800 Subject: [PATCH] feat(seer): Return denialReason on out-of-credits 402s --- src/sentry/seer/autofix/exceptions.py | 9 ++ src/sentry/seer/autofix/feature/dispatch.py | 16 +-- .../autofix/pr_iteration/manual_trigger.py | 22 +++-- src/sentry/seer/credits.py | 25 +++-- src/sentry/seer/endpoints/group_ai_autofix.py | 42 ++++---- .../endpoints/organization_seer_agent_chat.py | 29 +++--- .../organization_seer_credit_check.py | 19 ++-- src/sentry/seer/endpoints/utils.py | 10 ++ .../seer/autofix/feature/test_dispatch.py | 8 +- .../seer/endpoints/test_group_ai_autofix.py | 99 ++++++++++++------- .../test_organization_seer_agent_chat.py | 77 ++++++++++----- tests/sentry/seer/test_credits.py | 17 ++++ 12 files changed, 240 insertions(+), 133 deletions(-) diff --git a/src/sentry/seer/autofix/exceptions.py b/src/sentry/seer/autofix/exceptions.py index 7ee853708745..fce198f8b961 100644 --- a/src/sentry/seer/autofix/exceptions.py +++ b/src/sentry/seer/autofix/exceptions.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from sentry.seer.credits import SeerCreditDenialReason + + class NoSeerQuotaException(Exception): """Raised when an Autofix run cannot consume Seer quota.""" @@ -5,6 +10,10 @@ class NoSeerQuotaException(Exception): class SeerOutOfCreditsException(Exception): """Raised when the organization doesn't have enough Seer credits for an Autofix operation.""" + def __init__(self, denial_reason: SeerCreditDenialReason | None = None) -> None: + super().__init__(denial_reason) + self.denial_reason = denial_reason + class IssueSummaryUnavailable(Exception): """Raised when an issue summary cannot be generated for an expected reason.""" diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 4a7ff82e933f..6b309a65ab46 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -49,7 +49,12 @@ get_event_matched_project_repo_ids, is_free_cohort_org, ) -from sentry.seer.credits import SeerFeature, SeerOperation, has_enough_seer_credits +from sentry.seer.credits import ( + SeerFeature, + SeerOperation, + check_seer_operation, + get_denial_reason, +) from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -187,12 +192,9 @@ def check_autofix_billing( """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if features.has( - "organizations:seer-credit-checks", organization - ) and not has_enough_seer_credits( - organization, seer_feature_for_referrer(referrer), operation - ): - raise SeerOutOfCreditsException() + credits = check_seer_operation(organization, seer_feature_for_referrer(referrer), operation) + if not credits.has_enough_credits: + raise SeerOutOfCreditsException(get_denial_reason(credits)) return False # Check quota for non-token-billed orgs once per run. diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index dcb19b4f4750..c4753df9fc9c 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -25,11 +25,16 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource from sentry.seer.autofix.pr_iteration.pause import PauseReason, get_pause_reason from sentry.seer.autofix.pr_iteration.queue import enqueue_autofix_feedback -from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import ( + SeerFeature, + SeerOperation, + check_seer_operation, + get_denial_reason, +) from sentry.seer.endpoints.utils import ( SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, - SEER_OUT_OF_CREDITS_CODE, SEER_PERMISSION_DENIED, + seer_out_of_credits_response, ) from sentry.seer.models import SeerPermissionError from sentry.tasks.seer.pr_iteration import trigger_consume_pr_iteration_feedback @@ -87,15 +92,12 @@ def handle_ui_feedback( ) # Check credits here, since the queued feedback is consumed after this request returns. - if not can_run_seer_operation( + credits = check_seer_operation( group.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_PR_ITERATION - ): - return Response( - { - "detail": SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, - "code": SEER_OUT_OF_CREDITS_CODE, - }, - status=status.HTTP_402_PAYMENT_REQUIRED, + ) + if not credits.has_enough_credits: + return seer_out_of_credits_response( + SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, get_denial_reason(credits) ) serialized_users = user_service.serialize_many( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 2841452a9073..d0ca6fbc3fb4 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -2,6 +2,7 @@ from collections.abc import Callable from enum import StrEnum +from typing import Literal from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsRequest, @@ -15,6 +16,16 @@ OUT_OF_CREDITS = "out_of_credits" +SeerCreditDenialReason = Literal["not_enough_credits", "daily_cap_exceeded"] + +# Unspecified and unknown reasons map to None, so a reason added to the proto +# before this mapping doesn't leak an enum name into the API. +_DENIAL_REASONS: dict[CheckSeerCreditsResponse.DenialReason.ValueType, SeerCreditDenialReason] = { + CheckSeerCreditsResponse.DENIAL_REASON_NOT_ENOUGH_CREDITS: "not_enough_credits", + CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED: "daily_cap_exceeded", +} + + class SeerOperation(StrEnum): AGENT_CHAT_MESSAGE = "agent_chat_message" AUTOFIX_ROOT_CAUSE = "autofix_root_cause" @@ -66,16 +77,6 @@ def _check_seer_credits( ) -def has_enough_seer_credits( - organization: Organization, feature: SeerFeature, operation: SeerOperation -) -> bool: - """ - Checks whether the organization has enough Seer credits to run the operation, - for callers that already checked the credit check and token-billed flags. - """ - return _check_seer_credits(organization, feature, operation).has_enough_credits - - def check_seer_operation( organization: Organization, feature: SeerFeature, operation: SeerOperation ) -> CheckSeerCreditsResponse: @@ -91,3 +92,7 @@ def can_run_seer_operation( organization: Organization, feature: SeerFeature, operation: SeerOperation ) -> bool: return check_seer_operation(organization, feature, operation).has_enough_credits + + +def get_denial_reason(response: CheckSeerCreditsResponse) -> SeerCreditDenialReason | None: + return _DENIAL_REASONS.get(response.denial_reason) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 58f69081b32d..571edcb78436 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -84,18 +84,19 @@ CodingAgentProviderType, has_project_connected_repos, ) +from sentry.seer.credits import SeerCreditDenialReason from sentry.seer.endpoints.organization_seer_onboarding_check import ( has_supported_scm_integration, ) from sentry.seer.endpoints.utils import ( SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL, - SEER_OUT_OF_CREDITS_CODE, SEER_OUT_OF_QUOTA_CODE, SEER_PERMISSION_DENIED, SeerPaymentRequiredCode, get_seer_run, resolve_seer_run, + seer_out_of_credits_response, ) from sentry.seer.models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.types.activity import ActivityType @@ -164,6 +165,16 @@ class AutofixPaymentRequiredResponse(TypedDict): "`seer_out_of_quota` when it has no Seer budget" ), ] + denialReason: NotRequired[ + Annotated[ + SeerCreditDenialReason | None, + ApiDescription( + "Set with `seer_out_of_credits`. `not_enough_credits` when the organization's " + "credit balance doesn't cover the operation, `daily_cap_exceeded` when it would " + "pass the organization's daily spend cap, null when no reason is known" + ), + ] + ] class GroupAutofixPermission(GroupAiPermission): @@ -510,13 +521,9 @@ def post( ), user=acting_user(request.user), ) - except SeerOutOfCreditsException: - return Response( - { - "detail": SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, - "code": SEER_OUT_OF_CREDITS_CODE, - }, - status=status.HTTP_402_PAYMENT_REQUIRED, + except SeerOutOfCreditsException as e: + return seer_out_of_credits_response( + SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, e.denial_reason ) except SeerPermissionError: return Response(status=status.HTTP_404_NOT_FOUND) @@ -649,17 +656,14 @@ def post( {"detail": "No budget for Seer Autofix.", "code": SEER_OUT_OF_QUOTA_CODE}, status=status.HTTP_402_PAYMENT_REQUIRED, ) - except SeerOutOfCreditsException: - return Response( - { - "detail": ( - SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL - if resolved_run_id is None - else SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL - ), - "code": SEER_OUT_OF_CREDITS_CODE, - }, - status=status.HTTP_402_PAYMENT_REQUIRED, + except SeerOutOfCreditsException as e: + return seer_out_of_credits_response( + ( + SEER_AUTOFIX_START_OUT_OF_CREDITS_DETAIL + if resolved_run_id is None + else SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL + ), + e.denial_reason, ) except SeerPermissionError as e: if _is_unknown_run_id_error(e): diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 60c42123a53e..9c78b29c1a8b 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -40,13 +40,18 @@ from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.prompt_redaction import redact_autofix_prompts from sentry.seer.autofix.single_run import is_single_run -from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import ( + SeerFeature, + SeerOperation, + check_seer_operation, + get_denial_reason, +) from sentry.seer.endpoints.group_ai_autofix import RUN_IN_FLIGHT_CODE from sentry.seer.endpoints.utils import ( SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, - SEER_OUT_OF_CREDITS_CODE, ResolvedSeerRun, resolve_seer_run, + seer_out_of_credits_response, ) from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun @@ -276,11 +281,8 @@ def _maybe_reply_to_single_run( can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), referrer=AutofixReferrer.SEER_EXPLORER, ) - except SeerOutOfCreditsException: - return Response( - {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, - status=402, - ) + except SeerOutOfCreditsException as e: + return seer_out_of_credits_response(SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, e.denial_reason) except SeerApiError as e: if e.status == 404: return Response({"session": None}, status=404) @@ -452,13 +454,14 @@ def post( if not has_access and not is_dashboard_generate_run: raise PermissionDenied(error) # Do not bill for dashboard generation. - if not is_dashboard_generate_run and not can_run_seer_operation( - organization, SeerFeature.SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE - ): - return Response( - {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, - status=402, + if not is_dashboard_generate_run: + credits = check_seer_operation( + organization, SeerFeature.SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ) + if not credits.has_enough_credits: + return seer_out_of_credits_response( + SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, get_denial_reason(credits) + ) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index c1e215d8f3d6..806523d525a3 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -3,9 +3,6 @@ from rest_framework import serializers from rest_framework.request import Request from rest_framework.response import Response -from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( - CheckSeerCreditsResponse, -) from sentry.api.api_owners import ApiOwner from sentry.api.api_publish_status import ApiPublishStatus @@ -13,16 +10,14 @@ from sentry.api.bases.organization import OrganizationEndpoint from sentry.models.organization import Organization from sentry.ratelimits.config import RateLimitConfig -from sentry.seer.credits import SeerFeature, SeerOperation, check_seer_operation +from sentry.seer.credits import ( + SeerFeature, + SeerOperation, + check_seer_operation, + get_denial_reason, +) from sentry.types.ratelimit import RateLimit, RateLimitCategory -# Unspecified and unknown reasons map to None, so a reason added to the proto -# before this mapping doesn't leak an enum name into the API. -_DENIAL_REASONS: dict[CheckSeerCreditsResponse.DenialReason.ValueType, str] = { - CheckSeerCreditsResponse.DENIAL_REASON_NOT_ENOUGH_CREDITS: "not_enough_credits", - CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED: "daily_cap_exceeded", -} - class SeerCreditCheckSerializer(serializers.Serializer[dict[str, str]]): operation = serializers.ChoiceField(choices=[operation.value for operation in SeerOperation]) @@ -66,6 +61,6 @@ def get(self, request: Request, organization: Organization) -> Response: return Response( { "hasEnoughCredits": credits.has_enough_credits, - "denialReason": _DENIAL_REASONS.get(credits.denial_reason), + "denialReason": get_denial_reason(credits), } ) diff --git a/src/sentry/seer/endpoints/utils.py b/src/sentry/seer/endpoints/utils.py index e306540c8e97..0da0ddbfb6fa 100644 --- a/src/sentry/seer/endpoints/utils.py +++ b/src/sentry/seer/endpoints/utils.py @@ -8,6 +8,7 @@ from rest_framework.response import Response from sentry.api.bases.organization import OrganizationPermission +from sentry.seer.credits import SeerCreditDenialReason from sentry.seer.models.run import SeerRun, SeerRunMirrorStatus from sentry.utils.numbers import validate_bigint @@ -34,6 +35,15 @@ ) +def seer_out_of_credits_response( + detail: str, denial_reason: SeerCreditDenialReason | None +) -> Response: + return Response( + {"detail": detail, "code": SEER_OUT_OF_CREDITS_CODE, "denialReason": denial_reason}, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) + + class OrganizationTraceExplorerAIPermission(OrganizationPermission): scope_map = { "GET": ["org:read"], diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 87d691b1e8a4..69d790bc1da6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -346,7 +346,10 @@ def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse( + has_enough_credits=False, + denial_reason=CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED, + ), ) def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: with ( @@ -355,7 +358,7 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ): mock_quotas.backend.check_seer_quota.return_value = True - with pytest.raises(SeerOutOfCreditsException): + with pytest.raises(SeerOutOfCreditsException) as excinfo: trigger_autofix_feature( self.group, AutofixFeatureArgs( @@ -365,6 +368,7 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ), ) + assert excinfo.value.denial_reason == "daily_cap_exceeded" MockClient.return_value.start_feature_run.assert_not_called() mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 2f6f6c2e4916..d3d0c4f3d740 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -6,6 +6,9 @@ from django.test import override_settings from django.utils import timezone +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.integrations.services.integration import RpcIntegration from sentry.integrations.types import ExternalProviders @@ -39,7 +42,7 @@ from sentry.seer.autofix.prompt_redaction import AUTOFIX_PROMPT_PLACEHOLDER from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerFeature, SeerOperation +from sentry.seer.credits import SEER_OPERATION_ESTIMATES, SeerFeature, SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -72,6 +75,13 @@ def _user_context_length_calls(mock_distribution: Mock) -> list[Any]: ] +DENIAL_REASON_CASES = [ + (CheckSeerCreditsResponse.DENIAL_REASON_NOT_ENOUGH_CREDITS, "not_enough_credits"), + (CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED, "daily_cap_exceeded"), + (CheckSeerCreditsResponse.DENIAL_REASON_UNSPECIFIED, None), +] + + @override_settings(SENTRY_SELF_HOSTED=False) class GroupAutofixEndpointTest(APITestCase, SnubaTestCase): def _get_url(self, group_id: int) -> str: @@ -1715,14 +1725,11 @@ def test_pr_iteration_rejected_when_paused( assert response.data["detail"] == "Seer can no longer iterate on this pull request" mock_enqueue.assert_not_called() - @patch( - "sentry.seer.autofix.pr_iteration.manual_trigger.can_run_seer_operation", - return_value=False, - ) + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") def test_pr_iteration_rejected_when_out_of_credits( - self, mock_run_state: MagicMock, mock_enqueue: MagicMock, mock_can_run: MagicMock + self, mock_run_state: MagicMock, mock_enqueue: MagicMock ) -> None: group = self.create_group() mock_run_state.return_value = SeerRunState( @@ -1734,20 +1741,34 @@ def test_pr_iteration_rejected_when_out_of_credits( ) self.login_as(user=self.user) - response = self.client.post( - self._get_url(group.id), - data={"step": "pr_iteration", "run_id": 123, "user_context": "please fix this"}, - format="json", - ) + for denial_reason, expected in DENIAL_REASON_CASES: + with ( + self.subTest(denial_reason=expected), + patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse( + has_enough_credits=False, denial_reason=denial_reason + ), + ) as mock_check_seer_credits, + ): + response = self.client.post( + self._get_url(group.id), + data={"step": "pr_iteration", "run_id": 123, "user_context": "please fix this"}, + format="json", + ) - assert response.status_code == 402, response.data - assert response.data == { - "detail": "Your organization does not have enough credits to continue your Autofix run.", - "code": "seer_out_of_credits", - } - mock_can_run.assert_called_once_with( - group.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_PR_ITERATION - ) + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run.", + "code": "seer_out_of_credits", + "denialReason": expected, + } + request = mock_check_seer_credits.call_args.args[0] + assert request.feature_id == SeerFeature.AUTOFIX_WEB + assert ( + request.predicted_credit_usage + == SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_PR_ITERATION]() + ) mock_enqueue.assert_not_called() @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @@ -1834,7 +1855,7 @@ def test_post_returns_402_when_out_of_seer_credits( self, mock_trigger_explorer: MagicMock ) -> None: """POST returns 402 Payment Required when the credit check fails.""" - mock_trigger_explorer.side_effect = SeerOutOfCreditsException() + mock_trigger_explorer.side_effect = SeerOutOfCreditsException("not_enough_credits") group = self.create_group() self.login_as(user=self.user) @@ -1848,6 +1869,7 @@ def test_post_returns_402_when_out_of_seer_credits( assert response.data == { "detail": "Your organization does not have enough credits to start an Autofix run.", "code": "seer_out_of_credits", + "denialReason": "not_enough_credits", } @patch("sentry.seer.endpoints.group_ai_autofix.get_autofix_run_state") @@ -1877,6 +1899,7 @@ def test_post_returns_402_when_out_of_seer_credits_continuing_a_run( assert response.data == { "detail": "Your organization does not have enough credits to continue your Autofix run.", "code": "seer_out_of_credits", + "denialReason": None, } def test_post_coding_agent_handoff_errors_with_both_provider_and_integration_id(self) -> None: @@ -2089,23 +2112,33 @@ def test_open_pr_permission_error(self, mock_explorer_state_request: MagicMock) assert response.status_code == 404, response.data - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_push_changes") - def test_open_pr_out_of_credits(self, mock_push_changes: MagicMock) -> None: - mock_push_changes.side_effect = SeerOutOfCreditsException() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + def test_open_pr_out_of_credits(self) -> None: self.login_as(user=self.user) group = self.create_group() - response = self.client.post( - self._get_url(group.id), - data={"step": "open_pr", "run_id": 123}, - format="json", - ) + for denial_reason, expected in DENIAL_REASON_CASES: + with ( + self.subTest(denial_reason=expected), + patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse( + has_enough_credits=False, denial_reason=denial_reason + ), + ), + ): + response = self.client.post( + self._get_url(group.id), + data={"step": "open_pr", "run_id": 123}, + format="json", + ) - assert response.status_code == 402, response.data - assert response.data == { - "detail": "Your organization does not have enough credits to continue your Autofix run.", - "code": "seer_out_of_credits", - } + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run.", + "code": "seer_out_of_credits", + "denialReason": expected, + } def test_open_pr_coding_disabled(self) -> None: self.login_as(user=self.user) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index f8e169746a28..2e67e366e759 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -16,7 +16,6 @@ UsageAccumulator, ) from sentry.seer.agent.client_utils import snapshot_to_markdown -from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.prompt_redaction import AUTOFIX_PROMPT_PLACEHOLDER from sentry.seer.endpoints.organization_seer_agent_chat import SeerAgentChatSerializer @@ -27,6 +26,12 @@ from sentry.utils import json from sentry.utils.security.orgauthtoken_token import generate_token, hash_token +DENIAL_REASON_CASES = [ + (CheckSeerCreditsResponse.DENIAL_REASON_NOT_ENOUGH_CREDITS, "not_enough_credits"), + (CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED, "daily_cap_exceeded"), + (CheckSeerCreditsResponse.DENIAL_REASON_UNSPECIFIED, None), +] + @with_feature("organizations:seer-explorer") @override_settings(SENTRY_SELF_HOSTED=False) @@ -592,25 +597,33 @@ def test_reply_to_single_run_while_it_is_processing_is_409( assert response.data["code"] == "run_in_flight" mock_continue.assert_not_called() - @patch( - "sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run", - side_effect=SeerOutOfCreditsException(), - ) + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_reply_to_single_run_without_credits_is_402( - self, mock_client_class: MagicMock, mock_continue: MagicMock - ) -> None: + def test_reply_to_single_run_without_credits_is_402(self, mock_client_class: MagicMock) -> None: run = self._single_run(user_id=self.user.id) - response = self.client.post( - f"{self.url}{run.seer_run_state_id}/", {"query": "make a code fix"}, format="json" - ) + for denial_reason, expected in DENIAL_REASON_CASES: + with ( + self.subTest(denial_reason=expected), + patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse( + has_enough_credits=False, denial_reason=denial_reason + ), + ), + ): + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", + {"query": "make a code fix"}, + format="json", + ) - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer.", - "code": "seer_out_of_credits", - } + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer.", + "code": "seer_out_of_credits", + "denialReason": expected, + } @patch("sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run") @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") @@ -1039,21 +1052,30 @@ def test_get_denied_without_seer_access(self) -> None: assert response.status_code == 403 @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_new_conversation_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + self, mock_client_class: MagicMock ) -> None: - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + for denial_reason, expected in DENIAL_REASON_CASES: + with ( + self.subTest(denial_reason=expected), + patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse( + has_enough_credits=False, denial_reason=denial_reason + ), + ), + ): + response = self.client.post( + self.url, {"query": "What is this error about?"}, format="json" + ) - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer.", - "code": "seer_out_of_credits", - } + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer.", + "code": "seer_out_of_credits", + "denialReason": expected, + } mock_client_class.assert_not_called() @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @@ -1095,6 +1117,7 @@ def test_post_continue_without_credits_returns_402( assert response.data == { "detail": "Your organization does not have enough credits left for you to message Seer.", "code": "seer_out_of_credits", + "denialReason": None, } mock_client_class.assert_not_called() diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index 750ceaa285fa..b638afac3db4 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock, patch +import pytest from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsRequest, CheckSeerCreditsResponse, @@ -12,6 +13,7 @@ SeerFeature, SeerOperation, can_run_seer_operation, + get_denial_reason, ) from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -79,3 +81,18 @@ def test_can_run_seer_operation_returns_true_when_billing_has_credits( assert can_run_seer_operation( self.organization, SeerFeature.SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ) + + +@pytest.mark.parametrize( + ("denial_reason", "expected"), + [ + (CheckSeerCreditsResponse.DENIAL_REASON_NOT_ENOUGH_CREDITS, "not_enough_credits"), + (CheckSeerCreditsResponse.DENIAL_REASON_DAILY_CAP_EXCEEDED, "daily_cap_exceeded"), + (CheckSeerCreditsResponse.DENIAL_REASON_UNSPECIFIED, None), + (99, None), + ], +) +def test_get_denial_reason(denial_reason: int, expected: str | None) -> None: + response = CheckSeerCreditsResponse(has_enough_credits=False) + response.denial_reason = denial_reason # type: ignore[assignment] + assert get_denial_reason(response) == expected