From cbdde3a0098a58a2743926f9d2ff063e214802d2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:42:25 -0800 Subject: [PATCH 01/45] feat(seer): Gate agent chat messages on Seer credits Return a 402 before sending a chat message when a token-billed org does not have enough Seer credits. Dashboard generation runs are not gated. can_run_seer_operation now passes orgs that are not token-billed. Refs CW-3061 --- src/sentry/seer/credits.py | 14 +++- .../endpoints/organization_seer_agent_chat.py | 18 +++- .../test_organization_seer_agent_chat.py | 84 +++++++++++++++++++ .../test_organization_seer_credit_check.py | 4 +- tests/sentry/seer/test_credits.py | 4 +- 5 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 6908b5ba4d85..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,8 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization has enough Seer credits to run the operation.""" +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -75,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index ae787654d804..269705ba3b88 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -28,6 +28,7 @@ has_seer_agent_access_with_detail, snapshot_to_markdown, ) +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import ResolvedSeerRun, resolve_seer_run from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun @@ -356,10 +357,21 @@ def post( return result resolved = result - if not has_access and not ( - resolved is not None and _is_dashboard_generate_run(organization, resolved) - ): + is_dashboard_generate_run = resolved is not None and _is_dashboard_generate_run( + organization, resolved + ) + if not has_access and not is_dashboard_generate_run: raise PermissionDenied(error) + # Do not bill for dashboard generation. + if not is_dashboard_generate_run and not can_run_seer_operation( + organization, SeerOperation.AGENT_CHAT_MESSAGE + ): + return Response( + { + "detail": "Your organization does not have enough credits left for you to message Seer." + }, + status=402, + ) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index f9cf4b9eadf8..1b33d749ef45 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.seer.agent.client_models import ( MemoryBlock, @@ -493,6 +496,87 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=555, user_id=self.user.id + ) + + response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 199a106443b3..90aaf513233d 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -19,7 +19,7 @@ def setUp(self) -> None: super().setUp() self.login_as(user=self.user) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), @@ -39,7 +39,7 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index ba587cd18881..afd37295c106 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -27,7 +27,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -45,7 +45,7 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( ) ) - @with_feature("organizations:seer-credit-checks") + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=True), From c2473282edb707762f2ba74673fc5f8c36b06b98 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:43:12 -0800 Subject: [PATCH 02/45] feat(seer): Check Seer credits before Autofix steps and Open PR Token-billed orgs are checked for Seer credits before each Autofix step and before opening a PR, and get SeerOutOfCreditsException when they can't pay. Other orgs keep the seat-based quota check. Agentic triage runs are not gated. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 50 ++++--- src/sentry/seer/autofix/exceptions.py | 4 + src/sentry/seer/autofix/feature/dispatch.py | 90 ++++++++++-- src/sentry/seer/autofix/steps.py | 14 ++ .../seer/autofix/feature/test_dispatch.py | 86 ++++++++++- .../sentry/seer/autofix/test_autofix_agent.py | 136 +++++++++++++++++- 6 files changed, 342 insertions(+), 38 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 6515cef5ab52..7ce38f75c09a 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -9,7 +9,7 @@ from scm.types import GetBranchProtocol, GetRepositoryProtocol from sentry_sdk import traces -from sentry import features, quotas +from sentry import features from sentry.analytics.events.autofix_events import ( AiAutofixAgentHandoffEvent, AiAutofixCodeChangesCompletedEvent, @@ -23,7 +23,7 @@ AiAutofixSolutionCompletedEvent, AiAutofixSolutionStartedEvent, ) -from sentry.constants import ENABLE_SEER_CODING_DEFAULT, DataCategory +from sentry.constants import ENABLE_SEER_CODING_DEFAULT from sentry.integrations.services.integration import integration_service from sentry.models.group import Group from sentry.seer.agent.client import SeerAgentClient @@ -31,10 +31,11 @@ from sentry.seer.autofix.analytics import record_autofix_event from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import ( AutofixFeatureArgs, autofix_run_targets_other_issue, + check_autofix_billing, require_autofix_run_for_group, trigger_autofix_feature, ) @@ -62,6 +63,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) +from sentry.seer.credits import SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -460,19 +462,13 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - if run_id is None: - skip_quota_check = allow_free_cohort and is_free_cohort_org(group.organization) - if not skip_quota_check: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - raise NoSeerQuotaException() + check_autofix_billing( + group, + step.seer_operation, + referrer, + is_new_run=run_id is None, + skip_quota_for_free_cohort=allow_free_cohort, + ) if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) @@ -547,7 +543,12 @@ def trigger_autofix_agent( actor_user_id=actor_user_id, ) return feature_run - except (NoSeerQuotaException, SeerPermissionError, PrIterationNoPullRequestException): + except ( + NoSeerQuotaException, + SeerOutOfCreditsException, + SeerPermissionError, + PrIterationNoPullRequestException, + ): outcome = "declined" raise except BaseException: @@ -834,6 +835,17 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Open PR always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_OPEN_PR, + seer_run.referrer if seer_run else None, + is_new_run=False, + ) + client = get_autofix_agent_client(group, user=user) if state is None: @@ -851,10 +863,6 @@ def trigger_push_changes( ) ) - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/exceptions.py b/src/sentry/seer/autofix/exceptions.py index 74332026f4dc..7ee853708745 100644 --- a/src/sentry/seer/autofix/exceptions.py +++ b/src/sentry/seer/autofix/exceptions.py @@ -2,6 +2,10 @@ class NoSeerQuotaException(Exception): """Raised when an Autofix run cannot consume Seer quota.""" +class SeerOutOfCreditsException(Exception): + """Raised when the organization doesn't have enough Seer credits for an Autofix operation.""" + + class IssueSummaryUnavailable(Exception): """Raised when an issue summary cannot be generated for an expected reason.""" diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 5ec273daaaec..02b7f3f5c49b 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -22,7 +22,7 @@ ) from sentry.seer.agent.on_completion_hook import extract_hook_definition from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import ( FEATURE_ID, LEGACY_FEATURE_ID, @@ -41,6 +41,7 @@ get_automation_handoff, is_free_cohort_org, ) +from sentry.seer.credits import SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -140,18 +141,42 @@ def _single_run_pr_internal_settings( ) -def trigger_autofix_feature( - group: Group, - args: AutofixFeatureArgs, -) -> SeerRun: - # Avoid a circular import through the legacy Autofix dispatcher. - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook +def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: + """ + Raise if the organization can't afford the operation under token-based + credits. Agentic triage runs stay free. + """ + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + return + if not has_enough_seer_credits(group.organization, operation): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() - is_new_run = args.existing_run_id is None + +def _check_autofix_quota( + group: Group, + referrer: str | None, + *, + is_new_run: bool, + allow_free_cohort: bool, +) -> bool: + """ + Raise if the organization has no seat-based Seer quota left. Returns whether + the step uses quota, which is recorded once the run starts. + """ # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and args.allow_free_cohort and is_free_cohort_org(group.organization) - if is_new_run and not skip_quota: + skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) + uses_quota = is_new_run and not skip_quota + if uses_quota: has_budget: bool = quotas.backend.check_seer_quota( org_id=group.organization.id, data_category=DataCategory.SEER_AUTOFIX, @@ -162,11 +187,50 @@ def trigger_autofix_feature( extra={ "group_id": group.id, "organization_id": group.organization.id, - "referrer": args.referrer.value, + "referrer": referrer, }, ) raise NoSeerQuotaException() + return uses_quota + + +def check_autofix_billing( + group: Group, + operation: SeerOperation, + referrer: str | None, + *, + is_new_run: bool, + skip_quota_for_free_cohort: bool = False, +) -> bool: + """ + Raise if the organization can't pay for the operation. Token-billed orgs are + checked for credits, everyone else for seat-based quota. Returns whether the + operation uses quota. + """ + if features.has("organizations:token-billed-seer-enabled", group.organization): + _check_autofix_credits(group, operation, referrer) + return False + return _check_autofix_quota( + group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + ) + + +def trigger_autofix_feature( + group: Group, + args: AutofixFeatureArgs, +) -> SeerRun: + # Avoid a circular import through the legacy Autofix dispatcher. + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=args.existing_run_id is None, + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + single_run = should_use_single_run( group, args.step, @@ -259,7 +323,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if is_new_run: + if args.existing_run_id is None: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), @@ -285,7 +349,7 @@ def trigger_autofix_feature( else: raise Exception("Unhandled run_id branch, this should never happen") - if is_new_run and not skip_quota: + if uses_quota: quotas.backend.record_seer_run( group.organization.id, group.project.id, DataCategory.SEER_AUTOFIX ) diff --git a/src/sentry/seer/autofix/steps.py b/src/sentry/seer/autofix/steps.py index b35a0d86ebea..8ec531cfde9e 100644 --- a/src/sentry/seer/autofix/steps.py +++ b/src/sentry/seer/autofix/steps.py @@ -3,6 +3,7 @@ from enum import StrEnum from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerOperation class AutofixStep(StrEnum): @@ -13,6 +14,19 @@ class AutofixStep(StrEnum): CODE_CHANGES = "code_changes" PR_ITERATION = "pr_iteration" + @property + def seer_operation(self) -> SeerOperation: + """The Seer credit check operation corresponding to this step.""" + match self: + case AutofixStep.ROOT_CAUSE: + return SeerOperation.AUTOFIX_ROOT_CAUSE + case AutofixStep.SOLUTION: + return SeerOperation.AUTOFIX_SOLUTION + case AutofixStep.CODE_CHANGES: + return SeerOperation.AUTOFIX_CODE_CHANGES + case AutofixStep.PR_ITERATION: + return SeerOperation.AUTOFIX_PR_ITERATION + @staticmethod def from_autofix_stopping_point( autofix_stopping_point: AutofixStoppingPoint, diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 06ddaf9ca71b..9827183d33c6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -2,10 +2,13 @@ from unittest.mock import MagicMock, patch import pytest +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.constants import DataCategory from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature from sentry.seer.autofix.feature.models import ( FEATURE_ID, @@ -21,6 +24,7 @@ from sentry.seer.autofix.utils import AutofixStoppingPoint from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.testutils.pytest.fixtures import django_db_all @@ -275,6 +279,86 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + MockClient.return_value.start_feature_run.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits_for_a_continued_step( + self, mock_check_seer_credits: MagicMock + ) -> None: + """Unlike quota, credits are checked for every step, not only new runs.""" + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.SOLUTION, + existing_run_id=123, + step_args=SolutionStepArgs(), + ), + ) + + mock_quotas.backend.check_seer_quota.assert_not_called() + MockClient.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 477b8bf89ffa..dd598ac0a01e 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings from rest_framework.exceptions import PermissionDenied +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent from sentry.constants import DataCategory @@ -27,7 +30,7 @@ ) from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer, SeerAutomationSource -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.pr_iteration.feedback import Feedback @@ -38,6 +41,7 @@ from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature from sentry.types.activity import ActivityType from sentry.utils import json @@ -720,6 +724,30 @@ def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMoc ) mock_feature.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + def test_when_out_of_credits( + self, + mock_check_quota: MagicMock, + mock_feature: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.UNKNOWN, + run_id=None, + ) + mock_check_seer_credits.assert_called_once() + mock_check_quota.assert_not_called() + mock_feature.assert_not_called() + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -732,7 +760,7 @@ def test_free_cohort_skips_quota_check( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=True), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=True), ): trigger_autofix_agent( group=self.group, @@ -756,7 +784,7 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( with ( patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.autofix_agent.is_free_cohort_org", return_value=False), + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), ): trigger_autofix_agent( group=self.group, @@ -767,6 +795,35 @@ def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( mock_check_quota.assert_called_once() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.quotas.backend.check_seer_quota", return_value=True) + @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") + def test_agentic_triage_skips_credit_check( + self, + mock_feature: MagicMock, + mock_broadcast: MagicMock, + mock_check_quota: MagicMock, + mock_check_seer_credits: MagicMock, + ) -> None: + mock_feature.return_value = self.create_seer_run( + organization=self.group.organization, seer_run_state_id=12345 + ) + + trigger_autofix_agent( + group=self.group, + step=AutofixStep.CODE_CHANGES, + referrer=AutofixReferrer.AGENTIC_TRIAGE, + ) + + mock_check_seer_credits.assert_not_called() + mock_check_quota.assert_not_called() + mock_feature.assert_called_once() + @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -975,6 +1032,35 @@ def test_counts_declined_when_no_quota( } ] + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_counts_declined_when_out_of_credits( + self, + mock_check_seer_credits: MagicMock, + mock_client_class: MagicMock, + mock_incr: MagicMock, + mock_check_quota: MagicMock, + mock_record_run: MagicMock, + mock_broadcast: MagicMock, + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_autofix_agent( + self.group, + AutofixStep.ROOT_CAUSE, + AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + ) + + assert _trigger_attempts(mock_incr) == [ + { + "referrer": "issue_summary.post_process_fixability", + "step": "root_cause", + "outcome": "declined", + } + ] + @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") def test_counts_error_when_seer_rejects_run( self, @@ -1674,6 +1760,50 @@ def test_raises_permission_denied_when_coding_disabled(self) -> None: referrer=AutofixReferrer.UNKNOWN, ) + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_raises_when_out_of_credits( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + with pytest.raises(SeerOutOfCreditsException): + trigger_push_changes( + group=self.group, + run_id=123, + referrer=AutofixReferrer.UNKNOWN, + ) + + mock_post.assert_not_called() + + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_agentic_triage_run_skips_credit_check( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + # The push is called with another referrer; the run's own referrer decides. + self.create_seer_run( + organization=self.organization, + seer_run_state_id=123, + referrer=AutofixReferrer.AGENTIC_TRIAGE.value, + ) + + self._push( + mock_post, + features={ + "organizations:seer-credit-checks": True, + "organizations:token-billed-seer-enabled": True, + }, + ) + + mock_check_seer_credits.assert_not_called() + mock_post.assert_called_once() + @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock From 12cebafa43ecda04a1175efc40a34ff720a6afd3 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:44:02 -0800 Subject: [PATCH 03/45] feat(seer): Handle SeerOutOfCreditsException at Autofix callers The issue page returns a 402, Slack shows an error, the admin endpoints report it, and automatic paths (new issue automation, completion hook, PR iteration) log and stop. Refs CW-3061 --- src/sentry/seer/autofix/issue_summary.py | 3 +- src/sentry/seer/autofix/on_completion_hook.py | 29 ++++++-- .../seer/autofix/pr_iteration/completion.py | 7 ++ .../seer/endpoints/admin_autofix_retry.py | 3 + .../endpoints/admin_autofix_single_run.py | 6 +- src/sentry/seer/endpoints/group_ai_autofix.py | 20 +++++- src/sentry/seer/entrypoints/operator.py | 12 +++- src/sentry/tasks/seer/pr_iteration.py | 18 +++-- .../endpoints/test_admin_autofix_retry.py | 16 +++++ .../test_admin_autofix_single_run.py | 11 ++- .../seer/endpoints/test_group_ai_autofix.py | 67 ++++++++++++++++++- 11 files changed, 175 insertions(+), 17 deletions(-) diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py index a4086f19bebc..33c2119afaf0 100644 --- a/src/sentry/seer/autofix/issue_summary.py +++ b/src/sentry/seer/autofix/issue_summary.py @@ -34,6 +34,7 @@ IssueSummaryHidden, IssueSummarySelfHosted, NoSeerQuotaException, + SeerOutOfCreditsException, ) from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import ( @@ -204,7 +205,7 @@ def _trigger_autofix_task( data={"referrer": referrer.value}, send_notification=False, ) - except NoSeerQuotaException: + except (NoSeerQuotaException, SeerOutOfCreditsException): pass if run and SeerAutofixOperator.has_access(organization=group.project.organization): diff --git a/src/sentry/seer/autofix/on_completion_hook.py b/src/sentry/seer/autofix/on_completion_hook.py index 5383a1919fd1..194e93b412eb 100644 --- a/src/sentry/seer/autofix/on_completion_hook.py +++ b/src/sentry/seer/autofix/on_completion_hook.py @@ -28,6 +28,7 @@ ) from sentry.seer.autofix.coding_agent import IntegrationNotFound from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import LEGACY_FEATURE_ID as LEGACY_AUTOFIX_FEATURE_ID from sentry.seer.autofix.pr_iteration.completion import ( @@ -786,12 +787,22 @@ def _maybe_continue_pipeline( "iteration_index": get_latest_iteration_index(state), }, ) - trigger_autofix_agent( - group=group, - step=next_step, - referrer=referrer, - run_id=run_id, - ) + try: + trigger_autofix_agent( + group=group, + step=next_step, + referrer=referrer, + run_id=run_id, + ) + except SeerOutOfCreditsException: + logger.info( + "autofix.on_completion_hook.next_step_out_of_credits", + extra={ + "run_id": run_id, + "organization_id": organization.id, + "next_step": next_step, + }, + ) @classmethod def _push_changes( @@ -841,6 +852,12 @@ def _push_changes( state=state, verify_content=True, ) + except SeerOutOfCreditsException: + logger.info( + "autofix.on_completion_hook.push_changes_out_of_credits", + extra={"run_id": run_id, "organization_id": group.organization.id}, + ) + return False except Exception: logger.exception( "autofix.on_completion_hook.push_changes_failed", diff --git a/src/sentry/seer/autofix/pr_iteration/completion.py b/src/sentry/seer/autofix/pr_iteration/completion.py index dbf1a07e83b9..617284756832 100644 --- a/src/sentry/seer/autofix/pr_iteration/completion.py +++ b/src/sentry/seer/autofix/pr_iteration/completion.py @@ -15,6 +15,7 @@ from sentry.seer.autofix.autofix_agent import trigger_push_changes from sentry.seer.autofix.commit_author import SeerCommitAuthor, parse_commit_author from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature from sentry.seer.autofix.feature.models import PrIterationStepArgs from sentry.seer.autofix.pr_iteration.current_iteration import triggered_iteration_id @@ -489,6 +490,9 @@ def _push_iteration_changes( # Never overwrite commits on the PR that Seer didn't push. allow_force_push=False, ) + except SeerOutOfCreditsException: + log_ctx.info("autofix.pr_iteration.push", outcome="failed", reason="out_of_credits") + return False except Exception as e: log_ctx.error( "autofix.pr_iteration.push", @@ -572,6 +576,9 @@ def _retry_push_on_moved_branch( ), ), ) + except SeerOutOfCreditsException: + log_ctx.info("autofix.pr_iteration.push", outcome="not_pushed", reason="out_of_credits") + return PrIterationOutcome.PR_CREATION_ERRORED except Exception as e: # End the iteration as errored, so the caller pauses the run and records the outcome. log_ctx.error( diff --git a/src/sentry/seer/endpoints/admin_autofix_retry.py b/src/sentry/seer/endpoints/admin_autofix_retry.py index 1329ce811b2a..7a45ad6108d9 100644 --- a/src/sentry/seer/endpoints/admin_autofix_retry.py +++ b/src/sentry/seer/endpoints/admin_autofix_retry.py @@ -18,6 +18,7 @@ trigger_autofix_agent, ) from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID, LEGACY_FEATURE_ID from sentry.seer.autofix.on_completion_hook import PIPELINE_ORDER from sentry.seer.models import SeerRun @@ -120,6 +121,8 @@ def _retry_run(organization: Organization, run_id: int) -> dict[str, Any]: run_id=run_id, insert_index=insert_index, ) + except SeerOutOfCreditsException: + return _skipped(run_id, "Organization does not have enough Seer credits") except Exception: logger.exception( "autofix.admin_retry.trigger_failed", diff --git a/src/sentry/seer/endpoints/admin_autofix_single_run.py b/src/sentry/seer/endpoints/admin_autofix_single_run.py index 5772b42f6e98..0ae26e851eb6 100644 --- a/src/sentry/seer/endpoints/admin_autofix_single_run.py +++ b/src/sentry/seer/endpoints/admin_autofix_single_run.py @@ -11,7 +11,7 @@ from sentry.models.organization import Organization from sentry.seer.autofix.autofix_agent import trigger_autofix_agent from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import get_automation_handoff from sentry.seer.models import SeerPermissionError @@ -75,6 +75,10 @@ def post(self, request: Request) -> Response: ) except NoSeerQuotaException: return Response({"detail": "Organization has no Seer quota"}, status=400) + except SeerOutOfCreditsException: + return Response( + {"detail": "Organization does not have enough Seer credits"}, status=400 + ) except SeerPermissionError as e: return Response({"detail": e.message}, status=400) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 26ef5abe9075..0afe8500bc00 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -59,7 +59,7 @@ ) from sentry.seer.autofix.commit_author import commit_author_for_user from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import project_stopping_point, should_use_single_run from sentry.seer.autofix.on_completion_hook import get_run_stopping_point from sentry.seer.autofix.pr_iteration.manual_trigger import handle_ui_feedback @@ -452,6 +452,13 @@ def post( ), user=acting_user(request.user), ) + except SeerOutOfCreditsException: + return Response( + { + "detail": "Your organization does not have enough credits to continue your Autofix run." + }, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) except SeerPermissionError: return Response(status=status.HTTP_404_NOT_FOUND) @@ -582,6 +589,17 @@ def post( return Response( "No budget for Seer Autofix.", status=status.HTTP_402_PAYMENT_REQUIRED ) + except SeerOutOfCreditsException: + return Response( + { + "detail": ( + "Your organization does not have enough credits to start an Autofix run." + if resolved_run_id is None + else "Your organization does not have enough credits to continue your Autofix run." + ) + }, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) except SeerPermissionError as e: if _is_unknown_run_id_error(e): return Response(status=status.HTTP_404_NOT_FOUND) diff --git a/src/sentry/seer/entrypoints/operator.py b/src/sentry/seer/entrypoints/operator.py index afd5762bbc11..1cc192357f89 100644 --- a/src/sentry/seer/entrypoints/operator.py +++ b/src/sentry/seer/entrypoints/operator.py @@ -209,7 +209,7 @@ def trigger_autofix_agent( trigger_autofix_agent, trigger_push_changes, ) - from sentry.seer.autofix.exceptions import NoSeerQuotaException + from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep event_lifecyle = SeerOperatorEventLifecycleMetric( @@ -310,6 +310,16 @@ def trigger_autofix_agent( self.entrypoint.on_trigger_autofix_error(error=error) lifecycle.record_failure(failure_reason=error) return + except SeerOutOfCreditsException: + with SeerOperatorEventLifecycleMetric( + interaction_type=SeerOperatorInteractionType.ENTRYPOINT_ON_TRIGGER_AUTOFIX_ERROR, + entrypoint_key=self.entrypoint.key, + ).capture(): + self.entrypoint.on_trigger_autofix_error( + error="Your organization does not have enough Seer credits" + ) + lifecycle.record_halt(halt_reason="not_enough_seer_credits") + return except Exception as e: with SeerOperatorEventLifecycleMetric( interaction_type=SeerOperatorInteractionType.ENTRYPOINT_ON_TRIGGER_AUTOFIX_ERROR, diff --git a/src/sentry/tasks/seer/pr_iteration.py b/src/sentry/tasks/seer/pr_iteration.py index 7462b2d65887..c95fce7757b6 100644 --- a/src/sentry/tasks/seer/pr_iteration.py +++ b/src/sentry/tasks/seer/pr_iteration.py @@ -52,6 +52,7 @@ from sentry.seer.autofix.autofix_agent import trigger_autofix_agent from sentry.seer.autofix.commit_author import commit_author_for_feedback from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.pr_iteration.bot_identity import bot_logins_for_feedback from sentry.seer.autofix.pr_iteration.cap_exhausted import assign_user_for_exhausted_cap from sentry.seer.autofix.pr_iteration.constants import PR_ITERATION_PROVIDER @@ -840,7 +841,11 @@ def _drain_queued_autofix_feedback( commit_author=commit_author_for_feedback(feedback_items, organization_id), iteration_id=iteration_id, ) - except (PrIterationNoPullRequestException, SeerPermissionError) as error: + except ( + PrIterationNoPullRequestException, + SeerPermissionError, + SeerOutOfCreditsException, + ) as error: log_ctx.info( "autofix.pr_iteration.consume_feedback.trigger_agent", outcome="skipped", @@ -849,10 +854,13 @@ def _drain_queued_autofix_feedback( trigger_id=trigger_id, trigger_source=trigger_source, ) - _record_drain_outcome( - "skipped_permission" if isinstance(error, SeerPermissionError) else "skipped_no_pr", - trigger_source, - ) + if isinstance(error, SeerPermissionError): + drain_outcome = "skipped_permission" + elif isinstance(error, SeerOutOfCreditsException): + drain_outcome = "skipped_out_of_credits" + else: + drain_outcome = "skipped_no_pr" + _record_drain_outcome(drain_outcome, trigger_source) # The drain popped the queue, so this iteration will never run. _discard_iteration(log_ctx, run_id, organization_id, iteration_id) return diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py index 5d3fc7abc0b6..e95ecb37cb35 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py @@ -6,6 +6,7 @@ from sentry.models.organization import Organization from sentry.seer.agent.client_models import MemoryBlock, Message, RepoPRState, SeerRunState from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID from sentry.seer.autofix.steps import AutofixStep from sentry.testutils.cases import APITestCase @@ -156,6 +157,21 @@ def test_trigger_failure(self, mock_fetch: MagicMock, mock_trigger: MagicMock) - {"run_id": 1, "retried": False, "reason": "Failed to trigger the step"} ] + def test_trigger_out_of_credits(self, mock_fetch: MagicMock, mock_trigger: MagicMock) -> None: + self.create_autofix_run(1) + mock_fetch.return_value = run_state(1, [block("root_cause")], group_id=self.group.id) + mock_trigger.side_effect = SeerOutOfCreditsException() + + response = self.get_success_response(run_ids=[1]) + + assert response.data["results"] == [ + { + "run_id": 1, + "retried": False, + "reason": "Organization does not have enough Seer credits", + } + ] + def test_invalid_payload(self, mock_fetch: MagicMock, mock_trigger: MagicMock) -> None: payloads: list[dict[str, Any]] = [ {}, diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py b/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py index 413ebe7e1ec3..5df4abcba8c0 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_single_run.py @@ -4,7 +4,7 @@ from rest_framework.response import Response from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.testutils.cases import APITestCase from sentry.viewer_context import ActorType, get_viewer_context @@ -90,6 +90,15 @@ def test_out_of_quota_returns_reason(self, mock_trigger: MagicMock) -> None: assert response.data["detail"] + def test_out_of_credits_returns_reason(self, mock_trigger: MagicMock) -> None: + mock_trigger.side_effect = SeerOutOfCreditsException() + + response = self.get_error_response( + organization_slug="sentry", short_id=self.group.qualified_short_id, status_code=400 + ) + + assert response.data["detail"] + def test_unknown_issue_returns_404(self, mock_trigger: MagicMock) -> None: self.get_error_response(organization_slug="sentry", short_id="NOPE-1", status_code=404) diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 9f85fe38d07c..5ff9b6d348c9 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -21,7 +21,7 @@ SeerRunState, ) from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.exceptions import NoSeerQuotaException +from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.github_perms import MissingGithubPermissions from sentry.seer.autofix.pr_iteration.feedback import Feedback from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource, Decision @@ -1650,6 +1650,54 @@ def test_post_returns_402_when_no_seer_quota(self, mock_trigger_explorer: MagicM assert response.status_code == 402, response.data assert response.data == "No budget for Seer Autofix." + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_returns_402_when_out_of_seer_credits( + self, mock_trigger_explorer: MagicMock + ) -> None: + """POST returns 402 Payment Required when the credit check fails.""" + mock_trigger_explorer.side_effect = SeerOutOfCreditsException() + group = self.create_group() + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause"}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to start an Autofix run." + } + + @patch("sentry.seer.endpoints.group_ai_autofix.get_autofix_run_state") + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_returns_402_when_out_of_seer_credits_continuing_a_run( + self, mock_trigger_explorer: MagicMock, mock_run_state: MagicMock + ) -> None: + """POST says the run can't continue when a later step fails the credit check.""" + mock_trigger_explorer.side_effect = SeerOutOfCreditsException() + mock_run_state.return_value = SeerRunState( + run_id=123, + blocks=[], + status="completed", + updated_at="2024-01-01T00:00:00Z", + repo_pr_states={}, + ) + group = self.create_group() + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "solution", "run_id": 123}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run." + } + def test_post_coding_agent_handoff_errors_with_both_provider_and_integration_id(self) -> None: """POST returns 400 when both provider and integration_id are specified for coding_agent_handoff.""" group = self.create_group() @@ -1856,6 +1904,23 @@ def test_open_pr_permission_error(self, mock_explorer_state_request: MagicMock) assert response.status_code == 404, response.data + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_push_changes") + def test_open_pr_out_of_credits(self, mock_push_changes: MagicMock) -> None: + mock_push_changes.side_effect = SeerOutOfCreditsException() + self.login_as(user=self.user) + group = self.create_group() + + response = self.client.post( + self._get_url(group.id), + data={"step": "open_pr", "run_id": 123}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run." + } + def test_open_pr_coding_disabled(self) -> None: self.login_as(user=self.user) group = self.create_group() From e2855ba1be5c9a28d483661f1ea99b9c1bc7cbef Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:48:24 -0800 Subject: [PATCH 04/45] ref(seer): Keep the token-billed check inside can_run_seer_operation --- src/sentry/seer/credits.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..17ae7eb017a6 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,11 +60,10 @@ class SeerOperationConfig: } -def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: - """ - Checks whether the organization has enough Seer credits to run the operation - for callers that already checked the org is token-billed. - """ +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + if not features.has("organizations:token-billed-seer-enabled", organization): + return True if not features.has("organizations:seer-credit-checks", organization): return True @@ -78,10 +77,3 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation ) ) return response.has_enough_credits - - -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - return not features.has( - "organizations:token-billed-seer-enabled", organization - ) or has_enough_seer_credits(organization, operation) From d70d6cbddd51750df5c0dd1aac152f630622a332 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:49:40 -0800 Subject: [PATCH 05/45] ref(seer): Split has_enough_seer_credits out of can_run_seer_operation Autofix billing checks the token-billed flag itself, so it only needs the credit check. --- src/sentry/seer/credits.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..5b47405b5243 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -60,10 +60,11 @@ class SeerOperationConfig: } -def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True +def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: + """ + Checks whether the organization has enough Seer credits to run the operation + for callers that already checked the org is token-billed. + """ if not features.has("organizations:seer-credit-checks", organization): return True @@ -77,3 +78,10 @@ def can_run_seer_operation(organization: Organization, operation: SeerOperation) ) ) return response.has_enough_credits + + +def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: + """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + return not features.has( + "organizations:token-billed-seer-enabled", organization + ) or has_enough_seer_credits(organization, operation) From 58a1b631b942cf732754ee84b6e5037efbc25268 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 10:54:24 -0800 Subject: [PATCH 06/45] ref(seer): Combine the credit check flag conditions --- src/sentry/seer/credits.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 17ae7eb017a6..ebf4c3fc1411 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has("organizations:token-billed-seer-enabled", organization): - return True - if not features.has("organizations:seer-credit-checks", organization): + if not features.has( + "organizations:token-billed-seer-enabled", organization + ) or not features.has("organizations:seer-credit-checks", organization): return True config = SEER_OPERATIONS[operation] From 0ab3b109a98367d1f3cee0159b7a89d71a3656bc Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:21:40 -0800 Subject: [PATCH 07/45] feat(seer): Check Seer credits before replying to an Autofix single run Chat replies to single runs continue the run through continue_single_run, which skipped billing. The run's own referrer decides, so agentic triage runs stay free. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 1e9e5496062b..46d533585a58 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5b47405b5243..a4915a46067e 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From 8552e6472dcc5b5e7e176bf2686a76f42f5573ee Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:22:48 -0800 Subject: [PATCH 08/45] feat(seer): Return 402 when a single run reply is out of credits --- .../endpoints/organization_seer_agent_chat.py | 14 +++++++------ .../test_organization_seer_agent_chat.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 288e90ef56c2..9435403379dc 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -34,6 +34,7 @@ PUBLISH_SCOPES, AutofixReferrer, ) +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import continue_single_run from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.credits import SeerOperation, can_run_seer_operation @@ -52,6 +53,10 @@ _DASHBOARD_GENERATE_RUN_SOURCE = "dashboard_generate" +_OUT_OF_CREDITS_DETAIL = ( + "Your organization does not have enough credits left for you to message Seer." +) + class SeerAgentChatStateResponse(BaseModel): session: SeerRunState @@ -266,6 +271,8 @@ def _maybe_reply_to_single_run( can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), referrer=AutofixReferrer.SEER_EXPLORER, ) + except SeerOutOfCreditsException: + return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) except SeerApiError as e: if e.status == 404: return Response({"session": None}, status=404) @@ -435,12 +442,7 @@ def post( if not is_dashboard_generate_run and not can_run_seer_operation( organization, SeerOperation.AGENT_CHAT_MESSAGE ): - return Response( - { - "detail": "Your organization does not have enough credits left for you to message Seer." - }, - status=402, - ) + return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 1ebadaf3aea7..b5bad6de749a 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -16,6 +16,7 @@ UsageAccumulator, ) from sentry.seer.agent.client_utils import snapshot_to_markdown +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.endpoints.organization_seer_agent_chat import SeerAgentChatSerializer from sentry.seer.models import SeerApiError @@ -475,6 +476,25 @@ def test_reply_to_single_run_while_it_is_processing_is_409( assert response.data["code"] == "run_in_flight" mock_continue.assert_not_called() + @patch( + "sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run", + side_effect=SeerOutOfCreditsException(), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_reply_to_single_run_without_credits_is_402( + self, mock_client_class: MagicMock, mock_continue: MagicMock + ) -> None: + run = self._single_run(user_id=self.user.id) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "make a code fix"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + @patch("sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run") @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_reply_to_single_run_with_no_seer_session_is_404( From a3dae0d0298e80374d49e598e1a4e129766c4a4d Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:38:52 -0800 Subject: [PATCH 09/45] ref(seer): Check the credit check flag first --- src/sentry/seer/credits.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index ebf4c3fc1411..ac40567af5cd 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,9 +62,9 @@ class SeerOperationConfig: def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: """Whether the organization is token-billed and has enough Seer credits to run the operation.""" - if not features.has( + if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization - ) or not features.has("organizations:seer-credit-checks", organization): + ): return True config = SEER_OPERATIONS[operation] From 94b979c0fadb6da8ca292f4165cf2e0dc5b2fe89 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 11:54:15 -0800 Subject: [PATCH 10/45] test(seer): Align credit check tests with their files Cover orgs that aren't token-billed, group the chat credit tests after the access tests, and match the neighboring tests' names and setup. --- .../test_organization_seer_agent_chat.py | 165 +++++++++--------- .../test_organization_seer_credit_check.py | 13 ++ tests/sentry/seer/test_credits.py | 12 ++ 3 files changed, 109 insertions(+), 81 deletions(-) diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 46b6c73afedf..26bd0eb24494 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -647,87 +647,6 @@ def test_post_continue_with_uuid_resolves(self, mock_client_class: MagicMock) -> assert response.data == {"run_id": 555, "sentry_run_id": str(run.uuid)} assert mock_client.continue_run.call_args.kwargs["run_id"] == 555 - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_new_conversation_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_without_credits_returns_402( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=555, user_id=self.user.id - ) - - response = self.client.post(f"{self.url}{run.uuid}/", {"query": "More"}, format="json") - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - mock_client_class.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_continue_dashboard_run_without_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" - run = self.create_seer_run( - organization=self.organization, seer_run_state_id=789, user_id=self.user.id - ) - self.create_seer_agent_run(run=run, source="dashboard_generate") - mock_client = MagicMock() - mock_client.continue_run.return_value = 789 - mock_client_class.return_value = mock_client - - response = self.client.post( - f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" - ) - - assert response.status_code == 200 - assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=True), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_post_with_enough_credits_calls_client( - self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - mock_client = MagicMock() - mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) - mock_client_class.return_value = mock_client - - response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") - - assert response.status_code == 200 - mock_client.start_run.assert_called_once() - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_continuing_another_users_conversation_is_denied( self, mock_client_class: MagicMock @@ -1007,6 +926,90 @@ def test_get_denied_without_seer_access(self) -> None: assert response.status_code == 403 + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_new_conversation_with_enough_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + mock_client = MagicMock() + mock_client.start_run.return_value = MagicMock(seer_run_state_id=456, uuid=uuid.uuid4()) + mock_client_class.return_value = mock_client + + response = self.client.post(self.url, {"query": "What is this error about?"}, format="json") + + assert response.status_code == 200 + mock_client.start_run.assert_called_once() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_without_credits_returns_402( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + mock_client_class.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_post_continue_dashboard_run_without_credits_calls_client( + self, mock_client_class: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + """Dashboard generation isn't gated on credits, so its follow-ups aren't either.""" + run = self.create_seer_run( + organization=self.organization, seer_run_state_id=789, user_id=self.user.id + ) + self.create_seer_agent_run(run=run, source="dashboard_generate") + mock_client = MagicMock() + mock_client.continue_run.return_value = 789 + mock_client_class.return_value = mock_client + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "Follow up question"}, format="json" + ) + + assert response.status_code == 200 + assert response.data == {"run_id": 789, "sentry_run_id": str(run.uuid)} + mock_check_seer_credits.assert_not_called() + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_post_json_on_page_context_converted_to_markdown( self, mock_client_class: MagicMock diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 90aaf513233d..9a7303e5b8d3 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -57,6 +57,19 @@ def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> Non assert response.data == {"hasEnoughCredits": True} + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_get_returns_has_enough_credits_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + response = self.get_success_response(self.organization.slug, operation="autofix_solution") + + assert response.data == {"hasEnoughCredits": True} + mock_check_seer_credits.assert_not_called() + def test_get_returns_400_when_operation_missing(self) -> None: self.get_error_response(self.organization.slug, status_code=400) diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index afd37295c106..0b730e6d390a 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -17,6 +17,7 @@ def test_every_operation_has_a_credit_estimate(self) -> None: for operation in SeerOperation: assert SEER_OPERATIONS[operation].estimate() > 0 + @with_feature("organizations:token-billed-seer-enabled") @patch( "sentry.seer_credit.backend.check_seer_credits", return_value=CheckSeerCreditsResponse(has_enough_credits=False), @@ -27,6 +28,17 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) mock_check_seer_credits.assert_not_called() + @with_feature("organizations:seer-credit-checks") + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_can_run_seer_operation_skips_billing_when_not_token_billed( + self, mock_check_seer_credits: MagicMock + ) -> None: + assert can_run_seer_operation(self.organization, SeerOperation.AUTOFIX_ROOT_CAUSE) + mock_check_seer_credits.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", From 6f7cb6f9492f6d1332b90b4d05a7fc34213af13a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:16:01 -0800 Subject: [PATCH 11/45] ref(seer): Leave single run replies out of credit gating Single runs are behind organizations:autofix-single-run and need their own estimate, so they're out of scope here. --- src/sentry/seer/autofix/feature/dispatch.py | 8 ----- src/sentry/seer/credits.py | 4 --- .../seer/autofix/feature/test_dispatch.py | 36 ------------------- 3 files changed, 48 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..408b573b403e 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,14 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..5a894919c0a3 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,7 +19,6 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" - AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -58,9 +57,6 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( - SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE - ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..7264c3e2dbc1 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,39 +692,3 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, - pytest.raises(SeerOutOfCreditsException), - ): - continue_single_run( - self.agent_run, - self.group, - "make a code fix", - user=self.user, - can_publish=True, - referrer=AutofixReferrer.SEER_EXPLORER, - ) - - mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() From 9248e4caef6e34bef5394d36b71a321a41ac71bf Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:16:11 -0800 Subject: [PATCH 12/45] Revert "feat(seer): Return 402 when a single run reply is out of credits" Single run replies are out of scope for these PRs. --- .../endpoints/organization_seer_agent_chat.py | 14 ++++++------- .../test_organization_seer_agent_chat.py | 20 ------------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 5c1bf1be3f2b..081e624fdcb2 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -34,7 +34,6 @@ PUBLISH_SCOPES, AutofixReferrer, ) -from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import continue_single_run from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run @@ -54,10 +53,6 @@ _DASHBOARD_GENERATE_RUN_SOURCE = "dashboard_generate" -_OUT_OF_CREDITS_DETAIL = ( - "Your organization does not have enough credits left for you to message Seer." -) - class SeerAgentChatStateResponse(BaseModel): session: SeerRunState @@ -273,8 +268,6 @@ def _maybe_reply_to_single_run( can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), referrer=AutofixReferrer.SEER_EXPLORER, ) - except SeerOutOfCreditsException: - return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) except SeerApiError as e: if e.status == 404: return Response({"session": None}, status=404) @@ -448,7 +441,12 @@ def post( if not is_dashboard_generate_run and not can_run_seer_operation( organization, SeerOperation.AGENT_CHAT_MESSAGE ): - return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) + return Response( + { + "detail": "Your organization does not have enough credits left for you to message Seer." + }, + status=402, + ) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 5a9a992d32ff..26bd0eb24494 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -16,7 +16,6 @@ UsageAccumulator, ) from sentry.seer.agent.client_utils import snapshot_to_markdown -from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.endpoints.organization_seer_agent_chat import SeerAgentChatSerializer from sentry.seer.models import SeerApiError @@ -501,25 +500,6 @@ def test_reply_to_single_run_while_it_is_processing_is_409( assert response.data["code"] == "run_in_flight" mock_continue.assert_not_called() - @patch( - "sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run", - side_effect=SeerOutOfCreditsException(), - ) - @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") - def test_reply_to_single_run_without_credits_is_402( - self, mock_client_class: MagicMock, mock_continue: MagicMock - ) -> None: - run = self._single_run(user_id=self.user.id) - - response = self.client.post( - f"{self.url}{run.seer_run_state_id}/", {"query": "make a code fix"}, format="json" - ) - - assert response.status_code == 402 - assert response.data == { - "detail": "Your organization does not have enough credits left for you to message Seer." - } - @patch("sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run") @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_reply_to_single_run_with_no_seer_session_is_404( From e35a52cccbe2bd9d5b0ada2fc89a7193be8f4f4a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:40:39 -0800 Subject: [PATCH 13/45] cleaning up diff --- src/sentry/seer/autofix/feature/dispatch.py | 5 +++-- src/sentry/seer/credits.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 408b573b403e..270962c8a610 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -287,11 +287,12 @@ def trigger_autofix_feature( # Avoid a circular import through the legacy Autofix dispatcher. from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + is_new_run = args.existing_run_id is None uses_quota = check_autofix_billing( group, args.step.seer_operation, args.referrer, - is_new_run=args.existing_run_id is None, + is_new_run=is_new_run, skip_quota_for_free_cohort=args.allow_free_cohort, ) @@ -383,7 +384,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if args.existing_run_id is None: + if is_new_run: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5a894919c0a3..8663442354ed 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -62,7 +62,7 @@ class SeerOperationConfig: def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: """ - Checks whether the organization has enough Seer credits to run the operation + Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. """ config = SEER_OPERATIONS[operation] @@ -78,7 +78,7 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): From 99eb2acddfff3a71f38b84b997d3d98c300fbf0b Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:43:19 -0800 Subject: [PATCH 14/45] fix(seer): Don't let API callers claim the agentic triage referrer Agentic triage runs skip the Seer credit check, and the Autofix endpoint accepted any referrer from the request body. Treat a client-supplied agentic triage referrer as unknown. --- src/sentry/seer/endpoints/group_ai_autofix.py | 7 ++++++- .../seer/endpoints/test_group_ai_autofix.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4767dde92f13..93010d501f0e 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,10 +165,15 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - return AutofixReferrer(raw) + referrer = AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN + # Agentic triage runs skip the credit check, so callers can't claim to be it. + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) + return AutofixReferrer.UNKNOWN + return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 10c804a02408..f4ab9f832b8c 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,6 +1198,25 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_agentic_triage_referrer_is_unknown( + self, mock_trigger_explorer: MagicMock + ) -> None: + """Agentic triage runs skip the credit check, so the API can't claim to be one.""" + group = self.create_group() + run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + mock_trigger_explorer.return_value = run + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, + format="json", + ) + + assert response.status_code == 202, response.data + assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause""" From e8f415e822486ba13a99ce8f40d9a8a0def89f58 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:58:04 -0800 Subject: [PATCH 15/45] ref(seer): Clarify why the endpoint ignores the agentic triage referrer --- src/sentry/seer/endpoints/group_ai_autofix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 93010d501f0e..4b6bfb155757 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,7 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage runs skip the credit check, so callers can't claim to be it. + # Real agentic triage runs never come through this endpoint, so don't trust a + # request claiming to be one. Agentic triage runs skip the Seer credit check. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 396779d7c000946eb2fba43bd4360e90f3824eb2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:59:06 -0800 Subject: [PATCH 16/45] feat(seer): Check Seer credits before replying to an Autofix single run Restores the single run reply check, using the placeholder estimate for now. --- src/sentry/seer/autofix/feature/dispatch.py | 8 +++++ src/sentry/seer/credits.py | 4 +++ .../seer/autofix/feature/test_dispatch.py | 36 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 408b573b403e..82b9a0b31dc8 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,6 +160,14 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + # A reply always continues a run, so it never uses quota. + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + enable_coding = _org_allows_coding(group) open_pr = enable_coding and can_publish payload = AutofixSingleRunFeaturePayload( diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 5a894919c0a3..b945dfb0ea83 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -19,6 +19,7 @@ class SeerOperation(StrEnum): AUTOFIX_CODE_CHANGES = "autofix_code_changes" AUTOFIX_OPEN_PR = "autofix_open_pr" AUTOFIX_PR_ITERATION = "autofix_pr_iteration" + AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" class SeerFeature(StrEnum): @@ -57,6 +58,9 @@ class SeerOperationConfig: SeerOperation.AUTOFIX_PR_ITERATION: SeerOperationConfig( SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE ), + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY: SeerOperationConfig( + SeerFeature.AUTOFIX, lambda: PLACEHOLDER_CREDIT_ESTIMATE + ), } diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 7264c3e2dbc1..ab47678c838f 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -692,3 +692,39 @@ def test_respects_org_coding_opt_out(self) -> None: assert kwargs["agent_run_options"]["enable_coding"] is False assert kwargs["agent_run_options"]["enable_write_pull_request"] is False assert kwargs["payload"]["pr_internal_settings"] is None + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> None: + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls, + pytest.raises(SeerOutOfCreditsException), + ): + continue_single_run( + self.agent_run, + self.group, + "make a code fix", + user=self.user, + can_publish=True, + referrer=AutofixReferrer.SEER_EXPLORER, + ) + + mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + """The run's own referrer decides, not the reply's.""" + self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) + + self._reply() + + mock_check_seer_credits.assert_not_called() From 3c0d97b57441e2abf58c6e2eb6933437f1a2ea05 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:59:35 -0800 Subject: [PATCH 17/45] feat(seer): Return 402 when a single run reply is out of credits --- .../endpoints/organization_seer_agent_chat.py | 14 +++++++------ .../test_organization_seer_agent_chat.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 081e624fdcb2..5c1bf1be3f2b 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -34,6 +34,7 @@ PUBLISH_SCOPES, AutofixReferrer, ) +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.dispatch import continue_single_run from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run @@ -53,6 +54,10 @@ _DASHBOARD_GENERATE_RUN_SOURCE = "dashboard_generate" +_OUT_OF_CREDITS_DETAIL = ( + "Your organization does not have enough credits left for you to message Seer." +) + class SeerAgentChatStateResponse(BaseModel): session: SeerRunState @@ -268,6 +273,8 @@ def _maybe_reply_to_single_run( can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), referrer=AutofixReferrer.SEER_EXPLORER, ) + except SeerOutOfCreditsException: + return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) except SeerApiError as e: if e.status == 404: return Response({"session": None}, status=404) @@ -441,12 +448,7 @@ def post( if not is_dashboard_generate_run and not can_run_seer_operation( organization, SeerOperation.AGENT_CHAT_MESSAGE ): - return Response( - { - "detail": "Your organization does not have enough credits left for you to message Seer." - }, - status=402, - ) + return Response({"detail": _OUT_OF_CREDITS_DETAIL}, status=402) try: # Seer only exposes write_pull_request when coding and bash mode are also on. diff --git a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py index 26bd0eb24494..5a9a992d32ff 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_agent_chat.py @@ -16,6 +16,7 @@ UsageAccumulator, ) from sentry.seer.agent.client_utils import snapshot_to_markdown +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.endpoints.organization_seer_agent_chat import SeerAgentChatSerializer from sentry.seer.models import SeerApiError @@ -500,6 +501,25 @@ def test_reply_to_single_run_while_it_is_processing_is_409( assert response.data["code"] == "run_in_flight" mock_continue.assert_not_called() + @patch( + "sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run", + side_effect=SeerOutOfCreditsException(), + ) + @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") + def test_reply_to_single_run_without_credits_is_402( + self, mock_client_class: MagicMock, mock_continue: MagicMock + ) -> None: + run = self._single_run(user_id=self.user.id) + + response = self.client.post( + f"{self.url}{run.seer_run_state_id}/", {"query": "make a code fix"}, format="json" + ) + + assert response.status_code == 402 + assert response.data == { + "detail": "Your organization does not have enough credits left for you to message Seer." + } + @patch("sentry.seer.endpoints.organization_seer_agent_chat.continue_single_run") @patch("sentry.seer.endpoints.organization_seer_agent_chat.SeerAgentClient") def test_reply_to_single_run_with_no_seer_session_is_404( From 90c5db5f8f301e460de7cd339f3e3358bf6c2ba6 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 12:40:39 -0800 Subject: [PATCH 18/45] cleaning up diff --- src/sentry/seer/autofix/feature/dispatch.py | 5 +++-- src/sentry/seer/credits.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 82b9a0b31dc8..326785f2f95c 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -295,11 +295,12 @@ def trigger_autofix_feature( # Avoid a circular import through the legacy Autofix dispatcher. from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + is_new_run = args.existing_run_id is None uses_quota = check_autofix_billing( group, args.step.seer_operation, args.referrer, - is_new_run=args.existing_run_id is None, + is_new_run=is_new_run, skip_quota_for_free_cohort=args.allow_free_cohort, ) @@ -391,7 +392,7 @@ def trigger_autofix_feature( extras["stopping_point"] = stopping_point.value user_org_context = collect_user_org_context(args.user, group.organization) - if args.existing_run_id is None: + if is_new_run: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, payload=payload.dict(), diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index b945dfb0ea83..d6cff3e7f32d 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -66,7 +66,7 @@ class SeerOperationConfig: def has_enough_seer_credits(organization: Organization, operation: SeerOperation) -> bool: """ - Checks whether the organization has enough Seer credits to run the operation + Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. """ config = SEER_OPERATIONS[operation] @@ -82,7 +82,7 @@ def has_enough_seer_credits(organization: Organization, operation: SeerOperation def can_run_seer_operation(organization: Organization, operation: SeerOperation) -> bool: - """Whether the organization is token-billed and has enough Seer credits to run the operation.""" + """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): From ff4faaef48b3f2ac633a425603f81f04d85996bd Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:05:09 -0800 Subject: [PATCH 19/45] ref(seer): Restore the billing comment in trigger_autofix_agent --- src/sentry/seer/autofix/autofix_agent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 588cec9a4705..275a475ee4d5 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,6 +463,11 @@ def trigger_autofix_agent( """ outcome = "success" try: + # check billing quota for triggering a new autofix run + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). The API endpoint never sets this flag, + # so manual triggers still require quota. + # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( group, step.seer_operation, From 2b3143545c14535c5d8063046f039ee03a7dc1de Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:07:57 -0800 Subject: [PATCH 20/45] fix comment --- src/sentry/seer/autofix/autofix_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 275a475ee4d5..896a0b230918 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,7 +463,7 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run + # Seat-based orgs are checked for billing quota when triggering a new autofix run. # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). The API endpoint never sets this flag, # so manual triggers still require quota. From b86335d8d96a012ea9b3d57d971a624168cabad9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:18:02 -0800 Subject: [PATCH 21/45] cleaning up --- src/sentry/seer/autofix/autofix_agent.py | 3 +-- src/sentry/seer/autofix/feature/dispatch.py | 1 - src/sentry/seer/endpoints/group_ai_autofix.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 275a475ee4d5..f7a2f66c5f2b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,7 +463,7 @@ def trigger_autofix_agent( """ outcome = "success" try: - # check billing quota for triggering a new autofix run + # Seat-based orgs are checked for billing quota when triggering a new autofix run. # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). The API endpoint never sets this flag, # so manual triggers still require quota. @@ -848,7 +848,6 @@ def trigger_push_changes( seer_run = SeerRun.objects.filter( organization_id=group.organization.id, seer_run_state_id=run_id ).first() - # Open PR always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_OPEN_PR, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 326785f2f95c..e2b7fda54543 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -160,7 +160,6 @@ def continue_single_run( ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - # A reply always continues a run, so it never uses quota. check_autofix_billing( group, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 4b6bfb155757..1d8af3cdc4cf 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -169,8 +169,8 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Real agentic triage runs never come through this endpoint, so don't trust a - # request claiming to be one. Agentic triage runs skip the Seer credit check. + # Agentic triage is not exposed via the API. Don't trust such a referrer since + # it may have token-based billing implications. if referrer == AutofixReferrer.AGENTIC_TRIAGE: logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN From 245aca91af2f299185fcdf5ee22924c413b5457a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:19:02 -0800 Subject: [PATCH 22/45] fix ordering --- src/sentry/seer/autofix/feature/dispatch.py | 120 ++++++++++---------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e2b7fda54543..112efc9ade08 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,66 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def continue_single_run( - agent_run: SeerAgentRun, - group: Group, - reply: str, - *, - user: User | RpcUser, - can_publish: bool, - referrer: AutofixReferrer, -) -> SeerRun: - from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - - check_autofix_billing( - group, - SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, - is_new_run=False, - ) - - enable_coding = _org_allows_coding(group) - open_pr = enable_coding and can_publish - payload = AutofixSingleRunFeaturePayload( - group_id=group.id, - project_id=group.project_id, - short_id=group.qualified_short_id or str(group.id), - title=group.title or "Unknown error", - culprit=group.culprit or "unknown", - on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), - should_run_repo_checks=True, - existing_run_id=agent_run.run.seer_run_state_id, - reply=reply, - pr_internal_settings=( - _single_run_pr_internal_settings(group, referrer) if open_pr else None - ), - ) - agent_run_options = AgentRunOptions( - is_context_engine_enabled=False, - enable_frontend_code_search=False, - enable_coding=enable_coding, - enable_pr_context_tools=False, - enable_write_pull_request=open_pr, - ) - - client = SeerAgentClient( - organization=group.organization, - project=group.project, - group=group, - user=user, - enable_coding=enable_coding, - enable_bash_mode=True, - ) - return client.continue_feature_run( - existing_agent_run=agent_run, - payload=payload.dict(), - referrer=referrer.value, - user_org_context=collect_user_org_context(user, group.organization), - proxy_headers=get_proxy_headers(), - agent_run_options=agent_run_options, - ) - - def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: """ Raise if the organization can't afford the operation under token-based @@ -287,6 +227,66 @@ def check_autofix_billing( ) +def continue_single_run( + agent_run: SeerAgentRun, + group: Group, + reply: str, + *, + user: User | RpcUser, + can_publish: bool, + referrer: AutofixReferrer, +) -> SeerRun: + from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook + + check_autofix_billing( + group, + SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + agent_run.run.referrer, + is_new_run=False, + ) + + enable_coding = _org_allows_coding(group) + open_pr = enable_coding and can_publish + payload = AutofixSingleRunFeaturePayload( + group_id=group.id, + project_id=group.project_id, + short_id=group.qualified_short_id or str(group.id), + title=group.title or "Unknown error", + culprit=group.culprit or "unknown", + on_completion_hook=extract_hook_definition(AutofixOnCompletionHook, call_on_failure=True), + should_run_repo_checks=True, + existing_run_id=agent_run.run.seer_run_state_id, + reply=reply, + pr_internal_settings=( + _single_run_pr_internal_settings(group, referrer) if open_pr else None + ), + ) + agent_run_options = AgentRunOptions( + is_context_engine_enabled=False, + enable_frontend_code_search=False, + enable_coding=enable_coding, + enable_pr_context_tools=False, + enable_write_pull_request=open_pr, + ) + + client = SeerAgentClient( + organization=group.organization, + project=group.project, + group=group, + user=user, + enable_coding=enable_coding, + enable_bash_mode=True, + ) + return client.continue_feature_run( + existing_agent_run=agent_run, + payload=payload.dict(), + referrer=referrer.value, + user_org_context=collect_user_org_context(user, group.organization), + proxy_headers=get_proxy_headers(), + agent_run_options=agent_run_options, + ) + + def trigger_autofix_feature( group: Group, args: AutofixFeatureArgs, From 80bba2393003f19a88955e554663a45d757ebc2a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:21:07 -0800 Subject: [PATCH 23/45] Restore comment --- src/sentry/seer/autofix/feature/dispatch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 112efc9ade08..419de6744f38 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -300,6 +300,8 @@ def trigger_autofix_feature( args.step.seer_operation, args.referrer, is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. skip_quota_for_free_cohort=args.allow_free_cohort, ) From 914a60cd07bc47d7d50df3f05db2a6f47a1ffda2 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:24:39 -0800 Subject: [PATCH 24/45] ref(seer): Keep all billing skip rules in check_autofix_billing --- src/sentry/seer/autofix/feature/dispatch.py | 89 +++++++++------------ 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 419de6744f38..ad74466886e0 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,61 +149,22 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_credits(group: Group, operation: SeerOperation, referrer: str | None) -> None: - """ - Raise if the organization can't afford the operation under token-based - credits. Agentic triage runs stay free. - """ - if ( - not features.has("organizations:seer-credit-checks", group.organization) - or referrer == AutofixReferrer.AGENTIC_TRIAGE - ): - return - if not has_enough_seer_credits(group.organization, operation): - logger.info( - "autofix_feature.dispatch.out_of_credits", +def _check_autofix_quota(group: Group, referrer: str | None) -> None: + """Raise if the organization has no seat-based Seer quota left.""" + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", extra={ "group_id": group.id, "organization_id": group.organization.id, - "operation": operation.value, "referrer": referrer, }, ) - raise SeerOutOfCreditsException() - - -def _check_autofix_quota( - group: Group, - referrer: str | None, - *, - is_new_run: bool, - allow_free_cohort: bool, -) -> bool: - """ - Raise if the organization has no seat-based Seer quota left. Returns whether - the step uses quota, which is recorded once the run starts. - """ - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota = is_new_run and allow_free_cohort and is_free_cohort_org(group.organization) - uses_quota = is_new_run and not skip_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - return uses_quota + raise NoSeerQuotaException() def check_autofix_billing( @@ -219,12 +180,34 @@ def check_autofix_billing( checked for credits, everyone else for seat-based quota. Returns whether the operation uses quota. """ - if features.has("organizations:token-billed-seer-enabled", group.organization): - _check_autofix_credits(group, operation, referrer) + organization = group.organization + if features.has("organizations:token-billed-seer-enabled", organization): + if ( + features.has("organizations:seer-credit-checks", organization) + and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. + and not has_enough_seer_credits(organization, operation) + ): + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": organization.id, + "operation": operation.value, + "referrer": referrer, + }, + ) + raise SeerOutOfCreditsException() return False - return _check_autofix_quota( - group, referrer, is_new_run=is_new_run, allow_free_cohort=skip_quota_for_free_cohort + + # Quota counts runs, so only new runs use it. + # Free cohort orgs bypass quota only when called from agentic triage + # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. + uses_quota = is_new_run and not ( + skip_quota_for_free_cohort and is_free_cohort_org(organization) ) + if uses_quota: + _check_autofix_quota(group, referrer) + return uses_quota def continue_single_run( From 28e59907cf76b2a9b874161de47b5bfaf9825327 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:33:09 -0800 Subject: [PATCH 25/45] refactor billing helper --- src/sentry/seer/autofix/feature/dispatch.py | 50 ++++++++++----------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index ad74466886e0..4c650cca6ad9 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -149,24 +149,6 @@ def _org_allows_coding(group: Group) -> bool: ) -def _check_autofix_quota(group: Group, referrer: str | None) -> None: - """Raise if the organization has no seat-based Seer quota left.""" - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() - - def check_autofix_billing( group: Group, operation: SeerOperation, @@ -177,10 +159,12 @@ def check_autofix_billing( ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are - checked for credits, everyone else for seat-based quota. Returns whether the - operation uses quota. + checked for credits, and everyone else is checked for quota once per run. + Returns whether the operation uses quota. """ organization = group.organization + + # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( features.has("organizations:seer-credit-checks", organization) @@ -199,14 +183,26 @@ def check_autofix_billing( raise SeerOutOfCreditsException() return False - # Quota counts runs, so only new runs use it. - # Free cohort orgs bypass quota only when called from agentic triage - # (skip_quota_for_free_cohort=True). Agentic triage is not exposed via the API. - uses_quota = is_new_run and not ( - skip_quota_for_free_cohort and is_free_cohort_org(organization) - ) + # Check quota for non-token-billed orgs once per run. + # Free cohort orgs skip quota when the caller passes skip_quota_for_free_cohort=True. + # Only agentic triage does, and it is not exposed via API. + skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) + uses_quota = is_new_run and not skips_quota if uses_quota: - _check_autofix_quota(group, referrer) + has_budget: bool = quotas.backend.check_seer_quota( + org_id=group.organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ) + if not has_budget: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": referrer, + }, + ) + raise NoSeerQuotaException() return uses_quota From 766bc10bc13f1811b71349e821d3726b331b7815 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:39:40 -0800 Subject: [PATCH 26/45] move logs out of helper --- src/sentry/seer/autofix/feature/dispatch.py | 69 +++++++++++---------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 4c650cca6ad9..f636c20c0b88 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -171,15 +171,6 @@ def check_autofix_billing( and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. and not has_enough_seer_credits(organization, operation) ): - logger.info( - "autofix_feature.dispatch.out_of_credits", - extra={ - "group_id": group.id, - "organization_id": organization.id, - "operation": operation.value, - "referrer": referrer, - }, - ) raise SeerOutOfCreditsException() return False @@ -188,21 +179,11 @@ def check_autofix_billing( # Only agentic triage does, and it is not exposed via API. skips_quota = skip_quota_for_free_cohort and is_free_cohort_org(organization) uses_quota = is_new_run and not skips_quota - if uses_quota: - has_budget: bool = quotas.backend.check_seer_quota( - org_id=group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - if not has_budget: - logger.warning( - "autofix_feature.dispatch.quota_denied", - extra={ - "group_id": group.id, - "organization_id": group.organization.id, - "referrer": referrer, - }, - ) - raise NoSeerQuotaException() + if uses_quota and not quotas.backend.check_seer_quota( + org_id=organization.id, + data_category=DataCategory.SEER_AUTOFIX, + ): + raise NoSeerQuotaException() return uses_quota @@ -274,15 +255,37 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None - uses_quota = check_autofix_billing( - group, - args.step.seer_operation, - args.referrer, - is_new_run=is_new_run, - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). Not exposed via the API. - skip_quota_for_free_cohort=args.allow_free_cohort, - ) + try: + uses_quota = check_autofix_billing( + group, + args.step.seer_operation, + args.referrer, + is_new_run=is_new_run, + # Free cohort orgs bypass quota only when called from agentic triage + # (allow_free_cohort=True). Not exposed via the API. + skip_quota_for_free_cohort=args.allow_free_cohort, + ) + except NoSeerQuotaException: + logger.warning( + "autofix_feature.dispatch.quota_denied", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "referrer": args.referrer.value, + }, + ) + raise + except SeerOutOfCreditsException: + logger.info( + "autofix_feature.dispatch.out_of_credits", + extra={ + "group_id": group.id, + "organization_id": group.organization.id, + "operation": args.step.seer_operation.value, + "referrer": args.referrer.value, + }, + ) + raise single_run = should_use_single_run( group, From 6c0043dddb83fe8e502df025c1719775a275a7be Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:45:45 -0800 Subject: [PATCH 27/45] small test fix: --- tests/sentry/seer/autofix/feature/test_dispatch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index ab47678c838f..2bf923774c12 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -348,7 +348,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, ): - mock_quotas.backend.check_seer_quota.return_value = True MockClient.return_value.start_feature_run.return_value = fake_run run = trigger_autofix_feature( @@ -362,6 +361,8 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") From 1c7c1906056cd11fc5818b36256c423ed39a38eb Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 13:49:40 -0800 Subject: [PATCH 28/45] cleaning up tests --- tests/sentry/seer/autofix/feature/test_dispatch.py | 7 ++----- tests/sentry/seer/autofix/test_autofix_agent.py | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 2bf923774c12..751f07dd2818 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -320,7 +320,7 @@ def test_raises_when_out_of_credits_for_a_continued_step( """Unlike quota, credits are checked for every step, not only new runs.""" with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): with pytest.raises(SeerOutOfCreditsException): trigger_autofix_feature( @@ -333,7 +333,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( ), ) - mock_quotas.backend.check_seer_quota.assert_not_called() MockClient.return_value.continue_feature_run.assert_not_called() @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @@ -346,7 +345,7 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.quotas"), ): MockClient.return_value.start_feature_run.return_value = fake_run @@ -361,8 +360,6 @@ def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicM assert run is fake_run mock_check_seer_credits.assert_not_called() - mock_quotas.backend.check_seer_quota.assert_not_called() - mock_quotas.backend.record_seer_run.assert_not_called() def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index dd598ac0a01e..af9eb7ef1951 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -821,8 +821,6 @@ def test_agentic_triage_skips_credit_check( ) mock_check_seer_credits.assert_not_called() - mock_check_quota.assert_not_called() - mock_feature.assert_called_once() @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") From 8a11dd1899df8754d130a01ad916ff4685b199dd Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 14:20:52 -0800 Subject: [PATCH 29/45] feat(seer-billing): Check Seer credits before queueing UI PR feedback The drain that consumes UI feedback runs after the request returns, so an out-of-credits org's feedback was dropped silently. Return 402 up front instead. --- .../autofix/pr_iteration/manual_trigger.py | 15 +++++++-- .../seer/endpoints/test_group_ai_autofix.py | 33 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index 8eab5630069d..1356bf5f430f 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -25,6 +25,7 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource from sentry.seer.autofix.pr_iteration.pause import PauseReason, get_pause_reason from sentry.seer.autofix.pr_iteration.queue import enqueue_autofix_feedback +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import SEER_PERMISSION_DENIED from sentry.seer.models import SeerPermissionError from sentry.tasks.seer.pr_iteration import trigger_consume_pr_iteration_feedback @@ -52,8 +53,9 @@ def handle_ui_feedback( Returns None once the feedback has been handed off, or the error response the endpoint should send back when the request cannot be acted on: 400 when - the feedback is empty or the run has no pull request yet, and 409 when - iteration on the run is paused. Raises ``PermissionDenied`` when Seer + the feedback is empty or the run has no pull request yet, 409 when + iteration on the run is paused, and 402 when the organization doesn't have + enough Seer credits. Raises ``PermissionDenied`` when Seer refuses to show us the run. """ if not user_feedback: @@ -80,6 +82,15 @@ def handle_ui_feedback( status=status.HTTP_409_CONFLICT, ) + # Check credits here, since the queued feedback is consumed after this request returns. + if not can_run_seer_operation(group.organization, SeerOperation.AUTOFIX_PR_ITERATION): + return Response( + { + "detail": "Your organization does not have enough credits to continue your Autofix run." + }, + status=status.HTTP_402_PAYMENT_REQUIRED, + ) + serialized_users = user_service.serialize_many( filter={"user_ids": [user_id]}, ) diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 10bb50f3c8e9..b0dad23022ee 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -38,6 +38,7 @@ from sentry.seer.autofix.pr_iteration.queue import QueuedAutofixFeedback from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -1684,6 +1685,38 @@ def test_pr_iteration_rejected_when_paused( assert response.data["detail"] == "Seer can no longer iterate on this pull request" mock_enqueue.assert_not_called() + @patch( + "sentry.seer.autofix.pr_iteration.manual_trigger.can_run_seer_operation", + return_value=False, + ) + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") + def test_pr_iteration_rejected_when_out_of_credits( + self, mock_run_state: MagicMock, mock_enqueue: MagicMock, mock_can_run: MagicMock + ) -> None: + group = self.create_group() + mock_run_state.return_value = SeerRunState( + run_id=123, + blocks=[], + status="completed", + updated_at="2024-01-01T00:00:00Z", + repo_pr_states={"owner/repo": RepoPRState(repo_name="owner/repo", pr_number=7)}, + ) + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "pr_iteration", "run_id": 123, "user_context": "please fix this"}, + format="json", + ) + + assert response.status_code == 402, response.data + assert response.data == { + "detail": "Your organization does not have enough credits to continue your Autofix run." + } + mock_can_run.assert_called_once_with(group.organization, SeerOperation.AUTOFIX_PR_ITERATION) + mock_enqueue.assert_not_called() + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") From c787f4dc48c33e72bd06daaaaecb7c7c330374e7 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Fri, 6 Nov 2026 15:09:56 -0800 Subject: [PATCH 30/45] fix(seer-billing): Don't pause PR iteration when a retry is out of credits PR_CREATION_ERRORED pauses iteration on the PR for good, but credits can be topped up. End the iteration as PUSH_FAILED instead. --- src/sentry/seer/autofix/pr_iteration/completion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/autofix/pr_iteration/completion.py b/src/sentry/seer/autofix/pr_iteration/completion.py index 617284756832..d084960a37e0 100644 --- a/src/sentry/seer/autofix/pr_iteration/completion.py +++ b/src/sentry/seer/autofix/pr_iteration/completion.py @@ -578,7 +578,8 @@ def _retry_push_on_moved_branch( ) except SeerOutOfCreditsException: log_ctx.info("autofix.pr_iteration.push", outcome="not_pushed", reason="out_of_credits") - return PrIterationOutcome.PR_CREATION_ERRORED + # Not PR_CREATION_ERRORED, which pauses iteration for good. Credits can be topped up. + return PrIterationOutcome.PUSH_FAILED except Exception as e: # End the iteration as errored, so the caller pauses the run and records the outcome. log_ctx.error( From 451f3bda8485afed7a78612d1894e2aeb4f5cc05 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:28:49 +0000 Subject: [PATCH 31/45] ref(seer-billing): Pass organization to check_autofix_billing Co-Authored-By: Sofia Rest --- src/sentry/seer/autofix/autofix_agent.py | 4 ++-- src/sentry/seer/autofix/feature/dispatch.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index f7a2f66c5f2b..9c138164c6f1 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -469,7 +469,7 @@ def trigger_autofix_agent( # so manual triggers still require quota. # Token-billed orgs are checked for credits on every step instead. check_autofix_billing( - group, + group.organization, step.seer_operation, referrer, is_new_run=run_id is None, @@ -849,7 +849,7 @@ def trigger_push_changes( organization_id=group.organization.id, seer_run_state_id=run_id ).first() check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_OPEN_PR, seer_run.referrer if seer_run else None, is_new_run=False, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index f636c20c0b88..e5fc4c3ca0af 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -13,6 +13,7 @@ DataCategory, ) from sentry.models.group import Group +from sentry.models.organization import Organization from sentry.models.project import Project from sentry.seer.agent.client import SeerAgentClient from sentry.seer.agent.client_utils import ( @@ -150,7 +151,7 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( - group: Group, + organization: Organization, operation: SeerOperation, referrer: str | None, *, @@ -162,8 +163,6 @@ def check_autofix_billing( checked for credits, and everyone else is checked for quota once per run. Returns whether the operation uses quota. """ - organization = group.organization - # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): if ( @@ -199,7 +198,7 @@ def continue_single_run( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook check_autofix_billing( - group, + group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, agent_run.run.referrer, is_new_run=False, @@ -257,7 +256,7 @@ def trigger_autofix_feature( is_new_run = args.existing_run_id is None try: uses_quota = check_autofix_billing( - group, + group.organization, args.step.seer_operation, args.referrer, is_new_run=is_new_run, From ab0d0aa2223dd6df6afab58d1c8809a448508ae9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 13:08:32 -0800 Subject: [PATCH 32/45] ref(seer-billing): Check Autofix billing only in dispatch trigger_autofix_agent checked billing, then called trigger_autofix_feature, which checked it again with the same arguments. Keep only the dispatch check, and test a token-billed org with credit checks off. --- src/sentry/seer/autofix/autofix_agent.py | 13 --- .../seer/autofix/feature/test_dispatch.py | 52 +++++++++++ .../sentry/seer/autofix/test_autofix_agent.py | 91 ------------------- 3 files changed, 52 insertions(+), 104 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 9c138164c6f1..1b8a2a6d609b 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -463,19 +463,6 @@ def trigger_autofix_agent( """ outcome = "success" try: - # Seat-based orgs are checked for billing quota when triggering a new autofix run. - # Free cohort orgs bypass quota only when called from agentic triage - # (allow_free_cohort=True). The API endpoint never sets this flag, - # so manual triggers still require quota. - # Token-billed orgs are checked for credits on every step instead. - check_autofix_billing( - group.organization, - step.seer_operation, - referrer, - is_new_run=run_id is None, - skip_quota_for_free_cohort=allow_free_cohort, - ) - if run_id is not None: _assert_existing_run_belongs_to_group(group, run_id) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 751f07dd2818..809188a99de6 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -283,6 +283,58 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: mock_quotas.backend.check_seer_quota.assert_not_called() mock_quotas.backend.record_seer_run.assert_not_called() + def test_non_free_cohort_org_checks_quota_and_records_usage(self) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), + ): + mock_quotas.backend.check_seer_quota.return_value = True + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + allow_free_cohort=True, + ), + ) + + assert run is fake_run + mock_quotas.backend.check_seer_quota.assert_called_once() + mock_quotas.backend.record_seer_run.assert_called_once() + + @with_feature("organizations:token-billed-seer-enabled") + @patch("sentry.seer_credit.backend.check_seer_credits") + def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas") as mock_quotas, + ): + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + mock_quotas.backend.check_seer_quota.assert_not_called() + mock_quotas.backend.record_seer_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index af9eb7ef1951..95e41640240d 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -9,7 +9,6 @@ ) from sentry.analytics.events.autofix_events import AiAutofixSolutionCompletedEvent -from sentry.constants import DataCategory from sentry.models.activity import Activity from sentry.models.group import Group from sentry.seer.agent.client_models import ( @@ -708,46 +707,6 @@ def test_pr_iteration_requires_existing_pr( mock_feature.assert_not_called() - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) - def test_when_no_quota(self, mock_check_quota: MagicMock, mock_feature: MagicMock) -> None: - with pytest.raises(NoSeerQuotaException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_quota.assert_called_once_with( - org_id=self.group.organization.id, - data_category=DataCategory.SEER_AUTOFIX, - ) - mock_feature.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - def test_when_out_of_credits( - self, - mock_check_quota: MagicMock, - mock_feature: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - with pytest.raises(SeerOutOfCreditsException): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.UNKNOWN, - run_id=None, - ) - mock_check_seer_credits.assert_called_once() - mock_check_quota.assert_not_called() - mock_feature.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") @@ -772,56 +731,6 @@ def test_free_cohort_skips_quota_check( mock_check_quota.assert_not_called() assert mock_feature.call_args.args[1].allow_free_cohort is True - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_allow_free_cohort_still_checks_quota_for_non_cohort_org( - self, mock_feature: MagicMock, mock_broadcast: MagicMock, mock_check_quota: MagicMock - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - with ( - patch("sentry.seer.autofix.autofix_agent.features.has", return_value=False), - patch("sentry.seer.autofix.feature.dispatch.is_free_cohort_org", return_value=False), - ): - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - allow_free_cohort=True, - ) - - mock_check_quota.assert_called_once() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.quotas.backend.check_seer_quota", return_value=True) - @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") - @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") - def test_agentic_triage_skips_credit_check( - self, - mock_feature: MagicMock, - mock_broadcast: MagicMock, - mock_check_quota: MagicMock, - mock_check_seer_credits: MagicMock, - ) -> None: - mock_feature.return_value = self.create_seer_run( - organization=self.group.organization, seer_run_state_id=12345 - ) - - trigger_autofix_agent( - group=self.group, - step=AutofixStep.CODE_CHANGES, - referrer=AutofixReferrer.AGENTIC_TRIAGE, - ) - - mock_check_seer_credits.assert_not_called() - @patch("sentry.quotas.backend.check_seer_quota", return_value=False) @patch("sentry.seer.autofix.autofix_agent.broadcast_webhooks_for_organization.delay") @patch("sentry.seer.autofix.autofix_agent.trigger_autofix_feature") From f271a389e9ae46aab353514b042870ec30df43c9 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 13:29:04 -0800 Subject: [PATCH 33/45] improve pr iteration behavior --- .../seer/autofix/pr_iteration/completion.py | 12 +++++- .../test_autofix_on_completion_hook.py | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/autofix/pr_iteration/completion.py b/src/sentry/seer/autofix/pr_iteration/completion.py index d084960a37e0..3a9f1120b206 100644 --- a/src/sentry/seer/autofix/pr_iteration/completion.py +++ b/src/sentry/seer/autofix/pr_iteration/completion.py @@ -492,6 +492,13 @@ def _push_iteration_changes( ) except SeerOutOfCreditsException: log_ctx.info("autofix.pr_iteration.push", outcome="failed", reason="out_of_credits") + # TODO(CW-3066): Resume after a top-up. Until then nothing retries the push, so pause + # the run so that new feedback is refused. + pause_pr_iteration( + run_id=run_id, + organization_id=group.organization.id, + reason=PauseReason.RUN_ERRORED, + ) return False except Exception as e: log_ctx.error( @@ -578,8 +585,9 @@ def _retry_push_on_moved_branch( ) except SeerOutOfCreditsException: log_ctx.info("autofix.pr_iteration.push", outcome="not_pushed", reason="out_of_credits") - # Not PR_CREATION_ERRORED, which pauses iteration for good. Credits can be topped up. - return PrIterationOutcome.PUSH_FAILED + # TODO(CW-3066): Resume after a top-up. Until then nothing retries this, so end the + # iteration as errored, and the caller pauses the run and records the outcome. + return PrIterationOutcome.PR_CREATION_ERRORED except Exception as e: # End the iteration as errored, so the caller pauses the run and records the outcome. log_ctx.error( diff --git a/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py b/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py index 7e131d10b7e7..351e7554a0c5 100644 --- a/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py +++ b/tests/sentry/seer/autofix/test_autofix_on_completion_hook.py @@ -1,6 +1,10 @@ from typing import Any, Literal, TypedDict from unittest.mock import MagicMock, patch +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) + from sentry.analytics.events.pr_iteration_events import ( AiAutofixPrIterationFeedbackBatchCompletedEvent, ) @@ -18,6 +22,7 @@ from sentry.seer.autofix.autofix_agent import _group_and_referrer_from_run, get_current_step from sentry.seer.autofix.commit_author import SeerCommitAuthor from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID, LEGACY_FEATURE_ID, SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.on_completion_hook import ( PIPELINE_ORDER, @@ -1316,6 +1321,41 @@ def test_a_failed_push_reaches_completion_details_as_that_outcome( assert mock_complete.call_args.kwargs["outcome"] == PrIterationOutcome.PUSH_FAILED.value + @patch(f"{COMPLETION_PATH}.complete_pr_iteration_details") + @patch(f"{COMPLETION_PATH}.consume_queued_feedback") + @patch(f"{COMPLETION_PATH}.trigger_push_changes", side_effect=SeerOutOfCreditsException()) + def test_a_push_refused_for_credits_pauses_as_an_error( + self, mock_push: MagicMock, mock_consume: MagicMock, mock_complete: MagicMock + ) -> None: + self.create_seer_run(organization=self.organization, seer_run_state_id=123) + + continue_pr_iteration( + self.organization, self.group, 123, self._unsynced(), AutofixReferrer.ON_COMPLETION_HOOK + ) + + assert ( + get_pause_reason(run_id=123, organization_id=self.organization.id) + == PauseReason.RUN_ERRORED + ) + assert mock_complete.call_args.kwargs["outcome"] == PrIterationOutcome.PUSH_FAILED.value + mock_consume.assert_not_called() + + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_a_branch_moved_retry_refused_for_credits_ends_the_iteration_as_errored( + self, mock_check_seer_credits: MagicMock + ) -> None: + with self.feature( + ["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"] + ): + outcome, retried = self._branch_moved_push(self._branch_moved()) + + assert outcome == PrIterationOutcome.PR_CREATION_ERRORED + assert retried is False + mock_check_seer_credits.assert_called_once() + class TestFailedRunCompletionHook(TestCase): """What the hook does with a run that Seer marked ``error``.""" From 89bdfdcc00ade7ac8fa9ff52c0b5b42c78ce0e5f Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 15:25:38 -0800 Subject: [PATCH 34/45] ref(seer-billing): Drop the agentic triage referrer exemption Credit checks no longer skip runs by referrer. Agentic triage runs are checked like any other until a follow-up marks them by feature instead. Credit checks are behind seer-credit-checks, so nothing changes in production. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 14 +++---- src/sentry/seer/autofix/feature/dispatch.py | 11 ++--- src/sentry/seer/endpoints/group_ai_autofix.py | 8 +--- .../seer/autofix/feature/test_dispatch.py | 41 ------------------- .../sentry/seer/autofix/test_autofix_agent.py | 26 ------------ .../seer/endpoints/test_group_ai_autofix.py | 19 --------- 6 files changed, 9 insertions(+), 110 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 1b8a2a6d609b..af99e4c3e05f 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -832,15 +832,7 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - check_autofix_billing( - group.organization, - SeerOperation.AUTOFIX_OPEN_PR, - seer_run.referrer if seer_run else None, - is_new_run=False, - ) + check_autofix_billing(group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False) client = get_autofix_agent_client(group, user=user) @@ -859,6 +851,10 @@ def trigger_push_changes( ) ) + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e5fc4c3ca0af..be25d1b1c476 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -153,7 +153,6 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( organization: Organization, operation: SeerOperation, - referrer: str | None, *, is_new_run: bool, skip_quota_for_free_cohort: bool = False, @@ -165,11 +164,9 @@ def check_autofix_billing( """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if ( - features.has("organizations:seer-credit-checks", organization) - and referrer != AutofixReferrer.AGENTIC_TRIAGE # Agentic triage runs stay free. - and not has_enough_seer_credits(organization, operation) - ): + if features.has( + "organizations:seer-credit-checks", organization + ) and not has_enough_seer_credits(organization, operation): raise SeerOutOfCreditsException() return False @@ -200,7 +197,6 @@ def continue_single_run( check_autofix_billing( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, - agent_run.run.referrer, is_new_run=False, ) @@ -258,7 +254,6 @@ def trigger_autofix_feature( uses_quota = check_autofix_billing( group.organization, args.step.seer_operation, - args.referrer, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 1d8af3cdc4cf..4767dde92f13 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -165,16 +165,10 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - referrer = AutofixReferrer(raw) + return AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN - # Agentic triage is not exposed via the API. Don't trust such a referrer since - # it may have token-based billing implications. - if referrer == AutofixReferrer.AGENTIC_TRIAGE: - logger.warning("group_ai_autofix.internal_referrer", extra={"referrer": raw}) - return AutofixReferrer.UNKNOWN - return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 809188a99de6..62f8d3c9e03c 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -387,32 +387,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( MockClient.return_value.continue_feature_run.assert_not_called() - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_skips_credit_check(self, mock_check_seer_credits: MagicMock) -> None: - fake_run = self.create_seer_run(organization=self.organization, type="feature_run") - - with ( - patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, - patch("sentry.seer.autofix.feature.dispatch.quotas"), - ): - MockClient.return_value.start_feature_run.return_value = fake_run - - run = trigger_autofix_feature( - self.group, - AutofixFeatureArgs( - referrer=AutofixReferrer.AGENTIC_TRIAGE, - step=AutofixStep.ROOT_CAUSE, - step_args=RCAStepArgs(), - ), - ) - - assert run is fake_run - mock_check_seer_credits.assert_not_called() - def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -763,18 +737,3 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ) mock_client_cls.return_value.continue_feature_run.assert_not_called() - - @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - def test_agentic_triage_run_skips_credit_check( - self, mock_check_seer_credits: MagicMock - ) -> None: - """The run's own referrer decides, not the reply's.""" - self.seer_run.update(referrer=AutofixReferrer.AGENTIC_TRIAGE) - - self._reply() - - mock_check_seer_credits.assert_not_called() diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 95e41640240d..555ef5924e20 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -1685,32 +1685,6 @@ def test_raises_when_out_of_credits( mock_post.assert_not_called() - @patch( - "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), - ) - @patch("sentry.seer.agent.client.make_agent_update_request") - def test_agentic_triage_run_skips_credit_check( - self, mock_post: MagicMock, mock_check_seer_credits: MagicMock - ) -> None: - # The push is called with another referrer; the run's own referrer decides. - self.create_seer_run( - organization=self.organization, - seer_run_state_id=123, - referrer=AutofixReferrer.AGENTIC_TRIAGE.value, - ) - - self._push( - mock_post, - features={ - "organizations:seer-credit-checks": True, - "organizations:token-billed-seer-enabled": True, - }, - ) - - mock_check_seer_credits.assert_not_called() - mock_post.assert_called_once() - @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index f4ab9f832b8c..10c804a02408 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1198,25 +1198,6 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") - def test_post_agentic_triage_referrer_is_unknown( - self, mock_trigger_explorer: MagicMock - ) -> None: - """Agentic triage runs skip the credit check, so the API can't claim to be one.""" - group = self.create_group() - run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) - mock_trigger_explorer.return_value = run - - self.login_as(user=self.user) - response = self.client.post( - self._get_url(group.id), - data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, - format="json", - ) - - assert response.status_code == 202, response.data - assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN - @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause""" From 5b62f643821f032a6840ea1ddb8a6b2d58769b97 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 16:09:55 -0800 Subject: [PATCH 35/45] feat(seer-billing): Skip credit checks on runs agentic triage starts Agentic triage isn't billed yet. It now starts Autofix runs with SeerFeature.AGENTIC_TRIAGE, stored on the run, and credit checks skip runs with that feature: every step, the PR push, and single-run replies. Post-process automation skips token-billed orgs, since agentic triage handles their automations. Refs CW-3061 --- src/sentry/seer/agentic_triage/delivery.py | 2 + src/sentry/seer/autofix/autofix_agent.py | 14 ++- src/sentry/seer/autofix/feature/dispatch.py | 42 +++++++-- src/sentry/seer/credits.py | 1 + src/sentry/tasks/post_process.py | 6 +- .../seer/agentic_triage/test_delivery.py | 6 ++ .../seer/autofix/feature/test_dispatch.py | 91 +++++++++++++++++++ .../sentry/seer/autofix/test_autofix_agent.py | 23 +++++ tests/sentry/tasks/test_post_process.py | 18 ++++ 9 files changed, 192 insertions(+), 11 deletions(-) diff --git a/src/sentry/seer/agentic_triage/delivery.py b/src/sentry/seer/agentic_triage/delivery.py index a1e4821bbe54..99986cd39d3f 100644 --- a/src/sentry/seer/agentic_triage/delivery.py +++ b/src/sentry/seer/agentic_triage/delivery.py @@ -29,6 +29,7 @@ is_seer_autotriggered_autofix_rate_limited_and_increment, is_seer_seat_based_tier_enabled, ) +from sentry.seer.credits import SeerFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -338,6 +339,7 @@ def _process_verdicts( user_context=user_context, allow_free_cohort=True, single_run=single_run, + seer_feature=SeerFeature.AGENTIC_TRIAGE, ) except Exception: logger.exception( diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index af99e4c3e05f..77542b983522 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -36,6 +36,7 @@ AutofixFeatureArgs, autofix_run_targets_other_issue, check_autofix_billing, + get_run_seer_feature, require_autofix_run_for_group, trigger_autofix_feature, ) @@ -64,7 +65,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) -from sentry.seer.credits import SeerOperation +from sentry.seer.credits import SeerFeature, SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -449,6 +450,7 @@ def trigger_autofix_agent( iteration_id: int | None = None, allow_free_cohort: bool = False, single_run: bool = False, + seer_feature: SeerFeature = SeerFeature.AUTOFIX, ) -> SeerRun: """ Start or continue an agent-based autofix run. @@ -460,6 +462,8 @@ def trigger_autofix_agent( stopping_point: Where to stop the automated pipeline (only used for new runs) allow_free_cohort: Internal-only flag set by agentic triage to bypass quota for free cohort orgs. Not exposed via the API. + seer_feature: Internal-only. Agentic triage starts runs with AGENTIC_TRIAGE. + Not exposed via the API. """ outcome = "success" try: @@ -503,6 +507,7 @@ def trigger_autofix_agent( user_context=user_context, stopping_point=stopping_point, allow_free_cohort=allow_free_cohort, + seer_feature=seer_feature, user=user, single_run=single_run, ) @@ -832,7 +837,12 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - check_autofix_billing(group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False) + check_autofix_billing( + group.organization, + SeerOperation.AUTOFIX_OPEN_PR, + is_new_run=False, + seer_feature=get_run_seer_feature(group, run_id), + ) client = get_autofix_agent_client(group, user=user) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index be25d1b1c476..31b75f2293a2 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -42,7 +42,7 @@ get_automation_handoff, is_free_cohort_org, ) -from sentry.seer.credits import SeerOperation, has_enough_seer_credits +from sentry.seer.credits import SeerFeature, SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -65,6 +65,7 @@ class AutofixFeatureArgs: user: User | RpcUser | AnonymousUser | None = None flush: bool = True single_run: bool = False + seer_feature: SeerFeature = SeerFeature.AUTOFIX def _run_mirrors(group: Group, run_id: int) -> QuerySet[SeerAgentRun]: @@ -78,6 +79,17 @@ def _mirror_matches_issue(group: Group) -> Q: return Q(group_id=group.id, project_id=group.project_id) +def _parse_seer_feature(raw: object) -> SeerFeature: + return SeerFeature(raw) if raw in SeerFeature else SeerFeature.AUTOFIX + + +def get_run_seer_feature(group: Group, run_id: int) -> SeerFeature: + """The feature the run was started under.""" + return _parse_seer_feature( + _run_mirrors(group, run_id).values_list("extras__seer_feature", flat=True).first() + ) + + def autofix_run_targets_other_issue(group: Group, run_id: int) -> bool: """True unless ``run_id`` is mirrored to this issue's group and project. @@ -156,6 +168,7 @@ def check_autofix_billing( *, is_new_run: bool, skip_quota_for_free_cohort: bool = False, + seer_feature: SeerFeature = SeerFeature.AUTOFIX, ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are @@ -164,9 +177,12 @@ def check_autofix_billing( """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if features.has( - "organizations:seer-credit-checks", organization - ) and not has_enough_seer_credits(organization, operation): + if ( + # Don't bill runs started by agentic triage (not exposed via the API). + seer_feature != SeerFeature.AGENTIC_TRIAGE + and features.has("organizations:seer-credit-checks", organization) + and not has_enough_seer_credits(organization, operation) + ): raise SeerOutOfCreditsException() return False @@ -198,6 +214,7 @@ def continue_single_run( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, is_new_run=False, + seer_feature=_parse_seer_feature((agent_run.extras or {}).get("seer_feature")), ) enable_coding = _org_allows_coding(group) @@ -250,6 +267,17 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None + existing_agent_run = ( + require_autofix_run_for_group(group, args.existing_run_id) + if args.existing_run_id is not None + else None + ) + # A new run starts under args.seer_feature. A continued run keeps the one it started under. + seer_feature = ( + args.seer_feature + if existing_agent_run is None + else _parse_seer_feature((existing_agent_run.extras or {}).get("seer_feature")) + ) try: uses_quota = check_autofix_billing( group.organization, @@ -258,6 +286,7 @@ def trigger_autofix_feature( # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. skip_quota_for_free_cohort=args.allow_free_cohort, + seer_feature=seer_feature, ) except NoSeerQuotaException: logger.warning( @@ -363,6 +392,7 @@ def trigger_autofix_feature( extras: dict[str, Any] = { "referrer": args.referrer.value, + "seer_feature": args.seer_feature.value, } # Store the stopping point here for delivery to use when advancing steps. if stopping_point is not None: @@ -381,9 +411,7 @@ def trigger_autofix_feature( flush=args.flush, extras=extras, ) - elif args.existing_run_id is not None: - existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) - + elif existing_agent_run is not None: run = client.continue_feature_run( existing_agent_run=existing_agent_run, payload=payload.dict(), diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index d6cff3e7f32d..d19d1271bcf5 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -25,6 +25,7 @@ class SeerOperation(StrEnum): class SeerFeature(StrEnum): """Seer features as tagged on LLM proxy calls, which billing maps to data categories.""" + AGENTIC_TRIAGE = "agentic_triage" # Not billed yet. Marks the Autofix runs it starts. AUTOFIX = "autofix" EXPLORER = "explorer" diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index c51b93d836d8..249165f96a5a 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -1652,7 +1652,9 @@ def kick_off_seer_automation(job: PostProcessJob) -> None: event = job["event"] group = event.group - if is_seer_seat_based_tier_enabled(group.organization): + if is_seer_seat_based_tier_enabled(group.organization) or features.has( + "organizations:token-billed-seer-enabled", group.organization + ): if group.seer_fixability_score is not None: return # Issues created in last 5 minutes only. This can be removed once this is live past 1 week. @@ -1660,7 +1662,7 @@ def kick_off_seer_automation(job: PostProcessJob) -> None: if (timezone.now() - group.first_seen).total_seconds() > 300: return # Generate issue summary and fixability score if not cached. - # Agentic Triage handles automations for seat-based orgs but still needs these. + # Agentic Triage handles automations for seat-based and token-billed orgs but still needs these. cache_key = get_issue_summary_cache_key(group.id) if cache.get(cache_key) is None: lock_key, lock_name = get_issue_summary_lock_key(group.id) diff --git a/tests/sentry/seer/agentic_triage/test_delivery.py b/tests/sentry/seer/agentic_triage/test_delivery.py index 4b78e696840a..3196c56687bc 100644 --- a/tests/sentry/seer/agentic_triage/test_delivery.py +++ b/tests/sentry/seer/agentic_triage/test_delivery.py @@ -15,6 +15,7 @@ deliver_agentic_triage_result, ) from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -219,6 +220,11 @@ def test_single_run_option_starts_single_runs(self) -> None: assert mock_trigger.call_args.kwargs["single_run"] is True + def test_starts_autofix_as_agentic_triage(self) -> None: + mock_trigger = self._deliver_autofix_verdict() + + assert mock_trigger.call_args.kwargs["seer_feature"] == SeerFeature.AGENTIC_TRIAGE + def test_dry_run_with_single_run_starts_nothing(self) -> None: mock_trigger = self._deliver_autofix_verdict(single_run=True, dry_run=True) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 62f8d3c9e03c..78316bf8151a 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -26,6 +26,7 @@ from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint +from sentry.seer.credits import SeerFeature from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -236,6 +237,7 @@ def test_starts_feature_run_and_records_quota(self) -> None: assert start_kwargs["flush"] is True assert start_kwargs["extras"] == { "referrer": AutofixReferrer.AGENTIC_TRIAGE.value, + "seer_feature": SeerFeature.AUTOFIX.value, "stopping_point": AutofixStoppingPoint.OPEN_PR.value, } @@ -370,6 +372,10 @@ def test_raises_when_out_of_credits_for_a_continued_step( self, mock_check_seer_credits: MagicMock ) -> None: """Unlike quota, credits are checked for every step, not only new runs.""" + fake_run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + self.create_seer_agent_run( + run=fake_run, source=FEATURE_ID, group=self.group, project=self.group.project + ) with ( patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, patch("sentry.seer.autofix.feature.dispatch.quotas"), @@ -387,6 +393,73 @@ def test_raises_when_out_of_credits_for_a_continued_step( MockClient.return_value.continue_feature_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_starts_agentic_triage_run_without_checking_credits( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas"), + ): + MockClient.return_value.start_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + seer_feature=SeerFeature.AGENTIC_TRIAGE, + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + extras = MockClient.return_value.start_feature_run.call_args.kwargs["extras"] + assert extras["seer_feature"] == SeerFeature.AGENTIC_TRIAGE + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_continues_agentic_triage_run_without_checking_credits( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + self.create_seer_agent_run( + run=fake_run, + source=FEATURE_ID, + group=self.group, + project=self.group.project, + extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}, + ) + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas"), + ): + MockClient.return_value.continue_feature_run.return_value = fake_run + + run = trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + step=AutofixStep.SOLUTION, + existing_run_id=123, + step_args=SolutionStepArgs(), + ), + ) + + assert run is fake_run + mock_check_seer_credits.assert_not_called() + def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -506,6 +579,10 @@ def test_pr_iteration_enables_coding_and_sends_its_step_args(self) -> None: def test_code_changes_rejected_when_coding_disabled(self) -> None: self.organization.update_option("sentry:enable_seer_coding", False) + fake_run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + self.create_seer_agent_run( + run=fake_run, source=FEATURE_ID, group=self.group, project=self.group.project + ) with pytest.raises( SeerPermissionError, match="Seer coding is not enabled for this organization" @@ -737,3 +814,17 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> ) mock_client_cls.return_value.continue_feature_run.assert_not_called() + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + def test_agentic_triage_run_skips_credit_check( + self, mock_check_seer_credits: MagicMock + ) -> None: + self.agent_run.update(extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}) + + self._reply() + + mock_check_seer_credits.assert_not_called() diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 555ef5924e20..1eccaaa9f819 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -36,6 +36,7 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.user_ui import UserUIFeedbackSource from sentry.seer.autofix.pr_iteration.iterations import PrIterationNoPullRequestException from sentry.seer.autofix.steps import AutofixStep +from sentry.seer.credits import SeerFeature from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType @@ -1685,6 +1686,28 @@ def test_raises_when_out_of_credits( mock_post.assert_not_called() + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), + ) + @patch("sentry.seer.agent.client.make_agent_update_request") + def test_agentic_triage_run_skips_credit_check( + self, mock_post: MagicMock, mock_check_seer_credits: MagicMock + ) -> None: + run = mirror_autofix_run(self, self.group, 123) + run.agent.update(extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}) + + self._push( + mock_post, + features={ + "organizations:seer-credit-checks": True, + "organizations:token-billed-seer-enabled": True, + }, + ) + + mock_check_seer_credits.assert_not_called() + mock_post.assert_called_once() + @patch("sentry.seer.agent.client.make_agent_update_request") def test_passes_pr_description_suffix_ready_state_and_author( self, mock_post: MagicMock diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index 6c8fc0492f42..1075f0fd3727 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -3809,6 +3809,24 @@ def test_seat_based_org_generates_summary_for_new_issues( ) mock_generate_summary_only.assert_called_once_with(require_group(event).id) + @patch("sentry.seer.autofix.utils.is_seer_seat_based_tier_enabled", return_value=False) + @patch("sentry.tasks.seer.autofix.generate_summary_and_run_automation.delay") + @patch("sentry.tasks.seer.autofix.generate_issue_summary_only.delay") + @override_settings(SENTRY_SELF_HOSTED=False) + def test_token_billed_org_generates_summary_for_new_issues( + self, + mock_generate_summary_only: MagicMock, + mock_generate_summary_and_run_automation: MagicMock, + mock_seat_based_tier: MagicMock, + ) -> None: + event = self.create_event(data={"message": "testing"}, project_id=self.project.id) + with self.feature("organizations:token-billed-seer-enabled"): + self.call_post_process_group( + is_new=True, is_regression=False, is_new_group_environment=True, event=event + ) + mock_generate_summary_only.assert_called_once_with(require_group(event).id) + mock_generate_summary_and_run_automation.assert_not_called() + def setUp(self) -> None: super().setUp() From 02aff6cd6b43a3f3cc9acde9c77664b4b80b640c Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 16:21:28 -0800 Subject: [PATCH 36/45] ref(seer-billing): Read the run's feature from rows already loaded Dispatch reads it from the run mirror it already loads, and the PR push joins the mirror into its existing run query, so neither makes an extra query. Refs CW-3061 --- src/sentry/seer/autofix/autofix_agent.py | 12 +++++---- src/sentry/seer/autofix/feature/dispatch.py | 28 +++++++-------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 77542b983522..afc230b9e19a 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -837,11 +837,17 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") + seer_run = ( + SeerRun.objects.filter(organization_id=group.organization.id, seer_run_state_id=run_id) + .select_related("agent") + .first() + ) + seer_agent_run = getattr(seer_run, "agent", None) check_autofix_billing( group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False, - seer_feature=get_run_seer_feature(group, run_id), + seer_feature=get_run_seer_feature(seer_agent_run), ) client = get_autofix_agent_client(group, user=user) @@ -861,10 +867,6 @@ def trigger_push_changes( ) ) - seer_run = SeerRun.objects.filter( - organization_id=group.organization.id, seer_run_state_id=run_id - ).first() - # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 31b75f2293a2..e2c8d10ed0be 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -79,15 +79,10 @@ def _mirror_matches_issue(group: Group) -> Q: return Q(group_id=group.id, project_id=group.project_id) -def _parse_seer_feature(raw: object) -> SeerFeature: - return SeerFeature(raw) if raw in SeerFeature else SeerFeature.AUTOFIX - - -def get_run_seer_feature(group: Group, run_id: int) -> SeerFeature: +def get_run_seer_feature(agent_run: SeerAgentRun | None) -> SeerFeature: """The feature the run was started under.""" - return _parse_seer_feature( - _run_mirrors(group, run_id).values_list("extras__seer_feature", flat=True).first() - ) + raw = agent_run.extras.get("seer_feature") if agent_run is not None else None + return SeerFeature(raw) if isinstance(raw, str) and raw in SeerFeature else SeerFeature.AUTOFIX def autofix_run_targets_other_issue(group: Group, run_id: int) -> bool: @@ -214,7 +209,7 @@ def continue_single_run( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, is_new_run=False, - seer_feature=_parse_seer_feature((agent_run.extras or {}).get("seer_feature")), + seer_feature=get_run_seer_feature(agent_run), ) enable_coding = _org_allows_coding(group) @@ -267,17 +262,12 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None - existing_agent_run = ( - require_autofix_run_for_group(group, args.existing_run_id) - if args.existing_run_id is not None - else None - ) # A new run starts under args.seer_feature. A continued run keeps the one it started under. - seer_feature = ( - args.seer_feature - if existing_agent_run is None - else _parse_seer_feature((existing_agent_run.extras or {}).get("seer_feature")) - ) + existing_agent_run: SeerAgentRun | None = None + seer_feature = args.seer_feature + if args.existing_run_id is not None: + existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) + seer_feature = get_run_seer_feature(existing_agent_run) try: uses_quota = check_autofix_billing( group.organization, From a0b60465ad327ba1112efa14be16086332f5f36b Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 16:26:29 -0800 Subject: [PATCH 37/45] clean up --- src/sentry/seer/autofix/feature/dispatch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index e2c8d10ed0be..52966a9a9fda 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -262,12 +262,12 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook is_new_run = args.existing_run_id is None - # A new run starts under args.seer_feature. A continued run keeps the one it started under. existing_agent_run: SeerAgentRun | None = None seer_feature = args.seer_feature - if args.existing_run_id is not None: + if not is_new_run: existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) seer_feature = get_run_seer_feature(existing_agent_run) + try: uses_quota = check_autofix_billing( group.organization, From 36fa96c4ccafa9ec562dbb0182c13f856449c06a Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 16:47:41 -0800 Subject: [PATCH 38/45] ref(seer-billing): Derive is_new_run from the run lookup Checking existing_run_id directly lets mypy narrow it, and is_new_run can't drift from the lookup. Refs CW-3061 --- src/sentry/seer/autofix/feature/dispatch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 52966a9a9fda..523d754ce118 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -261,12 +261,12 @@ def trigger_autofix_feature( # Avoid a circular import through the legacy Autofix dispatcher. from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook - is_new_run = args.existing_run_id is None existing_agent_run: SeerAgentRun | None = None seer_feature = args.seer_feature - if not is_new_run: + if args.existing_run_id is not None: existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) seer_feature = get_run_seer_feature(existing_agent_run) + is_new_run = existing_agent_run is None try: uses_quota = check_autofix_billing( From d15d1c090993aa003f5f6c054e1c9e891438b9c6 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Mon, 9 Nov 2026 16:54:03 -0800 Subject: [PATCH 39/45] feat(seer-billing): Skip the PR feedback credit check on agentic triage runs Feedback typed in the UI is credit-checked before it's queued. The run's feature is only read when the check refuses, so most requests make no extra query. Refs CW-3061 --- .../autofix/pr_iteration/manual_trigger.py | 15 ++++++- .../seer/endpoints/test_group_ai_autofix.py | 45 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index 1356bf5f430f..1a73cb502330 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -20,14 +20,16 @@ from sentry.models.group import Group from sentry.seer.autofix.autofix_agent import get_autofix_run_state from sentry.seer.autofix.constants import AutofixReferrer +from sentry.seer.autofix.feature.dispatch import get_run_seer_feature from sentry.seer.autofix.pr_iteration.emit import bootstrap_iteration from sentry.seer.autofix.pr_iteration.feedback import Feedback from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource from sentry.seer.autofix.pr_iteration.pause import PauseReason, get_pause_reason from sentry.seer.autofix.pr_iteration.queue import enqueue_autofix_feedback -from sentry.seer.credits import SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import SEER_PERMISSION_DENIED from sentry.seer.models import SeerPermissionError +from sentry.seer.models.run import SeerAgentRun from sentry.tasks.seer.pr_iteration import trigger_consume_pr_iteration_feedback from sentry.users.services.user.service import user_service @@ -83,7 +85,16 @@ def handle_ui_feedback( ) # Check credits here, since the queued feedback is consumed after this request returns. - if not can_run_seer_operation(group.organization, SeerOperation.AUTOFIX_PR_ITERATION): + # Runs agentic triage starts aren't billed yet. + if ( + not can_run_seer_operation(group.organization, SeerOperation.AUTOFIX_PR_ITERATION) + and get_run_seer_feature( + SeerAgentRun.objects.filter( + run__organization_id=group.organization.id, run__seer_run_state_id=run_id + ).first() + ) + != SeerFeature.AGENTIC_TRIAGE + ): return Response( { "detail": "Your organization does not have enough credits to continue your Autofix run." diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index b5f565be0b66..890a64490206 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -38,7 +38,7 @@ from sentry.seer.autofix.pr_iteration.queue import QueuedAutofixFeedback from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerOperation +from sentry.seer.credits import SeerFeature, SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -1698,6 +1698,49 @@ def test_pr_iteration_rejected_when_out_of_credits( mock_can_run.assert_called_once_with(group.organization, SeerOperation.AUTOFIX_PR_ITERATION) mock_enqueue.assert_not_called() + @patch( + "sentry.seer.autofix.pr_iteration.manual_trigger.can_run_seer_operation", + return_value=False, + ) + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") + def test_pr_iteration_allowed_on_agentic_triage_run_when_out_of_credits( + self, + mock_run_state: MagicMock, + mock_enqueue: MagicMock, + mock_consume: MagicMock, + mock_can_run: MagicMock, + ) -> None: + """Runs agentic triage starts aren't billed yet.""" + group = self.create_group() + run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + self.create_seer_agent_run( + run=run, + source="autofix", + group=group, + project=group.project, + extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}, + ) + mock_run_state.return_value = SeerRunState( + run_id=123, + blocks=[], + status="completed", + updated_at="2024-01-01T00:00:00Z", + repo_pr_states={"owner/repo": RepoPRState(repo_name="owner/repo", pr_number=7)}, + ) + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "pr_iteration", "run_id": 123, "user_context": "please fix this"}, + format="json", + ) + + assert response.status_code == 202, response.data + mock_enqueue.assert_called_once() + mock_consume.assert_called_once() + @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") From 2f524d88968acca5d52cb888168c3fe8e13cfcb5 Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 14:36:43 -0800 Subject: [PATCH 40/45] feat(seer-billing): Read the billing feature from Seer's run state Send billing_feature to Seer when an Autofix run starts, and read it back from the run state for later steps instead of storing it in extras. Rename SeerFeature to SeerBillingFeature to match the billing protos, and require billing_feature wherever Autofix billing is checked so a caller can't leave it out. Expose it in the autofix response and accept it on the credit check endpoint. --- src/sentry/seer/agent/client.py | 5 ++ src/sentry/seer/agent/client_models.py | 2 + src/sentry/seer/agent/client_utils.py | 1 + src/sentry/seer/agentic_triage/delivery.py | 4 +- src/sentry/seer/autofix/autofix_agent.py | 39 ++++++++------- src/sentry/seer/autofix/feature/dispatch.py | 48 +++++++++---------- src/sentry/seer/autofix/issue_summary.py | 2 + src/sentry/seer/autofix/on_completion_hook.py | 2 + .../seer/autofix/pr_iteration/completion.py | 7 ++- .../autofix/pr_iteration/manual_trigger.py | 18 ++----- src/sentry/seer/credits.py | 18 ++++--- .../seer/endpoints/admin_autofix_retry.py | 2 + .../endpoints/admin_autofix_single_run.py | 2 + src/sentry/seer/endpoints/group_ai_autofix.py | 11 ++++- .../endpoints/organization_seer_agent_chat.py | 19 ++++---- .../organization_seer_credit_check.py | 19 +++++--- src/sentry/seer/entrypoints/operator.py | 8 ++++ src/sentry/tasks/seer/pr_iteration.py | 2 + .../seer/agentic_triage/test_delivery.py | 4 +- .../seer/autofix/feature/test_dispatch.py | 40 ++++++++++++---- .../sentry/seer/autofix/test_autofix_agent.py | 35 ++++++++++---- .../seer/endpoints/test_group_ai_autofix.py | 26 +++++----- tests/sentry/seer/test_credits.py | 10 ++-- 23 files changed, 201 insertions(+), 123 deletions(-) diff --git a/src/sentry/seer/agent/client.py b/src/sentry/seer/agent/client.py index d2a90bb03c41..27075fb61519 100644 --- a/src/sentry/seer/agent/client.py +++ b/src/sentry/seer/agent/client.py @@ -568,6 +568,7 @@ def start_feature_run( payload: dict[str, Any], title: str, referrer: str, + billing_feature: str | None = None, flush: bool = True, extras: dict[str, Any] | None = None, on_run_created: Callable[[SeerRun], None] | None = None, @@ -593,6 +594,8 @@ def start_feature_run( Explicit agent_run_options override any options derived from organization configuration. + + billing_feature, if given, is the feature Seer bills the run to for its whole life. """ user_id = self._user_id @@ -622,6 +625,8 @@ def _create_agent_run(run: SeerRun) -> None: body["user_org_context"] = user_org_context if proxy_headers is not None: body["proxy_headers"] = proxy_headers + if billing_feature is not None: + body["billing_feature"] = billing_feature return enqueue_seer_run( organization=self.organization, diff --git a/src/sentry/seer/agent/client_models.py b/src/sentry/seer/agent/client_models.py index a6ad29a41a56..53ef655b2302 100644 --- a/src/sentry/seer/agent/client_models.py +++ b/src/sentry/seer/agent/client_models.py @@ -400,6 +400,8 @@ class SeerRunState(BaseModel): coding_agents: dict[str, CodingAgentState] = Field(default_factory=dict, exclude=True) usage: UsageAccumulator = Field(default_factory=UsageAccumulator, exclude=True) pr_push_log: list[PushLogEntry] | None = Field(default=None, exclude=True) + # The feature the run is billed to, set when it starts. + billing_feature: str | None = Field(default=None, exclude=True) class Config: extra = "ignore" diff --git a/src/sentry/seer/agent/client_utils.py b/src/sentry/seer/agent/client_utils.py index e6f303b01106..2e41e2ddbedf 100644 --- a/src/sentry/seer/agent/client_utils.py +++ b/src/sentry/seer/agent/client_utils.py @@ -156,6 +156,7 @@ class SeerFeatureRunRequest(TypedDict): user_org_context: NotRequired[UserOrgContext] proxy_headers: NotRequired[dict[str, str] | None] referrer: str + billing_feature: NotRequired[str] class SeerFeatureRunWireRequest(SeerFeatureRunRequest): diff --git a/src/sentry/seer/agentic_triage/delivery.py b/src/sentry/seer/agentic_triage/delivery.py index 99986cd39d3f..059dbb233dad 100644 --- a/src/sentry/seer/agentic_triage/delivery.py +++ b/src/sentry/seer/agentic_triage/delivery.py @@ -29,7 +29,7 @@ is_seer_autotriggered_autofix_rate_limited_and_increment, is_seer_seat_based_tier_enabled, ) -from sentry.seer.credits import SeerFeature +from sentry.seer.credits import SeerBillingFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -335,11 +335,11 @@ def _process_verdicts( group=group, step=AutofixStep.ROOT_CAUSE, referrer=referrer, + billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, stopping_point=stopping_point_by_project_id[group.project_id], user_context=user_context, allow_free_cohort=True, single_run=single_run, - seer_feature=SeerFeature.AGENTIC_TRIAGE, ) except Exception: logger.exception( diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 3c77a1782a88..1fbb23e80ad9 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -36,7 +36,7 @@ AutofixFeatureArgs, autofix_run_targets_other_issue, check_autofix_billing, - get_run_seer_feature, + get_run_billing_feature, require_autofix_run_for_group, trigger_autofix_feature, ) @@ -65,7 +65,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) -from sentry.seer.credits import SeerFeature, SeerOperation +from sentry.seer.credits import SeerBillingFeature, SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -439,7 +439,9 @@ def trigger_autofix_agent( group: Group, step: AutofixStep, referrer: AutofixReferrer, + *, run_id: int | None = None, + billing_feature: SeerBillingFeature, stopping_point: AutofixStoppingPoint | None = None, user_context: str | None = None, insert_index: int | None = None, @@ -450,7 +452,6 @@ def trigger_autofix_agent( iteration_id: int | None = None, allow_free_cohort: bool = False, single_run: bool = False, - seer_feature: SeerFeature = SeerFeature.AUTOFIX, ) -> SeerRun: """ Start or continue an agent-based autofix run. @@ -459,11 +460,11 @@ def trigger_autofix_agent( group: The Sentry group (issue) to analyze step: Which autofix step to run run_id: Existing run ID to continue, or None for new run + billing_feature: Internal-only. The feature the run is billed to for its whole life. + Not exposed via the API. stopping_point: Where to stop the automated pipeline (only used for new runs) allow_free_cohort: Internal-only flag set by agentic triage to bypass quota for free cohort orgs. Not exposed via the API. - seer_feature: Internal-only. Agentic triage starts runs with AGENTIC_TRIAGE. - Not exposed via the API. """ outcome = "success" try: @@ -501,13 +502,13 @@ def trigger_autofix_agent( args = AutofixFeatureArgs( step=step, referrer=referrer, + step_args=step_args, + billing_feature=billing_feature, existing_run_id=run_id, insert_index=insert_index, - step_args=step_args, user_context=user_context, stopping_point=stopping_point, allow_free_cohort=allow_free_cohort, - seer_feature=seer_feature, user=user, single_run=single_run, ) @@ -838,19 +839,6 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - seer_run = ( - SeerRun.objects.filter(organization_id=group.organization.id, seer_run_state_id=run_id) - .select_related("agent") - .first() - ) - seer_agent_run = getattr(seer_run, "agent", None) - check_autofix_billing( - group.organization, - SeerOperation.AUTOFIX_OPEN_PR, - is_new_run=False, - seer_feature=get_run_seer_feature(seer_agent_run), - ) - client = get_autofix_agent_client(group, user=user) if state is None: @@ -858,6 +846,13 @@ def trigger_push_changes( else: _validate_run_belongs_to_group(state, group) + check_autofix_billing( + group.organization, + get_run_billing_feature(state), + SeerOperation.AUTOFIX_OPEN_PR, + is_new_run=False, + ) + record_autofix_event( AiAutofixPrCreatedStartedEvent( organization_id=group.organization.id, @@ -868,6 +863,10 @@ def trigger_push_changes( ) ) + seer_run = SeerRun.objects.filter( + organization_id=group.organization.id, seer_run_state_id=run_id + ).first() + # Draft when review-request/CI-green is enabled; mark ready once green fires. client.push_changes( run_id, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 40d3656e2bea..f0d31d614e5c 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -16,6 +16,7 @@ from sentry.models.organization import Organization from sentry.models.project import Project from sentry.seer.agent.client import SeerAgentClient +from sentry.seer.agent.client_models import SeerRunState from sentry.seer.agent.client_utils import ( AgentRunOptions, collect_user_org_context, @@ -42,7 +43,7 @@ get_automation_handoff, is_free_cohort_org, ) -from sentry.seer.credits import SeerFeature, SeerOperation, has_enough_seer_credits +from sentry.seer.credits import SeerBillingFeature, SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -57,6 +58,7 @@ class AutofixFeatureArgs: step: AutofixStep referrer: AutofixReferrer step_args: RCAStepArgs | SolutionStepArgs | CodeChangesStepArgs | PrIterationStepArgs + billing_feature: SeerBillingFeature existing_run_id: int | None = None insert_index: int | None = None user_context: str | None = None @@ -65,7 +67,6 @@ class AutofixFeatureArgs: user: User | RpcUser | AnonymousUser | None = None flush: bool = True single_run: bool = False - seer_feature: SeerFeature = SeerFeature.AUTOFIX def _run_mirrors(group: Group, run_id: int) -> QuerySet[SeerAgentRun]: @@ -79,10 +80,15 @@ def _mirror_matches_issue(group: Group) -> Q: return Q(group_id=group.id, project_id=group.project_id) -def get_run_seer_feature(agent_run: SeerAgentRun | None) -> SeerFeature: - """The feature the run was started under.""" - raw = agent_run.extras.get("seer_feature") if agent_run is not None else None - return SeerFeature(raw) if isinstance(raw, str) and raw in SeerFeature else SeerFeature.AUTOFIX +def get_run_billing_feature(state: SeerRunState) -> SeerBillingFeature: + """The billable feature the run was started under. Runs from before Seer + stored it are Autofix.""" + raw = state.billing_feature + return ( + SeerBillingFeature(raw) + if raw is not None and raw in SeerBillingFeature + else SeerBillingFeature.AUTOFIX + ) def autofix_run_targets_other_issue(group: Group, run_id: int) -> bool: @@ -159,25 +165,22 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( organization: Organization, + billing_feature: SeerBillingFeature, operation: SeerOperation, *, is_new_run: bool, skip_quota_for_free_cohort: bool = False, - seer_feature: SeerFeature = SeerFeature.AUTOFIX, ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are - checked for credits, and everyone else is checked for quota once per run. - Returns whether the operation uses quota. + checked for credits, except for runs agentic triage starts, which aren't billed yet. + Everyone else is checked for quota once per run. Returns whether the operation uses quota. """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): - if ( - # Don't bill runs started by agentic triage (not exposed via the API). - seer_feature != SeerFeature.AGENTIC_TRIAGE - and features.has("organizations:seer-credit-checks", organization) - and not has_enough_seer_credits(organization, seer_feature, operation) - ): + if features.has( + "organizations:seer-credit-checks", organization + ) and not has_enough_seer_credits(organization, billing_feature, operation): raise SeerOutOfCreditsException() return False @@ -199,17 +202,18 @@ def continue_single_run( group: Group, reply: str, *, + referrer: AutofixReferrer, + billing_feature: SeerBillingFeature, user: User | RpcUser, can_publish: bool, - referrer: AutofixReferrer, ) -> SeerRun: from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook check_autofix_billing( group.organization, + billing_feature, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, is_new_run=False, - seer_feature=get_run_seer_feature(agent_run), ) enable_coding = _org_allows_coding(group) @@ -262,21 +266,19 @@ def trigger_autofix_feature( from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook existing_agent_run: SeerAgentRun | None = None - seer_feature = args.seer_feature if args.existing_run_id is not None: existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) - seer_feature = get_run_seer_feature(existing_agent_run) is_new_run = existing_agent_run is None try: uses_quota = check_autofix_billing( group.organization, + args.billing_feature, args.step.seer_operation, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. skip_quota_for_free_cohort=args.allow_free_cohort, - seer_feature=seer_feature, ) except NoSeerQuotaException: logger.warning( @@ -380,10 +382,7 @@ def trigger_autofix_feature( if open_pr: agent_run_options["enable_write_pull_request"] = True - extras: dict[str, Any] = { - "referrer": args.referrer.value, - "seer_feature": args.seer_feature.value, - } + extras: dict[str, Any] = {"referrer": args.referrer.value} # Store the stopping point here for delivery to use when advancing steps. if stopping_point is not None: extras["stopping_point"] = stopping_point.value @@ -400,6 +399,7 @@ def trigger_autofix_feature( title=f"Autofix — {short_id}" if single_run else f"Autofix RCA — {short_id}", flush=args.flush, extras=extras, + billing_feature=args.billing_feature.value, ) elif existing_agent_run is not None: run = client.continue_feature_run( diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py index 33c2119afaf0..85f0547d0c16 100644 --- a/src/sentry/seer/autofix/issue_summary.py +++ b/src/sentry/seer/autofix/issue_summary.py @@ -43,6 +43,7 @@ is_seer_autotriggered_autofix_rate_limited_and_increment, is_seer_seat_based_tier_enabled, ) +from sentry.seer.credits import SeerBillingFeature from sentry.seer.entrypoints.cache import SeerOperatorAutofixCache from sentry.seer.entrypoints.operator import SeerAutofixOperator from sentry.seer.models import IssueSummary, SummarizeIssueResponse @@ -196,6 +197,7 @@ def _trigger_autofix_task( step=AutofixStep.ROOT_CAUSE, referrer=referrer, run_id=None, + billing_feature=SeerBillingFeature.AUTOFIX, stopping_point=stopping_point, ) with action_context_scope(ActionSource.SYSTEM, SYSTEM_ACTOR): diff --git a/src/sentry/seer/autofix/on_completion_hook.py b/src/sentry/seer/autofix/on_completion_hook.py index 194e93b412eb..d297fa9445c5 100644 --- a/src/sentry/seer/autofix/on_completion_hook.py +++ b/src/sentry/seer/autofix/on_completion_hook.py @@ -29,6 +29,7 @@ from sentry.seer.autofix.coding_agent import IntegrationNotFound from sentry.seer.autofix.constants import AutofixReferrer from sentry.seer.autofix.exceptions import SeerOutOfCreditsException +from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import LEGACY_FEATURE_ID as LEGACY_AUTOFIX_FEATURE_ID from sentry.seer.autofix.pr_iteration.completion import ( @@ -793,6 +794,7 @@ def _maybe_continue_pipeline( step=next_step, referrer=referrer, run_id=run_id, + billing_feature=get_run_billing_feature(state), ) except SeerOutOfCreditsException: logger.info( diff --git a/src/sentry/seer/autofix/pr_iteration/completion.py b/src/sentry/seer/autofix/pr_iteration/completion.py index ef8997de3a32..143eb7623b0b 100644 --- a/src/sentry/seer/autofix/pr_iteration/completion.py +++ b/src/sentry/seer/autofix/pr_iteration/completion.py @@ -16,7 +16,11 @@ from sentry.seer.autofix.commit_author import SeerCommitAuthor, parse_commit_author from sentry.seer.autofix.constants import AutofixReferrer from sentry.seer.autofix.exceptions import SeerOutOfCreditsException -from sentry.seer.autofix.feature.dispatch import AutofixFeatureArgs, trigger_autofix_feature +from sentry.seer.autofix.feature.dispatch import ( + AutofixFeatureArgs, + get_run_billing_feature, + trigger_autofix_feature, +) from sentry.seer.autofix.feature.models import PrIterationStepArgs from sentry.seer.autofix.pr_iteration.current_iteration import triggered_iteration_id from sentry.seer.autofix.pr_iteration.emit import ( @@ -582,6 +586,7 @@ def _retry_push_on_moved_branch( pr.repo_name: pr.pr_url for pr in state.repo_pr_states.values() if pr.pr_url }, ), + billing_feature=get_run_billing_feature(state), ), ) except SeerOutOfCreditsException: diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index a8fb47f02dfc..d18990e76551 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -20,20 +20,19 @@ from sentry.models.group import Group from sentry.seer.autofix.autofix_agent import get_autofix_run_state from sentry.seer.autofix.constants import AutofixReferrer -from sentry.seer.autofix.feature.dispatch import get_run_seer_feature +from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.pr_iteration.emit import bootstrap_iteration from sentry.seer.autofix.pr_iteration.feedback import Feedback from sentry.seer.autofix.pr_iteration.feedback_sources.base import ConsumeTriggerSource from sentry.seer.autofix.pr_iteration.pause import PauseReason, get_pause_reason from sentry.seer.autofix.pr_iteration.queue import enqueue_autofix_feedback -from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.utils import ( SEER_AUTOFIX_CONTINUE_OUT_OF_CREDITS_DETAIL, SEER_OUT_OF_CREDITS_CODE, SEER_PERMISSION_DENIED, ) from sentry.seer.models import SeerPermissionError -from sentry.seer.models.run import SeerAgentRun from sentry.tasks.seer.pr_iteration import trigger_consume_pr_iteration_feedback from sentry.users.services.user.service import user_service @@ -89,17 +88,8 @@ def handle_ui_feedback( ) # Check credits here, since the queued feedback is consumed after this request returns. - # Runs agentic triage starts aren't billed yet. - if ( - not can_run_seer_operation( - group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION - ) - and get_run_seer_feature( - SeerAgentRun.objects.filter( - run__organization_id=group.organization.id, run__seer_run_state_id=run_id - ).first() - ) - != SeerFeature.AGENTIC_TRIAGE + if not can_run_seer_operation( + group.organization, get_run_billing_feature(run_state), SeerOperation.AUTOFIX_PR_ITERATION ): return Response( { diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index bb52b7c868d0..37242cac9ded 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -21,7 +21,7 @@ class SeerOperation(StrEnum): AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" -class SeerFeature(StrEnum): +class SeerBillingFeature(StrEnum): """Seer features as tagged on LLM proxy calls, which billing maps to data categories.""" AGENTIC_TRIAGE = "agentic_triage" # Not billed yet. Marks the Autofix runs it starts. @@ -47,16 +47,19 @@ class SeerFeature(StrEnum): def has_enough_seer_credits( - organization: Organization, feature: SeerFeature, operation: SeerOperation + organization: Organization, billing_feature: SeerBillingFeature, operation: SeerOperation ) -> bool: """ Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. + Agentic triage isn't billed yet, so its runs always have enough credits. """ + if billing_feature == SeerBillingFeature.AGENTIC_TRIAGE: + return True response = seer_credit.backend.check_seer_credits( CheckSeerCreditsRequest( organization_id=organization.id, - feature_id=feature.value, + feature_id=billing_feature.value, predicted_credit_usage=SEER_OPERATION_ESTIMATES[operation](), take_lease=False, ) @@ -65,11 +68,14 @@ def has_enough_seer_credits( def can_run_seer_operation( - organization: Organization, feature: SeerFeature, operation: SeerOperation + organization: Organization, billing_feature: SeerBillingFeature, operation: SeerOperation ) -> bool: - """Checks whether the organization is token-billed and has enough Seer credits to run the operation.""" + """ + Checks whether the organization is token-billed and has enough Seer credits to run the operation. + Agentic triage isn't billed yet, so its runs can always run. + """ if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization ): return True - return has_enough_seer_credits(organization, feature, operation) + return has_enough_seer_credits(organization, billing_feature, operation) diff --git a/src/sentry/seer/endpoints/admin_autofix_retry.py b/src/sentry/seer/endpoints/admin_autofix_retry.py index 7a45ad6108d9..fcd4074d8b7a 100644 --- a/src/sentry/seer/endpoints/admin_autofix_retry.py +++ b/src/sentry/seer/endpoints/admin_autofix_retry.py @@ -19,6 +19,7 @@ ) from sentry.seer.autofix.constants import AutofixReferrer from sentry.seer.autofix.exceptions import SeerOutOfCreditsException +from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.feature.models import FEATURE_ID, LEGACY_FEATURE_ID from sentry.seer.autofix.on_completion_hook import PIPELINE_ORDER from sentry.seer.models import SeerRun @@ -119,6 +120,7 @@ def _retry_run(organization: Organization, run_id: int) -> dict[str, Any]: step=step, referrer=step_referrer or run_referrer or AutofixReferrer.UNKNOWN, run_id=run_id, + billing_feature=get_run_billing_feature(state), insert_index=insert_index, ) except SeerOutOfCreditsException: diff --git a/src/sentry/seer/endpoints/admin_autofix_single_run.py b/src/sentry/seer/endpoints/admin_autofix_single_run.py index 0ae26e851eb6..296b2960fe43 100644 --- a/src/sentry/seer/endpoints/admin_autofix_single_run.py +++ b/src/sentry/seer/endpoints/admin_autofix_single_run.py @@ -14,6 +14,7 @@ from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import get_automation_handoff +from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerPermissionError from sentry.viewer_context import Actor, ActorType, ViewerContext, viewer_context_scope @@ -69,6 +70,7 @@ def post(self, request: Request) -> Response: group=group, step=AutofixStep.ROOT_CAUSE, referrer=referrer, + billing_feature=SeerBillingFeature.AUTOFIX, user=user, actor_user_id=actor.user_id, single_run=True, diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 1f35185cc203..761b7ab4eed3 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -60,7 +60,11 @@ from sentry.seer.autofix.commit_author import commit_author_for_user from sentry.seer.autofix.constants import PUBLISH_SCOPES, AutofixReferrer from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException -from sentry.seer.autofix.feature.dispatch import project_stopping_point, should_use_single_run +from sentry.seer.autofix.feature.dispatch import ( + get_run_billing_feature, + project_stopping_point, + should_use_single_run, +) from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.on_completion_hook import get_run_stopping_point @@ -78,6 +82,7 @@ CodingAgentProviderType, has_project_connected_repos, ) +from sentry.seer.credits import SeerBillingFeature from sentry.seer.endpoints.organization_seer_onboarding_check import ( has_supported_scm_integration, ) @@ -513,6 +518,7 @@ def post( status=status.HTTP_409_CONFLICT, ) + billing_feature = SeerBillingFeature.AUTOFIX if resolved_run_id is not None: try: run_state = get_autofix_run_state(group, resolved_run_id) @@ -520,6 +526,7 @@ def post( if _is_unknown_run_id_error(e): return Response(status=status.HTTP_404_NOT_FOUND) raise PermissionDenied(SEER_PERMISSION_DENIED) + billing_feature = get_run_billing_feature(run_state) # A continued run keeps the stopping point it started with, and the # completion hook acts on it after this step, so gate on it too. @@ -586,6 +593,7 @@ def post( AutofixStoppingPoint(stopping_point) if stopping_point else None ), run_id=resolved_run_id, + billing_feature=billing_feature, user_context=user_context, insert_index=data.get("insert_index"), user=acting_user(request.user), @@ -712,6 +720,7 @@ def get(self, request: Request, group: Group) -> Response[AutofixStateResponse]: else AUTOFIX_FEATURE_ID, "status": state.status, "failure_reason": state.failure_reason, + "billing_feature": state.billing_feature, "blocks": blocks, "updated_at": state.updated_at, "pending_user_input": ( diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index 12416ecab212..f707373c3788 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -35,10 +35,10 @@ AutofixReferrer, ) from sentry.seer.autofix.exceptions import SeerOutOfCreditsException -from sentry.seer.autofix.feature.dispatch import continue_single_run +from sentry.seer.autofix.feature.dispatch import continue_single_run, get_run_billing_feature from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run -from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerBillingFeature, SeerOperation, can_run_seer_operation from sentry.seer.endpoints.group_ai_autofix import RUN_IN_FLIGHT_CODE from sentry.seer.endpoints.utils import ( SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, @@ -251,12 +251,10 @@ def _maybe_reply_to_single_run( if not request.access.has_any_project_scope(project, AUTOFIX_RUN_SCOPES): raise PermissionDenied("You do not have access to this run.") try: - if ( - SeerAgentClient(organization, request.user) - .get_run(run_id=resolved.seer_run_state_id) - .status - == "processing" - ): + run_state = SeerAgentClient(organization, request.user).get_run( + run_id=resolved.seer_run_state_id + ) + if run_state.status == "processing": return Response( { "detail": "This run is still working", @@ -270,9 +268,10 @@ def _maybe_reply_to_single_run( single_run, single_run.group, reply, + referrer=AutofixReferrer.SEER_EXPLORER, + billing_feature=get_run_billing_feature(run_state), user=replier, can_publish=request.access.has_any_project_scope(project, PUBLISH_SCOPES), - referrer=AutofixReferrer.SEER_EXPLORER, ) except SeerOutOfCreditsException: return Response( @@ -450,7 +449,7 @@ def post( raise PermissionDenied(error) # Do not bill for dashboard generation. if not is_dashboard_generate_run and not can_run_seer_operation( - organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + organization, SeerBillingFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE ): return Response( {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index 16d580ddedfc..de07cf1b40d2 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -10,12 +10,15 @@ from sentry.api.bases.organization import OrganizationEndpoint from sentry.models.organization import Organization from sentry.ratelimits.config import RateLimitConfig -from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerBillingFeature, SeerOperation, can_run_seer_operation from sentry.types.ratelimit import RateLimit, RateLimitCategory class SeerCreditCheckSerializer(serializers.Serializer[dict[str, str]]): operation = serializers.ChoiceField(choices=[operation.value for operation in SeerOperation]) + billing_feature = serializers.ChoiceField( + choices=[feature.value for feature in SeerBillingFeature], required=False + ) @cell_silo_endpoint @@ -46,12 +49,14 @@ def get(self, request: Request, organization: Organization) -> Response: return Response(serializer.errors, status=400) operation = SeerOperation(serializer.validated_data["operation"]) - # TODO: Let the UI pass the feature once it can tell agentic triage runs apart. - feature = ( - SeerFeature.EXPLORER - if operation == SeerOperation.AGENT_CHAT_MESSAGE - else SeerFeature.AUTOFIX - ) + + if "billing_feature" in serializer.validated_data: + feature = SeerBillingFeature(serializer.validated_data["billing_feature"]) + elif operation == SeerOperation.AGENT_CHAT_MESSAGE: + feature = SeerBillingFeature.EXPLORER + else: + feature = SeerBillingFeature.AUTOFIX + return Response( {"hasEnoughCredits": can_run_seer_operation(organization, feature, operation)} ) diff --git a/src/sentry/seer/entrypoints/operator.py b/src/sentry/seer/entrypoints/operator.py index b2d9ef8e074c..baeb76b11b5b 100644 --- a/src/sentry/seer/entrypoints/operator.py +++ b/src/sentry/seer/entrypoints/operator.py @@ -210,8 +210,10 @@ def trigger_autofix_agent( trigger_push_changes, ) from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException + from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.steps import AutofixStep + from sentry.seer.credits import SeerBillingFeature event_lifecyle = SeerOperatorEventLifecycleMetric( interaction_type=SeerOperatorInteractionType.OPERATOR_TRIGGER_AUTOFIX, @@ -271,6 +273,7 @@ def trigger_autofix_agent( step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.SLACK, run_id=None, + billing_feature=SeerBillingFeature.AUTOFIX, user=user, ) run_id = run.seer_run_state_id @@ -302,6 +305,11 @@ def trigger_autofix_agent( step=AutofixStep.from_autofix_stopping_point(stopping_point), referrer=AutofixReferrer.SLACK, run_id=run_id, + billing_feature=( + get_run_billing_feature(existing_state) + if existing_state and existing_state.run_id == run_id + else SeerBillingFeature.AUTOFIX + ), user=user, ) except NoSeerQuotaException: diff --git a/src/sentry/tasks/seer/pr_iteration.py b/src/sentry/tasks/seer/pr_iteration.py index c95fce7757b6..718839c8ee17 100644 --- a/src/sentry/tasks/seer/pr_iteration.py +++ b/src/sentry/tasks/seer/pr_iteration.py @@ -53,6 +53,7 @@ from sentry.seer.autofix.commit_author import commit_author_for_feedback from sentry.seer.autofix.constants import AutofixReferrer from sentry.seer.autofix.exceptions import SeerOutOfCreditsException +from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.pr_iteration.bot_identity import bot_logins_for_feedback from sentry.seer.autofix.pr_iteration.cap_exhausted import assign_user_for_exhausted_cap from sentry.seer.autofix.pr_iteration.constants import PR_ITERATION_PROVIDER @@ -835,6 +836,7 @@ def _drain_queued_autofix_feedback( step=AutofixStep.PR_ITERATION, referrer=referrer, run_id=run_id, + billing_feature=get_run_billing_feature(state), user_context="\n\n".join(item.text for item in feedback_items), feedback=feedback_items, actor_user_id=actor_user_id, diff --git a/tests/sentry/seer/agentic_triage/test_delivery.py b/tests/sentry/seer/agentic_triage/test_delivery.py index 3196c56687bc..42c0fec2f7f8 100644 --- a/tests/sentry/seer/agentic_triage/test_delivery.py +++ b/tests/sentry/seer/agentic_triage/test_delivery.py @@ -15,7 +15,7 @@ deliver_agentic_triage_result, ) from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerFeature +from sentry.seer.credits import SeerBillingFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -223,7 +223,7 @@ def test_single_run_option_starts_single_runs(self) -> None: def test_starts_autofix_as_agentic_triage(self) -> None: mock_trigger = self._deliver_autofix_verdict() - assert mock_trigger.call_args.kwargs["seer_feature"] == SeerFeature.AGENTIC_TRIAGE + assert mock_trigger.call_args.kwargs["billing_feature"] == SeerBillingFeature.AGENTIC_TRIAGE def test_dry_run_with_single_run_starts_nothing(self) -> None: mock_trigger = self._deliver_autofix_verdict(single_run=True, dry_run=True) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 78316bf8151a..9ee8bdcab519 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -26,7 +26,7 @@ from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerFeature +from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -66,6 +66,7 @@ def test_continues_feature_run(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, existing_run_id=123, insert_index=4, @@ -162,6 +163,7 @@ def test_refuses_run_from_another_project(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, user_context="Find a solution", @@ -190,6 +192,7 @@ def test_refuses_run_without_project(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, user_context="Find a solution", @@ -213,6 +216,7 @@ def test_starts_feature_run_and_records_quota(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), stopping_point=AutofixStoppingPoint.OPEN_PR, @@ -237,9 +241,9 @@ def test_starts_feature_run_and_records_quota(self) -> None: assert start_kwargs["flush"] is True assert start_kwargs["extras"] == { "referrer": AutofixReferrer.AGENTIC_TRIAGE.value, - "seer_feature": SeerFeature.AUTOFIX.value, "stopping_point": AutofixStoppingPoint.OPEN_PR.value, } + assert start_kwargs["billing_feature"] == SeerBillingFeature.AUTOFIX.value def test_raises_when_out_of_budget(self) -> None: with ( @@ -253,6 +257,7 @@ def test_raises_when_out_of_budget(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -275,6 +280,7 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), allow_free_cohort=True, @@ -300,6 +306,7 @@ def test_non_free_cohort_org_checks_quota_and_records_usage(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), allow_free_cohort=True, @@ -327,6 +334,7 @@ def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -354,6 +362,7 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -385,6 +394,7 @@ def test_raises_when_out_of_credits_for_a_continued_step( self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, step_args=SolutionStepArgs(), @@ -415,14 +425,14 @@ def test_starts_agentic_triage_run_without_checking_credits( referrer=AutofixReferrer.AGENTIC_TRIAGE, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), - seer_feature=SeerFeature.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, ), ) assert run is fake_run mock_check_seer_credits.assert_not_called() - extras = MockClient.return_value.start_feature_run.call_args.kwargs["extras"] - assert extras["seer_feature"] == SeerFeature.AGENTIC_TRIAGE + start_kwargs = MockClient.return_value.start_feature_run.call_args.kwargs + assert start_kwargs["billing_feature"] == SeerBillingFeature.AGENTIC_TRIAGE @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( @@ -438,7 +448,6 @@ def test_continues_agentic_triage_run_without_checking_credits( source=FEATURE_ID, group=self.group, project=self.group.project, - extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}, ) with ( @@ -454,6 +463,7 @@ def test_continues_agentic_triage_run_without_checking_credits( step=AutofixStep.SOLUTION, existing_run_id=123, step_args=SolutionStepArgs(), + billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, ), ) @@ -474,6 +484,7 @@ def test_allows_async_dispatch(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), flush=False, @@ -497,6 +508,7 @@ def test_forwards_run_options_to_client(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), user=user, @@ -522,6 +534,7 @@ def test_code_changes_enables_coding(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.CODE_CHANGES, step_args=CodeChangesStepArgs(), ), @@ -554,6 +567,7 @@ def test_pr_iteration_enables_coding_and_sends_its_step_args(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GITHUB_PR_COMMENT, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.PR_ITERATION, existing_run_id=123, step_args=PrIterationStepArgs( @@ -591,6 +605,7 @@ def test_code_changes_rejected_when_coding_disabled(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.CODE_CHANGES, step_args=CodeChangesStepArgs(), existing_run_id=123, @@ -746,7 +761,12 @@ def setUp(self) -> None: self.project.update_option("sentry:seer_automated_run_stopping_point", "root_cause") self.organization.update_option("sentry:enable_seer_coding", True) - def _reply(self, *, can_publish: bool = True) -> MagicMock: + def _reply( + self, + *, + can_publish: bool = True, + billing_feature: SeerBillingFeature = SeerBillingFeature.AUTOFIX, + ) -> MagicMock: with patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls: mock_client_cls.return_value.continue_feature_run.return_value = self.seer_run continue_single_run( @@ -756,6 +776,7 @@ def _reply(self, *, can_publish: bool = True) -> MagicMock: user=self.user, can_publish=can_publish, referrer=AutofixReferrer.SEER_EXPLORER, + billing_feature=billing_feature, ) return mock_client_cls @@ -811,6 +832,7 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> user=self.user, can_publish=True, referrer=AutofixReferrer.SEER_EXPLORER, + billing_feature=SeerBillingFeature.AUTOFIX, ) mock_client_cls.return_value.continue_feature_run.assert_not_called() @@ -823,8 +845,6 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> def test_agentic_triage_run_skips_credit_check( self, mock_check_seer_credits: MagicMock ) -> None: - self.agent_run.update(extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}) - - self._reply() + self._reply(billing_feature=SeerBillingFeature.AGENTIC_TRIAGE) mock_check_seer_credits.assert_not_called() diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 1eccaaa9f819..223aaa851f71 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -36,7 +36,7 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.user_ui import UserUIFeedbackSource from sentry.seer.autofix.pr_iteration.iterations import PrIterationNoPullRequestException from sentry.seer.autofix.steps import AutofixStep -from sentry.seer.credits import SeerFeature +from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType @@ -346,6 +346,7 @@ def assert_activity_exists(**_kwargs: object) -> None: group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=None, ) @@ -388,6 +389,7 @@ def test_root_cause_routes_to_seer_feature( group=self.group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=None, ) @@ -435,6 +437,7 @@ def test_solution_routes_to_seer_feature( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Keep compatibility", insert_index=3, @@ -473,6 +476,7 @@ def test_solution_without_prior_run_routes_to_seer_feature( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert result == feature_run @@ -509,6 +513,7 @@ def test_code_changes_routes_to_seer_feature( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Keep compatibility", insert_index=3, @@ -558,6 +563,7 @@ def test_continued_solution_attributes_actor( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, actor_user_id=self.user.id, ) @@ -591,6 +597,7 @@ def test_root_cause_without_actor_is_unattributed( group=self.group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert mock_record_activity.call_args.kwargs["activity_attribution"] is None @@ -611,6 +618,7 @@ def test_code_changes_without_prior_run_routes_to_seer_feature( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert result == feature_run @@ -651,6 +659,7 @@ def test_pr_iteration_routes_to_seer_feature( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.GITHUB_PR_COMMENT, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, user_context="Rename the helper.", feedback=[ @@ -703,6 +712,7 @@ def test_pr_iteration_requires_existing_pr( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -726,6 +736,7 @@ def test_free_cohort_skips_quota_check( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.AGENTIC_TRIAGE, + billing_feature=SeerBillingFeature.AUTOFIX, allow_free_cohort=True, ) @@ -745,6 +756,7 @@ def test_continued_run_permitted_with_no_remaining_budget( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -768,6 +780,7 @@ def test_continued_run_requires_matching_group( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -789,6 +802,7 @@ def test_continued_run_rejects_when_not_mirrored( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -821,6 +835,7 @@ def test_continued_run_rejects_foreign_project_even_when_metadata_matches( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -846,6 +861,7 @@ def test_solution_rejects_run_from_another_project(self, mock_feature: MagicMock group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Find a solution", ) @@ -909,6 +925,7 @@ def test_counts_success( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -930,6 +947,7 @@ def test_counts_declined_when_no_quota( self.group, AutofixStep.ROOT_CAUSE, AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -959,6 +977,7 @@ def test_counts_declined_when_out_of_credits( self.group, AutofixStep.ROOT_CAUSE, AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -986,6 +1005,7 @@ def test_counts_error_when_seer_rejects_run( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.SLACK, + billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -1619,6 +1639,7 @@ def _push( self, mock_post: MagicMock, features: dict[str, bool] | None = None, + billing_feature: SeerBillingFeature | None = None, **kwargs: Any, ) -> dict[str, Any]: """Push with a minimal run state and return the payload sent to Seer.""" @@ -1631,6 +1652,7 @@ def _push( updated_at="2024-01-01T00:00:00Z", repo_pr_states={}, metadata={"group_id": self.group.id}, + billing_feature=billing_feature, ) with self.feature(features or {}): @@ -1678,11 +1700,7 @@ def test_raises_when_out_of_credits( self, mock_post: MagicMock, mock_check_seer_credits: MagicMock ) -> None: with pytest.raises(SeerOutOfCreditsException): - trigger_push_changes( - group=self.group, - run_id=123, - referrer=AutofixReferrer.UNKNOWN, - ) + self._push(mock_post) mock_post.assert_not_called() @@ -1694,15 +1712,13 @@ def test_raises_when_out_of_credits( def test_agentic_triage_run_skips_credit_check( self, mock_post: MagicMock, mock_check_seer_credits: MagicMock ) -> None: - run = mirror_autofix_run(self, self.group, 123) - run.agent.update(extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}) - self._push( mock_post, features={ "organizations:seer-credit-checks": True, "organizations:token-billed-seer-enabled": True, }, + billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, ) mock_check_seer_credits.assert_not_called() @@ -1863,6 +1879,7 @@ def test_single_run_skips_started_events( group=group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, + billing_feature=SeerBillingFeature.AUTOFIX, single_run=True, ) diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 5ffd78d3af20..8eb66788512b 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -6,6 +6,9 @@ from django.test import override_settings from django.utils import timezone +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, +) from sentry.integrations.services.integration import RpcIntegration from sentry.integrations.types import ExternalProviders @@ -38,7 +41,7 @@ from sentry.seer.autofix.pr_iteration.queue import QueuedAutofixFeedback from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerFeature, SeerOperation +from sentry.seer.credits import SeerBillingFeature, SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -1221,6 +1224,7 @@ def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, stopping_point=AutofixStoppingPoint.CODE_CHANGES, run_id=None, + billing_feature=SeerBillingFeature.AUTOFIX, user_context=None, insert_index=None, user=ANY, @@ -1258,6 +1262,7 @@ def test_insert_index_passed_through( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, stopping_point=None, run_id=42, + billing_feature=SeerBillingFeature.AUTOFIX, user_context=None, insert_index=3, user=ANY, @@ -1697,13 +1702,14 @@ def test_pr_iteration_rejected_when_out_of_credits( "code": "seer_out_of_credits", } mock_can_run.assert_called_once_with( - group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + group.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION ) mock_enqueue.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( - "sentry.seer.autofix.pr_iteration.manual_trigger.can_run_seer_operation", - return_value=False, + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=False), ) @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @@ -1713,24 +1719,19 @@ def test_pr_iteration_allowed_on_agentic_triage_run_when_out_of_credits( mock_run_state: MagicMock, mock_enqueue: MagicMock, mock_consume: MagicMock, - mock_can_run: MagicMock, + mock_check_seer_credits: MagicMock, ) -> None: """Runs agentic triage starts aren't billed yet.""" group = self.create_group() run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) - self.create_seer_agent_run( - run=run, - source="autofix", - group=group, - project=group.project, - extras={"seer_feature": SeerFeature.AGENTIC_TRIAGE.value}, - ) + self.create_seer_agent_run(run=run, source="autofix", group=group, project=group.project) mock_run_state.return_value = SeerRunState( run_id=123, blocks=[], status="completed", updated_at="2024-01-01T00:00:00Z", repo_pr_states={"owner/repo": RepoPRState(repo_name="owner/repo", pr_number=7)}, + billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, ) self.login_as(user=self.user) @@ -1741,6 +1742,7 @@ def test_pr_iteration_allowed_on_agentic_triage_run_when_out_of_credits( ) assert response.status_code == 202, response.data + mock_check_seer_credits.assert_not_called() mock_enqueue.assert_called_once() mock_consume.assert_called_once() diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index ab40c1306bec..b4dd194a654f 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -9,7 +9,7 @@ from sentry.seer.credits import ( SEER_OPERATION_ESTIMATES, - SeerFeature, + SeerBillingFeature, SeerOperation, can_run_seer_operation, ) @@ -31,7 +31,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -44,7 +44,7 @@ def test_can_run_seer_operation_skips_billing_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -57,7 +57,7 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert not can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( @@ -77,5 +77,5 @@ def test_can_run_seer_operation_returns_true_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + self.organization, SeerBillingFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE ) From 1f348ba73f01e15fd3566dc298d7b038757bf4cd Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 14:41:14 -0800 Subject: [PATCH 41/45] minimize diff --- src/sentry/seer/autofix/feature/dispatch.py | 4 +++- src/sentry/seer/credits.py | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index f0d31d614e5c..ba4ff450ff9b 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -382,7 +382,9 @@ def trigger_autofix_feature( if open_pr: agent_run_options["enable_write_pull_request"] = True - extras: dict[str, Any] = {"referrer": args.referrer.value} + extras: dict[str, Any] = { + "referrer": args.referrer.value, + } # Store the stopping point here for delivery to use when advancing steps. if stopping_point is not None: extras["stopping_point"] = stopping_point.value diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 37242cac9ded..2997030fca08 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -47,19 +47,19 @@ class SeerBillingFeature(StrEnum): def has_enough_seer_credits( - organization: Organization, billing_feature: SeerBillingFeature, operation: SeerOperation + organization: Organization, feature: SeerBillingFeature, operation: SeerOperation ) -> bool: """ Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. Agentic triage isn't billed yet, so its runs always have enough credits. """ - if billing_feature == SeerBillingFeature.AGENTIC_TRIAGE: + if feature == SeerBillingFeature.AGENTIC_TRIAGE: return True response = seer_credit.backend.check_seer_credits( CheckSeerCreditsRequest( organization_id=organization.id, - feature_id=billing_feature.value, + feature_id=feature.value, predicted_credit_usage=SEER_OPERATION_ESTIMATES[operation](), take_lease=False, ) @@ -68,7 +68,7 @@ def has_enough_seer_credits( def can_run_seer_operation( - organization: Organization, billing_feature: SeerBillingFeature, operation: SeerOperation + organization: Organization, feature: SeerBillingFeature, operation: SeerOperation ) -> bool: """ Checks whether the organization is token-billed and has enough Seer credits to run the operation. @@ -78,4 +78,4 @@ def can_run_seer_operation( "organizations:token-billed-seer-enabled", organization ): return True - return has_enough_seer_credits(organization, billing_feature, operation) + return has_enough_seer_credits(organization, feature, operation) From 98d9accc6f99346777113f35725b35f3ae284cbc Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 14:52:48 -0800 Subject: [PATCH 42/45] fix tests --- tests/sentry/seer/autofix/feature/test_dispatch.py | 1 + tests/sentry/seer/endpoints/test_admin_autofix_retry.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 9ee8bdcab519..bde6665c245c 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -624,6 +624,7 @@ def _start(self, **kwargs: Any) -> MagicMock: "referrer": AutofixReferrer.AGENTIC_TRIAGE, "step": AutofixStep.ROOT_CAUSE, "step_args": RCAStepArgs(), + "billing_feature": SeerBillingFeature.AUTOFIX, **kwargs, } fake_run = self.create_seer_run(organization=self.organization, type="feature_run") diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py index e95ecb37cb35..39dde9659351 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py @@ -9,6 +9,7 @@ from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID from sentry.seer.autofix.steps import AutofixStep +from sentry.seer.credits import SeerBillingFeature from sentry.testutils.cases import APITestCase from sentry.viewer_context import Actor, ActorType, ViewerContext, get_viewer_context @@ -93,6 +94,7 @@ def test_retries_failed_step_as_system( step=AutofixStep.SOLUTION, referrer=AutofixReferrer.NIGHT_SHIFT, run_id=1, + billing_feature=SeerBillingFeature.AUTOFIX, insert_index=2, ) assert viewer_contexts == [ From 58d915c19e8fdf01ff178418b373965de7c7e06d Mon Sep 17 00:00:00 2001 From: srest2021 Date: Tue, 10 Nov 2026 15:32:08 -0800 Subject: [PATCH 43/45] cut scope --- src/sentry/seer/autofix/feature/dispatch.py | 4 ++-- src/sentry/seer/credits.py | 6 +----- .../seer/autofix/feature/test_dispatch.py | 18 +++++++++--------- .../sentry/seer/autofix/test_autofix_agent.py | 6 +++--- .../seer/endpoints/test_group_ai_autofix.py | 8 ++++---- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index ba4ff450ff9b..95f8a3ed6696 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -173,8 +173,8 @@ def check_autofix_billing( ) -> bool: """ Raise if the organization can't pay for the operation. Token-billed orgs are - checked for credits, except for runs agentic triage starts, which aren't billed yet. - Everyone else is checked for quota once per run. Returns whether the operation uses quota. + checked for credits under the run's billing feature, and everyone else is checked for + quota once per run. Returns whether the operation uses quota. """ # Check credits for token-billed orgs. if features.has("organizations:token-billed-seer-enabled", organization): diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 2997030fca08..87a4e09376e1 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -24,7 +24,7 @@ class SeerOperation(StrEnum): class SeerBillingFeature(StrEnum): """Seer features as tagged on LLM proxy calls, which billing maps to data categories.""" - AGENTIC_TRIAGE = "agentic_triage" # Not billed yet. Marks the Autofix runs it starts. + AGENTIC_TRIAGE = "agentic_triage" # Marks the Autofix runs agentic triage starts. AUTOFIX = "autofix" EXPLORER = "explorer" @@ -52,10 +52,7 @@ def has_enough_seer_credits( """ Checks whether the organization has enough Seer credits to run the operation, for callers that already checked the credit check and token-billed flags. - Agentic triage isn't billed yet, so its runs always have enough credits. """ - if feature == SeerBillingFeature.AGENTIC_TRIAGE: - return True response = seer_credit.backend.check_seer_credits( CheckSeerCreditsRequest( organization_id=organization.id, @@ -72,7 +69,6 @@ def can_run_seer_operation( ) -> bool: """ Checks whether the organization is token-billed and has enough Seer credits to run the operation. - Agentic triage isn't billed yet, so its runs can always run. """ if not features.has("organizations:seer-credit-checks", organization) or not features.has( "organizations:token-billed-seer-enabled", organization diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index bde6665c245c..1c21021f782e 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -406,9 +406,9 @@ def test_raises_when_out_of_credits_for_a_continued_step( @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse(has_enough_credits=True), ) - def test_starts_agentic_triage_run_without_checking_credits( + def test_starts_agentic_triage_run_under_its_billing_feature( self, mock_check_seer_credits: MagicMock ) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -430,16 +430,16 @@ def test_starts_agentic_triage_run_without_checking_credits( ) assert run is fake_run - mock_check_seer_credits.assert_not_called() + assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" start_kwargs = MockClient.return_value.start_feature_run.call_args.kwargs assert start_kwargs["billing_feature"] == SeerBillingFeature.AGENTIC_TRIAGE @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse(has_enough_credits=True), ) - def test_continues_agentic_triage_run_without_checking_credits( + def test_continues_agentic_triage_run_under_its_billing_feature( self, mock_check_seer_credits: MagicMock ) -> None: fake_run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) @@ -468,7 +468,7 @@ def test_continues_agentic_triage_run_without_checking_credits( ) assert run is fake_run - mock_check_seer_credits.assert_not_called() + assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -841,11 +841,11 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse(has_enough_credits=True), ) - def test_agentic_triage_run_skips_credit_check( + def test_checks_credits_under_the_run_billing_feature( self, mock_check_seer_credits: MagicMock ) -> None: self._reply(billing_feature=SeerBillingFeature.AGENTIC_TRIAGE) - mock_check_seer_credits.assert_not_called() + assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index 223aaa851f71..fb12a238237f 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -1706,10 +1706,10 @@ def test_raises_when_out_of_credits( @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse(has_enough_credits=True), ) @patch("sentry.seer.agent.client.make_agent_update_request") - def test_agentic_triage_run_skips_credit_check( + def test_checks_credits_under_the_run_billing_feature( self, mock_post: MagicMock, mock_check_seer_credits: MagicMock ) -> None: self._push( @@ -1721,7 +1721,7 @@ def test_agentic_triage_run_skips_credit_check( billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, ) - mock_check_seer_credits.assert_not_called() + assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" mock_post.assert_called_once() @patch("sentry.seer.agent.client.make_agent_update_request") diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 8eb66788512b..7cb71aea0b6a 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1709,19 +1709,19 @@ def test_pr_iteration_rejected_when_out_of_credits( @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( "sentry.seer_credit.backend.check_seer_credits", - return_value=CheckSeerCreditsResponse(has_enough_credits=False), + return_value=CheckSeerCreditsResponse(has_enough_credits=True), ) @patch("sentry.seer.autofix.pr_iteration.manual_trigger.trigger_consume_pr_iteration_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.enqueue_autofix_feedback") @patch("sentry.seer.autofix.pr_iteration.manual_trigger.get_autofix_run_state") - def test_pr_iteration_allowed_on_agentic_triage_run_when_out_of_credits( + def test_pr_iteration_checks_credits_under_the_run_billing_feature( self, mock_run_state: MagicMock, mock_enqueue: MagicMock, mock_consume: MagicMock, mock_check_seer_credits: MagicMock, ) -> None: - """Runs agentic triage starts aren't billed yet.""" + """Billing decides whether the run's feature is charged.""" group = self.create_group() run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) self.create_seer_agent_run(run=run, source="autofix", group=group, project=group.project) @@ -1742,7 +1742,7 @@ def test_pr_iteration_allowed_on_agentic_triage_run_when_out_of_credits( ) assert response.status_code == 202, response.data - mock_check_seer_credits.assert_not_called() + assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" mock_enqueue.assert_called_once() mock_consume.assert_called_once() From f8c8047e894af48e752cfcb52e2b27dfab2d20ba Mon Sep 17 00:00:00 2001 From: srest2021 Date: Wed, 11 Nov 2026 10:24:55 -0800 Subject: [PATCH 44/45] ref(seer-billing): Build on the shared billing attribution sender Use the billing feature sent by the shared dispatch for new runs and the stored feature for continuations, so credit checks and Seer agree. --- src/sentry/seer/agent/client.py | 5 -- src/sentry/seer/agent/client_models.py | 2 - src/sentry/seer/agent/client_utils.py | 1 - src/sentry/seer/agentic_triage/delivery.py | 2 - src/sentry/seer/autofix/autofix_agent.py | 4 +- src/sentry/seer/autofix/feature/dispatch.py | 37 +++++----- src/sentry/seer/autofix/issue_summary.py | 2 - src/sentry/seer/credits.py | 28 +++++--- .../endpoints/admin_autofix_single_run.py | 2 - src/sentry/seer/endpoints/group_ai_autofix.py | 7 +- .../endpoints/organization_seer_agent_chat.py | 7 +- .../organization_seer_credit_check.py | 21 ++++-- src/sentry/seer/entrypoints/operator.py | 5 +- .../seer/agentic_triage/test_delivery.py | 6 -- .../seer/autofix/feature/test_dispatch.py | 71 ++++++++++++------- .../sentry/seer/autofix/test_autofix_agent.py | 31 ++------ .../endpoints/test_admin_autofix_retry.py | 6 +- .../seer/endpoints/test_group_ai_autofix.py | 18 ++--- .../test_organization_seer_credit_check.py | 4 +- tests/sentry/seer/test_credits.py | 21 +++--- 20 files changed, 140 insertions(+), 140 deletions(-) diff --git a/src/sentry/seer/agent/client.py b/src/sentry/seer/agent/client.py index e615505148fe..e35bfdca1156 100644 --- a/src/sentry/seer/agent/client.py +++ b/src/sentry/seer/agent/client.py @@ -572,7 +572,6 @@ def start_feature_run( payload: dict[str, Any], title: str, referrer: str, - billing_feature: str | None = None, flush: bool = True, extras: dict[str, Any] | None = None, on_run_created: Callable[[SeerRun], None] | None = None, @@ -599,8 +598,6 @@ def start_feature_run( Explicit agent_run_options override any options derived from organization configuration. - - billing_feature, if given, is the feature Seer bills the run to for its whole life. """ user_id = self._user_id @@ -632,8 +629,6 @@ def _create_agent_run(run: SeerRun) -> None: body["user_org_context"] = user_org_context if proxy_headers is not None: body["proxy_headers"] = proxy_headers - if billing_feature is not None: - body["billing_feature"] = billing_feature return enqueue_seer_run( organization=self.organization, diff --git a/src/sentry/seer/agent/client_models.py b/src/sentry/seer/agent/client_models.py index 740ff23f5353..59c57e0658fe 100644 --- a/src/sentry/seer/agent/client_models.py +++ b/src/sentry/seer/agent/client_models.py @@ -401,8 +401,6 @@ class SeerRunState(BaseModel): coding_agents: dict[str, CodingAgentState] = Field(default_factory=dict, exclude=True) usage: UsageAccumulator = Field(default_factory=UsageAccumulator, exclude=True) pr_push_log: list[PushLogEntry] | None = Field(default=None, exclude=True) - # The feature the run is billed to, set when it starts. - billing_feature: str | None = Field(default=None, exclude=True) class Config: extra = "ignore" diff --git a/src/sentry/seer/agent/client_utils.py b/src/sentry/seer/agent/client_utils.py index 49a67321d254..ae089649dca3 100644 --- a/src/sentry/seer/agent/client_utils.py +++ b/src/sentry/seer/agent/client_utils.py @@ -158,7 +158,6 @@ class SeerFeatureRunRequest(TypedDict): user_org_context: NotRequired[UserOrgContext] proxy_headers: NotRequired[dict[str, str] | None] referrer: str - billing_feature: NotRequired[str] class SeerFeatureRunWireRequest(SeerFeatureRunRequest): diff --git a/src/sentry/seer/agentic_triage/delivery.py b/src/sentry/seer/agentic_triage/delivery.py index 059dbb233dad..a1e4821bbe54 100644 --- a/src/sentry/seer/agentic_triage/delivery.py +++ b/src/sentry/seer/agentic_triage/delivery.py @@ -29,7 +29,6 @@ is_seer_autotriggered_autofix_rate_limited_and_increment, is_seer_seat_based_tier_enabled, ) -from sentry.seer.credits import SeerBillingFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -335,7 +334,6 @@ def _process_verdicts( group=group, step=AutofixStep.ROOT_CAUSE, referrer=referrer, - billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, stopping_point=stopping_point_by_project_id[group.project_id], user_context=user_context, allow_free_cohort=True, diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index 47df21b47ddb..e954513c397e 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -65,7 +65,7 @@ is_free_cohort_org, read_preference_from_sentry_db, ) -from sentry.seer.credits import SeerBillingFeature, SeerOperation +from sentry.seer.credits import SeerOperation from sentry.seer.entrypoints.operator import ( SeerActivityAttribution, SeerAutofixOperator, @@ -441,7 +441,6 @@ def trigger_autofix_agent( referrer: AutofixReferrer, *, run_id: int | None = None, - billing_feature: SeerBillingFeature, stopping_point: AutofixStoppingPoint | None = None, user_context: str | None = None, insert_index: int | None = None, @@ -504,7 +503,6 @@ def trigger_autofix_agent( step=step, referrer=referrer, step_args=step_args, - billing_feature=billing_feature, existing_run_id=run_id, insert_index=insert_index, user_context=user_context, diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 94c1cecbdf4a..4c3a522e78a3 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -44,7 +44,7 @@ get_automation_handoff, is_free_cohort_org, ) -from sentry.seer.credits import SeerBillingFeature, SeerOperation, has_enough_seer_credits +from sentry.seer.credits import SeerOperation, has_enough_seer_credits from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.seer.models.seer_api_models import UNKNOWN_RUN_ID_FOR_GROUP, SeerPermissionError from sentry.users.models.user import User @@ -67,7 +67,6 @@ class AutofixFeatureArgs: step: AutofixStep referrer: AutofixReferrer step_args: RCAStepArgs | SolutionStepArgs | CodeChangesStepArgs | PrIterationStepArgs - billing_feature: SeerBillingFeature existing_run_id: int | None = None insert_index: int | None = None user_context: str | None = None @@ -90,15 +89,10 @@ def _mirror_matches_issue(group: Group) -> Q: return Q(group_id=group.id, project_id=group.project_id) -def get_run_billing_feature(state: SeerRunState) -> SeerBillingFeature: +def get_run_billing_feature(state: SeerRunState) -> int: """The billable feature the run was started under. Runs from before Seer - stored it are Autofix.""" - raw = state.billing_feature - return ( - SeerBillingFeature(raw) - if raw is not None and raw in SeerBillingFeature - else SeerBillingFeature.AUTOFIX - ) + stored it are unattributed.""" + return state.billing_feature or billing.SEER_BILLING_FEATURE_UNSPECIFIED def autofix_run_targets_other_issue(group: Group, run_id: int) -> bool: @@ -175,7 +169,7 @@ def _org_allows_coding(group: Group) -> bool: def check_autofix_billing( organization: Organization, - billing_feature: SeerBillingFeature, + billing_feature: int, operation: SeerOperation, *, is_new_run: bool, @@ -213,7 +207,7 @@ def continue_single_run( reply: str, *, referrer: AutofixReferrer, - billing_feature: SeerBillingFeature, + billing_feature: int, user: User | RpcUser, can_publish: bool, ) -> SeerRun: @@ -280,10 +274,21 @@ def trigger_autofix_feature( existing_agent_run = require_autofix_run_for_group(group, args.existing_run_id) is_new_run = existing_agent_run is None + billing_feature = args.billing_feature + if billing_feature is None: + # Continuations without the stored feature are unattributed rather than guessed. + billing_feature = ( + _BILLING_FEATURE_BY_REFERRER.get( + args.referrer, billing.SEER_BILLING_FEATURE_AUTOFIX_HEADLESS + ) + if is_new_run + else billing.SEER_BILLING_FEATURE_UNSPECIFIED + ) + try: uses_quota = check_autofix_billing( group.organization, - args.billing_feature, + billing_feature, args.step.seer_operation, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage @@ -403,10 +408,7 @@ def trigger_autofix_feature( if is_new_run: run = client.start_feature_run( feature_id=SINGLE_RUN_FEATURE_ID if single_run else FEATURE_ID, - billing_feature=args.billing_feature - or _BILLING_FEATURE_BY_REFERRER.get( - args.referrer, billing.SEER_BILLING_FEATURE_AUTOFIX_HEADLESS - ), + billing_feature=billing_feature, payload=payload.dict(), referrer=args.referrer.value, user_org_context=user_org_context, @@ -415,7 +417,6 @@ def trigger_autofix_feature( title=f"Autofix — {short_id}" if single_run else f"Autofix RCA — {short_id}", flush=args.flush, extras=extras, - billing_feature=args.billing_feature.value, ) elif existing_agent_run is not None: run = client.continue_feature_run( diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py index 85f0547d0c16..33c2119afaf0 100644 --- a/src/sentry/seer/autofix/issue_summary.py +++ b/src/sentry/seer/autofix/issue_summary.py @@ -43,7 +43,6 @@ is_seer_autotriggered_autofix_rate_limited_and_increment, is_seer_seat_based_tier_enabled, ) -from sentry.seer.credits import SeerBillingFeature from sentry.seer.entrypoints.cache import SeerOperatorAutofixCache from sentry.seer.entrypoints.operator import SeerAutofixOperator from sentry.seer.models import IssueSummary, SummarizeIssueResponse @@ -197,7 +196,6 @@ def _trigger_autofix_task( step=AutofixStep.ROOT_CAUSE, referrer=referrer, run_id=None, - billing_feature=SeerBillingFeature.AUTOFIX, stopping_point=stopping_point, ) with action_context_scope(ActionSource.SYSTEM, SYSTEM_ACTOR): diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 87a4e09376e1..a52b3caec79c 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -3,6 +3,10 @@ from collections.abc import Callable from enum import StrEnum +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_UNSPECIFIED, + SeerBillingFeature, +) from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsRequest, ) @@ -21,12 +25,18 @@ class SeerOperation(StrEnum): AUTOFIX_SINGLE_RUN_REPLY = "autofix_single_run_reply" -class SeerBillingFeature(StrEnum): - """Seer features as tagged on LLM proxy calls, which billing maps to data categories.""" - - AGENTIC_TRIAGE = "agentic_triage" # Marks the Autofix runs agentic triage starts. - AUTOFIX = "autofix" - EXPLORER = "explorer" +def billing_feature_label(feature: int) -> str: + """ + The label LLM proxy usage and billing use for a SeerBillingFeature value: the enum name, + lowercased, without its prefix. Missing attribution is "unattributed", like the LLM proxy. + """ + if feature == SEER_BILLING_FEATURE_UNSPECIFIED: + return "unattributed" + return ( + SeerBillingFeature.Name(SeerBillingFeature.ValueType(feature)) + .removeprefix("SEER_BILLING_FEATURE_") + .lower() + ) # TODO(CW-2314): Fake value until we have real estimates. @@ -47,7 +57,7 @@ class SeerBillingFeature(StrEnum): def has_enough_seer_credits( - organization: Organization, feature: SeerBillingFeature, operation: SeerOperation + organization: Organization, feature: int, operation: SeerOperation ) -> bool: """ Checks whether the organization has enough Seer credits to run the operation, @@ -56,7 +66,7 @@ def has_enough_seer_credits( response = seer_credit.backend.check_seer_credits( CheckSeerCreditsRequest( organization_id=organization.id, - feature_id=feature.value, + feature_id=billing_feature_label(feature), predicted_credit_usage=SEER_OPERATION_ESTIMATES[operation](), take_lease=False, ) @@ -65,7 +75,7 @@ def has_enough_seer_credits( def can_run_seer_operation( - organization: Organization, feature: SeerBillingFeature, operation: SeerOperation + organization: Organization, feature: int, operation: SeerOperation ) -> bool: """ Checks whether the organization is token-billed and has enough Seer credits to run the operation. diff --git a/src/sentry/seer/endpoints/admin_autofix_single_run.py b/src/sentry/seer/endpoints/admin_autofix_single_run.py index 296b2960fe43..0ae26e851eb6 100644 --- a/src/sentry/seer/endpoints/admin_autofix_single_run.py +++ b/src/sentry/seer/endpoints/admin_autofix_single_run.py @@ -14,7 +14,6 @@ from sentry.seer.autofix.exceptions import NoSeerQuotaException, SeerOutOfCreditsException from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import get_automation_handoff -from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerPermissionError from sentry.viewer_context import Actor, ActorType, ViewerContext, viewer_context_scope @@ -70,7 +69,6 @@ def post(self, request: Request) -> Response: group=group, step=AutofixStep.ROOT_CAUSE, referrer=referrer, - billing_feature=SeerBillingFeature.AUTOFIX, user=user, actor_user_id=actor.user_id, single_run=True, diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 6421ec335bf5..175920553276 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -83,7 +83,6 @@ CodingAgentProviderType, has_project_connected_repos, ) -from sentry.seer.credits import SeerBillingFeature from sentry.seer.endpoints.organization_seer_onboarding_check import ( has_supported_scm_integration, ) @@ -519,7 +518,8 @@ def post( status=status.HTTP_409_CONFLICT, ) - billing_feature = SeerBillingFeature.AUTOFIX + # The client can supply referrer. Resolve attribution from the request instead. + billing_feature: int = billing.SEER_BILLING_FEATURE_AUTOFIX_HEADLESS if resolved_run_id is not None: try: run_state = get_autofix_run_state(group, resolved_run_id) @@ -569,8 +569,6 @@ def post( status=status.HTTP_409_CONFLICT, ) - # The client can supply referrer. Resolve attribution from the request instead. - billing_feature = billing.SEER_BILLING_FEATURE_AUTOFIX_HEADLESS if is_autofix_kickoff: actor = resolve_action_actor(request) source = resolve_action_source(request) @@ -602,7 +600,6 @@ def post( AutofixStoppingPoint(stopping_point) if stopping_point else None ), run_id=resolved_run_id, - billing_feature=billing_feature, user_context=user_context, insert_index=data.get("insert_index"), user=acting_user(request.user), diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index f707373c3788..e696a8799c43 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -11,6 +11,9 @@ from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_SEER_AGENT, +) from sentry import features from sentry.api.api_owners import ApiOwner @@ -38,7 +41,7 @@ from sentry.seer.autofix.feature.dispatch import continue_single_run, get_run_billing_feature from sentry.seer.autofix.feature.models import SINGLE_RUN_FEATURE_ID from sentry.seer.autofix.single_run import is_single_run -from sentry.seer.credits import SeerBillingFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.seer.endpoints.group_ai_autofix import RUN_IN_FLIGHT_CODE from sentry.seer.endpoints.utils import ( SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, @@ -449,7 +452,7 @@ def post( raise PermissionDenied(error) # Do not bill for dashboard generation. if not is_dashboard_generate_run and not can_run_seer_operation( - organization, SeerBillingFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + organization, SEER_BILLING_FEATURE_SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ): return Response( {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index de07cf1b40d2..3e36496c92ed 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -3,6 +3,12 @@ from rest_framework import serializers from rest_framework.request import Request from rest_framework.response import Response +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_AUTOFIX_WEB, + SEER_BILLING_FEATURE_SEER_AGENT, + SEER_BILLING_FEATURE_UNSPECIFIED, + SeerBillingFeature, +) from sentry.api.api_owners import ApiOwner from sentry.api.api_publish_status import ApiPublishStatus @@ -10,14 +16,19 @@ from sentry.api.bases.organization import OrganizationEndpoint from sentry.models.organization import Organization from sentry.ratelimits.config import RateLimitConfig -from sentry.seer.credits import SeerBillingFeature, SeerOperation, can_run_seer_operation +from sentry.seer.credits import SeerOperation, can_run_seer_operation from sentry.types.ratelimit import RateLimit, RateLimitCategory class SeerCreditCheckSerializer(serializers.Serializer[dict[str, str]]): operation = serializers.ChoiceField(choices=[operation.value for operation in SeerOperation]) billing_feature = serializers.ChoiceField( - choices=[feature.value for feature in SeerBillingFeature], required=False + choices=[ + value + for value in SeerBillingFeature.values() + if value != SEER_BILLING_FEATURE_UNSPECIFIED + ], + required=False, ) @@ -51,11 +62,11 @@ def get(self, request: Request, organization: Organization) -> Response: operation = SeerOperation(serializer.validated_data["operation"]) if "billing_feature" in serializer.validated_data: - feature = SeerBillingFeature(serializer.validated_data["billing_feature"]) + feature = serializer.validated_data["billing_feature"] elif operation == SeerOperation.AGENT_CHAT_MESSAGE: - feature = SeerBillingFeature.EXPLORER + feature = SEER_BILLING_FEATURE_SEER_AGENT else: - feature = SeerBillingFeature.AUTOFIX + feature = SEER_BILLING_FEATURE_AUTOFIX_WEB return Response( {"hasEnoughCredits": can_run_seer_operation(organization, feature, operation)} diff --git a/src/sentry/seer/entrypoints/operator.py b/src/sentry/seer/entrypoints/operator.py index 7d94eae10a60..3a50a3fc0329 100644 --- a/src/sentry/seer/entrypoints/operator.py +++ b/src/sentry/seer/entrypoints/operator.py @@ -3,6 +3,7 @@ from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( SEER_BILLING_FEATURE_SEER_AGENT_SLACK, + SEER_BILLING_FEATURE_UNSPECIFIED, ) from taskbroker_client.retry import Retry @@ -216,7 +217,6 @@ def trigger_autofix_agent( from sentry.seer.autofix.feature.dispatch import get_run_billing_feature from sentry.seer.autofix.feature.models import FEATURE_ID as AUTOFIX_FEATURE_ID from sentry.seer.autofix.steps import AutofixStep - from sentry.seer.credits import SeerBillingFeature event_lifecyle = SeerOperatorEventLifecycleMetric( interaction_type=SeerOperatorInteractionType.OPERATOR_TRIGGER_AUTOFIX, @@ -276,7 +276,6 @@ def trigger_autofix_agent( step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.SLACK, run_id=None, - billing_feature=SeerBillingFeature.AUTOFIX, user=user, ) run_id = run.seer_run_state_id @@ -311,7 +310,7 @@ def trigger_autofix_agent( billing_feature=( get_run_billing_feature(existing_state) if existing_state and existing_state.run_id == run_id - else SeerBillingFeature.AUTOFIX + else SEER_BILLING_FEATURE_UNSPECIFIED ), user=user, ) diff --git a/tests/sentry/seer/agentic_triage/test_delivery.py b/tests/sentry/seer/agentic_triage/test_delivery.py index 42c0fec2f7f8..4b78e696840a 100644 --- a/tests/sentry/seer/agentic_triage/test_delivery.py +++ b/tests/sentry/seer/agentic_triage/test_delivery.py @@ -15,7 +15,6 @@ deliver_agentic_triage_result, ) from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerBillingFeature from sentry.seer.models.agentic_triage import ( SeerAgenticTriageRunErrorType, SeerAgenticTriageRunResult, @@ -220,11 +219,6 @@ def test_single_run_option_starts_single_runs(self) -> None: assert mock_trigger.call_args.kwargs["single_run"] is True - def test_starts_autofix_as_agentic_triage(self) -> None: - mock_trigger = self._deliver_autofix_verdict() - - assert mock_trigger.call_args.kwargs["billing_feature"] == SeerBillingFeature.AGENTIC_TRIAGE - def test_dry_run_with_single_run_starts_nothing(self) -> None: mock_trigger = self._deliver_autofix_verdict(single_run=True, dry_run=True) diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 101f11c31517..870dfbaba48b 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -4,6 +4,7 @@ import pytest from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, + SEER_BILLING_FEATURE_UNSPECIFIED, ) from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsResponse, @@ -29,7 +30,6 @@ from sentry.seer.autofix.on_completion_hook import AutofixOnCompletionHook from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -69,7 +69,6 @@ def test_continues_feature_run(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, existing_run_id=123, insert_index=4, @@ -167,7 +166,6 @@ def test_refuses_run_from_another_project(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, user_context="Find a solution", @@ -196,7 +194,6 @@ def test_refuses_run_without_project(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, user_context="Find a solution", @@ -220,7 +217,6 @@ def test_starts_feature_run_and_records_quota(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), stopping_point=AutofixStoppingPoint.OPEN_PR, @@ -248,7 +244,6 @@ def test_starts_feature_run_and_records_quota(self) -> None: "referrer": AutofixReferrer.AGENTIC_TRIAGE.value, "stopping_point": AutofixStoppingPoint.OPEN_PR.value, } - assert start_kwargs["billing_feature"] == SeerBillingFeature.AUTOFIX.value def test_raises_when_out_of_budget(self) -> None: with ( @@ -262,7 +257,6 @@ def test_raises_when_out_of_budget(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -285,7 +279,6 @@ def test_free_cohort_skips_quota_check_and_usage_recording(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), allow_free_cohort=True, @@ -311,7 +304,6 @@ def test_non_free_cohort_org_checks_quota_and_records_usage(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), allow_free_cohort=True, @@ -339,7 +331,6 @@ def test_token_billed_without_credit_checks_skips_quota_and_usage_recording( self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -367,7 +358,6 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), ), @@ -399,7 +389,6 @@ def test_raises_when_out_of_credits_for_a_continued_step( self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.SOLUTION, existing_run_id=123, step_args=SolutionStepArgs(), @@ -430,14 +419,14 @@ def test_starts_agentic_triage_run_under_its_billing_feature( referrer=AutofixReferrer.AGENTIC_TRIAGE, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), - billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, + billing_feature=SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, ), ) assert run is fake_run - assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" start_kwargs = MockClient.return_value.start_feature_run.call_args.kwargs - assert start_kwargs["billing_feature"] == SeerBillingFeature.AGENTIC_TRIAGE + assert start_kwargs["billing_feature"] == SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( @@ -468,12 +457,46 @@ def test_continues_agentic_triage_run_under_its_billing_feature( step=AutofixStep.SOLUTION, existing_run_id=123, step_args=SolutionStepArgs(), - billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, + billing_feature=SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, ), ) assert run is fake_run - assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" + + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + def test_continues_run_without_billing_feature_as_unattributed( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + self.create_seer_agent_run( + run=fake_run, + source=FEATURE_ID, + group=self.group, + project=self.group.project, + ) + + with ( + patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient, + patch("sentry.seer.autofix.feature.dispatch.quotas"), + ): + MockClient.return_value.continue_feature_run.return_value = fake_run + + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.SLACK, + step=AutofixStep.SOLUTION, + existing_run_id=123, + step_args=SolutionStepArgs(), + ), + ) + + assert mock_check_seer_credits.call_args.args[0].feature_id == "unattributed" def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -489,7 +512,6 @@ def test_allows_async_dispatch(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), flush=False, @@ -513,7 +535,6 @@ def test_forwards_run_options_to_client(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.ROOT_CAUSE, step_args=RCAStepArgs(), user=user, @@ -539,7 +560,6 @@ def test_code_changes_enables_coding(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.CODE_CHANGES, step_args=CodeChangesStepArgs(), ), @@ -572,7 +592,6 @@ def test_pr_iteration_enables_coding_and_sends_its_step_args(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.GITHUB_PR_COMMENT, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.PR_ITERATION, existing_run_id=123, step_args=PrIterationStepArgs( @@ -610,7 +629,6 @@ def test_code_changes_rejected_when_coding_disabled(self) -> None: self.group, AutofixFeatureArgs( referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, step=AutofixStep.CODE_CHANGES, step_args=CodeChangesStepArgs(), existing_run_id=123, @@ -629,7 +647,6 @@ def _start(self, **kwargs: Any) -> MagicMock: "referrer": AutofixReferrer.AGENTIC_TRIAGE, "step": AutofixStep.ROOT_CAUSE, "step_args": RCAStepArgs(), - "billing_feature": SeerBillingFeature.AUTOFIX, **kwargs, } fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -771,7 +788,7 @@ def _reply( self, *, can_publish: bool = True, - billing_feature: SeerBillingFeature = SeerBillingFeature.AUTOFIX, + billing_feature: int = SEER_BILLING_FEATURE_UNSPECIFIED, ) -> MagicMock: with patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as mock_client_cls: mock_client_cls.return_value.continue_feature_run.return_value = self.seer_run @@ -838,7 +855,7 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> user=self.user, can_publish=True, referrer=AutofixReferrer.SEER_EXPLORER, - billing_feature=SeerBillingFeature.AUTOFIX, + billing_feature=SEER_BILLING_FEATURE_UNSPECIFIED, ) mock_client_cls.return_value.continue_feature_run.assert_not_called() @@ -851,6 +868,6 @@ def test_raises_when_out_of_credits(self, mock_check_seer_credits: MagicMock) -> def test_checks_credits_under_the_run_billing_feature( self, mock_check_seer_credits: MagicMock ) -> None: - self._reply(billing_feature=SeerBillingFeature.AGENTIC_TRIAGE) + self._reply(billing_feature=SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE) - assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" diff --git a/tests/sentry/seer/autofix/test_autofix_agent.py b/tests/sentry/seer/autofix/test_autofix_agent.py index fb12a238237f..4d301fe51661 100644 --- a/tests/sentry/seer/autofix/test_autofix_agent.py +++ b/tests/sentry/seer/autofix/test_autofix_agent.py @@ -4,6 +4,9 @@ import pytest from django.test import override_settings from rest_framework.exceptions import PermissionDenied +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, +) from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsResponse, ) @@ -36,7 +39,6 @@ from sentry.seer.autofix.pr_iteration.feedback_sources.user_ui import UserUIFeedbackSource from sentry.seer.autofix.pr_iteration.iterations import PrIterationNoPullRequestException from sentry.seer.autofix.steps import AutofixStep -from sentry.seer.credits import SeerBillingFeature from sentry.seer.models import SeerApiError, SeerPermissionError from sentry.seer.models.run import SeerAgentRun, SeerRun from sentry.sentry_apps.utils.webhooks import SeerActionType @@ -346,7 +348,6 @@ def assert_activity_exists(**_kwargs: object) -> None: group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=None, ) @@ -389,7 +390,6 @@ def test_root_cause_routes_to_seer_feature( group=self.group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=None, ) @@ -437,7 +437,6 @@ def test_solution_routes_to_seer_feature( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Keep compatibility", insert_index=3, @@ -476,7 +475,6 @@ def test_solution_without_prior_run_routes_to_seer_feature( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert result == feature_run @@ -513,7 +511,6 @@ def test_code_changes_routes_to_seer_feature( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Keep compatibility", insert_index=3, @@ -563,7 +560,6 @@ def test_continued_solution_attributes_actor( group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, actor_user_id=self.user.id, ) @@ -597,7 +593,6 @@ def test_root_cause_without_actor_is_unattributed( group=self.group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert mock_record_activity.call_args.kwargs["activity_attribution"] is None @@ -618,7 +613,6 @@ def test_code_changes_without_prior_run_routes_to_seer_feature( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert result == feature_run @@ -659,7 +653,6 @@ def test_pr_iteration_routes_to_seer_feature( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.GITHUB_PR_COMMENT, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, user_context="Rename the helper.", feedback=[ @@ -712,7 +705,6 @@ def test_pr_iteration_requires_existing_pr( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -736,7 +728,6 @@ def test_free_cohort_skips_quota_check( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.AGENTIC_TRIAGE, - billing_feature=SeerBillingFeature.AUTOFIX, allow_free_cohort=True, ) @@ -756,7 +747,6 @@ def test_continued_run_permitted_with_no_remaining_budget( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -780,7 +770,6 @@ def test_continued_run_requires_matching_group( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -802,7 +791,6 @@ def test_continued_run_rejects_when_not_mirrored( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -835,7 +823,6 @@ def test_continued_run_rejects_foreign_project_even_when_metadata_matches( group=self.group, step=AutofixStep.PR_ITERATION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=67890, ) @@ -861,7 +848,6 @@ def test_solution_rejects_run_from_another_project(self, mock_feature: MagicMock group=self.group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, run_id=777, user_context="Find a solution", ) @@ -925,7 +911,6 @@ def test_counts_success( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -947,7 +932,6 @@ def test_counts_declined_when_no_quota( self.group, AutofixStep.ROOT_CAUSE, AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -977,7 +961,6 @@ def test_counts_declined_when_out_of_credits( self.group, AutofixStep.ROOT_CAUSE, AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -1005,7 +988,6 @@ def test_counts_error_when_seer_rejects_run( group=self.group, step=AutofixStep.CODE_CHANGES, referrer=AutofixReferrer.SLACK, - billing_feature=SeerBillingFeature.AUTOFIX, ) assert _trigger_attempts(mock_incr) == [ @@ -1639,7 +1621,7 @@ def _push( self, mock_post: MagicMock, features: dict[str, bool] | None = None, - billing_feature: SeerBillingFeature | None = None, + billing_feature: int | None = None, **kwargs: Any, ) -> dict[str, Any]: """Push with a minimal run state and return the payload sent to Seer.""" @@ -1718,10 +1700,10 @@ def test_checks_credits_under_the_run_billing_feature( "organizations:seer-credit-checks": True, "organizations:token-billed-seer-enabled": True, }, - billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, + billing_feature=SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, ) - assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" mock_post.assert_called_once() @patch("sentry.seer.agent.client.make_agent_update_request") @@ -1879,7 +1861,6 @@ def test_single_run_skips_started_events( group=group, step=AutofixStep.ROOT_CAUSE, referrer=AutofixReferrer.UNKNOWN, - billing_feature=SeerBillingFeature.AUTOFIX, single_run=True, ) diff --git a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py index 39dde9659351..d68d075f916d 100644 --- a/tests/sentry/seer/endpoints/test_admin_autofix_retry.py +++ b/tests/sentry/seer/endpoints/test_admin_autofix_retry.py @@ -2,6 +2,9 @@ from unittest.mock import MagicMock, patch from rest_framework.response import Response +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_UNSPECIFIED, +) from sentry.models.organization import Organization from sentry.seer.agent.client_models import MemoryBlock, Message, RepoPRState, SeerRunState @@ -9,7 +12,6 @@ from sentry.seer.autofix.exceptions import SeerOutOfCreditsException from sentry.seer.autofix.feature.models import FEATURE_ID from sentry.seer.autofix.steps import AutofixStep -from sentry.seer.credits import SeerBillingFeature from sentry.testutils.cases import APITestCase from sentry.viewer_context import Actor, ActorType, ViewerContext, get_viewer_context @@ -94,7 +96,7 @@ def test_retries_failed_step_as_system( step=AutofixStep.SOLUTION, referrer=AutofixReferrer.NIGHT_SHIFT, run_id=1, - billing_feature=SeerBillingFeature.AUTOFIX, + billing_feature=SEER_BILLING_FEATURE_UNSPECIFIED, insert_index=2, ) assert viewer_contexts == [ diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 4c259e2d1545..0476d0137405 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -6,12 +6,14 @@ from django.test import override_settings from django.utils import timezone -from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( - CheckSeerCreditsResponse, -) from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, SEER_BILLING_FEATURE_AUTOFIX_HEADLESS, SEER_BILLING_FEATURE_AUTOFIX_WEB, + SEER_BILLING_FEATURE_UNSPECIFIED, +) +from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( + CheckSeerCreditsResponse, ) from sentry.integrations.services.integration import RpcIntegration @@ -45,7 +47,7 @@ from sentry.seer.autofix.pr_iteration.queue import QueuedAutofixFeedback from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerBillingFeature, SeerOperation +from sentry.seer.credits import SeerOperation from sentry.seer.models import SeerPermissionError from sentry.seer.models.run import SeerRun from sentry.testutils.cases import APITestCase, SnubaTestCase @@ -1267,7 +1269,7 @@ def test_insert_index_passed_through( group=group, step=AutofixStep.SOLUTION, referrer=AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, - billing_feature=SEER_BILLING_FEATURE_AUTOFIX_HEADLESS, + billing_feature=SEER_BILLING_FEATURE_UNSPECIFIED, stopping_point=None, run_id=42, user_context=None, @@ -1709,7 +1711,7 @@ def test_pr_iteration_rejected_when_out_of_credits( "code": "seer_out_of_credits", } mock_can_run.assert_called_once_with( - group.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + group.organization, SEER_BILLING_FEATURE_UNSPECIFIED, SeerOperation.AUTOFIX_PR_ITERATION ) mock_enqueue.assert_not_called() @@ -1738,7 +1740,7 @@ def test_pr_iteration_checks_credits_under_the_run_billing_feature( status="completed", updated_at="2024-01-01T00:00:00Z", repo_pr_states={"owner/repo": RepoPRState(repo_name="owner/repo", pr_number=7)}, - billing_feature=SeerBillingFeature.AGENTIC_TRIAGE, + billing_feature=SEER_BILLING_FEATURE_AUTOFIX_AGENTIC_TRIAGE, ) self.login_as(user=self.user) @@ -1749,7 +1751,7 @@ def test_pr_iteration_checks_credits_under_the_run_billing_feature( ) assert response.status_code == 202, response.data - assert mock_check_seer_credits.call_args.args[0].feature_id == "agentic_triage" + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" mock_enqueue.assert_called_once() mock_consume.assert_called_once() diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 92246f1f6a68..64c91421924a 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -33,7 +33,7 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, - feature_id="autofix", + feature_id="autofix_web", predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) @@ -50,7 +50,7 @@ def test_get_returns_not_enough_credits_when_billing_has_no_credits( response = self.get_success_response(self.organization.slug, operation="agent_chat_message") assert response.data == {"hasEnoughCredits": False} - assert mock_check_seer_credits.call_args.args[0].feature_id == "explorer" + assert mock_check_seer_credits.call_args.args[0].feature_id == "seer_agent" def test_get_returns_has_enough_credits_when_credit_checks_disabled(self) -> None: response = self.get_success_response(self.organization.slug, operation="autofix_solution") diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index b4dd194a654f..e3ec9e7ec8fb 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -2,17 +2,16 @@ from unittest.mock import MagicMock, patch +from sentry_protos.billing.v1.common.v1.seer_billing_feature_pb2 import ( + SEER_BILLING_FEATURE_AUTOFIX_WEB, + SEER_BILLING_FEATURE_SEER_AGENT, +) from sentry_protos.billing.v1.services.seer_feature_billing.v1.endpoint_check_seer_credits_pb2 import ( CheckSeerCreditsRequest, CheckSeerCreditsResponse, ) -from sentry.seer.credits import ( - SEER_OPERATION_ESTIMATES, - SeerBillingFeature, - SeerOperation, - can_run_seer_operation, -) +from sentry.seer.credits import SEER_OPERATION_ESTIMATES, SeerOperation, can_run_seer_operation from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -31,7 +30,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SEER_BILLING_FEATURE_AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -44,7 +43,7 @@ def test_can_run_seer_operation_skips_billing_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SEER_BILLING_FEATURE_AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -57,12 +56,12 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert not can_run_seer_operation( - self.organization, SeerBillingFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SEER_BILLING_FEATURE_AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, - feature_id="autofix", + feature_id="autofix_web", predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) @@ -77,5 +76,5 @@ def test_can_run_seer_operation_returns_true_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerBillingFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + self.organization, SEER_BILLING_FEATURE_SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ) From ab2eb19ce420705fc9e7ed64f16541ed70592e98 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:03:42 +0000 Subject: [PATCH 45/45] ref(seer-billing): Move the token-billed post-process skip to its own PR Co-Authored-By: sofia --- src/sentry/tasks/post_process.py | 6 ++---- tests/sentry/tasks/test_post_process.py | 18 ------------------ 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 249165f96a5a..c51b93d836d8 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -1652,9 +1652,7 @@ def kick_off_seer_automation(job: PostProcessJob) -> None: event = job["event"] group = event.group - if is_seer_seat_based_tier_enabled(group.organization) or features.has( - "organizations:token-billed-seer-enabled", group.organization - ): + if is_seer_seat_based_tier_enabled(group.organization): if group.seer_fixability_score is not None: return # Issues created in last 5 minutes only. This can be removed once this is live past 1 week. @@ -1662,7 +1660,7 @@ def kick_off_seer_automation(job: PostProcessJob) -> None: if (timezone.now() - group.first_seen).total_seconds() > 300: return # Generate issue summary and fixability score if not cached. - # Agentic Triage handles automations for seat-based and token-billed orgs but still needs these. + # Agentic Triage handles automations for seat-based orgs but still needs these. cache_key = get_issue_summary_cache_key(group.id) if cache.get(cache_key) is None: lock_key, lock_name = get_issue_summary_lock_key(group.id) diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index 1075f0fd3727..6c8fc0492f42 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -3809,24 +3809,6 @@ def test_seat_based_org_generates_summary_for_new_issues( ) mock_generate_summary_only.assert_called_once_with(require_group(event).id) - @patch("sentry.seer.autofix.utils.is_seer_seat_based_tier_enabled", return_value=False) - @patch("sentry.tasks.seer.autofix.generate_summary_and_run_automation.delay") - @patch("sentry.tasks.seer.autofix.generate_issue_summary_only.delay") - @override_settings(SENTRY_SELF_HOSTED=False) - def test_token_billed_org_generates_summary_for_new_issues( - self, - mock_generate_summary_only: MagicMock, - mock_generate_summary_and_run_automation: MagicMock, - mock_seat_based_tier: MagicMock, - ) -> None: - event = self.create_event(data={"message": "testing"}, project_id=self.project.id) - with self.feature("organizations:token-billed-seer-enabled"): - self.call_post_process_group( - is_new=True, is_regression=False, is_new_group_environment=True, event=event - ) - mock_generate_summary_only.assert_called_once_with(require_group(event).id) - mock_generate_summary_and_run_automation.assert_not_called() - def setUp(self) -> None: super().setUp()