From 78d465a9dd7c398e3bbd72965ecf355e42e0c0a8 Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Fri, 13 Nov 2026 17:08:52 -0800 Subject: [PATCH 1/2] fix(seer): Send billed feature labels in the Seer credit check --- src/sentry/seer/autofix/autofix_agent.py | 4 +- src/sentry/seer/autofix/feature/dispatch.py | 22 ++++++++- .../autofix/pr_iteration/manual_trigger.py | 2 +- src/sentry/seer/credits.py | 13 ++++-- .../endpoints/organization_seer_agent_chat.py | 2 +- .../organization_seer_credit_check.py | 4 +- src/sentry/seer/entrypoints/operator.py | 1 + src/sentry/seer/entrypoints/slack/tasks.py | 2 +- .../seer/autofix/feature/test_dispatch.py | 46 ++++++++++++++++++- .../seer/endpoints/test_group_ai_autofix.py | 2 +- .../test_organization_seer_credit_check.py | 4 +- .../seer/entrypoints/slack/test_tasks.py | 2 +- tests/sentry/seer/test_credits.py | 10 ++-- 13 files changed, 93 insertions(+), 21 deletions(-) diff --git a/src/sentry/seer/autofix/autofix_agent.py b/src/sentry/seer/autofix/autofix_agent.py index d167f8874868..159eaf6482e3 100644 --- a/src/sentry/seer/autofix/autofix_agent.py +++ b/src/sentry/seer/autofix/autofix_agent.py @@ -838,7 +838,9 @@ def trigger_push_changes( ): raise PermissionDenied("Code generation is disabled for this organization") - check_autofix_billing(group.organization, SeerOperation.AUTOFIX_OPEN_PR, is_new_run=False) + check_autofix_billing( + group.organization, SeerOperation.AUTOFIX_OPEN_PR, referrer=referrer, is_new_run=False + ) client = get_autofix_agent_client(group, user=user) diff --git a/src/sentry/seer/autofix/feature/dispatch.py b/src/sentry/seer/autofix/feature/dispatch.py index 767ce73ddffd..4a7ff82e933f 100644 --- a/src/sentry/seer/autofix/feature/dispatch.py +++ b/src/sentry/seer/autofix/feature/dispatch.py @@ -158,10 +158,25 @@ def _org_allows_coding(group: Group) -> bool: ) +def seer_feature_for_referrer(referrer: AutofixReferrer) -> SeerFeature: + match referrer: + case AutofixReferrer.AGENTIC_TRIAGE: + return SeerFeature.AUTOFIX_AGENTIC_TRIAGE + case AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY: + return SeerFeature.AUTOFIX_POST_PROCESS + case AutofixReferrer.SLACK: + return SeerFeature.AUTOFIX_INTEGRATIONS + case AutofixReferrer.WEB | AutofixReferrer.GROUP_AUTOFIX_ENDPOINT: + return SeerFeature.AUTOFIX_WEB + case _: + return SeerFeature.AUTOFIX_HEADLESS + + def check_autofix_billing( organization: Organization, operation: SeerOperation, *, + referrer: AutofixReferrer, is_new_run: bool, skip_quota_for_free_cohort: bool = False, ) -> bool: @@ -174,7 +189,9 @@ def check_autofix_billing( if features.has("organizations:token-billed-seer-enabled", organization): if features.has( "organizations:seer-credit-checks", organization - ) and not has_enough_seer_credits(organization, SeerFeature.AUTOFIX, operation): + ) and not has_enough_seer_credits( + organization, seer_feature_for_referrer(referrer), operation + ): raise SeerOutOfCreditsException() return False @@ -203,6 +220,7 @@ def continue_single_run( check_autofix_billing( group.organization, SeerOperation.AUTOFIX_SINGLE_RUN_REPLY, + referrer=referrer, is_new_run=False, ) enable_coding = _org_allows_coding(group) @@ -234,6 +252,7 @@ def continue_single_run_from_pr_feedback( check_autofix_billing( group.organization, SeerOperation.AUTOFIX_PR_ITERATION, + referrer=referrer, is_new_run=False, ) enable_coding = _org_allows_coding(group) @@ -327,6 +346,7 @@ def trigger_autofix_feature( uses_quota = check_autofix_billing( group.organization, args.step.seer_operation, + referrer=args.referrer, is_new_run=is_new_run, # Free cohort orgs bypass quota only when called from agentic triage # (allow_free_cohort=True). Not exposed via the API. diff --git a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py index c53e909e070e..dcb19b4f4750 100644 --- a/src/sentry/seer/autofix/pr_iteration/manual_trigger.py +++ b/src/sentry/seer/autofix/pr_iteration/manual_trigger.py @@ -88,7 +88,7 @@ def handle_ui_feedback( # Check credits here, since the queued feedback is consumed after this request returns. if not can_run_seer_operation( - group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + group.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_PR_ITERATION ): return Response( { diff --git a/src/sentry/seer/credits.py b/src/sentry/seer/credits.py index 9db8f1e921b0..2841452a9073 100644 --- a/src/sentry/seer/credits.py +++ b/src/sentry/seer/credits.py @@ -26,10 +26,15 @@ class SeerOperation(StrEnum): class SeerFeature(StrEnum): - """Seer features as tagged on LLM proxy calls, which billing maps to data categories.""" - - AUTOFIX = "autofix" - EXPLORER = "explorer" + """Seer billing labels, which billing maps to data categories.""" + + AUTOFIX_WEB = "autofix_web" + AUTOFIX_HEADLESS = "autofix_headless" + AUTOFIX_INTEGRATIONS = "autofix_integrations" + AUTOFIX_POST_PROCESS = "autofix_post_process" + AUTOFIX_AGENTIC_TRIAGE = "autofix_agentic_triage" + SEER_AGENT = "seer_agent" + SEER_AGENT_SLACK = "seer_agent_slack" # Credits an operation is expected to use, about the p90 of what one costs. A credit is one cent. diff --git a/src/sentry/seer/endpoints/organization_seer_agent_chat.py b/src/sentry/seer/endpoints/organization_seer_agent_chat.py index fbf2e2a11243..60c42123a53e 100644 --- a/src/sentry/seer/endpoints/organization_seer_agent_chat.py +++ b/src/sentry/seer/endpoints/organization_seer_agent_chat.py @@ -453,7 +453,7 @@ def post( raise PermissionDenied(error) # Do not bill for dashboard generation. if not is_dashboard_generate_run and not can_run_seer_operation( - organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + organization, SeerFeature.SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ): return Response( {"detail": SEER_AGENT_CHAT_OUT_OF_CREDITS_DETAIL, "code": SEER_OUT_OF_CREDITS_CODE}, diff --git a/src/sentry/seer/endpoints/organization_seer_credit_check.py b/src/sentry/seer/endpoints/organization_seer_credit_check.py index 396016a3bc3e..c1e215d8f3d6 100644 --- a/src/sentry/seer/endpoints/organization_seer_credit_check.py +++ b/src/sentry/seer/endpoints/organization_seer_credit_check.py @@ -58,9 +58,9 @@ def get(self, request: Request, organization: Organization) -> Response: operation = SeerOperation(serializer.validated_data["operation"]) # TODO: Let the UI pass the feature once it can tell agentic triage runs apart. feature = ( - SeerFeature.EXPLORER + SeerFeature.SEER_AGENT if operation == SeerOperation.AGENT_CHAT_MESSAGE - else SeerFeature.AUTOFIX + else SeerFeature.AUTOFIX_WEB ) credits = check_seer_operation(organization, feature, operation) return Response( diff --git a/src/sentry/seer/entrypoints/operator.py b/src/sentry/seer/entrypoints/operator.py index 727dc5503643..d313a4ffbea1 100644 --- a/src/sentry/seer/entrypoints/operator.py +++ b/src/sentry/seer/entrypoints/operator.py @@ -180,6 +180,7 @@ def can_trigger_autofix(cls, *, group: Group) -> bool: check_autofix_billing( group.organization, SeerOperation.AUTOFIX_ROOT_CAUSE, + referrer=AutofixReferrer.SLACK, is_new_run=True, # Free cohort orgs have no subscription, so the quota check always fails for them. skip_quota_for_free_cohort=True, diff --git a/src/sentry/seer/entrypoints/slack/tasks.py b/src/sentry/seer/entrypoints/slack/tasks.py index b075ea4d6068..016e4f54b5ef 100644 --- a/src/sentry/seer/entrypoints/slack/tasks.py +++ b/src/sentry/seer/entrypoints/slack/tasks.py @@ -171,7 +171,7 @@ def process_mention_for_slack( # Check credits up front so an org without them gets an answer right away. if not can_run_seer_operation( - organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + organization, SeerFeature.SEER_AGENT_SLACK, SeerOperation.AGENT_CHAT_MESSAGE ): lifecycle.record_halt(ProcessMentionHaltReason.OUT_OF_SEER_CREDITS) _send_out_of_credits_message( diff --git a/tests/sentry/seer/autofix/feature/test_dispatch.py b/tests/sentry/seer/autofix/feature/test_dispatch.py index 743b92e1f999..87d691b1e8a4 100644 --- a/tests/sentry/seer/autofix/feature/test_dispatch.py +++ b/tests/sentry/seer/autofix/feature/test_dispatch.py @@ -14,6 +14,7 @@ AutofixFeatureArgs, continue_single_run, continue_single_run_from_pr_feedback, + seer_feature_for_referrer, trigger_autofix_feature, ) from sentry.seer.autofix.feature.models import ( @@ -32,7 +33,7 @@ ) from sentry.seer.autofix.steps import AutofixStep from sentry.seer.autofix.utils import AutofixStoppingPoint -from sentry.seer.credits import SeerOperation +from sentry.seer.credits import SeerFeature, SeerOperation from sentry.seer.models import SeerPermissionError from sentry.testutils.cases import TestCase from sentry.testutils.helpers.features import with_feature @@ -394,6 +395,30 @@ def test_raises_when_out_of_credits_for_a_continued_step( MockClient.return_value.continue_feature_run.assert_not_called() + @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) + @patch( + "sentry.seer_credit.backend.check_seer_credits", + return_value=CheckSeerCreditsResponse(has_enough_credits=True), + ) + def test_agentic_triage_checks_credits_with_free_label( + self, mock_check_seer_credits: MagicMock + ) -> None: + fake_run = self.create_seer_run(organization=self.organization, type="feature_run") + + with patch("sentry.seer.autofix.feature.dispatch.SeerAgentClient") as MockClient: + MockClient.return_value.start_feature_run.return_value = fake_run + trigger_autofix_feature( + self.group, + AutofixFeatureArgs( + referrer=AutofixReferrer.AGENTIC_TRIAGE, + step=AutofixStep.ROOT_CAUSE, + step_args=RCAStepArgs(), + allow_free_cohort=True, + ), + ) + + assert mock_check_seer_credits.call_args.args[0].feature_id == "autofix_agentic_triage" + def test_allows_async_dispatch(self) -> None: fake_run = self.create_seer_run(organization=self.organization, type="feature_run") @@ -528,6 +553,25 @@ def test_code_changes_rejected_when_coding_disabled(self) -> None: ) +@pytest.mark.parametrize( + ("referrer", "feature"), + [ + (AutofixReferrer.AGENTIC_TRIAGE, SeerFeature.AUTOFIX_AGENTIC_TRIAGE), + ( + AutofixReferrer.ISSUE_SUMMARY_POST_PROCESS_FIXABILITY, + SeerFeature.AUTOFIX_POST_PROCESS, + ), + (AutofixReferrer.SLACK, SeerFeature.AUTOFIX_INTEGRATIONS), + (AutofixReferrer.WEB, SeerFeature.AUTOFIX_WEB), + (AutofixReferrer.GROUP_AUTOFIX_ENDPOINT, SeerFeature.AUTOFIX_WEB), + (AutofixReferrer.CLI, SeerFeature.AUTOFIX_HEADLESS), + (AutofixReferrer.ON_COMPLETION_HOOK, SeerFeature.AUTOFIX_HEADLESS), + ], +) +def test_seer_feature_for_referrer(referrer: AutofixReferrer, feature: SeerFeature) -> None: + assert seer_feature_for_referrer(referrer) == feature + + @django_db_all class TestSingleRunStart(TestCase): def setUp(self) -> None: diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index d242ce15556c..2f6f6c2e4916 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1746,7 +1746,7 @@ def test_pr_iteration_rejected_when_out_of_credits( "code": "seer_out_of_credits", } mock_can_run.assert_called_once_with( - group.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_PR_ITERATION + group.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_PR_ITERATION ) mock_enqueue.assert_not_called() diff --git a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py index 4e2c17a66b1b..341070749bd3 100644 --- a/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py +++ b/tests/sentry/seer/endpoints/test_organization_seer_credit_check.py @@ -33,7 +33,7 @@ def test_get_returns_has_enough_credits_when_billing_has_credits( mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, - feature_id="autofix", + feature_id="autofix_web", predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) @@ -50,7 +50,7 @@ def test_get_returns_not_enough_credits_when_billing_has_no_credits( response = self.get_success_response(self.organization.slug, operation="agent_chat_message") assert response.data == {"hasEnoughCredits": False, "denialReason": None} - assert mock_check_seer_credits.call_args.args[0].feature_id == "explorer" + assert mock_check_seer_credits.call_args.args[0].feature_id == "seer_agent" @with_feature(["organizations:seer-credit-checks", "organizations:token-billed-seer-enabled"]) @patch( diff --git a/tests/sentry/seer/entrypoints/slack/test_tasks.py b/tests/sentry/seer/entrypoints/slack/test_tasks.py index 83cef801d1f3..146988662e50 100644 --- a/tests/sentry/seer/entrypoints/slack/test_tasks.py +++ b/tests/sentry/seer/entrypoints/slack/test_tasks.py @@ -451,7 +451,7 @@ def test_out_of_credits_replies_instead_of_running( ) mock_can_run.assert_called_once_with( - self.organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + self.organization, SeerFeature.SEER_AGENT_SLACK, SeerOperation.AGENT_CHAT_MESSAGE ) mock_operator_cls.assert_not_called() self.mock_entrypoint.install.get_thread_history.assert_not_called() diff --git a/tests/sentry/seer/test_credits.py b/tests/sentry/seer/test_credits.py index ab40c1306bec..750ceaa285fa 100644 --- a/tests/sentry/seer/test_credits.py +++ b/tests/sentry/seer/test_credits.py @@ -31,7 +31,7 @@ def test_can_run_seer_operation_skips_billing_when_credit_checks_disabled( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -44,7 +44,7 @@ def test_can_run_seer_operation_skips_billing_when_not_token_billed( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_not_called() @@ -57,12 +57,12 @@ def test_can_run_seer_operation_returns_false_when_billing_has_no_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert not can_run_seer_operation( - self.organization, SeerFeature.AUTOFIX, SeerOperation.AUTOFIX_ROOT_CAUSE + self.organization, SeerFeature.AUTOFIX_WEB, SeerOperation.AUTOFIX_ROOT_CAUSE ) mock_check_seer_credits.assert_called_once_with( CheckSeerCreditsRequest( organization_id=self.organization.id, - feature_id="autofix", + feature_id="autofix_web", predicted_credit_usage=SEER_OPERATION_ESTIMATES[SeerOperation.AUTOFIX_ROOT_CAUSE](), take_lease=False, ) @@ -77,5 +77,5 @@ def test_can_run_seer_operation_returns_true_when_billing_has_credits( self, mock_check_seer_credits: MagicMock ) -> None: assert can_run_seer_operation( - self.organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE + self.organization, SeerFeature.SEER_AGENT, SeerOperation.AGENT_CHAT_MESSAGE ) From 084351743c9ab59e52bd5b37e148d04456eaf6a9 Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Fri, 13 Nov 2026 17:16:21 -0800 Subject: [PATCH 2/2] fix(seer): Don't let a client claim the agentic triage referrer --- src/sentry/seer/endpoints/group_ai_autofix.py | 6 +++++- .../seer/endpoints/test_group_ai_autofix.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/sentry/seer/endpoints/group_ai_autofix.py b/src/sentry/seer/endpoints/group_ai_autofix.py index 58f69081b32d..626626e928e2 100644 --- a/src/sentry/seer/endpoints/group_ai_autofix.py +++ b/src/sentry/seer/endpoints/group_ai_autofix.py @@ -221,10 +221,14 @@ def _parse_autofix_referrer(raw: str | None, request: Request) -> AutofixReferre return AutofixReferrer.MCP return AutofixReferrer.GROUP_AUTOFIX_ENDPOINT try: - return AutofixReferrer(raw) + referrer = AutofixReferrer(raw) except ValueError: logger.warning("group_ai_autofix.unknown_referrer", extra={"referrer": raw}) return AutofixReferrer.UNKNOWN + # Agentic triage runs aren't billed, so a client can't claim to be one. + if referrer == AutofixReferrer.AGENTIC_TRIAGE: + return AutofixReferrer.UNKNOWN + return referrer def _record_user_context_length(request_data: Any) -> None: diff --git a/tests/sentry/seer/endpoints/test_group_ai_autofix.py b/tests/sentry/seer/endpoints/test_group_ai_autofix.py index 2f6f6c2e4916..90a44beb1727 100644 --- a/tests/sentry/seer/endpoints/test_group_ai_autofix.py +++ b/tests/sentry/seer/endpoints/test_group_ai_autofix.py @@ -1248,6 +1248,24 @@ def test_post_seer_explorer_referrer_is_recognized( assert response.status_code == 202, response.data assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.SEER_EXPLORER + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") + def test_post_cannot_claim_agentic_triage_referrer( + self, mock_trigger_explorer: MagicMock + ) -> None: + group = self.create_group() + run = self.create_seer_run(organization=self.organization, seer_run_state_id=123) + mock_trigger_explorer.return_value = run + + self.login_as(user=self.user) + response = self.client.post( + self._get_url(group.id), + data={"step": "root_cause", "referrer": AutofixReferrer.AGENTIC_TRIAGE.value}, + format="json", + ) + + assert response.status_code == 202, response.data + assert mock_trigger_explorer.call_args.kwargs["referrer"] == AutofixReferrer.UNKNOWN + @patch("sentry.seer.endpoints.group_ai_autofix.trigger_autofix_agent") def test_stopping_point(self, mock_trigger_explorer: MagicMock) -> None: """Stopping point forces the step to be root_cause"""