diff --git a/src/sentry/integrations/slack/webhooks/actions/seer_agent.py b/src/sentry/integrations/slack/webhooks/actions/seer_agent.py index c3d0d16c04f9..ac8f022efe4e 100644 --- a/src/sentry/integrations/slack/webhooks/actions/seer_agent.py +++ b/src/sentry/integrations/slack/webhooks/actions/seer_agent.py @@ -249,6 +249,7 @@ def handle_seer_agent_write_approval( integration_id=slack_request.integration.id, thread={"channel_id": channel_id, "thread_ts": cached_message["thread_ts"]}, slack_user_id=slack_user_id, + user_id=identity_user.id, ), ) # Replace the card only after Seer accepts the input so a failed resume remains retryable. diff --git a/src/sentry/seer/agent/client_models.py b/src/sentry/seer/agent/client_models.py index a6ad29a41a56..436ff2988236 100644 --- a/src/sentry/seer/agent/client_models.py +++ b/src/sentry/seer/agent/client_models.py @@ -118,6 +118,7 @@ class RepoPRState(BaseModel): title: str | None = None description: str | None = None integration_id: str | None = None + idempotency_key: str | None = Field(default=None, exclude=True) class Config: extra = "ignore" diff --git a/src/sentry/seer/entrypoints/slack/entrypoint.py b/src/sentry/seer/entrypoints/slack/entrypoint.py index d9eb95e64015..039a85774d3c 100644 --- a/src/sentry/seer/entrypoints/slack/entrypoint.py +++ b/src/sentry/seer/entrypoints/slack/entrypoint.py @@ -22,6 +22,7 @@ from sentry.notifications.platform.service import NotificationService from sentry.notifications.platform.templates.seer import ( SeerAgentError, + SeerAgentOutOfCredits, SeerAgentResponse, SeerAgentWriteApproval, SeerAutofixError, @@ -144,6 +145,8 @@ class SlackAgentCachePayload(TypedDict): integration_id: int thread: SlackThreadDetails slack_user_id: NotRequired[str] + # The Sentry user linked to slack_user_id. + user_id: NotRequired[int | None] class SlackInvestigationCachePayload(TypedDict): @@ -695,6 +698,8 @@ def __init__( self.integration = integration self.install = SlackIntegration(model=integration, organization_id=organization_id) self.slack_user_id = slack_user_id + # Set by the caller once it has resolved the Slack user to a Sentry user. + self.user_id: int | None = None @staticmethod def has_access(organization: Organization | RpcOrganization) -> bool: @@ -717,6 +722,7 @@ def create_agent_cache_payload(self) -> SlackAgentCachePayload: organization_id=self.organization_id, integration_id=self.install.model.id, slack_user_id=self.slack_user_id, + user_id=self.user_id, ) @staticmethod @@ -775,6 +781,33 @@ def on_agent_update( }, ) + replies = SlackSeerAgentReplyCache.get(run_id=run_id) + # Keyed by push: a failed retry leaves the PR's number and commit unchanged. + pr_keys = [ + key + for pr in _get_repo_pr_states(state) + if _is_pr_out_of_credits(pr) + and (key := f"pr_out_of_credits:{pr.repo_name}:{pr.idempotency_key}") not in replies + ] + if pr_keys: + # A turn gets one notice in total, shared by its failed PRs and the run itself. + turn_key = _out_of_credits_key(state) + notice = replies.get(turn_key) + message_ts = ( + notice["message_ts"] + if notice + else send_thread_update( + install=install, + thread=thread, + data=_agent_out_of_credits(cache_payload), + ) + ) + if message_ts: + for key in (turn_key, *pr_keys): + SlackSeerAgentReplyCache.set( + run_id=run_id, key=key, reply=SlackSeerAgentReply(message_ts=message_ts) + ) + replies = SlackSeerAgentReplyCache.get(run_id=run_id) answers = [answer for turn in state.get_turns() if (answer := turn.answer) is not None] latest = next( @@ -824,6 +857,27 @@ def _get_repo_pr_states(state: SeerRunState) -> list[RepoPRState]: return [state.repo_pr_states[repo] for repo in sorted(state.repo_pr_states)] +def _agent_out_of_credits(cache_payload: SlackAgentCachePayload) -> SeerAgentOutOfCredits: + """The out-of-credits notice, written for whoever sent the message that started the run.""" + return SeerAgentOutOfCredits( + organization_id=cache_payload["organization_id"], + has_billing_access=user_has_billing_access( + organization_id=cache_payload["organization_id"], + user_id=cache_payload.get("user_id"), + ), + ) + + +def _is_pr_out_of_credits(pr: RepoPRState) -> bool: + return pr.pr_creation_status == "error" and pr.pr_creation_error_code == OUT_OF_CREDITS + + +def _out_of_credits_key(state: SeerRunState) -> str: + """The reply key of the latest turn's out-of-credits notice, shared by the run and its PRs.""" + turns = state.get_turns() + return f"out_of_credits:{turns[-1].blocks[0].id}" if turns else "out_of_credits" + + def _hash_pr_status(prs: Sequence[RepoPRState]) -> str: payload = orjson.dumps( [ @@ -842,7 +896,7 @@ def _next_agent_reply( cache_payload: SlackAgentCachePayload, state: SeerRunState, replies: Mapping[str, SlackSeerAgentReply], -) -> tuple[str, SeerAgentError | SeerAgentResponse] | None: +) -> tuple[str, SeerAgentError | SeerAgentResponse | SeerAgentOutOfCredits] | None: """The reply to post and its cache key, or None if the thread is up to date. Sends and records the write approval DM when a new approval is pending. @@ -899,6 +953,12 @@ def _next_agent_reply( "I sent you a message." ) repo_pr_states: list[RepoPRState] = [] + elif state.failure_reason == OUT_OF_CREDITS: + # Replace Seer's plain-text notice with the out-of-credits notification. + key = _out_of_credits_key(state) + if key in replies: + return None + return key, _agent_out_of_credits(cache_payload) else: turn = next((turn for turn in reversed(state.get_turns()) if turn.answer), None) if turn is None or turn.answer is None: diff --git a/src/sentry/seer/entrypoints/slack/tasks.py b/src/sentry/seer/entrypoints/slack/tasks.py index 1074d7751c9d..b77fec791a77 100644 --- a/src/sentry/seer/entrypoints/slack/tasks.py +++ b/src/sentry/seer/entrypoints/slack/tasks.py @@ -8,12 +8,14 @@ from taskbroker_client.retry import Retry from sentry import analytics +from sentry.auth.access import from_user from sentry.identity.services.identity import identity_service from sentry.integrations.messaging.metrics import SeerSlackHaltReason from sentry.integrations.services.integration.model import RpcIntegration from sentry.integrations.services.integration.service import integration_service from sentry.models.organization import Organization from sentry.notifications.platform.slack.provider import SlackRenderable +from sentry.notifications.platform.templates.seer import SeerAgentOutOfCredits from sentry.seer.credits import SeerFeature, SeerOperation, can_run_seer_operation from sentry.seer.entrypoints.metrics import ( SlackEntrypointEventLifecycleMetric, @@ -25,7 +27,11 @@ SlackSeerAgentResponded, ) from sentry.seer.entrypoints.slack.cache import SlackSeerAgentMessageCache -from sentry.seer.entrypoints.slack.entrypoint import EntrypointSetupError, SlackAgentEntrypoint +from sentry.seer.entrypoints.slack.entrypoint import ( + EntrypointSetupError, + SlackAgentEntrypoint, + SlackThreadDetails, +) from sentry.seer.entrypoints.slack.mention import ( SlackMessageLink, build_linked_messages_context, @@ -37,6 +43,7 @@ from sentry.seer.entrypoints.slack.messaging import ( send_halt_message, send_not_org_member_message, + send_thread_update, ) from sentry.seer.entrypoints.slack.metrics import ( ProcessMentionFailureReason, @@ -144,6 +151,7 @@ def process_mention_for_slack( status="", ) return + entrypoint.user_id = user.id if not organization.has_access(user): lifecycle.record_halt(ProcessMentionHaltReason.USER_NOT_ORG_MEMBER) @@ -161,11 +169,17 @@ def process_mention_for_slack( ) return - # Check credits before fetching context or starting a run. + # Check credits up front so an org without them gets an answer right away. if not can_run_seer_operation( organization, SeerFeature.EXPLORER, SeerOperation.AGENT_CHAT_MESSAGE ): lifecycle.record_halt(ProcessMentionHaltReason.OUT_OF_SEER_CREDITS) + _send_out_of_credits_message( + entrypoint=entrypoint, + organization=organization, + user=user, + thread_ts=thread_ts or ts, + ) # The webhook set "is thinking..." before queueing this task; clear it. entrypoint.install.set_thread_status( channel_id=channel_id, @@ -408,6 +422,28 @@ def _resolve_linked_messages( ) +def _send_out_of_credits_message( + *, + entrypoint: SlackAgentEntrypoint, + organization: Organization, + user: RpcUser, + thread_ts: str, +) -> None: + """Tell the user, in the thread and only to them, that the org is out of Seer credits.""" + try: + send_thread_update( + install=entrypoint.install, + thread=SlackThreadDetails(thread_ts=thread_ts, channel_id=entrypoint.channel_id), + data=SeerAgentOutOfCredits( + organization_id=organization.id, + has_billing_access=from_user(user, organization).has_scope("org:billing"), + ), + ephemeral_user_id=entrypoint.slack_user_id, + ) + except Exception as e: + _logger.warning("seer.slack.process_mention.send_out_of_credits_failed", exc_info=e) + + def _send_inaccessible_links_prompt( *, entrypoint: SlackAgentEntrypoint, diff --git a/tests/sentry/integrations/slack/webhooks/actions/test_seer_agent.py b/tests/sentry/integrations/slack/webhooks/actions/test_seer_agent.py index 5c84480b619b..e56ad194e4f9 100644 --- a/tests/sentry/integrations/slack/webhooks/actions/test_seer_agent.py +++ b/tests/sentry/integrations/slack/webhooks/actions/test_seer_agent.py @@ -153,6 +153,7 @@ def test_approve_grants_scopes_and_resumes_run( "thread_ts": "1702424381.221719", }, "slack_user_id": self.external_id, + "user_id": self.user.id, } assert self.mock_post.call_args.kwargs["replace_original"] is True assert self.mock_post.call_args.kwargs["text"] == ":white_check_mark: Access granted." diff --git a/tests/sentry/seer/entrypoints/slack/test_slack.py b/tests/sentry/seer/entrypoints/slack/test_slack.py index a70b1f55a1e6..005ab71ba16d 100644 --- a/tests/sentry/seer/entrypoints/slack/test_slack.py +++ b/tests/sentry/seer/entrypoints/slack/test_slack.py @@ -27,6 +27,7 @@ AUTOFIX_OUT_OF_CREDITS_BILLING_MESSAGE, AUTOFIX_OUT_OF_CREDITS_MEMBER_MESSAGE, SeerAgentError, + SeerAgentOutOfCredits, SeerAgentResponse, SeerAgentWriteApproval, SeerAutofixUpdate, @@ -91,6 +92,11 @@ from sentry.testutils.helpers.options import override_options from sentry.utils.locking import UnableToAcquireLock +SEER_OUT_OF_CREDITS_TEXT = ( + "Your organization doesn't have enough Seer credits for this, so I stopped here. " + "Add credits to continue." +) + class SlackAutofixEntrypointTest(TestCase): def setUp(self) -> None: @@ -1054,6 +1060,10 @@ def test_create_agent_cache_payload(self) -> None: assert payload["thread"]["thread_ts"] == self.thread_ts assert payload["thread"]["channel_id"] == self.channel_id assert payload["slack_user_id"] == self.slack_user_id + assert payload["user_id"] is None + + ep.user_id = self.user.id + assert ep.create_agent_cache_payload()["user_id"] == self.user.id @override_options({"notifications.tracking.sources": [NotificationSource.SEER_AGENT_RESPONSE]}) @patch("sentry.seer.entrypoints.slack.messaging.record_sent") @@ -1379,6 +1389,250 @@ def test_on_agent_update_missing_scope_url_cached_per_thread( second_data = mock_send.call_args.kwargs["data"] assert second_data.missing_scope_settings_url is None + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_out_of_credits_replies_once(self, mock_send: MagicMock) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state(SEER_OUT_OF_CREDITS_TEXT) + state.status = "error" + state.failure_reason = "out_of_credits" + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + mock_send.assert_called_once() + data = mock_send.call_args.kwargs["data"] + assert isinstance(data, SeerAgentOutOfCredits) + assert data.has_billing_access is False + + @patch( + "sentry.models.organizationmember.OrganizationMember.get_scopes", + return_value=frozenset({"org:billing"}), + ) + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_out_of_credits_is_written_for_billing_sender( + self, mock_send: MagicMock, _mock_get_scopes: MagicMock + ) -> None: + ep = self._get_entrypoint() + ep.user_id = self.user.id + cache_payload = ep.create_agent_cache_payload() + state = _agent_state(SEER_OUT_OF_CREDITS_TEXT) + state.status = "error" + state.failure_reason = "out_of_credits" + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + data = mock_send.call_args.kwargs["data"] + assert isinstance(data, SeerAgentOutOfCredits) + assert data.has_billing_access is True + + @patch( + "sentry.models.organizationmember.OrganizationMember.get_scopes", + return_value=frozenset({"org:billing"}), + ) + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_pr_out_of_credits_is_written_for_billing_sender( + self, mock_send: MagicMock, _mock_get_scopes: MagicMock + ) -> None: + ep = self._get_entrypoint() + ep.user_id = self.user.id + cache_payload = ep.create_agent_cache_payload() + state = _agent_state("I tried to open a PR.") + state.repo_pr_states["acme/web"] = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + ) + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + notice = mock_send.call_args.kwargs["data"] + assert isinstance(notice, SeerAgentOutOfCredits) + assert notice.has_billing_access is True + + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_pr_out_of_credits_posts_notice_once( + self, mock_send: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state("I tried to open a PR.") + pr = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + ) + state.repo_pr_states[pr.repo_name] = pr + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + sent = [call.kwargs["data"] for call in mock_send.call_args_list] + assert len(sent) == 2 + answer, notice = sent + assert isinstance(answer, SeerAgentResponse) + assert answer.summary == "I tried to open a PR." + assert answer.repo_pr_states == [pr] + assert isinstance(notice, SeerAgentOutOfCredits) + + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_prs_out_of_credits_share_one_notice( + self, mock_send: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state("I tried to open two PRs.") + for repo_name in ("acme/api", "acme/web"): + state.repo_pr_states[repo_name] = RepoPRState( + repo_name=repo_name, + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + ) + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + sent = [call.kwargs["data"] for call in mock_send.call_args_list] + assert [type(data) for data in sent] == [SeerAgentResponse, SeerAgentOutOfCredits] + + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_run_and_pr_out_of_credits_share_one_notice( + self, mock_send: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state(SEER_OUT_OF_CREDITS_TEXT) + state.status = "error" + state.failure_reason = "out_of_credits" + state.repo_pr_states["acme/web"] = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + ) + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + mock_send.assert_called_once() + assert isinstance(mock_send.call_args.kwargs["data"], SeerAgentOutOfCredits) + + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_out_of_credits_notice_once_per_turn( + self, mock_send: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + pr = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + ) + # The PR fails while the turn is still running, then the run stops for the same reason. + state = _agent_state(None) + state.status = "processing" + state.repo_pr_states[pr.repo_name] = pr + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + state = _agent_state(SEER_OUT_OF_CREDITS_TEXT) + state.status = "error" + state.failure_reason = "out_of_credits" + state.repo_pr_states[pr.repo_name] = pr + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + mock_send.assert_called_once() + assert isinstance(mock_send.call_args.kwargs["data"], SeerAgentOutOfCredits) + + # The next turn runs out too. Its notice is new, but the old PR failure isn't. + state.blocks += [ + MemoryBlock( + id="b3", + message=Message(role="user", content="try again"), + timestamp="2024-01-01T00:00:02Z", + ), + MemoryBlock( + id="b4", + message=Message(role="assistant", content=SEER_OUT_OF_CREDITS_TEXT), + timestamp="2024-01-01T00:00:03Z", + ), + ] + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + assert mock_send.call_count == 2 + assert isinstance(mock_send.call_args.kwargs["data"], SeerAgentOutOfCredits) + + @patch("sentry.seer.entrypoints.slack.entrypoint.edit_thread_message") + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_pr_out_of_credits_notice_per_push_attempt( + self, mock_send: MagicMock, mock_edit: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state("I tried to open a PR.") + state.repo_pr_states["acme/web"] = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + idempotency_key="push-1", + ) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + sent = [type(call.kwargs["data"]) for call in mock_send.call_args_list] + assert sent == [SeerAgentResponse, SeerAgentOutOfCredits] + + # The next turn doesn't retry the PR, so the failure still in the state isn't new. + mock_send.reset_mock() + state.blocks += [ + MemoryBlock( + id="b3", + message=Message(role="user", content="what changed?"), + timestamp="2024-01-01T00:00:02Z", + ), + MemoryBlock( + id="b4", + message=Message(role="assistant", content="Here is what changed."), + timestamp="2024-01-01T00:00:03Z", + ), + ] + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + sent = [type(call.kwargs["data"]) for call in mock_send.call_args_list] + assert sent == [SeerAgentResponse] + + # The turn after retries the PR and runs out again; only the push's key is new. + mock_send.reset_mock() + state.blocks += [ + MemoryBlock( + id="b5", + message=Message(role="user", content="try the PR again"), + timestamp="2024-01-01T00:00:04Z", + ), + MemoryBlock( + id="b6", + message=Message(role="assistant", content="I tried again."), + timestamp="2024-01-01T00:00:05Z", + ), + ] + state.repo_pr_states["acme/web"] = RepoPRState( + repo_name="acme/web", + pr_creation_status="error", + pr_creation_error_code="out_of_credits", + idempotency_key="push-2", + ) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + sent = [type(call.kwargs["data"]) for call in mock_send.call_args_list] + assert sent == [SeerAgentResponse, SeerAgentOutOfCredits] + + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") + def test_on_agent_update_pr_error_without_credits_posts_no_notice( + self, mock_send: MagicMock + ) -> None: + cache_payload = self._get_entrypoint().create_agent_cache_payload() + state = _agent_state("I tried to open a PR.") + pr = RepoPRState(repo_name="acme/web", pr_creation_status="error") + state.repo_pr_states[pr.repo_name] = pr + + SlackAgentEntrypoint.on_agent_update(cache_payload=cache_payload, run_id=12345, state=state) + + mock_send.assert_called_once() + assert isinstance(mock_send.call_args.kwargs["data"], SeerAgentResponse) + @patch("sentry.seer.entrypoints.slack.entrypoint.send_thread_update", return_value="1.1") def test_on_agent_update_does_not_duplicate_answers(self, mock_send: MagicMock) -> None: cache_payload = self._get_entrypoint().create_agent_cache_payload() diff --git a/tests/sentry/seer/entrypoints/slack/test_tasks.py b/tests/sentry/seer/entrypoints/slack/test_tasks.py index 97d1f137c0dc..fc5b367a9f7e 100644 --- a/tests/sentry/seer/entrypoints/slack/test_tasks.py +++ b/tests/sentry/seer/entrypoints/slack/test_tasks.py @@ -8,6 +8,10 @@ from django.test import override_settings from sentry.integrations.messaging.metrics import SeerSlackHaltReason +from sentry.notifications.platform.templates.seer import ( + AGENT_OUT_OF_CREDITS_BILLING_MESSAGE, + AGENT_OUT_OF_CREDITS_MEMBER_MESSAGE, +) from sentry.seer.credits import SeerFeature, SeerOperation from sentry.seer.entrypoints.slack.analytics import ( SlackSeerAgentConversation, @@ -26,6 +30,7 @@ process_mention_for_slack, process_reaction_for_slack, ) +from sentry.shared_integrations.exceptions import IntegrationError from sentry.testutils.asserts import assert_failure_metric, assert_halt_metric from sentry.testutils.cases import TestCase from sentry.testutils.helpers.analytics import ( @@ -94,6 +99,7 @@ def test_happy_path( call_kwargs = mock_operator.trigger_agent.call_args[1] assert call_kwargs["organization"] == self.organization assert call_kwargs["user"] is mock_user + assert mock_entrypoint.user_id == self.user.id assert call_kwargs["prompt"] == "What is causing this issue?" assert call_kwargs["on_page_context"] is None assert call_kwargs["category_key"] == "slack_thread" @@ -416,12 +422,21 @@ def _run_task( mock_agent_cls.return_value = self.mock_entrypoint process_mention_for_slack(**{**TASK_KWARGS, "organization_id": self.organization.id}) + def _sent_text(self) -> str: + send = self.mock_entrypoint.install.send_threaded_ephemeral_message + send.assert_called_once() + call_kwargs = send.call_args.kwargs + assert call_kwargs["slack_user_id"] == self.mock_entrypoint.slack_user_id + assert call_kwargs["channel_id"] == self.mock_entrypoint.channel_id + assert call_kwargs["thread_ts"] == "1234567890.123456" + return call_kwargs["renderable"]["text"] + @patch("sentry.integrations.utils.metrics.EventLifecycle.record_event") @patch("sentry.seer.entrypoints.slack.tasks.can_run_seer_operation", return_value=False) @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator") @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint") @patch("sentry.seer.entrypoints.slack.tasks._resolve_user") - def test_out_of_credits_halts_without_running( + def test_out_of_credits_replies_instead_of_running( self, mock_resolve_user: MagicMock, mock_agent_cls: MagicMock, @@ -440,6 +455,9 @@ def test_out_of_credits_halts_without_running( ) mock_operator_cls.assert_not_called() self.mock_entrypoint.install.get_thread_history.assert_not_called() + + assert self._sent_text() == AGENT_OUT_OF_CREDITS_MEMBER_MESSAGE + self.mock_entrypoint.install.set_thread_status.assert_called_once_with( channel_id="C1234567890", thread_ts="1234567890.123456", @@ -447,6 +465,57 @@ def test_out_of_credits_halts_without_running( ) assert_halt_metric(mock_record, ProcessMentionHaltReason.OUT_OF_SEER_CREDITS) + @patch( + "sentry.models.organizationmember.OrganizationMember.get_scopes", + return_value=frozenset({"org:billing"}), + ) + @patch("sentry.seer.entrypoints.slack.tasks.can_run_seer_operation", return_value=False) + @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator") + @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint") + @patch("sentry.seer.entrypoints.slack.tasks._resolve_user") + def test_out_of_credits_reply_is_written_for_billing_member( + self, + mock_resolve_user: MagicMock, + mock_agent_cls: MagicMock, + mock_operator_cls: MagicMock, + mock_can_run: MagicMock, + _mock_get_scopes: MagicMock, + ) -> None: + mock_operator_cls.has_access.return_value = True + mock_resolve_user.return_value = self.user + mock_agent_cls.return_value = self.mock_entrypoint + + process_mention_for_slack(**{**TASK_KWARGS, "organization_id": self.organization.id}) + + assert self._sent_text() == AGENT_OUT_OF_CREDITS_BILLING_MESSAGE + + @patch("sentry.seer.entrypoints.slack.tasks.can_run_seer_operation", return_value=False) + @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator") + @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint") + @patch("sentry.seer.entrypoints.slack.tasks._resolve_user") + def test_out_of_credits_still_clears_status_when_reply_fails( + self, + mock_resolve_user: MagicMock, + mock_agent_cls: MagicMock, + mock_operator_cls: MagicMock, + mock_can_run: MagicMock, + ) -> None: + mock_operator_cls.has_access.return_value = True + self.mock_entrypoint.install.send_threaded_ephemeral_message.side_effect = IntegrationError( + "slack down" + ) + + self._run_task( + user_id=self.user.id, mock_agent_cls=mock_agent_cls, mock_resolve_user=mock_resolve_user + ) + + mock_operator_cls.assert_not_called() + self.mock_entrypoint.install.set_thread_status.assert_called_once_with( + channel_id="C1234567890", + thread_ts="1234567890.123456", + status="", + ) + @patch("sentry.seer.credits.seer_credit.backend.check_seer_credits") @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator") @patch("sentry.seer.entrypoints.slack.tasks.SlackAgentEntrypoint") @@ -473,6 +542,7 @@ def test_credit_check_flag_on_with_enough_credits_runs( mock_check.assert_called_once() mock_operator_cls.return_value.trigger_agent.assert_called_once() + self.mock_entrypoint.install.send_threaded_ephemeral_message.assert_not_called() @patch("sentry.seer.credits.seer_credit.backend.check_seer_credits") @patch("sentry.seer.entrypoints.slack.tasks.SeerAgentOperator") @@ -497,8 +567,8 @@ def test_credit_check_flag_on_without_enough_credits_halts( mock_resolve_user=mock_resolve_user, ) - mock_check.assert_called_once() mock_operator_cls.assert_not_called() + assert self._sent_text() == AGENT_OUT_OF_CREDITS_MEMBER_MESSAGE class LinkedMessagesContextTest(TestCase):