From e4bbaa41f600166412482d0ae5d5bd2649c2a84c Mon Sep 17 00:00:00 2001 From: Fredrick Onyango Date: Sat, 6 Jun 2026 23:20:38 +0300 Subject: [PATCH] Add metrics + Sentry wiring: metrics_extra, sentry_init, instrument worker and main startup; add tests --- app/main.py | 3 +++ app/metrics_extra.py | 39 ++++++++++++++++++++++++++++++++++++ app/sentry_init.py | 19 ++++++++++++++++++ app/worker.py | 14 +++++++++++++ tests/test_metrics_sentry.py | 27 +++++++++++++++++++++++++ 5 files changed, 102 insertions(+) create mode 100644 app/metrics_extra.py create mode 100644 app/sentry_init.py create mode 100644 tests/test_metrics_sentry.py diff --git a/app/main.py b/app/main.py index c73b6be..aaed14b 100644 --- a/app/main.py +++ b/app/main.py @@ -7,6 +7,7 @@ from app.config import settings from app.worker import worker from app.idempotency import idempotency +from app.sentry_init import init_sentry logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -17,6 +18,8 @@ @app.on_event("startup") async def startup(): + # initialize optional Sentry + init_sentry() # start background worker await worker.start() diff --git a/app/metrics_extra.py b/app/metrics_extra.py new file mode 100644 index 0000000..4893458 --- /dev/null +++ b/app/metrics_extra.py @@ -0,0 +1,39 @@ +from prometheus_client import Counter, Gauge + +# LLM token usage (best-effort approximation) +LLM_TOKEN_USAGE = Counter("llm_token_usage_total", "Estimated LLM token usage (best-effort)", ["model", "success"]) +# LLM call success/failure +LLM_CALLS = Counter("llm_calls_total", "Number of LLM calls", ["success"]) +# Worker queue depth +WORKER_QUEUE_DEPTH = Gauge("worker_queue_depth", "Background worker queue depth") +# Rate limit events (global / per_sender) +RATE_LIMIT_EVENTS = Counter("rate_limit_events_total", "Rate limit events", ["type"]) +# Message send retries +MESSAGES_SEND_RETRIES = Counter("messages_send_retries_total", "Outbound message send retries", ["provider"]) + + +def inc_llm_calls(success: bool): + LLM_CALLS.labels(success=str(success).lower()).inc() + + +def inc_llm_tokens(model: str, tokens: int, success: bool): + try: + LLM_TOKEN_USAGE.labels(model=model, success=str(success).lower()).inc(tokens) + except Exception: + # metrics client may not accept large floats; ignore errors + pass + + +def set_worker_queue_depth(n: int): + try: + WORKER_QUEUE_DEPTH.set(n) + except Exception: + pass + + +def inc_rate_limit_event(kind: str = "global"): + RATE_LIMIT_EVENTS.labels(type=kind).inc() + + +def inc_message_retry(provider: str = "twilio"): + MESSAGES_SEND_RETRIES.labels(provider=provider).inc() diff --git a/app/sentry_init.py b/app/sentry_init.py new file mode 100644 index 0000000..cd5d91e --- /dev/null +++ b/app/sentry_init.py @@ -0,0 +1,19 @@ +import sentry_sdk +from sentry_sdk.integrations.starlette import StarletteIntegration +import logging +from app.config import settings + +logger = logging.getLogger(__name__) + + +def init_sentry(): + """Initialize Sentry if SENTRY_DSN is configured. Safe to call multiple times.""" + dsn = getattr(settings, "SENTRY_DSN", "") + if not dsn: + logger.debug("SENTRY_DSN not set; skipping Sentry init") + return + try: + sentry_sdk.init(dsn=dsn, traces_sample_rate=0.1, integrations=[StarletteIntegration()]) + logger.info("Sentry initialized") + except Exception: + logger.exception("Failed to initialize Sentry") diff --git a/app/worker.py b/app/worker.py index edb8dd8..b661cc3 100644 --- a/app/worker.py +++ b/app/worker.py @@ -3,6 +3,7 @@ from typing import Any, Dict from app.reply_generator import generate_reply from app.messaging import client as messaging_client +from app.metrics_extra import set_worker_queue_depth, inc_message_retry, inc_llm_calls logger = logging.getLogger(__name__) @@ -31,6 +32,11 @@ async def stop(self): async def _run(self): while True: + # update queue depth metric + try: + set_worker_queue_depth(self.queue.qsize()) + except Exception: + pass item = await self.queue.get() if item is None: # sentinel to stop @@ -54,8 +60,11 @@ async def _process_item(self, item: Dict[str, Any]): # Generate reply (may internally call blocking code via run_in_executor) try: reply = await generate_reply(body, history) + # mark llm call success + inc_llm_calls(True) except Exception: logger.exception("Failed to generate reply; using fallback") + inc_llm_calls(False) reply = "Thanks — I received your message. Can you tell me more?" # Send message (messaging_client.send may be blocking -> run in thread) @@ -63,6 +72,11 @@ async def _process_item(self, item: Dict[str, Any]): await asyncio.to_thread(messaging_client.send, to=from_number, body=reply) except Exception: logger.exception("Failed to send message to %s", from_number) + # increment retry counter (best-effort; sending retries should be implemented elsewhere) + try: + inc_message_retry(getattr(messaging_client, "_client", type(messaging_client)).__class__.__name__) + except Exception: + inc_message_retry("unknown") async def enqueue(self, item: Dict[str, Any]): await self.queue.put(item) diff --git a/tests/test_metrics_sentry.py b/tests/test_metrics_sentry.py new file mode 100644 index 0000000..4acfc60 --- /dev/null +++ b/tests/test_metrics_sentry.py @@ -0,0 +1,27 @@ +import pytest +from unittest.mock import patch + + +def test_metrics_registration(): + # Import module to ensure metrics are registered + import app.metrics_extra as me + assert hasattr(me, 'LLM_CALLS') + assert hasattr(me, 'WORKER_QUEUE_DEPTH') + + +def test_sentry_init(monkeypatch): + # Ensure sentry init is called when DSN is present + import app.sentry_init as si + called = {} + def fake_init(*args, **kwargs): + called['ok'] = True + monkeypatch.setattr('sentry_sdk.init', fake_init) + # monkeypatch settings + import app.config as cfg + old = cfg.settings.SENTRY_DSN + cfg.settings.SENTRY_DSN = 'http://example.com' + try: + si.init_sentry() + assert called.get('ok') + finally: + cfg.settings.SENTRY_DSN = old