From e6c0bfd204ba2114788ba856b2d5072bed16fa88 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:01:55 +0200 Subject: [PATCH 01/22] docs: runtime batch 2 design (#536 #537 #538 #539 #565) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../2026-10-21-runtime-batch-2-design.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-21-runtime-batch-2-design.md diff --git a/docs/superpowers/specs/2026-10-21-runtime-batch-2-design.md b/docs/superpowers/specs/2026-10-21-runtime-batch-2-design.md new file mode 100644 index 000000000..23e156b60 --- /dev/null +++ b/docs/superpowers/specs/2026-10-21-runtime-batch-2-design.md @@ -0,0 +1,81 @@ +# Runtime batch 2: task progress, retained logs, Sentry, leases, in-process outbox + +Approved 2026-10-21 for #536, #537, #538, #539, #565 with the defaults below. + +## #536 Progress and owner-scoped status (background_tasks) + +- `TaskExecution` gains `progress_current: int|None`, `progress_total: int|None`, + `progress_message: str(255)|None`, `progress_phase: str(64)|None`, `owner_user_id: str(64)|None` + (indexed). All progress fields go in `__audit_exclude_fields__`. +- Owner stamping: `stamp_owner(headers)` next to `stamp_tenant` (`tenant_context.py`), header + `sm_owner_id`, set from `current_user_id` in a request and propagated by tasks that enqueue tasks; + `on_task_publish` writes the column. +- `heartbeat(*, progress=(i, n), message=None, phase=None, force=False)`: latest progress buffered + under the existing throttle; `i == n` forces a write. +- `GET /api/background_tasks/tasks/{celery_task_id}/status` in a new router without `PERM_VIEW`: + owner + active tenant (or tenancy off) → `TaskStatusDTO` (status, progress fields, timestamps, + sanitized error type; never traceback/args/result); anyone else 404; holders of `PERM_VIEW` bypass + the owner check. +- Progress bar on `pages/Detail.tsx`. One migration (branch label `background_tasks`, off + `e878882924e7`). + +## #537 Retained task and app logs + +- Hosting: `LogSink` Protocol (`emit_batch(records)`) + `QueueSinkHandler` (QueueHandler + listener + thread; batch 200 / 1 s; bounded queue, drop-and-count when full; level threshold; skips its own + logger and `sqlalchemy.*`). Redaction filter masks Authorization, Cookie, password, token, + `secret=` before queueing. +- background_tasks: table `background_tasks_log_record` (id, ts, level int, logger, message ≤4 KB, + task_id, job_id, tenant_id, correlation_id, extra JSON), indexes (task_id, ts), (ts), (level, ts); + no MultiTenantMixin (like TaskExecution; admin views filter tenant_id). DB sink via + `sync_session`, audit excluded; installed in `on_startup` and worker init. +- DB-backed settings: `log_sink_enabled=False`, `log_sink_level="INFO"`, `log_retention_days=7`; + purge pass in `purge_old_executions`. +- UI: Logs tab on task detail (level filter, #513 pagination) and an admin `Logs.tsx` search page + with a menu item; `PERM_VIEW`. +- Migration chains on #536's. + +## #538 Optional Sentry + +- `simple_module_hosting[sentry]` extra (`sentry-sdk[fastapi,celery]`); `simple_module_hosting/ + sentry.py` `init_sentry(process: "web"|"worker")` returns before importing the SDK when + `SM_SENTRY_DSN` is unset. +- Env only (secret + pre-DB process wiring): `SM_SENTRY_DSN`, `SM_SENTRY_ENVIRONMENT` (default + `SM_ENVIRONMENT`), `SM_SENTRY_TRACES_SAMPLE_RATE` (default 0.0). +- Web: called in `create_app` before middleware; worker: on `worker_process_init` in + background_tasks. +- Event processor tags request_id (correlation id), tenant, user id (never email), module, task. + `before_send` scrubs Authorization/Cookie/Set-Cookie/X-CSRF-Token/X-Api-Key headers, `token`/`code` + query values and request bodies; `send_default_pii=False`. DSN never logged. + +## #539 Distributed TTL leases + +- Core `simple_module_core/leases.py`: `LeaseBackend` Protocol (acquire / refresh / release / + semaphore_acquire); `lease(name, ttl, owner=None)` handle (`.token`, `.fence`, `refresh()`, + async context manager with wait/blocking); `semaphore(name, slots, ttl)`; + `InProcessLeaseBackend`; `configure_leases(backend)` / `get_leases()` module-global (workers have + no app.state). +- Redis backend (hosting): `SET NX PX`; Lua compare-and-set refresh/release on the token; + semaphore ZSET scored by expiry using Redis `TIME`; `INCR {name}:fence` fencing token. +- DB backend: table `sm_lease(name, slot, holder, fence, expires_at)`, unique (name, slot), DB clock, + in `simple_module_db` (branch label `leases`). +- Selection: Redis when `SM_REDIS_URL` is set, else DB table, else in-process. An expired owner + cannot release. + +## #565 In-process outbox + +- background_tasks setting `outbox_mode: Literal["celery","inprocess"]` (restart-required), default + `"celery"` but resolves to `"inprocess"` automatically when `task_always_eager` is set. +- In-process: `on_startup` starts an asyncio loop per web process running `dispatch(ids=None)` every + `outbox_sweep_interval_seconds` with jitter; `on_shutdown` cancels and drains. Eager `schedule` + path kept. +- Celery import moved inside `WorkerOutboxTransport.schedule`. +- `tasks.py` stuck sweep and purge read `retention_days` etc. via `load_runner_settings()` (DB), + not raw env. + +## Slices + +Wave 1 in parallel: A #539 (core/hosting/db + `leases` migration), B #538 (hosting sentry.py + one +hook in background_tasks worker init), C #536 (background_tasks models/signals/tenant_context/ +heartbeat/schemas/api_status/Detail.tsx + migration), E #565 (outbox_transport/module/tasks/settings). +Wave 2: D #537 after C. Update `host/tests/test_migration_graph.py` for new heads/labels. From 02b371d45e3f0515c81cc99a3bb972179e13591c Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:13:02 +0200 Subject: [PATCH 02/22] feat(hosting): optional Sentry error reporting (#538) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 1 + docs/.vitepress/config.ts | 1 + docs/framework/sentry.md | 44 +++++ docs/reference/env-vars.md | 10 + framework/hosting/pyproject.toml | 2 + .../simple_module_hosting/_phase_helpers.py | 3 + .../simple_module_hosting/app_builder.py | 2 + .../hosting/simple_module_hosting/sentry.py | 180 ++++++++++++++++++ .../hosting/tests/test_hosting_sentry.py | 136 +++++++++++++ .../background_tasks/sync_db.py | 8 + pyproject.toml | 2 + uv.lock | 29 ++- 12 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 docs/framework/sentry.md create mode 100644 framework/hosting/simple_module_hosting/sentry.py create mode 100644 framework/hosting/tests/test_hosting_sentry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9807e03a..1631c0c53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project are documented in this file. The format is b - `host/client_app` no longer re-declares `@simple-module-py/ui`'s runtime dependencies, nor `@base-ui/react`, `@radix-ui/react-slot`, `autoprefixer` or `postcss` (nothing loaded them) (#521). ### Added +- Optional Sentry error reporting: `pip install simple_module_hosting[sentry]` and set `SM_SENTRY_DSN` (plus `SM_SENTRY_ENVIRONMENT`, `SM_SENTRY_TRACES_SAMPLE_RATE`). Web and Celery worker processes tag events with request id, tenant, user id, module and task, and scrub auth headers, cookies, `token`/`code` query values and request bodies. Without a DSN the SDK is never imported (#538). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index bb656edaa..4b10d94c1 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -127,6 +127,7 @@ export default defineConfig({ { text: "Public routes", link: "/framework/public-routes" }, { text: "Events", link: "/framework/events" }, { text: "Cache invalidation", link: "/framework/invalidation" }, + { text: "Sentry", link: "/framework/sentry" }, { text: "Multi-tenancy", link: "/framework/multi-tenancy" }, { text: "Internationalization", link: "/framework/i18n" }, ], diff --git a/docs/framework/sentry.md b/docs/framework/sentry.md new file mode 100644 index 000000000..15ee15aeb --- /dev/null +++ b/docs/framework/sentry.md @@ -0,0 +1,44 @@ +# Sentry + +Optional error reporting for the web process and Celery workers. It costs +nothing until you opt in: with `SM_SENTRY_DSN` unset, `init_sentry()` returns +before `sentry_sdk` is imported. + +```bash +pip install "simple_module_hosting[sentry]" +export SM_SENTRY_DSN=https://@o0.ingest.sentry.io/ +``` + +| Variable | Default | +|---|---| +| `SM_SENTRY_DSN` | unset (disabled) | +| `SM_SENTRY_ENVIRONMENT` | `SM_ENVIRONMENT` | +| `SM_SENTRY_TRACES_SAMPLE_RATE` | `0.0` | + +These are env only: the DSN is a secret and the SDK has to start before the DB +is open. The DSN is never logged. + +## Where it starts + +- Web: `create_app` calls `init_sentry("web")` before any middleware is built. +- Worker: background_tasks calls `init_sentry("worker")` on `worker_process_init`, + so every prefork child reports. + +## Tags + +| Tag | Source | +|---|---| +| `request_id` | the correlation id | +| `tenant` | the request's resolved tenant, or the task's tenant | +| `user_id` | the principal's id (`user.id` is set; never email, username or IP) | +| `module` | top-level package of the route endpoint, or the prefix of `.` | +| `task` | the Celery task name (worker events) | +| `process` | `web` or `worker` | + +## Scrubbing + +`send_default_pii` is off and local variables are not attached to stack frames +(they hold the ASGI scope). Before send, events lose the `Authorization`, +`Cookie`, `Set-Cookie`, `X-CSRF-Token` and `X-Api-Key` headers, request cookies +and bodies, and the values of `token` and `code` query parameters (in the query +string and the URL). diff --git a/docs/reference/env-vars.md b/docs/reference/env-vars.md index ef20a05fc..1a8284db9 100644 --- a/docs/reference/env-vars.md +++ b/docs/reference/env-vars.md @@ -22,6 +22,16 @@ This is the full reference. See [Configuration](/guide/configuration) for a narr | `SM_TRUSTED_PROXY` | unset | Comma-separated proxy IPs / CIDRs whose `X-Forwarded-*` headers are trusted, or `*` to trust any peer (correct when the container is only reachable through one proxy). Setting it installs uvicorn's `ProxyHeadersMiddleware` outermost, so request logs record the real client IP and `request.url.scheme` reflects `X-Forwarded-Proto`. **Recommended behind a TLS-terminating proxy**, where otherwise every request in the logs is attributed to the proxy's own address rather than the visitor's. It is no longer needed for Inertia: the page url is root-relative, so `pushState` can't see a cross-scheme url whatever the proxy sends (it used to throw a `SecurityError` on every page — GH #223). It is still needed for anything that builds an absolute url from `request.url.scheme`/`request.url_for(...)` — notably OAuth/OIDC's callback url and the locale cookie's `secure` flag — left unset behind such a proxy, an OAuth `redirect_uri` ships as `http://…` and most providers reject it. Forwarded headers are never trusted by default. | | `SM_SESSION_COOKIE_SECURE` | unset | Mark the session cookie `Secure`. Unset means on everywhere except `SM_ENVIRONMENT=development`/`testing`. Set `false` only for an install that serves browsers plain HTTP on purpose; behind a TLS terminator leave it unset and set `SM_TRUSTED_PROXY`. Restart to apply. | +## Sentry (optional) + +Install `simple_module_hosting[sentry]`. All three are env only (the DSN is a secret and the SDK starts before the DB opens). See [Sentry](/framework/sentry). + +| Variable | Default | Notes | +|---|---|---| +| `SM_SENTRY_DSN` | unset | Enables error reporting in the web and Celery worker processes. Unset: `sentry_sdk` is never imported. Never logged. | +| `SM_SENTRY_ENVIRONMENT` | `SM_ENVIRONMENT` | Sentry `environment` tag. | +| `SM_SENTRY_TRACES_SAMPLE_RATE` | `0.0` | Performance trace sampling, `0.0`-`1.0`. | + ## DB connection pool | Variable | Default | Notes | diff --git a/framework/hosting/pyproject.toml b/framework/hosting/pyproject.toml index 235dc836e..2a1318b29 100644 --- a/framework/hosting/pyproject.toml +++ b/framework/hosting/pyproject.toml @@ -40,6 +40,8 @@ dependencies = [ [project.optional-dependencies] # Shared rate-limiter store (SM_REDIS_URL). Without it the limiter counts per worker. redis = ["redis>=5"] +# Error reporting (SM_SENTRY_DSN). Never imported unless the DSN is set. +sentry = ["sentry-sdk[fastapi,celery]>=2.20,<3"] [project.scripts] sm-host = "simple_module_hosting.host_cli:app" diff --git a/framework/hosting/simple_module_hosting/_phase_helpers.py b/framework/hosting/simple_module_hosting/_phase_helpers.py index 06371f87d..930ff3004 100644 --- a/framework/hosting/simple_module_hosting/_phase_helpers.py +++ b/framework/hosting/simple_module_hosting/_phase_helpers.py @@ -25,6 +25,7 @@ from starlette.middleware.gzip import GZipMiddleware from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware +from simple_module_hosting import sentry from simple_module_hosting._body_limit import BodyLimitMiddleware from simple_module_hosting._error_handlers import ( http_exception_handler, @@ -213,6 +214,8 @@ def install_middleware( ) app.add_middleware(RequestLoggingMiddleware) app.add_middleware(CorrelationIdMiddleware) + if sentry.is_enabled(): + app.add_middleware(sentry.SentryContextMiddleware) # Outermost: rewrite scheme/client from X-Forwarded-* before anything else # reads them, so request logs see the real client IP rather than the # proxy's. Inertia no longer needs this: its page url is diff --git a/framework/hosting/simple_module_hosting/app_builder.py b/framework/hosting/simple_module_hosting/app_builder.py index 943fa2828..0c312fb17 100644 --- a/framework/hosting/simple_module_hosting/app_builder.py +++ b/framework/hosting/simple_module_hosting/app_builder.py @@ -52,6 +52,7 @@ from simple_module_hosting.health import router as health_router from simple_module_hosting.i18n_manifest import build_i18n_registry from simple_module_hosting.i18n_routes import router as i18n_router +from simple_module_hosting.sentry import PROCESS_WEB, init_sentry from simple_module_hosting.settings import Settings from simple_module_hosting.setup_gate import register_migration_step from simple_module_hosting.setup_wizard import mount_setup_wizard @@ -93,6 +94,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: settings = settings or merge_host_settings() assert_not_placeholder(settings) + init_sentry(PROCESS_WEB) # no-op (SDK not imported) without SM_SENTRY_DSN # ── Phase 1: Discover modules ────────────────────────── # Production: any bad module (import error, missing meta, wrong base diff --git a/framework/hosting/simple_module_hosting/sentry.py b/framework/hosting/simple_module_hosting/sentry.py new file mode 100644 index 000000000..3c8c269d7 --- /dev/null +++ b/framework/hosting/simple_module_hosting/sentry.py @@ -0,0 +1,180 @@ +"""Optional Sentry error reporting (#538). + +``init_sentry`` is the only entry point. With ``SM_SENTRY_DSN`` unset it +returns before importing ``sentry_sdk``, so the ``sentry`` extra stays +optional and costs nothing. The DSN is a secret and process wiring that must +be known before the DB opens, hence env only; it is never logged. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Mapping +from contextvars import ContextVar +from typing import Any, Literal +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from pydantic import SecretStr +from pydantic_settings import BaseSettings, SettingsConfigDict +from simple_module_core.correlation import correlation_id +from simple_module_db.listeners import current_user_id +from simple_module_db.tenancy import current_tenant_id +from starlette.types import ASGIApp, Receive, Scope, Send + +logger = logging.getLogger(__name__) + +Process = Literal["web", "worker"] +PROCESS_WEB: Process = "web" +PROCESS_WORKER: Process = "worker" + +REDACTED = "[Filtered]" +_SCRUBBED_HEADERS = frozenset( + {"authorization", "cookie", "set-cookie", "x-csrf-token", "x-api-key"} +) +_SCRUBBED_QUERY_KEYS = frozenset({"token", "code"}) +_TAG_REQUEST_ID = "request_id" +_TAG_TENANT = "tenant" +_TAG_USER_ID = "user_id" +_TAG_MODULE = "module" +_TAG_TASK = "task" +_TAG_PROCESS = "process" +_TASK_SEP = "." + +Event = dict[str, Any] + +# The ASGI scope of the request in flight. Tenant/auth ContextVars are reset +# by their middleware before an unhandled error reaches Sentry's capture point +# (outside the whole stack), but ``scope["state"]`` and ``scope["endpoint"]`` +# survive, so the processor reads those first. +_request_scope: ContextVar[Scope | None] = ContextVar("sentry_request_scope", default=None) +_enabled = False + + +def is_enabled() -> bool: + return _enabled + + +class SentryContextMiddleware: + """Expose the request scope to the Sentry event processor.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "http": + _request_scope.set(scope) + await self.app(scope, receive, send) + + +class SentrySettings(BaseSettings): + """Env-only Sentry wiring (``SM_SENTRY_*``).""" + + model_config = SettingsConfigDict(env_prefix="SM_", env_file=".env", extra="ignore") + + sentry_dsn: SecretStr | None = None + sentry_environment: str | None = None + sentry_traces_sample_rate: float = 0.0 + environment: str = "development" + + +def _scrub_query(query: str) -> str: + pairs = parse_qsl(query, keep_blank_values=True) + return urlencode([(k, REDACTED if k.lower() in _SCRUBBED_QUERY_KEYS else v) for k, v in pairs]) + + +def _scrub_request(request: dict[str, Any]) -> None: + headers = request.get("headers") + if isinstance(headers, dict): + request["headers"] = { + k: v for k, v in headers.items() if str(k).lower() not in _SCRUBBED_HEADERS + } + for key in ("cookies", "data"): + request.pop(key, None) + if isinstance(request.get("query_string"), str): + request["query_string"] = _scrub_query(request["query_string"]) + url = request.get("url") + if isinstance(url, str) and "?" in url: + parts = urlsplit(url) + request["url"] = urlunsplit(parts._replace(query=_scrub_query(parts.query))) + + +def _scope_info(process: Process) -> dict[str, str | None]: + scope = _request_scope.get() if process == PROCESS_WEB else None + state = (scope or {}).get("state", {}) + user = state.get("user") + endpoint = (scope or {}).get("endpoint") + return { + _TAG_TENANT: state.get("tenant_id"), + _TAG_REQUEST_ID: state.get("correlation_id"), + _TAG_USER_ID: getattr(user, "id", None), + _TAG_MODULE: getattr(endpoint, "__module__", "").split(_TASK_SEP, 1)[0] or None, + } + + +def _tags(event: Event, process: Process) -> dict[str, str]: + info = _scope_info(process) + info[_TAG_TENANT] = info[_TAG_TENANT] or current_tenant_id.get() + info[_TAG_REQUEST_ID] = info[_TAG_REQUEST_ID] or correlation_id.get() + info[_TAG_USER_ID] = info[_TAG_USER_ID] or current_user_id.get() + # Fallback: events are named after the endpoint (".endpoints.api.fn") + # or the task ("."); both lead with the module. + transaction = event.get("transaction") + if not info[_TAG_MODULE] and isinstance(transaction, str) and _TASK_SEP in transaction: + info[_TAG_MODULE] = transaction.split(_TASK_SEP, 1)[0] + if process == PROCESS_WORKER and isinstance(transaction, str): + info[_TAG_TASK] = transaction + info[_TAG_PROCESS] = process + return {k: str(v) for k, v in info.items() if v} + + +def build_before_send(process: Process) -> Callable[[Event, Mapping[str, Any]], Event]: + """Tag and scrub an event; used for errors and transactions alike.""" + + def before_send(event: Event, hint: Mapping[str, Any]) -> Event: + tags = _tags(event, process) + event.setdefault("tags", {}).update(tags) + if user_id := tags.get(_TAG_USER_ID): + event["user"] = {"id": user_id} # id only: never email/username/ip + else: + event.pop("user", None) + if isinstance(event.get("request"), dict): + _scrub_request(event["request"]) + return event + + return before_send + + +def init_sentry(process: Process, *, transport: Any = None) -> bool: + """Start Sentry for this process; return whether it was enabled.""" + settings = SentrySettings() + if settings.sentry_dsn is None or not settings.sentry_dsn.get_secret_value(): + return False + try: + import sentry_sdk + from sentry_sdk.integrations.celery import CeleryIntegration + from sentry_sdk.integrations.fastapi import FastApiIntegration + from sentry_sdk.integrations.starlette import StarletteIntegration + except ImportError: + logger.warning("SM_SENTRY_DSN is set but sentry-sdk is not installed; install the extra") + return False + before_send: Any = build_before_send(process) # sentry types Event as a TypedDict + sentry_sdk.init( + dsn=settings.sentry_dsn.get_secret_value(), + environment=settings.sentry_environment or settings.environment, + traces_sample_rate=settings.sentry_traces_sample_rate, + send_default_pii=False, + include_local_variables=False, # frame locals hold the ASGI scope: headers, tokens + max_request_body_size="never", + before_send=before_send, + before_send_transaction=before_send, + transport=transport, + integrations=[ + StarletteIntegration(transaction_style="endpoint"), + FastApiIntegration(transaction_style="endpoint"), + CeleryIntegration(), + ], + ) + global _enabled + _enabled = True + logger.info("Sentry enabled for %s process", process) + return True diff --git a/framework/hosting/tests/test_hosting_sentry.py b/framework/hosting/tests/test_hosting_sentry.py new file mode 100644 index 000000000..a07349f30 --- /dev/null +++ b/framework/hosting/tests/test_hosting_sentry.py @@ -0,0 +1,136 @@ +"""Optional Sentry (#538): tagging, scrubbing, and the no-DSN zero-cost path.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from collections.abc import Iterator +from typing import Any + +import httpx +import pytest +from fastapi import FastAPI, Request +from simple_module_db.tenancy import current_tenant_id +from simple_module_hosting import sentry as sm_sentry + +sentry_sdk = pytest.importorskip("sentry_sdk") +from sentry_sdk.transport import Transport # noqa: E402 + +_TENANT = "acme" +_USER_ID = "user-42" +_SECRET_HEADERS = ("authorization", "cookie", "x-csrf-token", "x-api-key") + + +class FakeTransport(Transport): + def __init__(self, options: dict[str, Any] | None = None) -> None: + super().__init__(options) + self.events: list[dict[str, Any]] = [] + + def capture_envelope(self, envelope: Any) -> None: + for item in envelope.items: + if item.type == "event": + self.events.append(item.payload.json) + + +class _User: + id = _USER_ID + email = "private@example.com" + + +def boom(request: Request) -> None: + # What the tenant/auth/correlation middlewares leave on the scope state. + request.state.tenant_id = _TENANT + request.state.user = _User() + request.state.correlation_id = "cid-1" + raise RuntimeError("route failure") + + +@pytest.fixture +def transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeTransport]: + monkeypatch.setenv("SM_SENTRY_DSN", "https://key@o0.ingest.sentry.io/1") + fake = FakeTransport() + assert sm_sentry.init_sentry(sm_sentry.PROCESS_WEB, transport=fake) + yield fake + sentry_sdk.get_global_scope().client.close() + + +async def test_route_exception_is_tagged_and_scrubbed(transport: FakeTransport) -> None: + app = FastAPI() + app.add_middleware(sm_sentry.SentryContextMiddleware) + app.add_api_route("/boom", boom, methods=["POST"]) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://test", + ) as client: + await client.post( + "/boom?token=s3cret&code=abc&keep=1", + headers={ + "Authorization": "Bearer x", + "Cookie": "sm=1", + "X-CSRF-Token": "t", + "X-Api-Key": "k", + }, + content=b'{"password": "hunter2"}', + ) + sentry_sdk.flush() + assert len(transport.events) == 1 + event = transport.events[0] + assert event["tags"]["tenant"] == _TENANT + assert event["tags"]["module"] == __name__.split(".")[0] + assert event["tags"]["request_id"] == "cid-1" + assert event["user"] == {"id": _USER_ID} + request = event["request"] + assert not {h.lower() for h in request.get("headers", {})} & set(_SECRET_HEADERS) + dumped = str(event) + for leaked in ("s3cret", "abc", "hunter2", "Bearer x", "private@example.com"): + assert leaked not in dumped + assert "keep=1" in dumped + + +def test_task_exception_is_tagged(monkeypatch: pytest.MonkeyPatch) -> None: + from celery import Celery + + monkeypatch.setenv("SM_SENTRY_DSN", "https://key@o0.ingest.sentry.io/1") + fake = FakeTransport() + assert sm_sentry.init_sentry(sm_sentry.PROCESS_WORKER, transport=fake) + celery = Celery("t", broker="memory://", task_always_eager=True) + + @celery.task(name="orders.explode") + def explode() -> None: + raise RuntimeError("task failure") + + try: + token = current_tenant_id.set(_TENANT) + try: + with pytest.raises(RuntimeError): + explode.apply(throw=True).get() + finally: + current_tenant_id.reset(token) + sentry_sdk.flush() + finally: + sentry_sdk.get_global_scope().client.close() + event = next(e for e in fake.events if "exception" in e) + assert event["tags"]["tenant"] == _TENANT + assert event["tags"]["module"] == "orders" + assert event["tags"]["task"] == "orders.explode" + + +def test_no_dsn_never_imports_sdk() -> None: + code = textwrap.dedent( + """ + import os, sys + os.environ.pop("SM_SENTRY_DSN", None) + from simple_module_hosting.app_builder import create_app + from simple_module_hosting.sentry import PROCESS_WEB, init_sentry + assert init_sentry(PROCESS_WEB) is False + create_app() + assert "sentry_sdk" not in sys.modules, "sentry_sdk imported without a DSN" + """ + ) + env = {k: v for k, v in __import__("os").environ.items() if k != "SM_SENTRY_DSN"} + env["SM_DATABASE_URL"] = "sqlite+aiosqlite:///:memory:" + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, env=env, check=False + ) + assert result.returncode == 0, result.stderr[-2000:] diff --git a/modules/background_tasks/background_tasks/sync_db.py b/modules/background_tasks/background_tasks/sync_db.py index 05865445e..194bbcd16 100644 --- a/modules/background_tasks/background_tasks/sync_db.py +++ b/modules/background_tasks/background_tasks/sync_db.py @@ -180,6 +180,14 @@ def _dispose_inherited_engine(**_: object) -> None: _drop_engine() +@worker_process_init.connect +def _init_sentry(**_: object) -> None: + """Start Sentry in each prefork child (no-op without ``SM_SENTRY_DSN``).""" + from simple_module_hosting.sentry import PROCESS_WORKER, init_sentry + + init_sentry(PROCESS_WORKER) + + @contextmanager def sync_session() -> Generator[Session]: """Open a short-lived sync session; commit on success, rollback on error.""" diff --git a/pyproject.toml b/pyproject.toml index 58cf2826b..e42b7e31c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,8 @@ dev = [ # needs this. Declared explicitly because it was otherwise only present as a # transitive dep of locust's geventhttpclient. "brotli>=1.1", + # Test-only: the optional ``simple_module_hosting[sentry]`` extra, exercised with a fake transport. + "sentry-sdk[fastapi,celery]>=2.20,<3", ] [tool.ruff] diff --git a/uv.lock b/uv.lock index c2ecc98f9..09d03c5df 100644 --- a/uv.lock +++ b/uv.lock @@ -3023,6 +3023,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f0/ae7ca09223a81a1d890b2557186ea015f6e0502e9b8cb8e1813f1d8cfa4e/s3transfer-0.14.0-py3-none-any.whl", hash = "sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456", size = 85712, upload-time = "2025-09-09T19:23:30.041Z" }, ] +[[package]] +name = "sentry-sdk" +version = "2.72.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/c1/d2bbb2b03bdc2ef14b35e02b647b8489156c50b6e0b8bacc5fa303902e23/sentry_sdk-2.72.0.tar.gz", hash = "sha256:0d2ffbc28ee2e63cbaf3d015e93b448cffcc3ef3be03a44704ad36ade72faa95", size = 1069699, upload-time = "2026-10-12T13:08:11.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/0b/0dd9686842d9dc49c3ee84cced31c54bceb0fb52a826be7d3d4fe30d882a/sentry_sdk-2.72.0-py3-none-any.whl", hash = "sha256:3e13ace4ffd0b3cc78288236edfc4e4bd90eb12a396cc5845fa10a58ff289f50", size = 544754, upload-time = "2026-10-12T13:08:01.619Z" }, +] + +[package.optional-dependencies] +celery = [ + { name = "celery" }, +] +fastapi = [ + { name = "fastapi" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -3329,6 +3350,9 @@ dependencies = [ redis = [ { name = "redis" }, ] +sentry = [ + { name = "sentry-sdk", extra = ["celery", "fastapi"] }, +] [package.metadata] requires-dist = [ @@ -3338,6 +3362,7 @@ requires-dist = [ { name = "itsdangerous", specifier = ">=2.0" }, { name = "jinja2", specifier = ">=3.1" }, { name = "redis", marker = "extra == 'redis'", specifier = ">=5" }, + { name = "sentry-sdk", extras = ["fastapi", "celery"], marker = "extra == 'sentry'", specifier = ">=2.20,<3" }, { name = "simple-module-core", editable = "framework/core" }, { name = "simple-module-db", editable = "framework/db" }, { name = "simple-module-inertia", editable = "framework/inertia" }, @@ -3345,7 +3370,7 @@ requires-dist = [ { name = "tomlkit", specifier = ">=0.13" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34" }, ] -provides-extras = ["redis"] +provides-extras = ["redis", "sentry"] [[package]] name = "simple-module-inertia" @@ -3449,6 +3474,7 @@ dev = [ { name = "pytest-playwright" }, { name = "pytest-xdist" }, { name = "ruff" }, + { name = "sentry-sdk", extra = ["celery", "fastapi"] }, { name = "tomlkit" }, { name = "ty" }, ] @@ -3472,6 +3498,7 @@ dev = [ { name = "pytest-playwright", specifier = ">=0.7.2" }, { name = "pytest-xdist", specifier = ">=3.6" }, { name = "ruff", specifier = ">=0.8" }, + { name = "sentry-sdk", extras = ["fastapi", "celery"], specifier = ">=2.20,<3" }, { name = "tomlkit", specifier = ">=0.13" }, { name = "ty", specifier = ">=0.0.29" }, ] From 698ad143e619def5716667a4e87b956c355f3a6e Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:18:45 +0200 Subject: [PATCH 03/22] feat(background_tasks): task progress and owner-scoped status (#536) heartbeat(progress=, message=, phase=) buffers under the existing throttle; TaskExecution gains progress_* and an indexed owner_user_id stamped from the sm_owner_id header (propagated through tasks that enqueue tasks, kept on retry). GET /api/background_tasks/tasks/{id}/status is readable by the owner in the active tenant or PERM_VIEW holders, 404 otherwise. Migration bb906e1a1089. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 9 ++ docs/modules/background_tasks.md | 23 +++ ...ackground_tasks_task_progress_and_owner.py | 48 ++++++ host/tests/test_migration_graph.py | 6 +- .../background_tasks/_signal_support.py | 51 +++++++ .../background_tasks/constants.py | 6 + .../background_tasks/contracts/schemas.py | 29 ++++ .../background_tasks/endpoints/api_status.py | 63 ++++++++ .../background_tasks/locales/en.json | 4 +- .../background_tasks/models.py | 18 +++ .../background_tasks/module.py | 2 + .../background_tasks/pages/Detail.tsx | 3 + .../pages/components/ProgressCard.tsx | 43 ++++++ .../background_tasks/pages/constants.ts | 5 + .../background_tasks/retry_service.py | 14 +- .../background_tasks/signals.py | 69 ++------- .../background_tasks/task_heartbeat.py | 57 ++++++- .../background_tasks/tenant_context.py | 86 ++++++++++- .../background_tasks_DetailPage.test.tsx | 17 +++ modules/background_tasks/tests/conftest.py | 7 +- .../tests/test_bg_task_progress.py | 139 ++++++++++++++++++ .../tests/test_bg_task_progress_audit.py | 53 +++++++ .../tests/test_bg_task_status_api.py | 119 +++++++++++++++ packages/i18n/src/generated-resources.ts | 2 + packages/i18n/src/keys.generated.ts | 2 + 25 files changed, 803 insertions(+), 72 deletions(-) create mode 100644 host/migrations/versions/bb906e1a1089_background_tasks_task_progress_and_owner.py create mode 100644 modules/background_tasks/background_tasks/endpoints/api_status.py create mode 100644 modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx create mode 100644 modules/background_tasks/tests/test_bg_task_progress.py create mode 100644 modules/background_tasks/tests/test_bg_task_progress_audit.py create mode 100644 modules/background_tasks/tests/test_bg_task_status_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9807e03a..ef54a965e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,15 @@ All notable changes to this project are documented in this file. The format is b - `host/client_app` no longer re-declares `@simple-module-py/ui`'s runtime dependencies, nor `@base-ui/react`, `@radix-ui/react-slot`, `autoprefixer` or `postcss` (nothing loaded them) (#521). ### Added +- `background_tasks`: tasks report progress with `heartbeat(progress=(i, n), message=, phase=)` + (buffered under the existing 30 s throttle, `i == n` writes at once, never + audited), and `GET /api/background_tasks/tasks/{celery_task_id}/status` lets the user who + enqueued a task read its status and progress without `background_tasks.view` + (owner in the active tenant, or `view` holders; anyone else gets 404; never + returns args, result or traceback). `TaskExecution` gains `progress_*` and an + indexed `owner_user_id` stamped from the new `sm_owner_id` message header; + retries keep the original owner. The detail page shows a progress bar. + Migration `bb906e1a1089` (first revision labelled `background_tasks`) (#536). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial diff --git a/docs/modules/background_tasks.md b/docs/modules/background_tasks.md index a68f07836..c1d09f89f 100644 --- a/docs/modules/background_tasks.md +++ b/docs/modules/background_tasks.md @@ -89,6 +89,25 @@ Celery copies extra decorator options onto the task class; the sweep reads `stuc Not provided: an automatic heartbeat thread. A task that blocks in one long call (a single huge query, an external process) cannot heartbeat from inside it — give it a `stuck_after` that covers the call instead. +## Progress and owner-scoped status + +A task can report how far along it is, and the user whose request enqueued it can poll that without any `background_tasks.*` permission: + +```python +@shared_task(name="reports.rebuild") +def rebuild(report_id: int) -> None: + chunks = list(chunks_of(report_id)) + for i, chunk in enumerate(chunks, start=1): + process(chunk) + heartbeat(progress=(i, len(chunks)), message=f"chunk {i}", phase="rebuild") +``` + +`progress=(done, total)`, `message` (255 characters) and `phase` (64) are stored on the execution row. They share the heartbeat's 30-second throttle: a throttled call keeps only the latest report and the next write carries it, and `done == total` writes at once so a finished task never shows 9/10. None of these columns is audited, so reporting progress adds no `audit_log` rows. + +The enqueuing user is recorded as `owner_user_id` from the `sm_owner_id` message header (stamped from `current_user_id` at publish, next to the tenant header). A task that enqueues another task passes the owner on, and a retry keeps the original owner rather than adopting the operator. Beat and other platform publishes have no owner. + +`GET /api/background_tasks/tasks/{celery_task_id}/status` returns a `TaskStatusDTO`: status, the progress fields, timestamps and the exception's type name. It never returns args, kwargs, result, traceback or an error message. It is visible to the owner while acting in the row's tenant (any tenant when multi-tenancy is off) and to holders of `background_tasks.view`; everyone else, and every unknown id, gets `404` so the route does not reveal which ids exist. The task detail page shows a progress bar once a report exists. + ## Scheduling periodic work (beat) A module schedules recurring work the same way it registers tasks — by shipping a `tasks.py` — and additionally exporting a module-level `BEAT_SCHEDULE` dict. `build_celery` merges every installed module's `BEAT_SCHEDULE` into the beat schedule at boot (identically in the web and worker processes), so `make beat` runs them alongside the built-ins: @@ -344,6 +363,7 @@ All require `background_tasks.view`; retry additionally needs `background_tasks. | `GET /api/background_tasks/admin/executions` | `?status=&task_name=&page=&per_page=` → `TaskExecutionListResponse` | per_page max 200 | | `GET /api/background_tasks/admin/executions/{execution_id}` | → `TaskExecutionDetail` | 404 if missing | | `POST /api/background_tasks/admin/executions/{execution_id}/retry` | → `TaskExecutionDetail` | 409 unless status is `failed` or `stuck` | +| `GET /api/background_tasks/tasks/{celery_task_id}/status` | → `TaskStatusDTO` | no permission gate: owner in the active tenant or `view`; otherwise 404 | | `GET /api/background_tasks/admin/workers` | → `WorkerSnapshot` | live ping; consumed by the `Workers` page | | `GET /api/background_tasks/admin/outbox` | `?status=&page=&per_page=` → `OutboxListResponse` | undelivered rows (pending, running, dead) and per-status counts | | `POST /api/background_tasks/admin/outbox/{id}/retry` | → `OutboxItem` | `manage`; 409 unless `dead`; resets attempts and dispatches after commit | @@ -397,6 +417,9 @@ from background_tasks.contracts.schemas import ( | `worker` | `str(255) \| None` | hostname that ran it | | `retries` | `int` | | | `retried_from_id` | `UUID \| None` | self-FK; tracks retry chains | +| `owner_user_id` | `str(64) \| None` | indexed; user whose request enqueued the task (#536) | +| `progress_current` / `progress_total` | `int \| None` | latest `heartbeat(progress=...)` report | +| `progress_message` / `progress_phase` | `str(255) \| None` / `str(64) \| None` | not audited | | `queued_at` / `started_at` / `finished_at` / `heartbeat_at` | `datetime` | | | audit | from `AuditMixin` | | diff --git a/host/migrations/versions/bb906e1a1089_background_tasks_task_progress_and_owner.py b/host/migrations/versions/bb906e1a1089_background_tasks_task_progress_and_owner.py new file mode 100644 index 000000000..f4d7c7de5 --- /dev/null +++ b/host/migrations/versions/bb906e1a1089_background_tasks_task_progress_and_owner.py @@ -0,0 +1,48 @@ +"""background_tasks: task progress and owner (GH #536) + +``progress_*`` hold the latest report a task body sends through +``heartbeat(progress=...)``; ``owner_user_id`` is the user whose request +enqueued the task, so that user can read its status without a task-admin +permission. All columns are nullable plain values: beat and platform +publishes have no owner and most tasks never report progress. The owner is +indexed (the status endpoint filters on it); the others are not. + +Starts the ``background_tasks`` label; it extends the mainline head like +``b5d3f08a6e17`` and ``e878882924e7`` did. + +Revision ID: bb906e1a1089 +Revises: d7b3a9e5c1f2 +Create Date: 2026-10-21 18:09:59.606581 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "bb906e1a1089" +down_revision: str | None = "d7b3a9e5c1f2" +branch_labels: str | Sequence[str] | None = ("background_tasks",) +depends_on: str | Sequence[str] | None = None + +_TABLE = "background_tasks_task_execution" +_OWNER_INDEX = "ix_background_tasks_task_execution_owner_user_id" + + +def upgrade() -> None: + op.add_column(_TABLE, sa.Column("owner_user_id", sa.String(length=64), nullable=True)) + op.add_column(_TABLE, sa.Column("progress_current", sa.Integer(), nullable=True)) + op.add_column(_TABLE, sa.Column("progress_total", sa.Integer(), nullable=True)) + op.add_column(_TABLE, sa.Column("progress_message", sa.String(length=255), nullable=True)) + op.add_column(_TABLE, sa.Column("progress_phase", sa.String(length=64), nullable=True)) + op.create_index(_OWNER_INDEX, _TABLE, ["owner_user_id"], unique=False) + + +def downgrade() -> None: + op.drop_index(_OWNER_INDEX, table_name=_TABLE) + op.drop_column(_TABLE, "progress_phase") + op.drop_column(_TABLE, "progress_message") + op.drop_column(_TABLE, "progress_total") + op.drop_column(_TABLE, "progress_current") + op.drop_column(_TABLE, "owner_user_id") diff --git a/host/tests/test_migration_graph.py b/host/tests/test_migration_graph.py index 0fc0ae4be..2deb94476 100644 --- a/host/tests/test_migration_graph.py +++ b/host/tests/test_migration_graph.py @@ -37,6 +37,7 @@ "e5f2a8c1d7b3": "core", # timezone-aware timestamps across modules "b5d3f08a6e17": "background_tasks", "e878882924e7": "background_tasks", + "bb906e1a1089": "background_tasks", # task progress + owner (#536) "51b2058d8afd": "api_tokens", "168a2882f443": "keycloak", "f6a3b9d2e8c4": "keycloak", @@ -51,7 +52,7 @@ # head revision -> module owning the tables it creates or alters. _HEADS = { "a7c2e91d4b58": "audit_log", - "d7b3a9e5c1f2": "mail", # reported as ``api_tokens`` (inherited label) + "bb906e1a1089": "background_tasks", # also carries the inherited ``api_tokens`` "f1a9c3d7b2e4": "tenants", "f6a3b9d2e8c4": "keycloak", } @@ -62,6 +63,7 @@ "51b2058d8afd": "api_tokens", "168a2882f443": "keycloak", "e36ab09c4a92": "tenants", + "bb906e1a1089": "background_tasks", } # Known mislabels: revisions that inherit a label from an ancestor owned by a @@ -82,6 +84,8 @@ "b5018b617142": "api_tokens", "c8e4f1a2b6d9": "api_tokens", "d7b3a9e5c1f2": "api_tokens", + "bb906e1a1089": "api_tokens", # also its own ``background_tasks`` + "51b2058d8afd": "api_tokens", # ancestors of a labelled revision inherit its label too } diff --git a/modules/background_tasks/background_tasks/_signal_support.py b/modules/background_tasks/background_tasks/_signal_support.py index d4605ee36..37be8af1c 100644 --- a/modules/background_tasks/background_tasks/_signal_support.py +++ b/modules/background_tasks/background_tasks/_signal_support.py @@ -7,14 +7,20 @@ from __future__ import annotations +import logging import traceback as tb_mod +from collections.abc import Callable from datetime import UTC, datetime from typing import Any from sqlalchemy import select from sqlalchemy.orm import Session +from background_tasks.constants import INTERNAL_TASK_SWEEP_OUTBOX, TaskStatus from background_tasks.models import TaskExecution +from background_tasks.sync_db import sync_session + +logger = logging.getLogger(__name__) def now_utc() -> datetime: @@ -103,3 +109,48 @@ def render_traceback(einfo: Any, exception: BaseException | None) -> str | None: if exception is not None: return "".join(tb_mod.format_exception(type(exception), exception, exception.__traceback__)) return None + + +# Beat ticks that fire every few seconds and are a no-op almost every time; +# recording each run buried real work (QA, GH #459). Only a run that went wrong +# is worth a row. +_UNRECORDED_WHEN_QUIET = frozenset({INTERNAL_TASK_SWEEP_OUTBOX}) +_QUIET_STATUSES = frozenset({None, TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.SUCCESS}) + + +def apply_signal( + handler: str, + *, + celery_task_id: str | None, + defaults: dict[str, Any], + after: Callable[[TaskExecution], None] | None = None, + tenant_id: str | None = None, + owner_user_id: str | None = None, +) -> TaskExecution | None: + """Open a sync session, upsert by celery_task_id, log on failure. + + Returns the upserted row (or ``None`` if the session raised) so callers + can read DB-assigned fields like ``id`` without a second round trip. + + ``tenant_id`` and ``owner_user_id`` (the message's) are written only when + known, so a signal whose message carries neither never blanks a row already + stamped. + """ + if ( + defaults.get("task_name") in _UNRECORDED_WHEN_QUIET + and defaults.get("status") in _QUIET_STATUSES + ): + return None + if tenant_id: + defaults = {**defaults, "tenant_id": tenant_id} + if owner_user_id: + defaults = {**defaults, "owner_user_id": owner_user_id} + try: + with sync_session() as session: + row = upsert_by_celery_id(session, celery_task_id=celery_task_id, defaults=defaults) + if after is not None: + after(row) + return row + except Exception: + logger.exception("%s failed for task_id=%s", handler, celery_task_id) + return None diff --git a/modules/background_tasks/background_tasks/constants.py b/modules/background_tasks/background_tasks/constants.py index e2d78bcc3..f116bd01c 100644 --- a/modules/background_tasks/background_tasks/constants.py +++ b/modules/background_tasks/background_tasks/constants.py @@ -41,6 +41,8 @@ VIEW_PREFIX = "/admin/background-tasks" ADMIN_ROUTER_PREFIX = "/admin" +STATUS_ROUTER_PREFIX = "/tasks" + # ── Menu ──────────────────────────────────────────────────────── MENU_LABEL = "Background tasks" MENU_ICON = "activity" @@ -172,3 +174,7 @@ def max_outbox_handler_timeout(lease_seconds: float) -> float: TENANT_ID_MAX_LENGTH = 50 # Filter value meaning "executions with no tenant" (beat / platform publishes). PLATFORM_TENANT_FILTER = "__platform__" + +OWNER_ID_MAX_LENGTH = 64 +PROGRESS_MESSAGE_MAX_LENGTH = 255 +PROGRESS_PHASE_MAX_LENGTH = 64 diff --git a/modules/background_tasks/background_tasks/contracts/schemas.py b/modules/background_tasks/background_tasks/contracts/schemas.py index 35dcb2027..e94f6e351 100644 --- a/modules/background_tasks/background_tasks/contracts/schemas.py +++ b/modules/background_tasks/background_tasks/contracts/schemas.py @@ -59,6 +59,35 @@ class TaskExecutionDetail(SQLModel): worker: str | None = None retries: int = 0 retried_from_id: uuid.UUID | None = None + owner_user_id: str | None = None + progress_current: int | None = None + progress_total: int | None = None + progress_message: str | None = None + progress_phase: str | None = None + queued_at: datetime | None = None + started_at: datetime | None = None + finished_at: datetime | None = None + heartbeat_at: datetime | None = None + + +class TaskStatusDTO(SQLModel): + """What the owner of a task may see of it (#536). + + Deliberately narrower than :class:`TaskExecutionDetail`: no args, kwargs, + result or traceback, and the error is the exception's *type name* only — + a message or traceback can carry other users' data or server internals. + """ + + model_config = ConfigDict(from_attributes=True) # ty: ignore[invalid-assignment] + + celery_task_id: str + task_name: str + status: TaskStatus + progress_current: int | None = None + progress_total: int | None = None + progress_message: str | None = None + progress_phase: str | None = None + exception_type: str | None = None queued_at: datetime | None = None started_at: datetime | None = None finished_at: datetime | None = None diff --git a/modules/background_tasks/background_tasks/endpoints/api_status.py b/modules/background_tasks/background_tasks/endpoints/api_status.py new file mode 100644 index 000000000..909c97fe3 --- /dev/null +++ b/modules/background_tasks/background_tasks/endpoints/api_status.py @@ -0,0 +1,63 @@ +"""Owner-scoped task status (#536). + +Mounted apart from the admin router because it carries no ``PERM_VIEW`` gate: +the person who started a job may follow it without being a task operator. The +check lives in the handler instead, and every refusal is a 404 so the route +does not reveal which task ids exist. +""" + +from __future__ import annotations + +import re + +from fastapi import APIRouter, Depends, HTTPException, Request +from simple_module_core.permissions import grants +from simple_module_db import RequestSession, current_tenant_id +from simple_module_db.deps import get_db +from simple_module_hosting.permissions import ensure_resolved_permissions +from sqlalchemy import select + +from background_tasks.constants import MODULE_NAME, PERM_VIEW, STATUS_ROUTER_PREFIX +from background_tasks.contracts.schemas import TaskStatusDTO +from background_tasks.models import TaskExecution + +router = APIRouter(prefix=STATUS_ROUTER_PREFIX, tags=[f"{MODULE_NAME}-status"]) + +_TYPE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]{0,254}$") +_NOT_FOUND = "Task not found" + + +async def _visible(request: Request, row: TaskExecution, user_id: str) -> bool: + if grants(await ensure_resolved_permissions(request), PERM_VIEW): + return True + if not row.owner_user_id or row.owner_user_id != user_id: + return False + if not bool(getattr(request.app.state.sm.settings, "multi_tenant", False)): + return True + tenant_id = current_tenant_id.get() + return tenant_id is not None and row.tenant_id == tenant_id + + +@router.get("/{celery_task_id}/status", response_model=TaskStatusDTO) +async def get_task_status( + celery_task_id: str, + request: Request, + db: RequestSession = Depends(get_db), +) -> TaskStatusDTO: + user = getattr(request.state, "user", None) + if user is None: + raise HTTPException(status_code=401, detail="Authentication required") + row = ( + await db.execute( + select(TaskExecution) + .where(TaskExecution.celery_task_id == celery_task_id) # ty: ignore[invalid-argument-type] + .order_by(TaskExecution.queued_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + if row is None or not await _visible(request, row, str(user.id)): + raise HTTPException(status_code=404, detail=_NOT_FOUND) + error = ( + row.exception_type if row.exception_type and _TYPE_NAME.match(row.exception_type) else None + ) + return TaskStatusDTO.model_validate(row, update={"exception_type": error}) diff --git a/modules/background_tasks/background_tasks/locales/en.json b/modules/background_tasks/background_tasks/locales/en.json index 9f8118f21..a54c57177 100644 --- a/modules/background_tasks/background_tasks/locales/en.json +++ b/modules/background_tasks/background_tasks/locales/en.json @@ -73,7 +73,9 @@ "copy": "Copy", "copied": "Copied", "no_traceback": "No traceback recorded.", - "tenant": "Tenant" + "tenant": "Tenant", + "progress": "Progress", + "progress_count": "{current} of {total} ({percent}%)" }, "retry_dialog": { "title": "Retry {name}?", diff --git a/modules/background_tasks/background_tasks/models.py b/modules/background_tasks/background_tasks/models.py index d5d0b80a6..0fd0c8a75 100644 --- a/modules/background_tasks/background_tasks/models.py +++ b/modules/background_tasks/background_tasks/models.py @@ -16,6 +16,9 @@ DEFAULT_QUEUE, MODULE_NAME, OUTBOX_KEY_MAX_LENGTH, + OWNER_ID_MAX_LENGTH, + PROGRESS_MESSAGE_MAX_LENGTH, + PROGRESS_PHASE_MAX_LENGTH, TABLE_OUTBOX, TABLE_TASK_EXECUTION, TENANT_ID_MAX_LENGTH, @@ -45,6 +48,10 @@ class TaskExecution(Base, AuditMixin, table=True): # ty: ignore[unsupported-bas "finished_at", "queued_at", "retries", + "progress_current", + "progress_total", + "progress_message", + "progress_phase", } id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) @@ -70,6 +77,17 @@ class TaskExecution(Base, AuditMixin, table=True): # ty: ignore[unsupported-bas # platform-wide and filter on this column. tenant_id: str | None = Field(default=None, index=True, max_length=TENANT_ID_MAX_LENGTH) + # The user whose request enqueued the task (``current_user_id`` at publish, + # carried through tasks that enqueue tasks); NULL for beat and platform + # publishes. It scopes who may read the task's status (#536). + owner_user_id: str | None = Field(default=None, index=True, max_length=OWNER_ID_MAX_LENGTH) + + # Progress the task body reports through ``heartbeat(progress=...)``. + progress_current: int | None = None + progress_total: int | None = None + progress_message: str | None = Field(default=None, max_length=PROGRESS_MESSAGE_MAX_LENGTH) + progress_phase: str | None = Field(default=None, max_length=PROGRESS_PHASE_MAX_LENGTH) + args: list[Any] = Field(default_factory=list, sa_column=Column(JSON)) kwargs: dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON)) result: dict[str, Any] | None = Field(default=None, sa_column=Column(JSON)) diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 343949e29..195acd828 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -123,11 +123,13 @@ def register_menu_items(self, registry: MenuRegistry) -> None: def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: from background_tasks.endpoints.api_admin import router as api_admin + from background_tasks.endpoints.api_status import router as api_status from background_tasks.endpoints.outbox_api import router as outbox_api from background_tasks.endpoints.outbox_views import router as outbox_views from background_tasks.endpoints.views import router as views api_router.include_router(api_admin) + api_router.include_router(api_status) api_router.include_router(outbox_api) # Before ``views``: its ``/{execution_id}`` would otherwise swallow ``/outbox``. view_router.include_router(outbox_views) diff --git a/modules/background_tasks/background_tasks/pages/Detail.tsx b/modules/background_tasks/background_tasks/pages/Detail.tsx index 84b7b808d..b40f60ec0 100644 --- a/modules/background_tasks/background_tasks/pages/Detail.tsx +++ b/modules/background_tasks/background_tasks/pages/Detail.tsx @@ -8,6 +8,7 @@ import { ArrowLeft, RefreshCcw } from 'lucide-react'; import { useState } from 'react'; import { DetailFacts } from './components/DetailFacts'; import { PayloadCards } from './components/PayloadCards'; +import { ProgressCard } from './components/ProgressCard'; import { RetryConfirmDialog } from './components/RetryConfirmDialog'; import { StatusPill } from './components/StatusPill'; import { TracebackCard } from './components/TracebackCard'; @@ -96,6 +97,8 @@ function Detail() { {/* `order` puts the traceback straight after the facts on a phone — the payload cards are reference material, the traceback is why anyone opened this page. On desktop the deck's order returns. */} + +
diff --git a/modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx b/modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx new file mode 100644 index 000000000..f525bac00 --- /dev/null +++ b/modules/background_tasks/background_tasks/pages/components/ProgressCard.tsx @@ -0,0 +1,43 @@ +import { keys, useT } from '@simple-module-py/i18n'; +import { Card } from '@simple-module-py/ui/components/ui/card'; +import { Progress } from '@simple-module-py/ui/components/ui/progress'; +import type { TaskDetail } from '../constants'; + +interface Props { + execution: Pick< + TaskDetail, + 'progress_current' | 'progress_total' | 'progress_message' | 'progress_phase' + >; + className?: string; +} + +/** The task's own progress report, when it sent one. */ +export function ProgressCard({ execution, className }: Props) { + const { t } = useT(); + const { progress_current: current, progress_total: total } = execution; + if (current == null || total == null) return null; + + const percent = total > 0 ? Math.min(100, Math.round((current / total) * 100)) : 0; + return ( + +
+ + {execution.progress_phase ?? t(keys.background_tasks.detail.progress)} + + + {t(keys.background_tasks.detail.progress_count, { current, total, percent })} + +
+
+ +
+ {execution.progress_message && ( +

{execution.progress_message}

+ )} + + ); +} diff --git a/modules/background_tasks/background_tasks/pages/constants.ts b/modules/background_tasks/background_tasks/pages/constants.ts index 681f70f3a..15db2d753 100644 --- a/modules/background_tasks/background_tasks/pages/constants.ts +++ b/modules/background_tasks/background_tasks/pages/constants.ts @@ -104,6 +104,11 @@ export interface TaskDetail { worker: string | null; retries: number; retried_from_id: string | null; + owner_user_id: string | null; + progress_current: number | null; + progress_total: number | null; + progress_message: string | null; + progress_phase: string | null; queued_at: string | null; started_at: string | null; finished_at: string | null; diff --git a/modules/background_tasks/background_tasks/retry_service.py b/modules/background_tasks/background_tasks/retry_service.py index 9e312d652..d7d5743f9 100644 --- a/modules/background_tasks/background_tasks/retry_service.py +++ b/modules/background_tasks/background_tasks/retry_service.py @@ -25,6 +25,7 @@ from background_tasks.filters import Conditions, bulk_retry_conditions from background_tasks.models import TaskExecution from background_tasks.sync_db import sync_session +from background_tasks.tenant_context import publish_as_owner if TYPE_CHECKING: from celery import Celery @@ -186,6 +187,7 @@ def _publisher(self, row: TaskExecution, task_id: str) -> Callable[[], str]: """ name = row.task_name tenant_id = row.tenant_id + owner_user_id = row.owner_user_id options: dict[str, Any] = { "args": list(row.args or []), "kwargs": dict(row.kwargs or {}), @@ -210,11 +212,14 @@ def _send() -> str: # tenant's job stamped with theirs (``stamp_tenant`` would refuse # the mismatch). A row with no tenant is a platform publish, so it # goes out with none bound. - if tenant_id: - with tenant_context(tenant_id): + # The original owner stays the owner, so the person who started the + # job can still follow its retry. + with publish_as_owner(owner_user_id): + if tenant_id: + with tenant_context(tenant_id): + return self.celery.send_task(name, **options).id + with all_tenants(): return self.celery.send_task(name, **options).id - with all_tenants(): - return self.celery.send_task(name, **options).id return publish @@ -226,6 +231,7 @@ def _new_attempt(self, row: TaskExecution, celery_task_id: str) -> TaskExecution status=TaskStatus.PENDING, queue=row.queue, tenant_id=row.tenant_id, + owner_user_id=row.owner_user_id, args=list(row.args or []), kwargs=dict(row.kwargs or {}), retried_from_id=row.id, diff --git a/modules/background_tasks/background_tasks/signals.py b/modules/background_tasks/background_tasks/signals.py index ab5f5e5b3..84334cd85 100644 --- a/modules/background_tasks/background_tasks/signals.py +++ b/modules/background_tasks/background_tasks/signals.py @@ -19,83 +19,39 @@ from __future__ import annotations -import logging -from collections.abc import Callable from typing import Any from celery import signals from background_tasks._signal_bus import bind_event_bus, publish_from_signal, unbind_event_bus from background_tasks._signal_support import ( + apply_signal, coerce_args_kwargs, jsonable_result, now_utc, render_traceback, task_id_from, task_name_of, - upsert_by_celery_id, ) from background_tasks.constants import ( DEFAULT_QUEUE, - INTERNAL_TASK_SWEEP_OUTBOX, TaskStatus, ) from background_tasks.contracts.events import TaskFailed from background_tasks.log_context import signal_task_finished, signal_task_started from background_tasks.models import TaskExecution -from background_tasks.sync_db import sync_session from background_tasks.task_heartbeat import forget_heartbeat from background_tasks.tenant_context import ( + message_owner, message_tenant, + published_owner, published_tenant, release_tenant, restore_tenant, + stamp_owner, stamp_tenant, ) -logger = logging.getLogger(__name__) - -# Beat ticks that fire every few seconds and are a no-op almost every time; -# recording each run buried real work (QA, GH #459). Only a run that went wrong -# is worth a row. -_UNRECORDED_WHEN_QUIET = frozenset({INTERNAL_TASK_SWEEP_OUTBOX}) -_QUIET_STATUSES = frozenset({None, TaskStatus.PENDING, TaskStatus.RUNNING, TaskStatus.SUCCESS}) - - -def _apply( - handler: str, - *, - celery_task_id: str | None, - defaults: dict[str, Any], - after: Callable[[TaskExecution], None] | None = None, - tenant_id: str | None = None, -) -> TaskExecution | None: - """Open a sync session, upsert by celery_task_id, log on failure. - - Returns the upserted row (or ``None`` if the session raised) so callers - can read DB-assigned fields like ``id`` without a second round trip. - - ``tenant_id`` (the message's) is written only when known, so a signal - whose message carries no tenant never blanks a row already stamped. - """ - if ( - defaults.get("task_name") in _UNRECORDED_WHEN_QUIET - and defaults.get("status") in _QUIET_STATUSES - ): - return None - if tenant_id: - defaults = {**defaults, "tenant_id": tenant_id} - try: - with sync_session() as session: - row = upsert_by_celery_id(session, celery_task_id=celery_task_id, defaults=defaults) - if after is not None: - after(row) - return row - except Exception: - logger.exception("%s failed for task_id=%s", handler, celery_task_id) - return None - - # ── Enqueue ───────────────────────────────────────────────────── @@ -109,6 +65,7 @@ def on_task_publish( ) -> None: """Record a row the moment a task is pushed onto the broker.""" stamp_tenant(headers) + stamp_owner(headers) task_id = (headers or {}).get("id") task_name = sender or (headers or {}).get("task") or "unknown" @@ -127,11 +84,12 @@ def on_task_publish( "kwargs": kwargs, "queued_at": now_utc(), } - _apply( + apply_signal( "on_task_publish", celery_task_id=task_id, defaults=defaults, tenant_id=published_tenant(headers), + owner_user_id=published_owner(headers), ) @@ -151,7 +109,7 @@ def on_task_prerun( args_n, kwargs_n = coerce_args_kwargs(args, kwargs) now = now_utc() name = task_name_of(sender, task) - _apply( + apply_signal( "on_task_prerun", celery_task_id=task_id, defaults={ @@ -163,6 +121,7 @@ def on_task_prerun( "heartbeat_at": now, }, tenant_id=message_tenant(task=task), + owner_user_id=message_owner(task=task), ) signal_task_started(task_id=task_id, task_name=name) restore_tenant(task_id=task_id, task=task) @@ -181,7 +140,7 @@ def on_task_postrun( ``task_retry`` which fire *before* postrun; postrun only refreshes the heartbeat so the sweep doesn't immediately flip a just-finished row. """ - _apply( + apply_signal( "on_task_postrun", celery_task_id=task_id, defaults={"task_name": task_name_of(sender, task), "heartbeat_at": now_utc()}, @@ -194,7 +153,7 @@ def on_task_postrun( @signals.task_success.connect def on_task_success(sender: Any = None, result: Any = None, **_k: Any) -> None: - _apply( + apply_signal( "on_task_success", celery_task_id=task_id_from(sender=sender), defaults={ @@ -220,7 +179,7 @@ def on_task_failure( task_name = task_name_of(sender) exception_type = type(exception).__name__ if exception is not None else None - row = _apply( + row = apply_signal( "on_task_failure", celery_task_id=task_id, defaults={ @@ -256,7 +215,7 @@ def _stamp_reason(row: TaskExecution) -> None: if reason is not None: row.traceback = f"retry: {reason!r}" - _apply( + apply_signal( "on_task_retry", celery_task_id=task_id_from(request=request), defaults={ @@ -272,7 +231,7 @@ def _stamp_reason(row: TaskExecution) -> None: @signals.task_revoked.connect def on_task_revoked(sender: Any = None, request: Any = None, **_k: Any) -> None: - _apply( + apply_signal( "on_task_revoked", celery_task_id=task_id_from(request=request), defaults={ diff --git a/modules/background_tasks/background_tasks/task_heartbeat.py b/modules/background_tasks/background_tasks/task_heartbeat.py index 97497cfc2..f52d49b89 100644 --- a/modules/background_tasks/background_tasks/task_heartbeat.py +++ b/modules/background_tasks/background_tasks/task_heartbeat.py @@ -7,6 +7,10 @@ from *inside* the task, every job longer than the threshold was flagged stuck and could be run a second time while the first was still working. +A task body also reports progress through it (#536): ``heartbeat(progress=(i, +n), message=..., phase=...)``. The latest report is buffered under the same +throttle and written with the next heartbeat; ``i == n`` forces the write. + A task body calls :func:`heartbeat` from its loop. Writes are throttled per task, in-process, so calling it on every iteration is cheap; and it is a no-op outside a running task, so shared helpers can call it unconditionally. @@ -22,13 +26,20 @@ from sqlalchemy import update from background_tasks._signal_support import now_utc -from background_tasks.constants import HEARTBEAT_MIN_INTERVAL_SECONDS, TaskStatus +from background_tasks.constants import ( + HEARTBEAT_MIN_INTERVAL_SECONDS, + PROGRESS_MESSAGE_MAX_LENGTH, + PROGRESS_PHASE_MAX_LENGTH, + TaskStatus, +) from background_tasks.models import TaskExecution from background_tasks.sync_db import sync_session logger = logging.getLogger(__name__) _last_beat: dict[str, float] = {} +# Progress reported but not yet written, per task: the latest report wins. +_pending: dict[str, dict[str, object]] = {} _lock = threading.Lock() @@ -48,14 +59,40 @@ def _current_task_id() -> str | None: return task_id if isinstance(task_id, str) and task_id else None -def heartbeat(*, force: bool = False) -> bool: - """Tell the stuck sweep this task is still alive. +def _progress_values( + progress: tuple[int, int] | None, message: str | None, phase: str | None +) -> dict[str, object]: + values: dict[str, object] = {} + if progress is not None: + current, total = int(progress[0]), int(progress[1]) + values["progress_current"] = max(current, 0) + values["progress_total"] = max(total, 0) + if message is not None: + values["progress_message"] = message[:PROGRESS_MESSAGE_MAX_LENGTH] + if phase is not None: + values["progress_phase"] = phase[:PROGRESS_PHASE_MAX_LENGTH] + return values + + +def heartbeat( + *, + progress: tuple[int, int] | None = None, + message: str | None = None, + phase: str | None = None, + force: bool = False, +) -> bool: + """Tell the stuck sweep this task is still alive, optionally with progress. Refreshes ``heartbeat_at`` on the current task's execution row, at most once per :data:`~background_tasks.constants.HEARTBEAT_MIN_INTERVAL_SECONDS` per task unless *force* is set. Call it more often than the task's stuck threshold — every loop iteration is fine. + *progress* is ``(done, total)``; *message* and *phase* are short labels. + A throttled call keeps the latest of them and the next write carries it; + reaching ``done == total`` writes immediately so a finished task never + shows 9/10. None of these columns is audited. + Returns whether a write was made. Never raises: outside a task it does nothing, and a failed write is logged rather than failing the job it is reporting on. @@ -63,12 +100,23 @@ def heartbeat(*, force: bool = False) -> bool: task_id = _current_task_id() if task_id is None: return False + try: + reported = _progress_values(progress, message, phase) + except (TypeError, ValueError, IndexError): + logger.warning("ignoring malformed progress %r for task_id=%s", progress, task_id) + reported = _progress_values(None, message, phase) + if progress is not None and "progress_current" in reported: + force = force or reported["progress_current"] == reported["progress_total"] now = time.monotonic() with _lock: + pending = {**_pending.get(task_id, {}), **reported} last = _last_beat.get(task_id) if not force and last is not None and now - last < HEARTBEAT_MIN_INTERVAL_SECONDS: + if pending: + _pending[task_id] = pending return False _last_beat[task_id] = now + _pending.pop(task_id, None) try: with sync_session() as session: session.execute( @@ -80,7 +128,7 @@ def heartbeat(*, force: bool = False) -> bool: # retried since) or rewrite a terminal one. TaskExecution.status == TaskStatus.RUNNING, # ty: ignore[invalid-argument-type] ) - .values(heartbeat_at=now_utc()) + .values(heartbeat_at=now_utc(), **pending) ) except Exception: logger.exception("heartbeat failed for task_id=%s", task_id) @@ -93,6 +141,7 @@ def forget_heartbeat(task_id: str | None) -> None: if task_id: with _lock: _last_beat.pop(task_id, None) + _pending.pop(task_id, None) __all__ = ["forget_heartbeat", "heartbeat"] diff --git a/modules/background_tasks/background_tasks/tenant_context.py b/modules/background_tasks/background_tasks/tenant_context.py index 229358655..05fe065f9 100644 --- a/modules/background_tasks/background_tasks/tenant_context.py +++ b/modules/background_tasks/background_tasks/tenant_context.py @@ -12,15 +12,26 @@ from __future__ import annotations import logging -from contextvars import Token +from collections.abc import Generator +from contextlib import contextmanager +from contextvars import ContextVar, Token from typing import Any from simple_module_db import TenantIsolationError, current_tenant_id, is_valid_tenant_id +from simple_module_db.listeners import current_user_id + +from background_tasks.constants import OWNER_ID_MAX_LENGTH TENANT_HEADER = "sm_tenant_id" +OWNER_HEADER = "sm_owner_id" _log = logging.getLogger(__name__) _tokens: dict[str, Token[str | None]] = {} +_owner_tokens: dict[str, Token[str | None]] = {} +# The owner of the task body running on this context, so tasks it enqueues +# carry the same owner. Kept apart from ``current_user_id`` on purpose: that +# one also stamps ``created_by`` on audited rows. +_task_owner: ContextVar[str | None] = ContextVar("sm_task_owner", default=None) # Single-tenant hosts (``default_tenant``): tasks with no tenant on the message # — beat jobs, tasks enqueued outside a request — run as that tenant. _default_tenant: str | None = None @@ -60,6 +71,41 @@ def stamp_tenant(headers: dict[str, Any] | None) -> None: headers[TENANT_HEADER] = tenant_id +def stamp_owner(headers: dict[str, Any] | None) -> None: + """Record the user whose request enqueues this message (publish side). + + A task that enqueues a task has no ``current_user_id`` in its worker, so + :func:`restore_tenant` binds the incoming message's owner for the body and + this stamps it onward; :func:`publish_as_owner` does the same for a retry, + which must keep the original owner rather than adopt the operator. Failing + both, the bound user is the owner. A header already on the message is + ignored when a user is bound (request code must not name another owner). + """ + if headers is None: + return + user_id = _task_owner.get() + if user_id is None: + user_id = current_user_id.get() + if user_id: + headers[OWNER_HEADER] = str(user_id)[:OWNER_ID_MAX_LENGTH] + + +@contextmanager +def publish_as_owner(owner_user_id: str | None) -> Generator[None]: + """Publish tasks as *owner_user_id* (``None``: no owner), whoever is bound.""" + token = _task_owner.set(owner_user_id or "") + try: + yield + finally: + _task_owner.reset(token) + + +def published_owner(headers: dict[str, Any] | None) -> str | None: + """The owner an outgoing message carries (call after :func:`stamp_owner`).""" + value = (headers or {}).get(OWNER_HEADER) + return str(value)[:OWNER_ID_MAX_LENGTH] if value else None + + def published_tenant(headers: dict[str, Any] | None) -> str | None: """The tenant an outgoing message is for: its header, else the publisher's. @@ -73,14 +119,28 @@ def _tenant_of(task: Any) -> str | None: return _tenant_of_request(getattr(task, "request", None)) -def _tenant_of_request(request: Any) -> str | None: - if request is None: - return None - value = getattr(request, TENANT_HEADER, None) +def _header_of_request(request: Any, name: str) -> Any: + value = getattr(request, name, None) # A worker ``Request`` (revoked signal) keeps headers in a dict instead. for attr in ("headers", "request_dict"): if value is None and isinstance(getattr(request, attr, None), dict): - value = getattr(request, attr).get(TENANT_HEADER) + value = getattr(request, attr).get(name) + return value + + +def message_owner(*, task: Any = None, request: Any = None) -> str | None: + """The owner named on a running task's message header, if any.""" + request = request if request is not None else getattr(task, "request", None) + if request is None: + return None + value = _header_of_request(request, OWNER_HEADER) + return str(value)[:OWNER_ID_MAX_LENGTH] if value else None + + +def _tenant_of_request(request: Any) -> str | None: + if request is None: + return None + value = _header_of_request(request, TENANT_HEADER) if not value: return None if not is_valid_tenant_id(value): @@ -107,6 +167,9 @@ def restore_tenant(*, task_id: str | None, task: Any) -> None: """Enter the message's tenant for the task body (prerun).""" if not task_id: return + owner = message_owner(task=task) + if owner is not None: + _owner_tokens[task_id] = _task_owner.set(owner) tenant_id = _tenant_of(task) or _default_tenant if tenant_id is not None: _tokens[task_id] = current_tenant_id.set(tenant_id) @@ -114,6 +177,12 @@ def restore_tenant(*, task_id: str | None, task: Any) -> None: def release_tenant(*, task_id: str | None) -> None: """Leave the tenant entered by :func:`restore_tenant` (postrun).""" + owner_token = _owner_tokens.pop(task_id, None) if task_id else None + if owner_token is not None: + try: + _task_owner.reset(owner_token) + except ValueError: + _task_owner.set(None) token = _tokens.pop(task_id, None) if task_id else None if token is None: return @@ -127,12 +196,17 @@ def release_tenant(*, task_id: str | None) -> None: __all__ = [ + "OWNER_HEADER", "TENANT_HEADER", "default_tenant", + "message_owner", "message_tenant", + "publish_as_owner", + "published_owner", "published_tenant", "release_tenant", "restore_tenant", "set_default_tenant", + "stamp_owner", "stamp_tenant", ] diff --git a/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx b/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx index 63609a313..a9c5193e3 100644 --- a/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx +++ b/modules/background_tasks/tests-js/background_tasks_DetailPage.test.tsx @@ -41,6 +41,11 @@ function execution(over: Partial = {}): TaskDetail { worker: 'w1', retries: 2, retried_from_id: null, + owner_user_id: null, + progress_current: null, + progress_total: null, + progress_message: null, + progress_phase: null, queued_at: '2026-01-01T00:00:00Z', started_at: '2026-01-01T00:00:01Z', finished_at: '2026-01-01T00:00:02Z', @@ -66,6 +71,18 @@ describe('background_tasks Detail page', () => { expect(screen.getByText('Traceback (most recent call last)')).toBeInTheDocument(); }); + it('shows a progress bar only when the task reported progress', () => { + const { unmount } = render(); + expect(screen.queryByTestId('task-progress')).toBeNull(); + unmount(); + + setProps({ progress_current: 3, progress_total: 10, progress_message: 'copying rows' }); + render(); + expect(screen.getByTestId('task-progress')).toBeInTheDocument(); + expect(screen.getByText('copying rows')).toBeInTheDocument(); + expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '30'); + }); + it('offers no retry to a viewer without the manage permission', () => { setProps({}, []); render(); diff --git a/modules/background_tasks/tests/conftest.py b/modules/background_tasks/tests/conftest.py index 4316528d9..640aaa81f 100644 --- a/modules/background_tasks/tests/conftest.py +++ b/modules/background_tasks/tests/conftest.py @@ -83,9 +83,12 @@ async def _seed( queue: str = "default", queued_at: datetime | None = None, tenant_id: str | None = None, + owner_user_id: str | None = None, + celery_task_id: str | None = None, + **fields: object, ) -> TaskExecution: row = TaskExecution( - celery_task_id=str(uuid.uuid4()), + celery_task_id=celery_task_id or str(uuid.uuid4()), task_name=task_name, status=status, queue=queue, @@ -93,6 +96,8 @@ async def _seed( kwargs={}, queued_at=queued_at or datetime.now(UTC), tenant_id=tenant_id, + owner_user_id=owner_user_id, + **fields, ) async with app.state.sm.db.session_factory() as session: session.add(row) diff --git a/modules/background_tasks/tests/test_bg_task_progress.py b/modules/background_tasks/tests/test_bg_task_progress.py new file mode 100644 index 000000000..a7118d5af --- /dev/null +++ b/modules/background_tasks/tests/test_bg_task_progress.py @@ -0,0 +1,139 @@ +"""Progress reporting and owner stamping on task executions (#536).""" + +from __future__ import annotations + +import uuid +from pathlib import Path +from types import SimpleNamespace + +import pytest +from background_tasks import heartbeat, sync_db +from background_tasks import task_heartbeat as heartbeat_mod +from background_tasks.constants import TaskStatus +from background_tasks.models import TaskExecution +from background_tasks.signals import on_task_postrun, on_task_prerun, on_task_publish +from background_tasks.tenant_context import OWNER_HEADER, publish_as_owner +from simple_module_db.listeners import current_user_id +from sqlalchemy import select + + +def _row(celery_id: str) -> TaskExecution: + with sync_db.get_sync_session_factory()() as session: + return session.execute( + select(TaskExecution).where(TaskExecution.celery_task_id == celery_id) + ).scalar_one() + + +@pytest.fixture +def running(sync_sqlite: Path, monkeypatch: pytest.MonkeyPatch) -> str: + heartbeat_mod._last_beat.clear() + heartbeat_mod._pending.clear() + celery_id = str(uuid.uuid4()) + with sync_db.get_sync_session_factory()() as session: + session.add( + TaskExecution(celery_task_id=celery_id, task_name="t", status=TaskStatus.RUNNING) + ) + session.commit() + monkeypatch.setattr(heartbeat_mod, "_current_task_id", lambda: celery_id) + return celery_id + + +class TestProgress: + def test_a_report_is_stored(self, running: str): + assert heartbeat(progress=(3, 10), message="rows", phase="copy") is True + row = _row(running) + assert (row.progress_current, row.progress_total) == (3, 10) + assert (row.progress_message, row.progress_phase) == ("rows", "copy") + + def test_throttled_reports_are_buffered_and_the_final_one_is_forced(self, running: str): + assert heartbeat(progress=(1, 10)) is True + assert heartbeat(progress=(2, 10), message="two") is False + assert heartbeat(progress=(3, 10)) is False + assert _row(running).progress_current == 1 + + assert heartbeat(progress=(10, 10)) is True + + row = _row(running) + assert row.progress_current == 10 + assert row.progress_message == "two" # buffered label survives the later report + + def test_the_buffered_report_rides_the_next_unthrottled_beat(self, running: str): + heartbeat(progress=(1, 10)) + heartbeat(progress=(5, 10)) + assert heartbeat(force=True) is True + assert _row(running).progress_current == 5 + + def test_labels_are_truncated_to_the_column(self, running: str): + heartbeat(progress=(1, 2), message="m" * 400, phase="p" * 100) + row = _row(running) + assert len(row.progress_message or "") == 255 + assert len(row.progress_phase or "") == 64 + + def test_malformed_progress_does_not_raise(self, running: str): + assert heartbeat(progress=("x",), message="ok") is True + assert _row(running).progress_message == "ok" + + def test_progress_columns_are_not_audited(self): + excluded = TaskExecution.__audit_exclude_fields__ + assert { + "progress_current", + "progress_total", + "progress_message", + "progress_phase", + } <= excluded + + +def _publish(headers: dict | None = None) -> str: + task_id = str(uuid.uuid4()) + headers = {"id": task_id, "task": "demo.echo", **(headers or {})} + on_task_publish(sender="demo.echo", headers=headers, body=([], {}, {})) + return task_id + + +class TestOwner: + def test_publish_stamps_the_bound_user(self, sync_sqlite: Path): + token = current_user_id.set("u-1") + try: + task_id = _publish() + finally: + current_user_id.reset(token) + assert _row(task_id).owner_user_id == "u-1" + + def test_platform_publish_has_no_owner(self, sync_sqlite: Path): + assert _row(_publish()).owner_user_id is None + + def test_a_request_cannot_name_another_owner(self, sync_sqlite: Path): + token = current_user_id.set("u-1") + try: + task_id = _publish({OWNER_HEADER: "someone-else"}) + finally: + current_user_id.reset(token) + assert _row(task_id).owner_user_id == "u-1" + + def test_a_retry_keeps_the_original_owner(self, sync_sqlite: Path): + token = current_user_id.set("operator") + try: + with publish_as_owner("original"): + task_id = _publish() + with publish_as_owner(None): + platform = _publish() + finally: + current_user_id.reset(token) + assert _row(task_id).owner_user_id == "original" + assert _row(platform).owner_user_id is None + + def test_the_owner_survives_a_task_that_enqueues_a_task(self, sync_sqlite: Path): + parent_id = str(uuid.uuid4()) + task = SimpleNamespace( + name="demo.parent", request=SimpleNamespace(id=parent_id, **{OWNER_HEADER: "u-1"}) + ) + on_task_prerun(sender=task, task_id=parent_id, task=task) + try: + assert current_user_id.get() is None # a worker has no request user + child_id = _publish() + finally: + on_task_postrun(sender=task, task_id=parent_id, task=task) + + assert _row(parent_id).owner_user_id == "u-1" + assert _row(child_id).owner_user_id == "u-1" + assert _row(_publish()).owner_user_id is None # released after the body diff --git a/modules/background_tasks/tests/test_bg_task_progress_audit.py b/modules/background_tasks/tests/test_bg_task_progress_audit.py new file mode 100644 index 000000000..841c60fc7 --- /dev/null +++ b/modules/background_tasks/tests/test_bg_task_progress_audit.py @@ -0,0 +1,53 @@ +"""Progress writes leave no audit trail (#536).""" + +from __future__ import annotations + +import uuid +from collections.abc import Iterator + +import pytest +from background_tasks import heartbeat, sync_db +from background_tasks import task_heartbeat as heartbeat_mod +from background_tasks.constants import TaskStatus +from background_tasks.models import TaskExecution +from simple_module_db.audit_sinks import discover_audit_callback +from sqlalchemy import create_engine, func, select + +audit_models = pytest.importorskip("audit_log.models") +AuditEntry = audit_models.AuditEntry + + +@pytest.fixture +def audited_db(tmp_path, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(sync_db, "discover_audit_callback", discover_audit_callback) + url = f"sqlite:///{tmp_path}/progress_audit.db" + engine = create_engine(url) + TaskExecution.metadata.create_all(engine) + AuditEntry.metadata.create_all(engine) + engine.dispose() + sync_db.set_database_url(url) + yield + sync_db.dispose_sync_engine() + + +def _audit_count() -> int: + with sync_db.sync_session() as s: + return s.scalar(select(func.count()).select_from(AuditEntry)) or 0 + + +def test_reporting_progress_adds_no_audit_row(audited_db, monkeypatch: pytest.MonkeyPatch): + celery_id = str(uuid.uuid4()) + with sync_db.sync_session() as s: + s.add(TaskExecution(celery_task_id=celery_id, task_name="t", status=TaskStatus.RUNNING)) + before = _audit_count() + monkeypatch.setattr(heartbeat_mod, "_current_task_id", lambda: celery_id) + heartbeat_mod._last_beat.clear() + + for i in range(1, 6): + heartbeat(progress=(i, 5), message=f"step {i}", force=True) + with sync_db.sync_session() as s: # the ORM path signals use + row = s.scalars(select(TaskExecution)).one() + row.progress_current = 2 + row.progress_message = "via orm" + + assert _audit_count() == before diff --git a/modules/background_tasks/tests/test_bg_task_status_api.py b/modules/background_tasks/tests/test_bg_task_status_api.py new file mode 100644 index 000000000..398b4d0a0 --- /dev/null +++ b/modules/background_tasks/tests/test_bg_task_status_api.py @@ -0,0 +1,119 @@ +"""Owner-scoped task status endpoint (#536).""" + +from __future__ import annotations + +import httpx +import pytest +from background_tasks.constants import TaskStatus +from simple_module_core.tenancy import TenantRole + +URL = "/api/background_tasks/tasks/{}/status" +SECRETS = {"args", "kwargs", "result", "traceback", "owner_user_id", "tenant_id"} + + +def _seed(seed_execution, tenant, owner, **fields): + return seed_execution( + status=TaskStatus.RUNNING, + tenant_id=tenant, + owner_user_id=owner, + progress_current=3, + progress_total=10, + progress_message="copying rows", + progress_phase="copy", + **fields, + ) + + +async def test_the_owner_sees_the_progress(seed_execution, tenant_client): + async with tenant_client(TenantRole.MEMBER) as me: + row = await _seed(seed_execution, me.tenant_id, me.user_id) + + resp = await me.client.get(URL.format(row.celery_task_id)) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert (body["progress_current"], body["progress_total"]) == (3, 10) + assert (body["progress_message"], body["progress_phase"]) == ("copying rows", "copy") + assert body["status"] == "running" + assert SECRETS.isdisjoint(body) + + +async def test_another_member_of_the_tenant_gets_404(seed_execution, tenant_client): + async with ( + tenant_client(TenantRole.MEMBER) as owner, + tenant_client(TenantRole.MEMBER, tenant_id=owner.tenant_id) as other, + ): + row = await _seed(seed_execution, owner.tenant_id, owner.user_id) + + resp = await other.client.get(URL.format(row.celery_task_id)) + + assert resp.status_code == 404 + + +async def test_the_owner_acting_in_another_tenant_gets_404(seed_execution, tenant_client): + async with tenant_client() as home, tenant_client() as elsewhere: + row = await _seed(seed_execution, home.tenant_id, elsewhere.user_id) + + resp = await elsewhere.client.get(URL.format(row.celery_task_id)) + + assert resp.status_code == 404 + + +async def test_a_task_nobody_owns_is_not_visible_to_members(seed_execution, tenant_client): + async with tenant_client(TenantRole.MEMBER) as me: + row = await _seed(seed_execution, me.tenant_id, None) + + assert (await me.client.get(URL.format(row.celery_task_id))).status_code == 404 + + +async def test_an_unknown_task_is_404(tenant_client): + async with tenant_client() as me: + assert (await me.client.get(URL.format("nope"))).status_code == 404 + + +async def test_a_task_viewer_sees_any_task(seed_execution, authenticated_client: httpx.AsyncClient): + row = await _seed(seed_execution, "acme", "someone-else") + + resp = await authenticated_client.get(URL.format(row.celery_task_id)) + + assert resp.status_code == 200 + assert resp.json()["progress_current"] == 3 + + +async def test_the_error_is_a_type_name_never_a_traceback( + seed_execution, authenticated_client: httpx.AsyncClient +): + row = await seed_execution( + status=TaskStatus.FAILED, + owner_user_id="u", + exception_type="ValueError", + traceback="Traceback ... secret-token", + result={"password": "x"}, + ) + + resp = await authenticated_client.get(URL.format(row.celery_task_id)) + + assert resp.json()["exception_type"] == "ValueError" + assert "secret-token" not in resp.text + assert "password" not in resp.text + + +@pytest.mark.parametrize( + "bad", + [ + pytest.param("Boom: leaked message", id="message"), + pytest.param("x" * 300, id="too-long"), + ], +) +async def test_a_malformed_exception_type_is_dropped( + bad, seed_execution, authenticated_client: httpx.AsyncClient +): + row = await seed_execution(status=TaskStatus.FAILED, exception_type=bad) + + resp = await authenticated_client.get(URL.format(row.celery_task_id)) + + assert resp.json()["exception_type"] is None + + +async def test_anonymous_callers_are_refused(client: httpx.AsyncClient): + assert (await client.get(URL.format("abc"))).status_code in {302, 401} diff --git a/packages/i18n/src/generated-resources.ts b/packages/i18n/src/generated-resources.ts index 61ee34058..aa35513dc 100644 --- a/packages/i18n/src/generated-resources.ts +++ b/packages/i18n/src/generated-resources.ts @@ -143,6 +143,8 @@ export default { 'background_tasks.detail.kwargs': '', 'background_tasks.detail.meta': '', 'background_tasks.detail.no_traceback': '', + 'background_tasks.detail.progress': '', + 'background_tasks.detail.progress_count': '', 'background_tasks.detail.queue': '', 'background_tasks.detail.queued_at': '', 'background_tasks.detail.result': '', diff --git a/packages/i18n/src/keys.generated.ts b/packages/i18n/src/keys.generated.ts index fab915734..7c3269b58 100644 --- a/packages/i18n/src/keys.generated.ts +++ b/packages/i18n/src/keys.generated.ts @@ -199,6 +199,8 @@ export const keys = { kwargs: 'background_tasks.detail.kwargs', meta: 'background_tasks.detail.meta', no_traceback: 'background_tasks.detail.no_traceback', + progress: 'background_tasks.detail.progress', + progress_count: 'background_tasks.detail.progress_count', queue: 'background_tasks.detail.queue', queued_at: 'background_tasks.detail.queued_at', result: 'background_tasks.detail.result', From 38e7865b9c51e7a06d91b18a7d4e461eb7f8c1a0 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:20:17 +0200 Subject: [PATCH 04/22] feat(leases): distributed TTL leases and semaphores (#539) simple_module_core.leases: lease(name, ttl, owner=None) handle with token, strictly increasing fence, refresh()/release() and an async context manager (blocking/wait); semaphore(name, slots, ttl); LeaseBackend Protocol, InProcessLeaseBackend and configure_leases()/get_leases(). Backends: Redis (SET NX PX, Lua compare-and-set refresh/release, semaphore ZSET scored by Redis TIME, INCR fence) in simple_module_hosting; the framework-owned sm_lease table (DB clock, ON CONFLICT claims) in simple_module_db with migration d263ea727663, a new root labelled "leases". create_app and the Celery worker bootstrap select Redis, else the table, else in-process. An expired owner can neither refresh nor release. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 7 + CLAUDE.md | 7 +- docs/.vitepress/config.ts | 1 + docs/database/migrations.md | 4 +- docs/framework/leases.md | 66 +++++ framework/core/simple_module_core/leases.py | 275 ++++++++++++++++++ framework/core/tests/test_core_leases.py | 162 +++++++++++ framework/db/simple_module_db/leases.py | 188 ++++++++++++ framework/db/simple_module_db/migrations.py | 4 +- .../simple_module_hosting/_lease_redis.py | 148 ++++++++++ .../simple_module_hosting/app_builder.py | 2 + .../hosting/simple_module_hosting/leases.py | 55 ++++ .../tests/test_hosting_lease_backends.py | 151 ++++++++++ .../tests/test_hosting_lease_selection.py | 48 +++ .../testing/simple_module_test/_schema.py | 1 + .../versions/d263ea727663_leases_table.py | 36 +++ host/tests/test_migration_graph.py | 5 +- .../background_tasks/runner.py | 3 + .../tests/test_bg_runner_settings.py | 6 + 19 files changed, 1162 insertions(+), 7 deletions(-) create mode 100644 docs/framework/leases.md create mode 100644 framework/core/simple_module_core/leases.py create mode 100644 framework/core/tests/test_core_leases.py create mode 100644 framework/db/simple_module_db/leases.py create mode 100644 framework/hosting/simple_module_hosting/_lease_redis.py create mode 100644 framework/hosting/simple_module_hosting/leases.py create mode 100644 framework/hosting/tests/test_hosting_lease_backends.py create mode 100644 framework/hosting/tests/test_hosting_lease_selection.py create mode 100644 host/migrations/versions/d263ea727663_leases_table.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9807e03a..40efeca92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ All notable changes to this project are documented in this file. The format is b - `host/client_app` no longer re-declares `@simple-module-py/ui`'s runtime dependencies, nor `@base-ui/react`, `@radix-ui/react-slot`, `autoprefixer` or `postcss` (nothing loaded them) (#521). ### Added +- Distributed TTL leases in `simple_module_core.leases`: `lease(name, ttl)` (an + owner-checked lock with `token`, a strictly increasing `fence`, `refresh()`, + `release()` and `async with` waiting) and `semaphore(name, slots, ttl)` (at + most N holders across processes). Backed by Redis when `SM_REDIS_URL` is set, + else the new framework table `sm_lease` (migration `d263ea727663`, a new root + labelled `leases`), else in-process; `create_app` and the Celery worker + bootstrap install it. An expired owner can neither refresh nor release (#539). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial diff --git a/CLAUDE.md b/CLAUDE.md index f6ef422da..f7ae4641b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -294,9 +294,10 @@ development and testing (`session_cookie_secure`). - **Migrations** live in `host/migrations/versions/`, never in module packages. `host/migrations/env.py` builds metadata from every installed module and runs `ensure_db_extensions` first. The history is **not** one linear chain: it has - two roots (`77162e7b184b` initial schema, `e36ab09c4a92` tenants) and four - heads (`audit_log`, `tenants`, `keycloak`, and the users trunk, which Alembic - reports as `api_tokens`). Apply with `upgrade heads` (what `make migrate` + three roots (`77162e7b184b` initial schema, `e36ab09c4a92` tenants, + `d263ea727663` the framework's `sm_lease`) and five heads (`audit_log`, + `tenants`, `keycloak`, `leases`, and the users trunk, which Alembic reports + as `api_tokens`). Apply with `upgrade heads` (what `make migrate` does); plain `revision --autogenerate` refuses with "Multiple heads", so name the branch: diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index bb656edaa..112db705c 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -127,6 +127,7 @@ export default defineConfig({ { text: "Public routes", link: "/framework/public-routes" }, { text: "Events", link: "/framework/events" }, { text: "Cache invalidation", link: "/framework/invalidation" }, + { text: "Leases & semaphores", link: "/framework/leases" }, { text: "Multi-tenancy", link: "/framework/multi-tenancy" }, { text: "Internationalization", link: "/framework/i18n" }, ], diff --git a/docs/database/migrations.md b/docs/database/migrations.md index 71f560fa4..581ec035d 100644 --- a/docs/database/migrations.md +++ b/docs/database/migrations.md @@ -16,12 +16,14 @@ The history is **not** a single linear chain. It has several roots and several h |---|---| | `77162e7b184b` initial schema | the users trunk (also holds file_storage, background_tasks, api_tokens revisions) | | `e36ab09c4a92` tenants initial schema | tenants (`f1a9c3d7b2e4` head) | +| `d263ea727663` framework `sm_lease` table | leases (its own head; [leases](../framework/leases.md)) | | Head | Owning module | Label Alembic reports | |---|---|---| | `c8e4f1a2b6d9` | users (login lockout) | `api_tokens` | | `a7c2e91d4b58` | audit_log | `audit_log` | | `f1a9c3d7b2e4` | tenants | `tenants` | +| `d263ea727663` | framework (`sm_lease`) | `leases` | | `f6a3b9d2e8c4` | keycloak | `keycloak` | `audit_log` forks off the trunk at `41cf2c53660e`, and `keycloak` forks off the users revision `873ca2015033`. @@ -30,7 +32,7 @@ A module's first migration sets `branch_labels = ("",)`. Alembic ** ### Known mislabels -Only four of the revisions set `branch_labels` (`audit_log`, `api_tokens`, `keycloak`, `tenants`). The graph carries these defects; they are documented rather than rewritten, because changing labels on revisions that installed databases already applied is not worth the risk: +Only five of the revisions set `branch_labels` (`audit_log`, `api_tokens`, `keycloak`, `tenants`, `leases`). The graph carries these defects; they are documented rather than rewritten, because changing labels on revisions that installed databases already applied is not worth the risk: - `51b2058d8afd_api_tokens_initial` sits in the middle of the trunk, so every revision above it (file_storage, background_tasks including the outbox, and the users revisions through `c8e4f1a2b6d9`) reports the `api_tokens` label. The `api_tokens` head is therefore really a users revision. - `keycloak` branches off a users revision, so a Keycloak-only install still builds the users schema. diff --git a/docs/framework/leases.md b/docs/framework/leases.md new file mode 100644 index 000000000..bf8e6b5ea --- /dev/null +++ b/docs/framework/leases.md @@ -0,0 +1,66 @@ +# Leases and semaphores + +`simple_module_core.leases` gives every process the same two primitives ([#539](https://github.com/antosubash/simple_module_python/issues/539)): + +- **`lease(name, ttl)`**: an exclusive lock that expires on its own. Use it to deduplicate work ("only one worker imports dataset 42") or to debounce. +- **`semaphore(name, slots, ttl)`**: at most `slots` concurrent holders, for example at most 4 downloads per remote host across every web and Celery worker. + +Both are *leases*: whoever acquires one gets a random `token`, and only that token can refresh or release it. Once the TTL lapses, the old holder can do neither, even if no one else has taken it yet. + +## Usage + +```python +from simple_module_core.leases import LeaseNotAcquiredError, lease, semaphore + +# Exclusive. Waits up to 5 s for the current holder, then raises. +async with lease("import:42", ttl=60, wait=5) as held: + for chunk in chunks: + await process(chunk) + if not await held.refresh(): # extend the TTL from now + raise RuntimeError("lost the lease") # it expired; stop writing + +# Try once, never wait. +job = lease("rebuild-index", ttl=300, blocking=False) +if await job.acquire(): + try: + await rebuild() + finally: + await job.release() + +# Counted slots, shared across processes. +async with semaphore("dl:example.org", slots=4, ttl=120, wait=30): + await download(url) +``` + +| Member | Meaning | +|---|---| +| `token` | The owner's identity. Random unless you pass `owner=` (for example a Celery task id, so a retry of the same task can re-acquire its own lease). | +| `fence` | A number that strictly increases each time the name is acquired. Store it next to whatever the holder writes and refuse a write that carries a smaller fence: that rejects a holder that stalled past its TTL and woke up late. | +| `acquire(blocking=True, wait=None)` | Returns `True` once held. Non-blocking tries once; blocking polls with jittered back-off until it gets the lease or `wait` seconds pass (`None` waits forever). | +| `refresh(ttl=None)` | Extends the TTL from now. Returns `False` if the lease expired or moved to another owner. | +| `release()` | Returns `False` if the lease had already expired or moved on; it never releases someone else's lease. | +| `async with` | Acquires with the handle's `blocking`/`wait`, raises `LeaseNotAcquiredError` if it cannot, and releases on exit. | + +A lease does not refresh itself. Pick a TTL longer than the work, or call `refresh()` from the loop that does it. + +## Backends + +The backend is process-global because a Celery worker has no `app.state`. `create_app` installs it when the app is built, and the worker bootstrap (`background_tasks.runner.build_runner_celery`) installs the same kind from the same settings: + +| Configured | Backend | Notes | +|---|---|---| +| `SM_REDIS_URL` | `RedisLeaseBackend` (`simple_module_hosting`) | `SET NX PX` plus Lua compare-and-set for refresh and release; a semaphore is a sorted set scored by expiry against Redis `TIME`; `INCR :fence` for the fence. | +| a database, no Redis | `DbLeaseBackend` (`simple_module_db`) | Rows in `sm_lease (name, slot, holder, fence, expires_at)`, primary key `(name, slot)`. Every comparison uses the database clock, so clock skew between hosts does not matter. | +| neither | `InProcessLeaseBackend` | One process only. Also what tests and eager Celery get when nothing else is configured. | + +Backend errors propagate. A lease that the backend could not confirm is never reported as held. This is unlike the rate limiter, which falls back to per-process counters. + +Override the backend with `configure_leases(backend)`, or pass `backend=` to a single `lease()`/`semaphore()` call. `configure_leases(None)` restores a fresh in-process backend, which is useful in test teardown. + +### The `sm_lease` table + +The table belongs to the framework, like `settings_setting`, so it exists whether or not any module uses leases. Its migration `d263ea727663` is its own root with the `leases` branch label (see [Migrations](../database/migrations.md)). `make migrate` (`upgrade heads`) applies it. + +Rows are never deleted: a released lease is a row whose `expires_at` is in the past. That is what keeps `fence` increasing for the life of the name. A lease uses slot `0`; a semaphore uses slots `0..n-1` and keeps its fence counter in slot `-1`. Use a bounded set of names: `"dl:"` is fine, a fresh UUID per request is not. + +On SQLite each attempt takes the database write lock before it reads, so processes queue on `busy_timeout` instead of racing. The table works on SQLite, but production deployments with several hosts should use Postgres or Redis. diff --git a/framework/core/simple_module_core/leases.py b/framework/core/simple_module_core/leases.py new file mode 100644 index 000000000..e230e6b81 --- /dev/null +++ b/framework/core/simple_module_core/leases.py @@ -0,0 +1,275 @@ +"""Distributed TTL leases: owner-checked locks and counted slots (GH #539). + +A *lease* is a lock that expires on its own. Whoever acquires it gets a random +``token`` and a ``fence`` number; only the holder of the token can refresh or +release it, and once the TTL lapses the old holder cannot do either, even if +someone else has not taken it yet. ``fence`` strictly increases per name, so a +downstream write can reject a stale holder that woke up late:: + + from simple_module_core.leases import lease, semaphore + + async with lease("import:42", ttl=60) as held: + ... + await held.refresh() # extend before the TTL runs out + + # At most 4 concurrent downloads per host, across every process. + async with semaphore("dl:example.org", slots=4, ttl=120, wait=30): + ... + +The backend is process-global (Celery workers have no ``app.state``): the host +installs one with :func:`configure_leases` when the app is built and when a +worker boots — Redis when ``SM_REDIS_URL`` is set, else the ``sm_lease`` table, +else :class:`InProcessLeaseBackend`, which only coordinates one process. +Backend errors propagate: a lease that cannot be confirmed is never reported +as held. +""" + +from __future__ import annotations + +import asyncio +import random +import threading +import time +import uuid +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Protocol, Self + +__all__ = [ + "InProcessLeaseBackend", + "Lease", + "LeaseBackend", + "LeaseNotAcquiredError", + "configure_leases", + "get_leases", + "lease", + "semaphore", +] + +# Seconds between attempts while waiting for a held lease. +_POLL = 0.05 +_POLL_MAX = 0.5 + + +class LeaseNotAcquiredError(RuntimeError): + """``async with lease(...)`` could not take the lease (held, or the wait ran out).""" + + +class LeaseBackend(Protocol): + """Storage for leases. ``ttl`` is in seconds; every call is atomic. + + ``acquire``/``semaphore_acquire`` return the new fence number, or ``None`` + when the lease (or every slot) is held by someone else. ``refresh`` and + ``release`` return ``False`` when *token* no longer holds it — expired, + released, or taken by another owner. + """ + + async def acquire(self, name: str, token: str, ttl: float) -> int | None: ... + + async def refresh(self, name: str, token: str, ttl: float) -> bool: ... + + async def release(self, name: str, token: str) -> bool: ... + + async def semaphore_acquire( + self, name: str, token: str, slots: int, ttl: float + ) -> int | None: ... + + async def semaphore_refresh(self, name: str, token: str, ttl: float) -> bool: ... + + async def semaphore_release(self, name: str, token: str) -> bool: ... + + +@dataclass +class _NameState: + fence: int = 0 + holders: dict[str, float] = field(default_factory=dict) # token -> expiry + + +class InProcessLeaseBackend: + """Leases held in this process only: correct within one worker, nothing more. + + The default when neither Redis nor a database is configured, and what tests + and eager-mode Celery use. Thread-safe, so sync code in worker threads and + the event loop see the same state. + """ + + def __init__(self, *, clock: Callable[[], float] = time.monotonic) -> None: + self._clock = clock + self._lock = threading.Lock() + self._names: dict[str, _NameState] = {} + + def _live(self, name: str) -> _NameState: + state = self._names.setdefault(name, _NameState()) + now = self._clock() + state.holders = {t: exp for t, exp in state.holders.items() if exp > now} + return state + + def _take(self, name: str, token: str, ttl: float, slots: int) -> int | None: + with self._lock: + state = self._live(name) + if token not in state.holders and len(state.holders) >= slots: + return None + state.holders[token] = self._clock() + ttl + state.fence += 1 + return state.fence + + def _extend(self, name: str, token: str, ttl: float) -> bool: + with self._lock: + state = self._live(name) + if token not in state.holders: + return False + state.holders[token] = self._clock() + ttl + return True + + def _drop(self, name: str, token: str) -> bool: + with self._lock: + return self._live(name).holders.pop(token, None) is not None + + async def acquire(self, name: str, token: str, ttl: float) -> int | None: + return self._take(name, token, ttl, 1) + + async def refresh(self, name: str, token: str, ttl: float) -> bool: + return self._extend(name, token, ttl) + + async def release(self, name: str, token: str) -> bool: + return self._drop(name, token) + + async def semaphore_acquire(self, name: str, token: str, slots: int, ttl: float) -> int | None: + return self._take(name, token, ttl, slots) + + async def semaphore_refresh(self, name: str, token: str, ttl: float) -> bool: + return self._extend(name, token, ttl) + + async def semaphore_release(self, name: str, token: str) -> bool: + return self._drop(name, token) + + +_backend: LeaseBackend = InProcessLeaseBackend() + + +def configure_leases(backend: LeaseBackend | None) -> None: + """Install the process-wide backend; ``None`` restores a fresh in-process one.""" + global _backend + _backend = backend if backend is not None else InProcessLeaseBackend() + + +def get_leases() -> LeaseBackend: + """The process-wide backend :func:`lease` and :func:`semaphore` use.""" + return _backend + + +class Lease: + """One owner's handle on a lease or a semaphore slot. + + Not acquired until :meth:`acquire` returns ``True`` (or ``async with`` + enters). ``fence`` is the number handed out at the last acquire. + """ + + def __init__( + self, + name: str, + ttl: float, + *, + owner: str | None = None, + slots: int | None = None, + blocking: bool = True, + wait: float | None = None, + backend: LeaseBackend | None = None, + ) -> None: + if ttl <= 0: + raise ValueError("ttl must be positive") + if slots is not None and slots < 1: + raise ValueError("slots must be at least 1") + self.name = name + self.ttl = ttl + self.slots = slots + self.token = owner or uuid.uuid4().hex + self.fence: int | None = None + self._blocking = blocking + self._wait = wait + self._backend = backend + + @property + def backend(self) -> LeaseBackend: + return self._backend if self._backend is not None else get_leases() + + @property + def acquired(self) -> bool: + return self.fence is not None + + async def _try(self) -> int | None: + if self.slots is None: + return await self.backend.acquire(self.name, self.token, self.ttl) + return await self.backend.semaphore_acquire(self.name, self.token, self.slots, self.ttl) + + async def acquire(self, *, blocking: bool | None = None, wait: float | None = None) -> bool: + """Take the lease. Non-blocking tries once; blocking polls until it is + free, giving up after *wait* seconds (``None`` waits forever).""" + blocking = self._blocking if blocking is None else blocking + wait = self._wait if wait is None else wait + deadline = None if wait is None else time.monotonic() + wait + delay = _POLL + while True: + fence = await self._try() + if fence is not None: + self.fence = fence + return True + if not blocking: + return False + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + return False + pause = delay * (0.5 + random.random()) # jitter, not crypto + await asyncio.sleep(pause if remaining is None else min(pause, remaining)) + delay = min(delay * 2, _POLL_MAX) + + async def refresh(self, ttl: float | None = None) -> bool: + """Extend the TTL from now; ``False`` once the lease is no longer ours.""" + ttl = self.ttl if ttl is None else ttl + if self.slots is None: + return await self.backend.refresh(self.name, self.token, ttl) + return await self.backend.semaphore_refresh(self.name, self.token, ttl) + + async def release(self) -> bool: + """Give the lease up; ``False`` if it had already expired or moved on.""" + if self.slots is None: + released = await self.backend.release(self.name, self.token) + else: + released = await self.backend.semaphore_release(self.name, self.token) + self.fence = None + return released + + async def __aenter__(self) -> Self: + if not await self.acquire(): + raise LeaseNotAcquiredError(f"lease {self.name!r} is held elsewhere") + return self + + async def __aexit__(self, *_exc: object) -> None: + await self.release() + + +def lease( + name: str, + ttl: float, + owner: str | None = None, + *, + blocking: bool = True, + wait: float | None = None, + backend: LeaseBackend | None = None, +) -> Lease: + """An exclusive lease on *name* for *ttl* seconds (see the module docstring).""" + return Lease(name, ttl, owner=owner, blocking=blocking, wait=wait, backend=backend) + + +def semaphore( + name: str, + slots: int, + ttl: float, + owner: str | None = None, + *, + blocking: bool = True, + wait: float | None = None, + backend: LeaseBackend | None = None, +) -> Lease: + """One of *slots* concurrent leases on *name*; expired holders are reclaimed.""" + return Lease(name, ttl, owner=owner, slots=slots, blocking=blocking, wait=wait, backend=backend) diff --git a/framework/core/tests/test_core_leases.py b/framework/core/tests/test_core_leases.py new file mode 100644 index 000000000..cc7b7ff8e --- /dev/null +++ b/framework/core/tests/test_core_leases.py @@ -0,0 +1,162 @@ +"""The lease handle and the in-process backend (GH #539), on a fake clock.""" + +from __future__ import annotations + +import asyncio + +import pytest +from simple_module_core.leases import ( + InProcessLeaseBackend, + LeaseNotAcquiredError, + configure_leases, + get_leases, + lease, + semaphore, +) + + +class _Clock: + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock() -> _Clock: + return _Clock() + + +@pytest.fixture +def backend(clock: _Clock) -> InProcessLeaseBackend: + return InProcessLeaseBackend(clock=clock) + + +@pytest.fixture(autouse=True) +def _restore_default(): + yield + configure_leases(None) + + +async def test_racing_acquirers_exactly_one_wins(backend: InProcessLeaseBackend) -> None: + handles = [lease("import:42", 10, blocking=False, backend=backend) for _ in range(10)] + results = await asyncio.gather(*(h.acquire() for h in handles)) + assert results.count(True) == 1 + + +async def test_after_ttl_old_owner_cannot_release( + backend: InProcessLeaseBackend, clock: _Clock +) -> None: + old = lease("job", 10, blocking=False, backend=backend) + assert await old.acquire() + clock.now += 11 + new = lease("job", 10, blocking=False, backend=backend) + assert await new.acquire() + + assert await old.release() is False + assert await old.refresh() is False + assert await lease("job", 10, blocking=False, backend=backend).acquire() is False + assert await new.release() is True + + +async def test_refresh_extends_from_now(backend: InProcessLeaseBackend, clock: _Clock) -> None: + held = lease("job", 10, blocking=False, backend=backend) + assert await held.acquire() + clock.now += 8 + assert await held.refresh() + clock.now += 8 # 16 s after acquire, 8 s after refresh + assert await lease("job", 10, blocking=False, backend=backend).acquire() is False + + +async def test_fence_strictly_increases_across_owners( + backend: InProcessLeaseBackend, clock: _Clock +) -> None: + fences = [] + for _ in range(3): + held = lease("job", 1, blocking=False, backend=backend) + assert await held.acquire() + fences.append(held.fence) + clock.now += 2 # let it lapse rather than release + assert fences == sorted(set(fences)) and len(fences) == 3 + + +async def test_semaphore_caps_concurrent_holders(backend: InProcessLeaseBackend) -> None: + active = 0 + peak = 0 + + async def worker() -> None: + nonlocal active, peak + async with semaphore("dl:example.org", 4, 30, backend=backend): + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) # intentional-sleep: overlap the holders + active -= 1 + + await asyncio.gather(*(worker() for _ in range(8))) + assert peak == 4 + + +async def test_semaphore_expired_slot_is_reclaimed( + backend: InProcessLeaseBackend, clock: _Clock +) -> None: + holders = [semaphore("s", 2, 5, blocking=False, backend=backend) for _ in range(3)] + assert [await h.acquire() for h in holders] == [True, True, False] + clock.now += 6 + assert await holders[2].acquire() + assert await holders[0].release() is False + + +async def test_context_manager_raises_when_wait_runs_out(backend: InProcessLeaseBackend) -> None: + async with lease("busy", 30, backend=backend): + with pytest.raises(LeaseNotAcquiredError): + async with lease("busy", 30, wait=0.05, backend=backend): + pass # pragma: no cover + + +async def test_blocking_acquire_waits_for_release(backend: InProcessLeaseBackend) -> None: + first = lease("q", 30, backend=backend) + await first.acquire() + + async def release_soon() -> None: + await asyncio.sleep(0.05) # intentional-sleep: the holder finishes later + await first.release() + + task = asyncio.create_task(release_soon()) + second = lease("q", 30, wait=5, backend=backend) + assert await second.acquire() + await task + + +async def test_non_blocking_handle_fails_fast(backend: InProcessLeaseBackend) -> None: + async with lease("x", 30, backend=backend): + with pytest.raises(LeaseNotAcquiredError): + async with lease("x", 30, blocking=False, backend=backend): + pass # pragma: no cover + + +async def test_owner_names_the_token(backend: InProcessLeaseBackend) -> None: + handle = lease("x", 30, owner="task-1", backend=backend) + assert handle.token == "task-1" + assert await handle.acquire() + # The same owner from another handle (a retried task) re-acquires its own lease. + again = lease("x", 30, owner="task-1", blocking=False, backend=backend) + assert await again.acquire() + assert again.fence > handle.fence + + +def test_validation() -> None: + with pytest.raises(ValueError, match="ttl"): + lease("x", 0) + with pytest.raises(ValueError, match="slots"): + semaphore("x", 0, 10) + + +async def test_configure_leases_sets_the_process_default(backend: InProcessLeaseBackend) -> None: + configure_leases(backend) + assert get_leases() is backend + handle = lease("y", 30) + assert await handle.acquire() + assert await backend.release("y", handle.token) is True + configure_leases(None) + assert get_leases() is not backend diff --git a/framework/db/simple_module_db/leases.py b/framework/db/simple_module_db/leases.py new file mode 100644 index 000000000..f30f62099 --- /dev/null +++ b/framework/db/simple_module_db/leases.py @@ -0,0 +1,188 @@ +"""Database-backed :class:`~simple_module_core.leases.LeaseBackend` (GH #539). + +The ``sm_lease`` table holds one row per ``(name, slot)``: a lease uses slot 0, +a semaphore slots ``0..n-1``, and slot ``-1`` keeps a semaphore's fence +counter. Rows are never deleted — a released or expired row is simply one whose +``expires_at`` has passed — so ``fence`` keeps increasing for the name's +lifetime. + +Every comparison uses the *database* clock (read at the start of each +transaction), never this process's, so workers with skewed clocks agree on +who holds what. Atomicity comes from ``INSERT … ON CONFLICT DO NOTHING`` and +conditional ``UPDATE``\\ s, which serialise on the row on Postgres and on the +database lock on SQLite. +""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncGenerator +from datetime import UTC, datetime, timedelta +from typing import Any, Final + +from sqlalchemy import and_, func, select, text, update +from sqlalchemy.dialects import postgresql, sqlite +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine +from sqlmodel import Field + +from simple_module_db.base import create_module_base +from simple_module_db.types import AwareDateTime + +LEASE_METADATA_NAME: Final = "sm" +TABLE_LEASE: Final = "sm_lease" +NAME_MAX_LENGTH: Final = 255 +HOLDER_MAX_LENGTH: Final = 255 +_COUNTER_SLOT: Final = -1 +_FREE: Final = "" +_EPOCH: Final = datetime(1970, 1, 1, tzinfo=UTC) +_SQLITE_NOW: Final = text("strftime('%Y-%m-%d %H:%M:%f', 'now')") +# A write that matches nothing: on SQLite it takes the write lock (waiting out +# ``busy_timeout``) *before* the transaction reads, so two processes never both +# read and then race to upgrade a read lock into a write. +_SQLITE_WRITE_LOCK: Final = text("UPDATE sm_lease SET fence = fence WHERE 1 = 0") + +Base = create_module_base(LEASE_METADATA_NAME) + + +class LeaseRow(Base, table=True): # ty: ignore[unsupported-base] + """One lease slot. ``holder`` is the owner's token, ``""`` when never held.""" + + __tablename__ = TABLE_LEASE + + name: str = Field(primary_key=True, max_length=NAME_MAX_LENGTH) + slot: int = Field(primary_key=True) + holder: str = Field(default=_FREE, max_length=HOLDER_MAX_LENGTH) + fence: int = Field(default=0) + expires_at: datetime = Field(sa_type=AwareDateTime()) + + +_T = LeaseRow.__table__ + + +def _insert(dialect: str) -> Any: + if dialect == "postgresql": + return postgresql.insert(_T) + if dialect == "sqlite": + return sqlite.insert(_T) + raise NotImplementedError(f"sm_lease supports postgresql and sqlite, not {dialect}") + + +async def _db_now(conn: AsyncConnection) -> datetime: + if conn.dialect.name == "sqlite": + raw = (await conn.execute(select(_SQLITE_NOW))).scalar_one() + return datetime.fromisoformat(raw).replace(tzinfo=UTC) + value = (await conn.execute(select(func.clock_timestamp()))).scalar_one() + return value if value.tzinfo else value.replace(tzinfo=UTC) + + +def _row(name: str, slot: int) -> Any: + return and_(_T.c.name == name, _T.c.slot == slot) + + +class DbLeaseBackend: + """Leases in the ``sm_lease`` table, shared by every process on the database.""" + + def __init__(self, engine: AsyncEngine) -> None: + self._engine = engine + self._sqlite = engine.dialect.name == "sqlite" + self._lock: asyncio.Lock | None = None + self._lock_loop: asyncio.AbstractEventLoop | None = None + + def _guard(self) -> contextlib.AbstractAsyncContextManager[Any]: + """Serialise this process's SQLite transactions (an in-memory database + shares one connection); Postgres needs no process-side lock.""" + if not self._sqlite: + return contextlib.nullcontext() + loop = asyncio.get_running_loop() + if self._lock is None or self._lock_loop is not loop: + self._lock, self._lock_loop = asyncio.Lock(), loop + return self._lock + + @contextlib.asynccontextmanager + async def _tx(self) -> AsyncGenerator[tuple[AsyncConnection, datetime]]: + async with self._guard(), self._engine.begin() as conn: + if self._sqlite: + await conn.execute(_SQLITE_WRITE_LOCK) + yield conn, await _db_now(conn) + + async def _take_slot( + self, + conn: AsyncConnection, + name: str, + slot: int, + token: str, + expires: datetime, + now: datetime, + ) -> int | None: + """Claim *slot* if free (missing or expired) or already ours; return its fence.""" + values = {"holder": token, "expires_at": expires} + ins = _insert(conn.dialect.name).values(name=name, slot=slot, fence=1, **values) + taken = (await conn.execute(ins.on_conflict_do_nothing())).rowcount == 1 + if not taken: + stmt = ( + update(_T) + .where(_row(name, slot), (_T.c.expires_at <= now) | (_T.c.holder == token)) + .values(fence=_T.c.fence + 1, **values) + ) + taken = (await conn.execute(stmt)).rowcount == 1 + if not taken: + return None + return (await conn.execute(select(_T.c.fence).where(_row(name, slot)))).scalar_one() + + async def _next_counter(self, conn: AsyncConnection, name: str) -> int: + ins = _insert(conn.dialect.name).values( + name=name, slot=_COUNTER_SLOT, holder=_FREE, fence=1, expires_at=_EPOCH + ) + await conn.execute( + ins.on_conflict_do_update( + index_elements=[_T.c.name, _T.c.slot], set_={"fence": _T.c.fence + 1} + ) + ) + stmt = select(_T.c.fence).where(_row(name, _COUNTER_SLOT)) + return (await conn.execute(stmt)).scalar_one() + + async def acquire(self, name: str, token: str, ttl: float) -> int | None: + async with self._tx() as (conn, now): + return await self._take_slot(conn, name, 0, token, now + timedelta(seconds=ttl), now) + + async def semaphore_acquire(self, name: str, token: str, slots: int, ttl: float) -> int | None: + async with self._tx() as (conn, now): + expires = now + timedelta(seconds=ttl) + mine = await self._slot_of(conn, name, token, now) + order = [mine] if mine is not None else range(slots) + for slot in order: + if await self._take_slot(conn, name, slot, token, expires, now) is not None: + return await self._next_counter(conn, name) + return None + + async def _slot_of( + self, conn: AsyncConnection, name: str, token: str, now: datetime + ) -> int | None: + stmt = select(_T.c.slot).where( + _T.c.name == name, _T.c.slot >= 0, _T.c.holder == token, _T.c.expires_at > now + ) + return (await conn.execute(stmt.limit(1))).scalar_one_or_none() + + async def _held_update( + self, name: str, token: str, *, ttl: float | None, any_slot: bool + ) -> bool: + """Extend (``ttl``) or end (``None``) *token*'s unexpired hold.""" + async with self._tx() as (conn, now): + where = [_T.c.name == name, _T.c.holder == token, _T.c.expires_at > now] + where.append(_T.c.slot >= 0 if any_slot else _T.c.slot == 0) + expires = now if ttl is None else now + timedelta(seconds=ttl) + result = await conn.execute(update(_T).where(*where).values(expires_at=expires)) + return result.rowcount > 0 + + async def refresh(self, name: str, token: str, ttl: float) -> bool: + return await self._held_update(name, token, ttl=ttl, any_slot=False) + + async def release(self, name: str, token: str) -> bool: + return await self._held_update(name, token, ttl=None, any_slot=False) + + async def semaphore_refresh(self, name: str, token: str, ttl: float) -> bool: + return await self._held_update(name, token, ttl=ttl, any_slot=True) + + async def semaphore_release(self, name: str, token: str) -> bool: + return await self._held_update(name, token, ttl=None, any_slot=True) diff --git a/framework/db/simple_module_db/migrations.py b/framework/db/simple_module_db/migrations.py index 9b83a3778..ef1548638 100644 --- a/framework/db/simple_module_db/migrations.py +++ b/framework/db/simple_module_db/migrations.py @@ -68,8 +68,8 @@ def build_module_metadata(modules: Sequence[ModuleBase] | None = None) -> MetaDa """ if modules is None: modules = discover_modules() - # The framework owns ``settings_setting`` (#500): collect it even when the - # ``settings`` plugin is not installed, since the host reads it at boot. + # Framework-owned tables, plugin or not: settings_setting (#500), sm_lease (#539). + import simple_module_db.leases import simple_module_db.stored_settings # noqa: F401 for mod in modules: diff --git a/framework/hosting/simple_module_hosting/_lease_redis.py b/framework/hosting/simple_module_hosting/_lease_redis.py new file mode 100644 index 000000000..605d36104 --- /dev/null +++ b/framework/hosting/simple_module_hosting/_lease_redis.py @@ -0,0 +1,148 @@ +"""Redis :class:`~simple_module_core.leases.LeaseBackend` (GH #539). + +* A lease is ``SET key token NX PX ttl``; refresh and release are Lua scripts + that compare the stored token first, so only the live owner can touch it — + an expired key is simply gone. +* A semaphore is a sorted set of tokens scored by expiry in milliseconds of + Redis' own ``TIME`` (never this process's clock); every attempt first drops + members whose score has passed, then admits while ``ZCARD < slots``. +* ``INCR :fence`` hands out the fencing token, inside the same script. + +Unlike the rate limiter there is no per-process fallback: a lease that Redis +could not confirm must not be reported as held, so errors propagate. +""" + +from __future__ import annotations + +import asyncio +import os +from typing import Any + +_KEY_PREFIX = "sm:lease:" +_SEM_PREFIX = "sm:sem:" +_FENCE_SUFFIX = ":fence" + +_ACQUIRE = """ +if redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', ARGV[2]) then + return redis.call('INCR', KEYS[2]) +end +if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('PEXPIRE', KEYS[1], ARGV[2]) + return redis.call('INCR', KEYS[2]) +end +return 0 +""" + +_REFRESH = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) +end +return 0 +""" + +_RELEASE = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" + +# Shared prologue: Redis' clock in ms, expired members dropped. +_SEM_NOW = """ +local t = redis.call('TIME') +local now = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000) +redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now) +local function keep(ttl) + if redis.call('PTTL', KEYS[1]) < ttl then redis.call('PEXPIRE', KEYS[1], ttl) end +end +""" + +_SEM_ACQUIRE = ( + _SEM_NOW + + """ +local ttl = tonumber(ARGV[3]) +if redis.call('ZSCORE', KEYS[1], ARGV[1]) + or redis.call('ZCARD', KEYS[1]) < tonumber(ARGV[2]) then + redis.call('ZADD', KEYS[1], now + ttl, ARGV[1]) + keep(ttl) + return redis.call('INCR', KEYS[2]) +end +return 0 +""" +) + +_SEM_REFRESH = ( + _SEM_NOW + + """ +local ttl = tonumber(ARGV[2]) +if not redis.call('ZSCORE', KEYS[1], ARGV[1]) then return 0 end +redis.call('ZADD', KEYS[1], 'XX', now + ttl, ARGV[1]) +keep(ttl) +return 1 +""" +) + +_SEM_RELEASE = _SEM_NOW + "return redis.call('ZREM', KEYS[1], ARGV[1])" + + +def _ms(ttl: float) -> int: + return max(1, int(ttl * 1000)) + + +class RedisLeaseBackend: + """Leases shared by every process on one Redis. + + redis-py's asyncio connections belong to the event loop (and process) that + opened them, and a Celery worker may run each task under a fresh + ``asyncio.run``; the client is therefore rebuilt per loop and per process. + """ + + def __init__(self, url: str, *, factory: Any = None) -> None: + self._url = url + self._factory = factory + self._client: Any = None + self._owner: tuple[int, int] | None = None + + def _redis(self) -> Any: + owner = (os.getpid(), id(asyncio.get_running_loop())) + if self._client is None or self._owner != owner: + if self._factory is not None: + self._client = self._factory() + else: + import redis.asyncio as aioredis + + self._client = aioredis.from_url(self._url) + self._owner = owner + return self._client + + async def _eval(self, script: str, keys: list[str], *args: Any) -> int: + return int(await self._redis().eval(script, len(keys), *keys, *args)) + + @staticmethod + def _lease_keys(name: str) -> list[str]: + key = _KEY_PREFIX + name + return [key, key + _FENCE_SUFFIX] + + @staticmethod + def _sem_keys(name: str) -> list[str]: + key = _SEM_PREFIX + name + return [key, key + _FENCE_SUFFIX] + + async def acquire(self, name: str, token: str, ttl: float) -> int | None: + return await self._eval(_ACQUIRE, self._lease_keys(name), token, _ms(ttl)) or None + + async def refresh(self, name: str, token: str, ttl: float) -> bool: + return bool(await self._eval(_REFRESH, self._lease_keys(name)[:1], token, _ms(ttl))) + + async def release(self, name: str, token: str) -> bool: + return bool(await self._eval(_RELEASE, self._lease_keys(name)[:1], token)) + + async def semaphore_acquire(self, name: str, token: str, slots: int, ttl: float) -> int | None: + keys = self._sem_keys(name) + return await self._eval(_SEM_ACQUIRE, keys, token, slots, _ms(ttl)) or None + + async def semaphore_refresh(self, name: str, token: str, ttl: float) -> bool: + return bool(await self._eval(_SEM_REFRESH, self._sem_keys(name)[:1], token, _ms(ttl))) + + async def semaphore_release(self, name: str, token: str) -> bool: + return bool(await self._eval(_SEM_RELEASE, self._sem_keys(name)[:1], token)) diff --git a/framework/hosting/simple_module_hosting/app_builder.py b/framework/hosting/simple_module_hosting/app_builder.py index 943fa2828..60e28dd97 100644 --- a/framework/hosting/simple_module_hosting/app_builder.py +++ b/framework/hosting/simple_module_hosting/app_builder.py @@ -52,6 +52,7 @@ from simple_module_hosting.health import router as health_router from simple_module_hosting.i18n_manifest import build_i18n_registry from simple_module_hosting.i18n_routes import router as i18n_router +from simple_module_hosting.leases import configure_process_leases from simple_module_hosting.settings import Settings from simple_module_hosting.setup_gate import register_migration_step from simple_module_hosting.setup_wizard import mount_setup_wizard @@ -238,6 +239,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: if not settings.multi_tenant and settings.default_tenant: db_state.default_tenant_id = settings.default_tenant register_listeners(db_state) + configure_process_leases(settings.redis_url, engine=db_state.engine) # GH #539 # The host's own readiness signal, and the only probe-safe check in a # default install — module checks reach third parties and are on-demand. register_database_check(health_registry, db_state) diff --git a/framework/hosting/simple_module_hosting/leases.py b/framework/hosting/simple_module_hosting/leases.py new file mode 100644 index 000000000..dd90acc37 --- /dev/null +++ b/framework/hosting/simple_module_hosting/leases.py @@ -0,0 +1,55 @@ +"""Pick and install the process-wide lease backend (GH #539). + +Redis when ``SM_REDIS_URL`` is set, else the ``sm_lease`` table on the app's +database, else in-process. :func:`create_app` installs it from the engine it +builds; a Celery worker never builds the app, so its bootstrap calls +:func:`configure_process_leases` with the URLs it resolved. +""" + +from __future__ import annotations + +import logging + +from simple_module_core.leases import InProcessLeaseBackend, LeaseBackend, configure_leases +from simple_module_db.leases import DbLeaseBackend +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlalchemy.pool import NullPool + +from simple_module_hosting._lease_redis import RedisLeaseBackend + +__all__ = ["RedisLeaseBackend", "build_lease_backend", "configure_process_leases"] + +logger = logging.getLogger(__name__) + + +def build_lease_backend( + redis_url: str | None, + *, + engine: AsyncEngine | None = None, + database_url: str | None = None, +) -> LeaseBackend: + """Redis → *engine* (or a pool-less engine on *database_url*) → in-process.""" + if redis_url: + return RedisLeaseBackend(redis_url) + if engine is None and database_url: + from simple_module_db.session import init_db + + # No pool: a worker may run each task in its own event loop, and a + # pooled async connection cannot cross loops. + engine = init_db(database_url, poolclass=NullPool).engine + if engine is not None: + return DbLeaseBackend(engine) + return InProcessLeaseBackend() + + +def configure_process_leases( + redis_url: str | None, + *, + engine: AsyncEngine | None = None, + database_url: str | None = None, +) -> LeaseBackend: + """Build the backend and make it what ``lease()`` / ``semaphore()`` use.""" + backend = build_lease_backend(redis_url, engine=engine, database_url=database_url) + configure_leases(backend) + logger.debug("Leases use %s", type(backend).__name__) + return backend diff --git a/framework/hosting/tests/test_hosting_lease_backends.py b/framework/hosting/tests/test_hosting_lease_backends.py new file mode 100644 index 000000000..464fd353a --- /dev/null +++ b/framework/hosting/tests/test_hosting_lease_backends.py @@ -0,0 +1,151 @@ +"""Every lease backend honours the same contract (GH #539). + +In-process, the ``sm_lease`` table (SQLite here; Postgres under +``make test-py-pg``) and Redis (``redis_server``: skipped locally without +``redis-server``, required on CI) run the same tests. +""" + +from __future__ import annotations + +import asyncio +import itertools +import uuid +from collections.abc import AsyncIterator + +import pytest +from simple_module_core.leases import InProcessLeaseBackend, LeaseBackend, lease, semaphore +from simple_module_db.leases import DbLeaseBackend, LeaseRow +from simple_module_hosting._lease_redis import RedisLeaseBackend +from simple_module_test import wait_for + +_SHORT_TTL = 0.3 +_LONG_TTL = 30.0 + + +@pytest.fixture(params=["inprocess", "db", "redis"]) +async def backend(request: pytest.FixtureRequest, engine) -> AsyncIterator[LeaseBackend]: + if request.param == "inprocess": + yield InProcessLeaseBackend() + elif request.param == "db": + async with engine.begin() as conn: + await conn.run_sync(LeaseRow.metadata.create_all) + yield DbLeaseBackend(engine) + else: + url = request.getfixturevalue("redis_server") + redis_backend = RedisLeaseBackend(url) + yield redis_backend + client = redis_backend._redis() + keys = [k async for k in client.scan_iter(match="sm:*:t-*")] + if keys: + await client.delete(*keys) + await client.aclose() + + +def _name() -> str: + return f"t-{uuid.uuid4().hex}" + + +async def test_two_racing_acquirers_one_wins(backend: LeaseBackend) -> None: + name = _name() + handles = [lease(name, _LONG_TTL, blocking=False, backend=backend) for _ in range(2)] + results = await asyncio.gather(*(h.acquire() for h in handles)) + assert sorted(results) == [False, True] + + +async def test_expired_owner_cannot_release_or_refresh(backend: LeaseBackend) -> None: + name = _name() + old = lease(name, _SHORT_TTL, blocking=False, backend=backend) + assert await old.acquire() + new = lease(name, _LONG_TTL, blocking=False, backend=backend) + assert await wait_for(new.acquire, timeout=5) + + assert await old.refresh() is False + assert await old.release() is False + # The new owner still holds it: nobody else gets in. + assert await lease(name, _LONG_TTL, blocking=False, backend=backend).acquire() is False + assert await new.refresh() is True + assert await new.release() is True + + +async def test_expired_owner_cannot_release_even_when_unclaimed(backend: LeaseBackend) -> None: + held = lease(_name(), _SHORT_TTL, blocking=False, backend=backend) + assert await held.acquire() + # Probing with acquire() would claim it; the point is that nobody did. + await asyncio.sleep(_SHORT_TTL * 2) # intentional-sleep: let the TTL lapse unclaimed + assert await held.release() is False + + +async def test_fence_strictly_increases(backend: LeaseBackend) -> None: + name = _name() + fences = [] + for _ in range(4): + async with lease(name, _LONG_TTL, backend=backend) as held: + fences.append(held.fence) + assert all(a < b for a, b in itertools.pairwise(fences)), fences + + +async def test_released_lease_is_free_again(backend: LeaseBackend) -> None: + name = _name() + first = lease(name, _LONG_TTL, blocking=False, backend=backend) + assert await first.acquire() + assert await first.release() is True + assert await first.release() is False + assert await lease(name, _LONG_TTL, blocking=False, backend=backend).acquire() + + +async def test_semaphore_never_exceeds_its_slots(backend: LeaseBackend) -> None: + name = _name() + active = 0 + peak = 0 + + async def worker() -> None: + nonlocal active, peak + async with semaphore(name, 4, _LONG_TTL, wait=10, backend=backend): + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.2) # intentional-sleep: hold the slot so holders overlap + active -= 1 + + await asyncio.gather(*(worker() for _ in range(8))) + assert peak == 4 + + +async def test_semaphore_admits_n_then_refuses(backend: LeaseBackend) -> None: + name = _name() + holders = [semaphore(name, 4, _LONG_TTL, blocking=False, backend=backend) for _ in range(5)] + results = await asyncio.gather(*(h.acquire() for h in holders)) + assert sorted(results) == [False, True, True, True, True] + fences = [h.fence for h in holders if h.acquired] + assert len(set(fences)) == 4 + + winner = next(h for h in holders if h.acquired) + assert await winner.refresh() is True + assert await winner.release() is True + loser = next(h for h in holders if not h.acquired) + assert await loser.acquire() is True + + +async def test_semaphore_reclaims_expired_holders(backend: LeaseBackend) -> None: + name = _name() + stale = semaphore(name, 1, _SHORT_TTL, blocking=False, backend=backend) + assert await stale.acquire() + fresh = semaphore(name, 1, _LONG_TTL, blocking=False, backend=backend) + assert await wait_for(fresh.acquire, timeout=5) + assert await stale.refresh() is False + assert await stale.release() is False + assert await fresh.release() is True + + +@pytest.mark.postgres +async def test_postgres_lease_race_across_connections(engine) -> None: + """Real parallel transactions on separate connections: exactly one winner.""" + async with engine.begin() as conn: + await conn.run_sync(LeaseRow.metadata.create_all) + backend = DbLeaseBackend(engine) + name = _name() + handles = [lease(name, _LONG_TTL, blocking=False, backend=backend) for _ in range(12)] + results = await asyncio.gather(*(h.acquire() for h in handles)) + assert results.count(True) == 1 + + sem = [semaphore(name + "s", 4, _LONG_TTL, blocking=False, backend=backend) for _ in range(12)] + assert (await asyncio.gather(*(h.acquire() for h in sem))).count(True) == 4 diff --git a/framework/hosting/tests/test_hosting_lease_selection.py b/framework/hosting/tests/test_hosting_lease_selection.py new file mode 100644 index 000000000..679f85ef7 --- /dev/null +++ b/framework/hosting/tests/test_hosting_lease_selection.py @@ -0,0 +1,48 @@ +"""Which lease backend a process gets (GH #539).""" + +from __future__ import annotations + +import pytest +from simple_module_core.leases import InProcessLeaseBackend, configure_leases, get_leases +from simple_module_db.leases import DbLeaseBackend +from simple_module_hosting.leases import ( + RedisLeaseBackend, + build_lease_backend, + configure_process_leases, +) + + +@pytest.fixture(autouse=True) +def _restore_default(): + yield + configure_leases(None) + + +def test_redis_wins_when_configured(engine) -> None: + backend = build_lease_backend("redis://localhost:6379/0", engine=engine) + assert isinstance(backend, RedisLeaseBackend) + + +def test_database_engine_without_redis(engine) -> None: + assert isinstance(build_lease_backend(None, engine=engine), DbLeaseBackend) + + +async def test_database_url_builds_its_own_engine(tmp_path) -> None: + backend = build_lease_backend(None, database_url=f"sqlite+aiosqlite:///{tmp_path}/l.db") + assert isinstance(backend, DbLeaseBackend) + await backend._engine.dispose() + + +def test_nothing_configured_is_in_process() -> None: + assert isinstance(build_lease_backend(None), InProcessLeaseBackend) + + +def test_configure_process_leases_installs_it(engine) -> None: + backend = configure_process_leases(None, engine=engine) + assert get_leases() is backend + + +async def test_create_app_installs_the_database_backend(app) -> None: + backend = get_leases() + assert isinstance(backend, DbLeaseBackend) + assert backend._engine is app.state.sm.db.engine diff --git a/framework/testing/simple_module_test/_schema.py b/framework/testing/simple_module_test/_schema.py index 5408508bc..dcf206ec4 100644 --- a/framework/testing/simple_module_test/_schema.py +++ b/framework/testing/simple_module_test/_schema.py @@ -19,6 +19,7 @@ @lru_cache(maxsize=1) def _ensure_models_imported() -> list: """Import all module models so all_module_bases is populated (cached).""" + import simple_module_db.leases # framework-owned table (#539) import simple_module_db.stored_settings # noqa: F401 framework-owned table (#500) for mod in discover_modules(): diff --git a/host/migrations/versions/d263ea727663_leases_table.py b/host/migrations/versions/d263ea727663_leases_table.py new file mode 100644 index 000000000..d5b4d8836 --- /dev/null +++ b/host/migrations/versions/d263ea727663_leases_table.py @@ -0,0 +1,36 @@ +"""leases table (GH #539) + +Framework-owned ``sm_lease`` for the database lease backend. Its own root and +``leases`` label: it depends on no module's tables. + +Revision ID: d263ea727663 +Revises: +Create Date: 2026-10-21 18:07:40.451232 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "d263ea727663" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = ("leases",) +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "sm_lease", + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slot", sa.Integer(), nullable=False), + sa.Column("holder", sa.String(length=255), nullable=False), + sa.Column("fence", sa.Integer(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("name", "slot", name=op.f("pk_sm_lease")), + ) + + +def downgrade() -> None: + op.drop_table("sm_lease") diff --git a/host/tests/test_migration_graph.py b/host/tests/test_migration_graph.py index 0fc0ae4be..d03a6d4e2 100644 --- a/host/tests/test_migration_graph.py +++ b/host/tests/test_migration_graph.py @@ -44,9 +44,10 @@ "f1a9c3d7b2e4": "tenants", "70786227af4c": "audit_log", "a7c2e91d4b58": "audit_log", + "d263ea727663": "leases", # framework-owned sm_lease (#539) } -_ROOTS = {"77162e7b184b", "e36ab09c4a92"} +_ROOTS = {"77162e7b184b", "e36ab09c4a92", "d263ea727663"} # head revision -> module owning the tables it creates or alters. _HEADS = { @@ -54,6 +55,7 @@ "d7b3a9e5c1f2": "mail", # reported as ``api_tokens`` (inherited label) "f1a9c3d7b2e4": "tenants", "f6a3b9d2e8c4": "keycloak", + "d263ea727663": "leases", } # Labels written in the revision files themselves. @@ -62,6 +64,7 @@ "51b2058d8afd": "api_tokens", "168a2882f443": "keycloak", "e36ab09c4a92": "tenants", + "d263ea727663": "leases", } # Known mislabels: revisions that inherit a label from an ancestor owned by a diff --git a/modules/background_tasks/background_tasks/runner.py b/modules/background_tasks/background_tasks/runner.py index 41c145418..801e799c1 100644 --- a/modules/background_tasks/background_tasks/runner.py +++ b/modules/background_tasks/background_tasks/runner.py @@ -51,6 +51,7 @@ def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: # Imported after ``.env`` is in the environment, so nothing below can # resolve a setting before it is. from simple_module_hosting import merge_host_settings + from simple_module_hosting.leases import configure_process_leases from background_tasks.celery_app import build_celery from background_tasks.sync_db import set_database_url @@ -66,6 +67,8 @@ def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: host.database_url, tenant_strict=host.multi_tenant, default_tenant=default_tenant ) set_default_tenant(default_tenant) + # Task bodies share the web process's leases (GH #539): same Redis or table. + configure_process_leases(host.redis_url, database_url=host.database_url) # Hydrated from the DB like the web process, so an admin-edited setting # reaches the worker at boot, not only the web app. return build_celery(load_runner_settings()) diff --git a/modules/background_tasks/tests/test_bg_runner_settings.py b/modules/background_tasks/tests/test_bg_runner_settings.py index 6c060b9fc..ab5602d89 100644 --- a/modules/background_tasks/tests/test_bg_runner_settings.py +++ b/modules/background_tasks/tests/test_bg_runner_settings.py @@ -110,19 +110,25 @@ def test_shared_runner_builds_celery_from_db_settings(db_url, tmp_path, monkeypa """GH #490: the entry point scaffolded apps use must hydrate from the DB.""" from background_tasks import tenant_context from background_tasks.runner import build_runner_celery + from simple_module_core import leases + from simple_module_db.leases import DbLeaseBackend # The runner sets process-wide state; don't leak it into later tests. monkeypatch.setattr(tenant_context, "_default_tenant", tenant_context._default_tenant) + monkeypatch.setattr(leases, "_backend", leases._backend) for name in ("_tenant_strict", "_default_tenant"): monkeypatch.setattr(sync_db, name, getattr(sync_db, name)) _put(db_url, "background_tasks.task_default_queue", "from-db", "string") monkeypatch.setenv("SM_PROJECT_ROOT", str(tmp_path)) monkeypatch.setenv("SM_DATABASE_URL", db_url.replace("sqlite://", "sqlite+aiosqlite://")) monkeypatch.delenv("SM_BG_TASKS_TASK_DEFAULT_QUEUE", raising=False) + monkeypatch.delenv("SM_REDIS_URL", raising=False) celery = build_runner_celery() assert celery.conf.task_default_queue == "from-db" + # GH #539: task bodies share the web process's leases (no Redis here: the table). + assert isinstance(leases.get_leases(), DbLeaseBackend) def _forget_after_test(monkeypatch, *names: str) -> None: From c2bb4dd67bd6c4b9c0859ab5389131e3e99760a4 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:21:04 +0200 Subject: [PATCH 05/22] docs: note the trunk head's background_tasks label (#536) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index f7ae4641b..756745d33 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -297,7 +297,7 @@ development and testing (`session_cookie_secure`). three roots (`77162e7b184b` initial schema, `e36ab09c4a92` tenants, `d263ea727663` the framework's `sm_lease`) and five heads (`audit_log`, `tenants`, `keycloak`, `leases`, and the users trunk, which Alembic reports - as `api_tokens`). Apply with `upgrade heads` (what `make migrate` + as `api_tokens` and, since #536, `background_tasks`). Apply with `upgrade heads` (what `make migrate` does); plain `revision --autogenerate` refuses with "Multiple heads", so name the branch: From d39c318f9cf8c088cc50447dc556e99102a785b9 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:21:43 +0200 Subject: [PATCH 06/22] refactor(hosting): one process-services hook for Sentry and leases keeps app_builder under the cap Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 From 0c1e844deb5496ec1c91783d9d7bdfda2eccf1fc Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:23:40 +0200 Subject: [PATCH 07/22] fix(sentry): scrub cookies, auth-like headers, breadcrumbs, task args and nested secrets from events Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 1 + docs/framework/sentry.md | 24 ++- .../hosting/simple_module_hosting/sentry.py | 39 +--- .../simple_module_hosting/sentry_scrub.py | 168 +++++++++++++++ .../tests/test_hosting_sentry_scrub.py | 202 ++++++++++++++++++ 5 files changed, 399 insertions(+), 35 deletions(-) create mode 100644 framework/hosting/simple_module_hosting/sentry_scrub.py create mode 100644 framework/hosting/tests/test_hosting_sentry_scrub.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1631c0c53..4564209bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ All notable changes to this project are documented in this file. The format is b ### Added - Optional Sentry error reporting: `pip install simple_module_hosting[sentry]` and set `SM_SENTRY_DSN` (plus `SM_SENTRY_ENVIRONMENT`, `SM_SENTRY_TRACES_SAMPLE_RATE`). Web and Celery worker processes tag events with request id, tenant, user id, module and task, and scrub auth headers, cookies, `token`/`code` query values and request bodies. Without a DSN the SDK is never imported (#538). +- Sentry events no longer leak secrets the SDK's exact-name scrubber misses: auth-, session- and token-like headers (`X-Auth-Token`, `Proxy-Authorization`, `X-Forwarded-For`), nested secrets in `extra`/`contexts`, tokens in log and HTTP breadcrumbs, credentials in exception messages, and Celery task args/kwargs are scrubbed for errors and transactions alike (#538). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial diff --git a/docs/framework/sentry.md b/docs/framework/sentry.md index 15ee15aeb..9588b7699 100644 --- a/docs/framework/sentry.md +++ b/docs/framework/sentry.md @@ -38,7 +38,23 @@ is open. The DSN is never logged. ## Scrubbing `send_default_pii` is off and local variables are not attached to stack frames -(they hold the ASGI scope). Before send, events lose the `Authorization`, -`Cookie`, `Set-Cookie`, `X-CSRF-Token` and `X-Api-Key` headers, request cookies -and bodies, and the values of `token` and `code` query parameters (in the query -string and the URL). +(they hold the ASGI scope). The SDK's own scrubber matches exact key names +only, so `simple_module_hosting.sentry_scrub` runs a stricter pass from both +`before_send` and `before_send_transaction`: + +- Any key, at any depth of `request`, `extra`, `contexts`, breadcrumbs and + span data, whose name contains `password`, `secret`, `token`, `auth` + (`Authorization`, `X-Auth-*`), `cookie`, `session`, `api_key`, `csrf`, + `credential`, `signature`, `dsn` or a client-address word (`forwarded`, + `real_ip`, `remote_addr`, …) is replaced with `[Filtered]`; such request + headers are removed outright. +- Request cookies, bodies and `env` (`REMOTE_ADDR`) are dropped; the event + user is reduced to `{"id": …}`. +- Free text (exception messages, log messages and params, breadcrumb messages + and URLs, span descriptions, query strings, source context) loses + `key=value` / `key: value` pairs with a sensitive key, the OAuth `code` + query parameter, `Bearer`/`Basic` credentials and URL userinfo. +- Celery task `args`, `kwargs` and return values are dropped (the Celery + integration and Celery's own trace log records both attach them). +- URL-sourced transaction names and request URL paths have id- and + token-shaped segments replaced with `*`. diff --git a/framework/hosting/simple_module_hosting/sentry.py b/framework/hosting/simple_module_hosting/sentry.py index 3c8c269d7..c8b452358 100644 --- a/framework/hosting/simple_module_hosting/sentry.py +++ b/framework/hosting/simple_module_hosting/sentry.py @@ -12,7 +12,6 @@ from collections.abc import Callable, Mapping from contextvars import ContextVar from typing import Any, Literal -from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from pydantic import SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict @@ -21,17 +20,14 @@ from simple_module_db.tenancy import current_tenant_id from starlette.types import ASGIApp, Receive, Scope, Send +from simple_module_hosting.sentry_scrub import scrub_event + logger = logging.getLogger(__name__) Process = Literal["web", "worker"] PROCESS_WEB: Process = "web" PROCESS_WORKER: Process = "worker" -REDACTED = "[Filtered]" -_SCRUBBED_HEADERS = frozenset( - {"authorization", "cookie", "set-cookie", "x-csrf-token", "x-api-key"} -) -_SCRUBBED_QUERY_KEYS = frozenset({"token", "code"}) _TAG_REQUEST_ID = "request_id" _TAG_TENANT = "tenant" _TAG_USER_ID = "user_id" @@ -77,27 +73,6 @@ class SentrySettings(BaseSettings): environment: str = "development" -def _scrub_query(query: str) -> str: - pairs = parse_qsl(query, keep_blank_values=True) - return urlencode([(k, REDACTED if k.lower() in _SCRUBBED_QUERY_KEYS else v) for k, v in pairs]) - - -def _scrub_request(request: dict[str, Any]) -> None: - headers = request.get("headers") - if isinstance(headers, dict): - request["headers"] = { - k: v for k, v in headers.items() if str(k).lower() not in _SCRUBBED_HEADERS - } - for key in ("cookies", "data"): - request.pop(key, None) - if isinstance(request.get("query_string"), str): - request["query_string"] = _scrub_query(request["query_string"]) - url = request.get("url") - if isinstance(url, str) and "?" in url: - parts = urlsplit(url) - request["url"] = urlunsplit(parts._replace(query=_scrub_query(parts.query))) - - def _scope_info(process: Process) -> dict[str, str | None]: scope = _request_scope.get() if process == PROCESS_WEB else None state = (scope or {}).get("state", {}) @@ -128,7 +103,11 @@ def _tags(event: Event, process: Process) -> dict[str, str]: def build_before_send(process: Process) -> Callable[[Event, Mapping[str, Any]], Event]: - """Tag and scrub an event; used for errors and transactions alike.""" + """Tag and scrub an event; used for errors and transactions alike. + + Scrubbing is ours, not the SDK's: its ``EventScrubber`` matches exact key + names only and skips ``contexts`` and free text (``sentry_scrub``). + """ def before_send(event: Event, hint: Mapping[str, Any]) -> Event: tags = _tags(event, process) @@ -137,9 +116,7 @@ def before_send(event: Event, hint: Mapping[str, Any]) -> Event: event["user"] = {"id": user_id} # id only: never email/username/ip else: event.pop("user", None) - if isinstance(event.get("request"), dict): - _scrub_request(event["request"]) - return event + return scrub_event(event) return before_send diff --git a/framework/hosting/simple_module_hosting/sentry_scrub.py b/framework/hosting/simple_module_hosting/sentry_scrub.py new file mode 100644 index 000000000..bc77d3e9e --- /dev/null +++ b/framework/hosting/simple_module_hosting/sentry_scrub.py @@ -0,0 +1,168 @@ +"""Scrub secrets and PII out of a serialized Sentry event before it is sent. + +The SDK's own ``EventScrubber`` matches exact key names, is not recursive and +never looks at ``contexts`` or free text, so this runs a second, stricter +pass from ``before_send``/``before_send_transaction``: + +* every mapping key matching :data:`SENSITIVE_KEY` (``password``, ``token``, + ``authorization``, ``cookie``, ``session``, ``api_key``, ``csrf``, …, plus + client-IP headers) has its value replaced, at any depth; +* free text (exception messages, log messages, breadcrumb data, span + descriptions, URLs, query strings) has ``key=value`` pairs with a sensitive + key, ``Bearer``/``Basic`` credentials and URL userinfo redacted; +* request cookies, bodies and the WSGI/ASGI ``env`` (``REMOTE_ADDR``) are + dropped, sensitive headers removed outright; +* Celery task ``args``/``kwargs``/return values are dropped wherever they + appear (the Celery integration's ``extra["celery-job"]`` and the + ``extra["data"]`` the ``celery.app.trace`` log records carry); +* frame locals are dropped even though ``include_local_variables=False``. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +REDACTED = "[Filtered]" +_MAX_DEPTH = 12 +_PATH_SEP = "/" +_PATH_PLACEHOLDER = "*" +_SOURCE_URL = "url" + +_SENSITIVE_WORDS = ( + r"passw(?:or)?d|secret|token|authori[sz]ation|(?["']?[\w.\-]*(?:{_SENSITIVE_WORDS})[\w.\-]*["']?\s*[=:]\s*)""" + r"""(?P["']?)(?P(?:Bearer\s+|Basic\s+)?[^\s&,;"'}\]]+)""", + re.IGNORECASE, +) +_SCHEME_CREDENTIAL = re.compile(r"\b(Bearer|Basic|Token)\s+[A-Za-z0-9._~+/=\-]+", re.IGNORECASE) +_URL_USERINFO = re.compile(r"(?P[a-z][a-z0-9+.\-]*://)[^/@\s]+@", re.IGNORECASE) +_ID_SEGMENT = re.compile( + r"^(?:\d+|[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}" + r"|(?=[\w\-]*\d)(?=[\w\-]*[A-Za-z])[\w\-]{20,})$", + re.IGNORECASE, +) + +Event = dict[str, Any] + + +def is_sensitive_key(key: object) -> bool: + return isinstance(key, str) and (key.lower() in _EXACT_KEYS or bool(SENSITIVE_KEY.search(key))) + + +def scrub_text(text: str) -> str: + """Redact credentials embedded in free text.""" + text = _URL_USERINFO.sub(rf"\g{REDACTED}@", text) + text = _QUERY_CODE.sub(rf"\g<1>{REDACTED}", text) + text = _KEY_VALUE.sub(rf"\g\g{REDACTED}", text) + return _SCHEME_CREDENTIAL.sub(rf"\1 {REDACTED}", text) + + +def scrub(value: Any, depth: int = 0) -> Any: + """Recursively redact sensitive keys and scrub every string.""" + if depth > _MAX_DEPTH: + return REDACTED + if isinstance(value, Mapping): + return { + k: REDACTED if is_sensitive_key(k) else scrub(v, depth + 1) + for k, v in value.items() + if k not in _TASK_PAYLOAD_KEYS + } + if isinstance(value, list | tuple): + return [scrub(v, depth + 1) for v in value] + if isinstance(value, str): + return scrub_text(value) + return value + + +def scrub_path(path: str) -> str: + """Replace id- and token-shaped path segments with ``*``.""" + return _PATH_SEP.join( + _PATH_PLACEHOLDER if _ID_SEGMENT.match(seg) else seg for seg in path.split(_PATH_SEP) + ) + + +def _scrub_url(url: str) -> str: + parts = urlsplit(url) + return scrub_text(urlunsplit(parts._replace(path=scrub_path(parts.path)))) + + +def _scrub_request(request: dict[str, Any]) -> None: + for key in ("cookies", "data", "env"): + request.pop(key, None) + headers = request.get("headers") + if isinstance(headers, Mapping): + request["headers"] = { + k: scrub(v) for k, v in headers.items() if not is_sensitive_key(str(k)) + } + query = request.get("query_string") + if isinstance(query, str): + request["query_string"] = scrub_text(query) + elif query is not None: + request["query_string"] = scrub(query) + if isinstance(request.get("url"), str): + request["url"] = _scrub_url(request["url"]) + + +def _scrub_exceptions(event: Event) -> None: + for exc in (event.get("exception") or {}).get("values") or []: + if isinstance(exc.get("value"), str): + exc["value"] = scrub_text(exc["value"]) + _scrub_frames(exc) + for thread in (event.get("threads") or {}).get("values") or []: + _scrub_frames(thread) + + +def _scrub_frames(holder: Mapping[str, Any]) -> None: + """Drop frame locals; scrub source context (a hard-coded secret is still one).""" + for frame in (holder.get("stacktrace") or {}).get("frames") or []: + frame.pop("vars", None) + for key in ("pre_context", "context_line", "post_context"): + if frame.get(key) is not None: + frame[key] = scrub(frame[key]) + + +def _scrub_spans(event: Event) -> None: + for span in event.get("spans") or []: + if isinstance(span.get("description"), str): + span["description"] = scrub_text(span["description"]) + if isinstance(span.get("data"), Mapping): + span["data"] = scrub(span["data"]) + + +def _scrub_transaction(event: Event) -> None: + transaction = event.get("transaction") + source = (event.get("transaction_info") or {}).get("source") + if isinstance(transaction, str) and source == _SOURCE_URL: + event["transaction"] = scrub_text(scrub_path(transaction)) + + +def scrub_event(event: Event) -> Event: + """Scrub ``event`` in place (and return it).""" + if isinstance(event.get("request"), dict): + _scrub_request(event["request"]) + for key in ("extra", "contexts"): + if isinstance(event.get(key), Mapping): + event[key] = scrub(event[key]) + # Breadcrumbs: log messages and HTTP URLs/queries; logentry: log params. + for key in ("breadcrumbs", "logentry", "message"): + if event.get(key) is not None: + event[key] = scrub(event[key]) + _scrub_exceptions(event) + _scrub_spans(event) + _scrub_transaction(event) + return event diff --git a/framework/hosting/tests/test_hosting_sentry_scrub.py b/framework/hosting/tests/test_hosting_sentry_scrub.py new file mode 100644 index 000000000..79810706e --- /dev/null +++ b/framework/hosting/tests/test_hosting_sentry_scrub.py @@ -0,0 +1,202 @@ +"""Sentry events never carry secrets or client addresses (#538 follow-up). + +Each test drives the real SDK through a fake transport and asserts the secret +is absent from the serialized envelope payload. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Iterator +from typing import Any + +import httpx +import pytest +from fastapi import FastAPI, Request +from simple_module_hosting import sentry as sm_sentry +from simple_module_hosting.sentry_scrub import REDACTED, scrub_event, scrub_text + +sentry_sdk = pytest.importorskip("sentry_sdk") +from sentry_sdk.transport import Transport # noqa: E402 + +_DSN_KEY = "dsnpublickey0123" +_DSN = f"https://{_DSN_KEY}@o0.ingest.sentry.io/1" +_LEAKS = ( + "c00kie-val", + "203.0.113.9", + "proxy-cred", + "hdr-t0ken", + "sess-1d", + "q-t0ken", + "exc-passw0rd", + "bearer-abc", + "log-s3cret", + "up-k3y", + "nested-s3cret", + "ctx-s3cret", + "frame-l0cal", +) + + +class FakeTransport(Transport): + def __init__(self, options: dict[str, Any] | None = None) -> None: + super().__init__(options) + self.payloads: list[dict[str, Any]] = [] + + def capture_envelope(self, envelope: Any) -> None: + for item in envelope.items: + if item.type in ("event", "transaction"): + self.payloads.append(item.payload.json) + + def of(self, kind: str) -> list[dict[str, Any]]: + return [p for p in self.payloads if p.get("type", "event") == kind] + + +def _start(monkeypatch: pytest.MonkeyPatch, process: sm_sentry.Process) -> FakeTransport: + monkeypatch.setenv("SM_SENTRY_DSN", _DSN) + monkeypatch.setenv("SM_SENTRY_TRACES_SAMPLE_RATE", "1.0") + fake = FakeTransport() + assert sm_sentry.init_sentry(process, transport=fake) + return fake + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeTransport]: + fake = _start(monkeypatch, sm_sentry.PROCESS_WEB) + yield fake + sentry_sdk.get_global_scope().client.close() + + +def leak(request: Request) -> None: + # Built at runtime so only a frame-locals capture (not source context) could leak it. + frame_local = "frame-" + "l0cal" # noqa: F841 + logging.getLogger("app.audit").warning("refresh failed token=log-s3cret for user") + upstream = httpx.Client(transport=httpx.MockTransport(lambda _: httpx.Response(200))) + upstream.get("http://up.example/v1?api_key=up-k3y&page=2") + sentry_sdk.set_extra("job", {"config": {"db_password": "nested-s3cret", "size": 3}}) + sentry_sdk.set_context("oauth", {"refresh_token": "ctx-s3cret", "provider": "github"}) + raise RuntimeError("login rejected password=exc-passw0rd header Bearer bearer-abc") + + +async def _hit(app: FastAPI) -> None: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app, raise_app_exceptions=False), + base_url="http://test", + ) as client: + await client.get( + "/leak?access_token=q-t0ken&page=1", + headers={ + "Cookie": "sm_session=c00kie-val", + "X-Forwarded-For": "203.0.113.9", + "X-Real-IP": "203.0.113.9", + "Proxy-Authorization": "Basic proxy-cred", + "X-Auth-Token": "hdr-t0ken", + "X-Session-Id": "sess-1d", + "X-Request-Id": "keep-me", + }, + ) + sentry_sdk.flush() + + +def _app() -> FastAPI: + app = FastAPI() + app.add_middleware(sm_sentry.SentryContextMiddleware) + app.add_api_route("/leak", leak, methods=["GET"]) + return app + + +async def test_error_event_carries_no_secret(web: FakeTransport) -> None: + await _hit(_app()) + (event,) = web.of("event") + dumped = json.dumps(event) + for secret in _LEAKS: + assert secret not in dumped, secret + request = event["request"] + assert "cookies" not in request + assert "env" not in request # REMOTE_ADDR + headers = {h.lower() for h in request["headers"]} + assert not headers & {"cookie", "x-forwarded-for", "x-real-ip", "proxy-authorization"} + assert not headers & {"x-auth-token", "x-session-id"} + assert request["headers"].get("x-request-id") == "keep-me" + assert "ip_address" not in event.get("user", {}) + frames = event["exception"]["values"][0]["stacktrace"]["frames"] + assert all("vars" not in f for f in frames) + # Non-secret signal survives. + assert "page=1" in request["query_string"] + assert "page=2" in dumped + assert event["extra"]["job"]["config"] == {"db_password": REDACTED, "size": 3} + assert event["contexts"]["oauth"]["provider"] == "github" + + +async def test_transaction_event_is_scrubbed_too(web: FakeTransport) -> None: + await _hit(_app()) + (transaction,) = web.of("transaction") + dumped = json.dumps(transaction) + for secret in ("c00kie-val", "203.0.113.9", "hdr-t0ken", "q-t0ken", "up-k3y"): + assert secret not in dumped, secret + assert transaction["tags"]["process"] == sm_sentry.PROCESS_WEB + + +def test_task_args_and_kwargs_are_dropped(monkeypatch: pytest.MonkeyPatch) -> None: + from celery import Celery + + fake = _start(monkeypatch, sm_sentry.PROCESS_WORKER) + celery = Celery("t", broker="memory://", task_always_eager=True) + + @celery.task(name="orders.charge") + def charge(card: str, *, api_secret: str) -> None: + raise RuntimeError("charge failed") + + card, secret = "4111" + "-arg-card", "kw-" + "s3cret" # runtime-only values + try: + with pytest.raises(RuntimeError): + charge.apply(args=(card,), kwargs={"api_secret": secret}, throw=True) + sentry_sdk.flush() + finally: + sentry_sdk.get_global_scope().client.close() + events = [e for e in fake.of("event") if "exception" in e] + assert events + for event in events: + dumped = json.dumps(event) + assert "4111-arg-card" not in dumped + assert "kw-s3cret" not in dumped + job = event.get("extra", {}).get("celery-job", {}) + assert "args" not in job and "kwargs" not in job + + +def test_dsn_is_never_logged( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.DEBUG) + _start(monkeypatch, sm_sentry.PROCESS_WEB) + sentry_sdk.get_global_scope().client.close() + assert _DSN_KEY not in caplog.text + + +@pytest.mark.parametrize( + ("raw", "secret"), + [ + ("GET /cb?code=x&refresh_token=rt-1&state=s", "rt-1"), + ('{"client_secret": "cs-1", "id": 4}', "cs-1"), + ("Authorization: Bearer ab.cd.ef", "ab.cd.ef"), + ("postgresql://app:pg-pass@db:5432/x", "pg-pass"), + ("X-API-Key=ak-1", "ak-1"), + ("csrf-token: cf-1", "cf-1"), + ], +) +def test_scrub_text_redacts_inline_credentials(raw: str, secret: str) -> None: + assert secret not in scrub_text(raw) + + +def test_url_transaction_names_and_paths_lose_ids() -> None: + event: dict[str, Any] = { + "transaction": "/invite/accept/3f2b9c1d8e7a6b5c4d3e2f1a0b9c8d7e6f5a", + "transaction_info": {"source": "url"}, + "request": {"url": "http://h/orders/12345/items?session=s1"}, + "breadcrumbs": {"values": [{"message": "x", "data": {"http.query": "token=bt-1"}}]}, + } + scrubbed = scrub_event(event) + assert scrubbed["transaction"] == "/invite/accept/*" + assert scrubbed["request"]["url"] == f"http://h/orders/*/items?session={REDACTED}" + assert "bt-1" not in json.dumps(scrubbed["breadcrumbs"]) From 02bf929bfb8f6a4d69a2cb3d7275586ee7955c9e Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:27:07 +0200 Subject: [PATCH 08/22] feat(background_tasks): in-process outbox sweep, lazy Celery import, DB-backed retention (#565) Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 2 + docs/modules/background_tasks.md | 15 +- .../background_tasks/__init__.py | 24 ++- .../background_tasks/after_commit.py | 13 +- .../background_tasks/constants.py | 7 +- .../background_tasks/module.py | 15 +- .../background_tasks/outbox_sweep.py | 58 ++++++ .../background_tasks/outbox_transport.py | 15 +- .../background_tasks/services.py | 4 + .../background_tasks/settings.py | 12 +- .../background_tasks/tasks.py | 12 +- .../tests/test_bg_inprocess_sweep.py | 171 ++++++++++++++++++ 12 files changed, 320 insertions(+), 28 deletions(-) create mode 100644 modules/background_tasks/background_tasks/outbox_sweep.py create mode 100644 modules/background_tasks/tests/test_bg_inprocess_sweep.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9807e03a..0d00c936e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project are documented in this file. The format is b - `host/client_app` no longer re-declares `@simple-module-py/ui`'s runtime dependencies, nor `@base-ui/react`, `@radix-ui/react-slot`, `autoprefixer` or `postcss` (nothing loaded them) (#521). ### Added +- In-process durable outbox delivery and lifespan-owned recovery sweeps, automatically enabled in eager mode (#565). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial @@ -287,6 +288,7 @@ All notable changes to this project are documented in this file. The format is b previously served, and anonymous, in production. ### Fixed +- Background task stuck sweeps and retention purges now use DB-backed settings so admin edits apply instead of reading only environment variables (#565). - User directory follow-ups (#501): `audit_log` installs its activity feed in `on_startup` (cleared on shutdown) instead of `register_settings`, which tripped `SM012`; `resolve_user_labels` omits users whose label is empty so diff --git a/docs/modules/background_tasks.md b/docs/modules/background_tasks.md index a68f07836..5eeff0a8a 100644 --- a/docs/modules/background_tasks.md +++ b/docs/modules/background_tasks.md @@ -175,6 +175,16 @@ Each field defaults to `None`, which keeps Celery's own default; all require a w ## Durable events (outbox) +`outbox_mode` defaults to `celery`: workers deliver committed rows and beat +recovers missed dispatches. Set it to `inprocess` to deliver on the web app's +async engine without a worker for durable events. `task_always_eager` also +selects `inprocess` automatically. Each web process sweeps pending rows and +expired leases every `outbox_sweep_interval_seconds` (default 5, minimum 1), +with small jitter and at most one retry per failed sweep. Shutdown cancels +and joins the sweep, then drains scheduled deliveries. Mode and interval +changes require a restart. Ordinary Celery tasks still use their configured queue. + + `bus.publish` is in-process: a crash, a restart or a failing handler loses the event. For side effects that must happen — an invoice email, a search-index update, a call to another system — subscribe a **durable** handler and publish @@ -209,7 +219,7 @@ How it is delivered: - After the commit, `background_tasks.dispatch_outbox(ids)` is sent to the broker. If that never happens (the process died between commit and publish, or the broker was down), the beat task `background_tasks.sweep_outbox` delivers the row - within `outbox_sweep_interval_seconds` (30 s). + within `outbox_sweep_interval_seconds` (5 s). - A worker **claims** a row with one conditional `UPDATE` (`pending → running`, `attempts + 1`, `claim_seq + 1`, `locked_until = now + outbox_lease_seconds`); on Postgres the candidate select uses `FOR UPDATE SKIP LOCKED`. Two workers @@ -434,7 +444,8 @@ DB-backed; defaults are in `BackgroundTasksSettings`. Several are marked `requir | `retention_days` | `14` | how long to keep terminal rows; range 1-3650 | | `max_retries` | `3` | informational; tasks define their own policies; range 0-100 | | `outbox_max_attempts` | `8` | durable-event delivery attempts before `dead`; range 1-100 | -| `outbox_sweep_interval_seconds` | `30` | restart required; beat cadence of `sweep_outbox` | +| `outbox_mode` | `"celery"` | restart required; `celery` or `inprocess`; eager forces `inprocess` | +| `outbox_sweep_interval_seconds` | `5` | restart required; sweep cadence in either mode; minimum 1 | | `outbox_lease_seconds` | `300` | how long a claimed delivery may run before another worker reclaims it; range 10-86400 — keep it above your slowest durable handler | | `outbox_handler_timeout_seconds` | unset | per-attempt handler timeout; unset means the lease minus a safety margin (10 %, at least 5 s), which is also the most it may be set to | diff --git a/modules/background_tasks/background_tasks/__init__.py b/modules/background_tasks/background_tasks/__init__.py index 49913f19d..5a4bf365d 100644 --- a/modules/background_tasks/background_tasks/__init__.py +++ b/modules/background_tasks/background_tasks/__init__.py @@ -1,13 +1,33 @@ """BackgroundTasks module — Celery + Redis task queue with admin UI.""" +from typing import TYPE_CHECKING, Any + from background_tasks.after_commit import enqueue_after_commit from background_tasks.log_context import ( bind_task_context, get_log_context, install_log_filter, ) -from background_tasks.task_heartbeat import heartbeat -from background_tasks.worker_settings import settings_for + +if TYPE_CHECKING: + from background_tasks.task_heartbeat import heartbeat + from background_tasks.worker_settings import settings_for + +_LAZY_HEARTBEAT = "heartbeat" +_LAZY_SETTINGS = "settings_for" + + +def __getattr__(name: str) -> Any: + if name == _LAZY_HEARTBEAT: + from background_tasks.task_heartbeat import heartbeat + + return heartbeat + if name == _LAZY_SETTINGS: + from background_tasks.worker_settings import settings_for + + return settings_for + raise AttributeError(name) + __all__ = [ "bind_task_context", diff --git a/modules/background_tasks/background_tasks/after_commit.py b/modules/background_tasks/background_tasks/after_commit.py index 87e7c5c8b..9922d1eff 100644 --- a/modules/background_tasks/background_tasks/after_commit.py +++ b/modules/background_tasks/background_tasks/after_commit.py @@ -30,13 +30,24 @@ from collections.abc import Callable, Sequence from typing import Any -from celery import current_app from simple_module_db import RequestSession, bind_current_tenant, mark_written from simple_module_db.transaction import SESSION_START_KEY from sqlalchemy import event from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session, SessionTransaction + +class _LazyCurrentApp: + """``celery.current_app``, imported on first use so importing this module stays light.""" + + def __getattr__(self, name: str) -> Any: + from celery import current_app as _current_app + + return getattr(_current_app, name) + + +current_app: Any = _LazyCurrentApp() + logger = logging.getLogger(__name__) type Publish = Callable[[], object] diff --git a/modules/background_tasks/background_tasks/constants.py b/modules/background_tasks/background_tasks/constants.py index e2d78bcc3..767373726 100644 --- a/modules/background_tasks/background_tasks/constants.py +++ b/modules/background_tasks/background_tasks/constants.py @@ -140,7 +140,7 @@ class OutboxScope(StrEnum): DEFAULT_OUTBOX_MAX_ATTEMPTS = 8 -DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS = 30 +DEFAULT_OUTBOX_SWEEP_INTERVAL_SECONDS = 5 # How long a claimed row stays ``running`` before another worker may reclaim it. # Durable handlers are cut off ``outbox_lease_margin`` before it, so no run can # outlive its claim and overlap the rerun a reclaim would start. @@ -172,3 +172,8 @@ def max_outbox_handler_timeout(lease_seconds: float) -> float: TENANT_ID_MAX_LENGTH = 50 # Filter value meaning "executions with no tenant" (beat / platform publishes). PLATFORM_TENANT_FILTER = "__platform__" + + +class OutboxMode(StrEnum): + CELERY = "celery" + INPROCESS = "inprocess" diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 343949e29..fb1d6f40c 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -202,13 +202,22 @@ async def on_startup(self, app: FastAPI) -> None: @staticmethod def _install_durable_transport(app: FastAPI) -> None: """Make ``bus.publish_durable`` write to this module's outbox (GH #459).""" + from background_tasks.constants import OutboxMode + from background_tasks.outbox_sweep import start_sweep from background_tasks.outbox_transport import OutboxTransport from background_tasks.outbox_worker import use_bus bus = app.state.sm.event_bus - bus.durable_transport = OutboxTransport( - services=app.state.background_tasks, bus=bus, db_state=app.state.sm.db + services = app.state.background_tasks + mode = ( + OutboxMode.INPROCESS + if services.celery.conf.task_always_eager + else OutboxMode(services.settings.outbox_mode) ) + transport = OutboxTransport(services=services, bus=bus, db_state=app.state.sm.db, mode=mode) + bus.durable_transport = transport + if mode == OutboxMode.INPROCESS: + start_sweep(services, app.state.sm.db.session_factory, bus, transport.policy) # An eager in-process dispatch delivers with the app's bus, which already # carries every module's durable subscriptions. use_bus(bus) @@ -251,10 +260,12 @@ async def _start_invalidation_transport(app: FastAPI) -> None: ) async def on_shutdown(self, app: FastAPI) -> None: + from background_tasks.outbox_sweep import stop_sweep from background_tasks.outbox_worker import reset_worker_state from background_tasks.signals import unbind_event_bus from background_tasks.sync_db import dispose_sync_engine + await stop_sweep(app.state.background_tasks) bus = getattr(app.state.sm, "event_bus", None) if bus is not None: drain = getattr(bus.durable_transport, "drain", None) diff --git a/modules/background_tasks/background_tasks/outbox_sweep.py b/modules/background_tasks/background_tasks/outbox_sweep.py new file mode 100644 index 000000000..0231b7857 --- /dev/null +++ b/modules/background_tasks/background_tasks/outbox_sweep.py @@ -0,0 +1,58 @@ +"""App-owned recovery sweeps for in-process durable delivery (#565).""" + +from __future__ import annotations + +import asyncio +import logging +import random +from collections.abc import Callable + +from simple_module_core.events import EventBus + +from background_tasks.outbox_claim import OutboxPolicy, SessionFactory +from background_tasks.outbox_dispatch import dispatch +from background_tasks.services import BackgroundTasksServices + +logger = logging.getLogger(__name__) +JITTER_FRACTION = 0.1 +MAX_TICK_ATTEMPTS = 2 + + +async def sweep_loop( + factory: SessionFactory, + bus: EventBus, + policy: Callable[[], OutboxPolicy], + interval: float, +) -> None: + """Sweep serially, with at most one retry after an infrastructure failure.""" + while True: + await asyncio.sleep(interval * random.uniform(1 - JITTER_FRACTION, 1)) + for _ in range(MAX_TICK_ATTEMPTS): + try: + await dispatch(factory, bus, policy(), ids=None) + except Exception: + logger.exception("outbox: in-process sweep failed") + else: + break + + +def start_sweep( + services: BackgroundTasksServices, + factory: SessionFactory, + bus: EventBus, + policy: Callable[[], OutboxPolicy], +) -> None: + """Install at most one recovery task on this app's services.""" + if services.outbox_sweep_task is None or services.outbox_sweep_task.done(): + services.outbox_sweep_task = asyncio.create_task( + sweep_loop(factory, bus, policy, services.settings.outbox_sweep_interval_seconds) + ) + + +async def stop_sweep(services: BackgroundTasksServices) -> None: + """Cancel and join before the app drains deliveries and disposes engines.""" + task = services.outbox_sweep_task + if task is not None: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + services.outbox_sweep_task = None diff --git a/modules/background_tasks/background_tasks/outbox_transport.py b/modules/background_tasks/background_tasks/outbox_transport.py index dd96d0451..aaecd5dde 100644 --- a/modules/background_tasks/background_tasks/outbox_transport.py +++ b/modules/background_tasks/background_tasks/outbox_transport.py @@ -31,7 +31,6 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any -from celery import current_app from simple_module_core.durable_events import event_key, serialize_event from simple_module_db import RequestSession, current_tenant_id from simple_module_db.tenancy import is_all_tenants @@ -39,8 +38,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session -from background_tasks.after_commit import after_commit -from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxScope +from background_tasks.after_commit import after_commit, current_app +from background_tasks.constants import INTERNAL_TASK_DISPATCH_OUTBOX, OutboxMode, OutboxScope from background_tasks.models import OutboxMessage from background_tasks.outbox_claim import OutboxPolicy from background_tasks.outbox_clock import db_now @@ -119,8 +118,14 @@ class OutboxTransport(_OutboxWriter): """Outbox-backed durable delivery for the web app (GH #459).""" def __init__( - self, *, services: BackgroundTasksServices, bus: EventBus, db_state: DatabaseState + self, + *, + services: BackgroundTasksServices, + bus: EventBus, + db_state: DatabaseState, + mode: OutboxMode = OutboxMode.CELERY, ) -> None: + self._mode = mode self._services = services self._bus = bus self._db_state = db_state @@ -141,7 +146,7 @@ def schedule(self, db: AsyncSession | Session, ids: Sequence[uuid.UUID]) -> None if not ids: return id_list = list(ids) - if self._services.settings.task_always_eager: + if self._mode == OutboxMode.INPROCESS or self._services.settings.task_always_eager: if isinstance(db, RequestSession) and SESSION_START_KEY in db.info: db.on_commit(lambda: self._dispatch(id_list)) else: diff --git a/modules/background_tasks/background_tasks/services.py b/modules/background_tasks/background_tasks/services.py index 9488bf750..12796a738 100644 --- a/modules/background_tasks/background_tasks/services.py +++ b/modules/background_tasks/background_tasks/services.py @@ -17,6 +17,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import asyncio + from celery import Celery from background_tasks.contracts.schemas import WorkerSnapshot @@ -40,3 +42,5 @@ class BackgroundTasksServices: # framework bus, kept only so ``on_shutdown`` can stop its listener task. # ``None`` when ``broadcast_invalidations`` is off. invalidation_transport: RedisInvalidationTransport | None = None + + outbox_sweep_task: asyncio.Task[None] | None = None diff --git a/modules/background_tasks/background_tasks/settings.py b/modules/background_tasks/background_tasks/settings.py index 94cba1027..1d10c1c08 100644 --- a/modules/background_tasks/background_tasks/settings.py +++ b/modules/background_tasks/background_tasks/settings.py @@ -27,18 +27,15 @@ ``@model_validator`` to refuse a localhost broker in production — that's a host-level setting, not a background_tasks-module field. -The Celery-critical fields (``broker_url``, ``result_backend``, -``task_default_queue``) are marked ``requires_restart=True`` via -``json_schema_extra`` because workers read these once at process start, so -DB changes can't be hot-reloaded: the admin UI should surface that bumping -these values requires a worker restart. +Process wiring, including ``outbox_mode`` and sweep cadence, requires a restart; +``json_schema_extra`` exposes that requirement in the settings UI. """ from __future__ import annotations import logging import os -from typing import Any +from typing import Any, Literal from pydantic import Field, ValidationInfo, field_validator, model_validator from pydantic_core import PydanticCustomError @@ -219,6 +216,9 @@ def _vet_task_routes(cls, value: object) -> object: ) # ── Durable event outbox (GH #459) ────────────────────── + outbox_mode: Literal["celery", "inprocess"] = Field( + default="celery", json_schema_extra=_CELERY_RESTART + ) outbox_max_attempts: int = Field( default=DEFAULT_OUTBOX_MAX_ATTEMPTS, ge=1, diff --git a/modules/background_tasks/background_tasks/tasks.py b/modules/background_tasks/background_tasks/tasks.py index 4a3c7268a..f26cb117b 100644 --- a/modules/background_tasks/background_tasks/tasks.py +++ b/modules/background_tasks/background_tasks/tasks.py @@ -14,7 +14,6 @@ from __future__ import annotations import logging -import os from collections import defaultdict from collections.abc import Mapping from datetime import UTC, datetime, timedelta @@ -24,8 +23,6 @@ from sqlalchemy import ColumnElement, and_, delete, or_, update from background_tasks.constants import ( - DEFAULT_RETENTION_DAYS, - DEFAULT_STUCK_AFTER_SECONDS, DEMO_ECHO_TASK, INTERNAL_TASK_DISPATCH_OUTBOX, INTERNAL_TASK_PURGE_OLD, @@ -39,6 +36,7 @@ from background_tasks.models import OutboxMessage, TaskExecution from background_tasks.settings import BackgroundTasksSettings from background_tasks.sync_db import sync_session +from background_tasks.worker_settings import load_runner_settings logger = logging.getLogger(__name__) @@ -112,11 +110,7 @@ def sweep_stuck_tasks(self: Task) -> int: Returns the number of rows flipped — handy for metrics/log scraping. """ - # Avoid import-time coupling to settings (beat schedules this task - # before the worker has hydrated settings in some edge cases). - stuck_after = int( - os.environ.get("SM_BG_TASKS_STUCK_AFTER_SECONDS", DEFAULT_STUCK_AFTER_SECONDS) - ) + stuck_after = load_runner_settings().stuck_after_seconds now = datetime.now(UTC) overrides = stuck_thresholds(self.app.tasks) @@ -139,7 +133,7 @@ def sweep_stuck_tasks(self: Task) -> int: @shared_task(name=INTERNAL_TASK_PURGE_OLD) def purge_old_executions() -> int: """Delete terminal rows older than the retention window.""" - retention_days = int(os.environ.get("SM_BG_TASKS_RETENTION_DAYS", DEFAULT_RETENTION_DAYS)) + retention_days = load_runner_settings().retention_days cutoff = datetime.now(UTC) - timedelta(days=retention_days) with sync_session() as session: diff --git a/modules/background_tasks/tests/test_bg_inprocess_sweep.py b/modules/background_tasks/tests/test_bg_inprocess_sweep.py new file mode 100644 index 000000000..11a50daec --- /dev/null +++ b/modules/background_tasks/tests/test_bg_inprocess_sweep.py @@ -0,0 +1,171 @@ +"""Recovery and lifetime of the web process outbox sweeper (#565).""" + +import asyncio +import subprocess +import sys +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta + +import pytest +from background_tasks.constants import OutboxStatus, TaskStatus +from background_tasks.models import OutboxMessage, TaskExecution +from background_tasks.outbox_sweep import start_sweep, stop_sweep +from background_tasks.tasks import purge_old_executions +from simple_module_core.durable_events import event_key, serialize_event +from simple_module_core.events import Event +from simple_module_db.stored_settings import Setting +from simple_module_test import wait_for +from sqlalchemy import select + + +@pytest.fixture(autouse=True) +def eager_mode(monkeypatch): + monkeypatch.setenv("SM_BG_TASKS_TASK_ALWAYS_EAGER", "true") + + +@dataclass +class SweepPing(Event): + value: int + + +async def seed(app, *, crashed=False): + now = datetime.now(UTC) + row = OutboxMessage( + event_key=event_key(SweepPing), + payload=serialize_event(SweepPing(7)), + handler_name="sweep-test", + next_attempt_at=now, + created_at=now, + status=OutboxStatus.RUNNING if crashed else OutboxStatus.PENDING, + locked_until=now - timedelta(seconds=60) if crashed else None, + ) + async with app.state.sm.db.session_factory() as db: + db.add(row) + await db.commit() + return row.id + + +async def restart_short(app): + services = app.state.background_tasks + await stop_sweep(services) + # Validation stays >= 1 in production; shorten only this test's clock. + services.settings.outbox_sweep_interval_seconds = 0.05 + start_sweep( + services, + app.state.sm.db.session_factory, + app.state.sm.event_bus, + app.state.sm.event_bus.durable_transport.policy, + ) + return services.outbox_sweep_task + + +@pytest.mark.parametrize("crashed", [False, True]) +async def test_eager_sweep_recovers_without_schedule(app, crashed): + assert app.state.background_tasks.celery.conf.task_always_eager + assert app.state.background_tasks.outbox_sweep_task is not None + seen = [] + + async def handler(event, ctx): + seen.append(event.value) + + app.state.sm.event_bus.subscribe_durable(SweepPing, handler, name="sweep-test") + row_id = await seed(app, crashed=crashed) + task = await restart_short(app) + + async def delivered(): + async with app.state.sm.db.session_factory() as db: + row = await db.get(OutboxMessage, row_id) + return row.status == OutboxStatus.DELIVERED + + await wait_for(delivered, timeout=10) + await stop_sweep(app.state.background_tasks) + assert task.done() + assert seen == [7] + + +async def test_loop_lifespan_is_idempotent(app): + services = app.state.background_tasks + task = services.outbox_sweep_task + assert task is not None + start_sweep( + services, + app.state.sm.db.session_factory, + app.state.sm.event_bus, + app.state.sm.event_bus.durable_transport.policy, + ) + assert services.outbox_sweep_task is task + from background_tasks.module import BackgroundTasksModule + + await BackgroundTasksModule().on_shutdown(app) + assert task.done() + assert services.outbox_sweep_task is None + + +def test_transport_import_does_not_load_celery(): + subprocess.run( + [ + sys.executable, + "-c", + "import sys; import background_tasks.outbox_transport; " + "assert 'celery' not in sys.modules", + ], + check=True, + ) + + +def test_purge_uses_db_retention(sync_sqlite, monkeypatch): + from background_tasks.sync_db import get_sync_session_factory, sync_session + + Setting.metadata.create_all(get_sync_session_factory().kw["bind"]) + + monkeypatch.delenv("SM_BG_TASKS_RETENTION_DAYS", raising=False) + with sync_session() as db: + db.add(Setting(key="background_tasks.retention_days", value="2", value_type="int")) + db.add( + TaskExecution( + task_name="old", + status=TaskStatus.SUCCESS, + finished_at=datetime.now(UTC) - timedelta(days=3), + ) + ) + db.add( + TaskExecution( + task_name="recent", + status=TaskStatus.SUCCESS, + finished_at=datetime.now(UTC) - timedelta(days=1), + ) + ) + assert purge_old_executions.run() == 1 + with sync_session() as db: + assert list(db.scalars(select(TaskExecution.task_name))) == ["recent"] + + +@pytest.mark.postgres +async def test_two_web_loops_deliver_once(app): + if app.state.sm.db.engine.dialect.name != "postgresql": + pytest.skip("requires PostgreSQL") + from background_tasks.outbox_sweep import sweep_loop + + seen = [] + + async def handler(event, ctx): + seen.append(event.value) + + bus = app.state.sm.event_bus + bus.subscribe_durable(SweepPing, handler, name="sweep-test") + await stop_sweep(app.state.background_tasks) + await seed(app) + loops = [ + asyncio.create_task( + sweep_loop(app.state.sm.db.session_factory, bus, bus.durable_transport.policy, 0.05) + ) + for _ in range(2) + ] + try: + await wait_for(lambda: bool(seen), timeout=5) + await asyncio.sleep(0.15) # intentional-sleep: window for a duplicate delivery to show up + assert seen == [7] + finally: + for task in loops: + task.cancel() + await asyncio.gather(*loops, return_exceptions=True) From eae6b9ab7455cf8197a145a968b148c997346b13 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:36:02 +0200 Subject: [PATCH 09/22] feat(background_tasks): retained task and app logs (#537) Opt-in DB log sink (log_sink_enabled), LogSink/QueueSinkHandler in hosting, Logs tab + admin Logs page, retention in purge_old_executions. Migration 1f5985daa10b. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 8 + docs/.vitepress/config.ts | 1 + docs/framework/logging.md | 57 +++++ docs/modules/background_tasks.md | 20 +- .../simple_module_hosting/log_sinks.py | 232 ++++++++++++++++++ .../hosting/tests/test_hosting_log_sinks.py | 154 ++++++++++++ ...b_background_tasks_retained_log_records.py | 50 ++++ host/tests/test_migration_graph.py | 4 +- .../background_tasks/constants.py | 15 ++ .../background_tasks/contracts/logs.py | 31 +++ .../background_tasks/background_tasks/deps.py | 9 + .../background_tasks/endpoints/logs_api.py | 62 +++++ .../background_tasks/endpoints/logs_views.py | 44 ++++ .../background_tasks/locales/en.json | 27 +- .../background_tasks/log_purge.py | 32 +++ .../background_tasks/log_service.py | 85 +++++++ .../background_tasks/log_sink.py | 127 ++++++++++ .../background_tasks/models.py | 37 +++ .../background_tasks/module.py | 24 ++ .../background_tasks/pages/Detail.tsx | 30 ++- .../background_tasks/pages/Logs.tsx | 93 +++++++ .../pages/components/LogLevelSelect.tsx | 29 +++ .../pages/components/LogTable.tsx | 76 ++++++ .../pages/components/LogsPanel.tsx | 46 ++++ .../background_tasks/pages/logs.ts | 50 ++++ .../background_tasks/runner.py | 8 +- .../background_tasks/settings.py | 10 +- .../background_tasks/settings_logs.py | 45 ++++ .../background_tasks/tasks.py | 4 +- .../background_tasks_LogsTab.test.tsx | 56 +++++ .../background_tasks/tests/test_bg_log_api.py | 118 +++++++++ .../tests/test_bg_log_purge.py | 50 ++++ .../tests/test_bg_log_sink.py | 102 ++++++++ packages/i18n/src/generated-resources.ts | 23 ++ packages/i18n/src/keys.generated.ts | 25 ++ 35 files changed, 1767 insertions(+), 17 deletions(-) create mode 100644 docs/framework/logging.md create mode 100644 framework/hosting/simple_module_hosting/log_sinks.py create mode 100644 framework/hosting/tests/test_hosting_log_sinks.py create mode 100644 host/migrations/versions/1f5985daa10b_background_tasks_retained_log_records.py create mode 100644 modules/background_tasks/background_tasks/contracts/logs.py create mode 100644 modules/background_tasks/background_tasks/endpoints/logs_api.py create mode 100644 modules/background_tasks/background_tasks/endpoints/logs_views.py create mode 100644 modules/background_tasks/background_tasks/log_purge.py create mode 100644 modules/background_tasks/background_tasks/log_service.py create mode 100644 modules/background_tasks/background_tasks/log_sink.py create mode 100644 modules/background_tasks/background_tasks/pages/Logs.tsx create mode 100644 modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx create mode 100644 modules/background_tasks/background_tasks/pages/components/LogTable.tsx create mode 100644 modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx create mode 100644 modules/background_tasks/background_tasks/pages/logs.ts create mode 100644 modules/background_tasks/background_tasks/settings_logs.py create mode 100644 modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx create mode 100644 modules/background_tasks/tests/test_bg_log_api.py create mode 100644 modules/background_tasks/tests/test_bg_log_purge.py create mode 100644 modules/background_tasks/tests/test_bg_log_sink.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ef54a965e..cd51c1eba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ All notable changes to this project are documented in this file. The format is b - `host/client_app` no longer re-declares `@simple-module-py/ui`'s runtime dependencies, nor `@base-ui/react`, `@radix-ui/react-slot`, `autoprefixer` or `postcss` (nothing loaded them) (#521). ### Added +- `background_tasks`: opt-in retained logs. With `log_sink_enabled` (default off; + `log_sink_level` default `INFO`, `log_retention_days` default 7) web and worker + log lines are kept in the new `background_tasks_log_record` table, batched off + the logging path by the new `simple_module_hosting.log_sinks` (`LogSink` + protocol, `QueueSinkHandler`: bounded queue that drops and counts instead of + blocking, secrets masked before queueing). The task detail page gains a Logs + tab, `/admin/background-tasks/logs` searches all retained lines, and + `purge_old_executions` enforces retention. Migration `1f5985daa10b` (#537). - `background_tasks`: tasks report progress with `heartbeat(progress=(i, n), message=, phase=)` (buffered under the existing 30 s throttle, `i == n` writes at once, never audited), and `GET /api/background_tasks/tasks/{celery_task_id}/status` lets the user who diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index bb656edaa..6c743fc06 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -129,6 +129,7 @@ export default defineConfig({ { text: "Cache invalidation", link: "/framework/invalidation" }, { text: "Multi-tenancy", link: "/framework/multi-tenancy" }, { text: "Internationalization", link: "/framework/i18n" }, + { text: "Retained logging", link: "/framework/logging" }, ], }, ], diff --git a/docs/framework/logging.md b/docs/framework/logging.md new file mode 100644 index 000000000..6ceaa003d --- /dev/null +++ b/docs/framework/logging.md @@ -0,0 +1,57 @@ +# Retained logging + +`simple_module_hosting.log_sinks` lets a module keep log lines somewhere durable +without putting that work on the logging call. The `background_tasks` module +ships the database sink; any module can add another. + +## The sink protocol + +```python +class LogSink(Protocol): + def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: ... +``` + +`emit_batch` runs on a background thread, never in a request, and receives +records that have already been redacted. Raising is allowed: the batch is +counted in `QueueSinkHandler.failed_batches`, a warning is logged, and the +records are lost. + +## Installing one + +```python +from simple_module_hosting.log_sinks import install_sink, uninstall_sink + +handler = install_sink(MySink(), level=logging.INFO) # attaches to the root logger +... +uninstall_sink(handler) # detaches and flushes +``` + +`QueueSinkHandler` is a `logging.handlers.QueueHandler` plus a listener thread: + +| Behaviour | Detail | +|---|---| +| Batching | up to 200 records, or whatever arrived within 1 second | +| Bounded | 10,000 queued records; when full a record is dropped and counted in `handler.dropped`. Logging never blocks and cannot deadlock on a slow sink | +| Level | `level` is a threshold on the handler; the logger it is attached to must also pass the record (a root logger at `WARNING` never offers `INFO` lines) | +| Self-exclusion | records from `simple_module_hosting.log_sinks` and `sqlalchemy.*` are skipped, so a database sink cannot log about itself | +| Correlation | `record.correlation_id` is filled from the request context when unset | + +Handler filters run on the emitting thread, where the request or task context +is bound. Add a filter to the handler (as `background_tasks` does for the task +id and the tenant) to capture context; the listener thread cannot see it. + +## Redaction + +`redact(text)` and `redact_record(record)` mask `Authorization`, +`Proxy-Authorization`, `Cookie` and `Set-Cookie` values, bearer tokens, and the +value of any `key=value` / `"key": "value"` whose key contains `password`, +`passwd`, `token`, `secret` or `api_key`. Extra attributes whose *name* +matches are replaced outright. `QueueSinkHandler` redacts a copy of the record +before queueing it, so the original, shared with every other handler, is never +changed. Redaction is pattern-based: it cannot recognise a secret that is +logged without a label. + +## Database sink + +See [Retained task and app logs](/modules/background_tasks#retained-task-and-app-logs) +for the table, settings, retention and the admin screens. diff --git a/docs/modules/background_tasks.md b/docs/modules/background_tasks.md index c1d09f89f..03d08d215 100644 --- a/docs/modules/background_tasks.md +++ b/docs/modules/background_tasks.md @@ -108,6 +108,16 @@ The enqueuing user is recorded as `owner_user_id` from the `sm_owner_id` message `GET /api/background_tasks/tasks/{celery_task_id}/status` returns a `TaskStatusDTO`: status, the progress fields, timestamps and the exception's type name. It never returns args, kwargs, result, traceback or an error message. It is visible to the owner while acting in the row's tenant (any tenant when multi-tenancy is off) and to holders of `background_tasks.view`; everyone else, and every unknown id, gets `404` so the route does not reveal which ids exist. The task detail page shows a progress bar once a report exists. +## Retained task and app logs + +Opt-in: with `log_sink_enabled` on, every log line at or above `log_sink_level` (default `INFO`) from the web process and from workers is kept in `background_tasks_log_record` for `log_retention_days` (default 7). It is off by default because it writes a row per line. The three settings are in the **Logs** group; changing the first two needs a restart of the web process and the workers. + +Lines are queued in memory and written in batches (200 records or 1 second) by a listener thread through the worker-style sync session, so logging never waits on the database; if the queue fills, lines are dropped and counted rather than blocking. Secrets (`Authorization`, `Cookie`, bearer tokens, and `password` / `token` / `secret` / `api_key` values) are masked before a line is queued. Messages are truncated to 4096 characters. The mechanism (`LogSink`, `QueueSinkHandler`) lives in the framework: see [Retained logging](/framework/logging). + +Each row carries the Celery `task_id` and the task's `job_id` when bound through `bind_task_context`, the tenant active when the line was emitted, the request correlation id, and the record's extra attributes as JSON. `tenant_id` is a plain column (no `MultiTenantMixin`) because worker and platform lines have no tenant. The admin views show a platform admin every tenant's lines and anyone else only the active tenant's. The table is excluded from auditing, and `purge_old_executions` deletes rows past retention in the same pass as terminal executions. + +The task detail page has a **Logs** tab (minimum-level filter, paginated), and `/admin/background-tasks/logs` searches across everything retained by message text, task id and level. + ## Scheduling periodic work (beat) A module schedules recurring work the same way it registers tasks — by shipping a `tasks.py` — and additionally exporting a module-level `BEAT_SCHEDULE` dict. `build_celery` merges every installed module's `BEAT_SCHEDULE` into the beat schedule at boot (identically in the web and worker processes), so `make beat` runs them alongside the built-ins: @@ -362,6 +372,8 @@ All require `background_tasks.view`; retry additionally needs `background_tasks. |---|---|---| | `GET /api/background_tasks/admin/executions` | `?status=&task_name=&page=&per_page=` → `TaskExecutionListResponse` | per_page max 200 | | `GET /api/background_tasks/admin/executions/{execution_id}` | → `TaskExecutionDetail` | 404 if missing | +| `GET /api/background_tasks/admin/executions/{execution_id}/logs` | `?level=&page=&per_page=` → `LogListResponse` | the task's retained lines, newest first; `level` is a minimum | +| `GET /api/background_tasks/admin/logs` | `?q=&level=&task_id=&logger=&page=&per_page=` → `LogListResponse` | search across retained lines | | `POST /api/background_tasks/admin/executions/{execution_id}/retry` | → `TaskExecutionDetail` | 409 unless status is `failed` or `stuck` | | `GET /api/background_tasks/tasks/{celery_task_id}/status` | → `TaskStatusDTO` | no permission gate: owner in the active tenant or `view`; otherwise 404 | | `GET /api/background_tasks/admin/workers` | → `WorkerSnapshot` | live ping; consumed by the `Workers` page | @@ -376,6 +388,7 @@ All require `background_tasks.view`; retry additionally needs `background_tasks. | `GET /admin/background-tasks/` | `BackgroundTasks/Index` | `background_tasks.view` | | `GET /admin/background-tasks/workers` | `BackgroundTasks/Workers` | `background_tasks.view` | | `GET /admin/background-tasks/outbox` | `BackgroundTasks/Outbox` | `background_tasks.view` | +| `GET /admin/background-tasks/logs` | `BackgroundTasks/Logs` | `background_tasks.view` | | `GET /admin/background-tasks/{execution_id}` | `BackgroundTasks/Detail` | `background_tasks.view` | ## Public contracts @@ -461,6 +474,10 @@ DB-backed; defaults are in `BackgroundTasksSettings`. Several are marked `requir | `outbox_lease_seconds` | `300` | how long a claimed delivery may run before another worker reclaims it; range 10-86400 — keep it above your slowest durable handler | | `outbox_handler_timeout_seconds` | unset | per-attempt handler timeout; unset means the lease minus a safety margin (10 %, at least 5 s), which is also the most it may be set to | +| `log_sink_enabled` | `False` | restart required; keep log lines in the database — see [Retained task and app logs](#retained-task-and-app-logs) | +| `log_sink_level` | `"INFO"` | restart required; lowest level kept | +| `log_retention_days` | `7` | how long to keep log lines; range 1-3650 | + Bootstrap env-var equivalents (`SM_BG_TASKS_*`) only seed pydantic defaults at first boot — once a value lives in the DB, it's authoritative. ## Permissions @@ -529,11 +546,12 @@ Production deployments running multiple workers can write concurrently because e - `BackgroundTasks/Detail.tsx` — full execution view with retry button (`background_tasks.manage`). - `BackgroundTasks/Workers.tsx` — live worker dashboard. - `BackgroundTasks/Outbox.tsx` — undelivered durable events with retry / discard. +- `BackgroundTasks/Logs.tsx` — search across retained log lines. - Components: `ExecutionRow.tsx`, `RetryConfirmDialog.tsx`. ## Locales -Top-level keys in `background_tasks/locales/en.json`: `index`, `filters`, `status` (per-state labels), `table`, `detail`, `retry_dialog`, `toasts`, `outbox`. +Top-level keys in `background_tasks/locales/en.json`: `index`, `filters`, `status` (per-state labels), `table`, `detail`, `retry_dialog`, `toasts`, `outbox`, `logs`. ## Reading DB-backed settings in a task diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py new file mode 100644 index 000000000..9fcb6b8fa --- /dev/null +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -0,0 +1,232 @@ +"""Pluggable retained-log sinks (GH #537). + +A :class:`LogSink` receives batches of already-redacted records. The +:class:`QueueSinkHandler` sits on a logger (normally the root) and moves +records to the sink on a background thread, so logging never waits on a +database or a network call. + +Guarantees, in order of importance: + +* **Never blocks.** The queue is bounded; when it is full the record is dropped + and counted (:attr:`QueueSinkHandler.dropped`). A slow or dead sink costs log + lines, never request latency. +* **Secrets are masked before queueing.** :func:`redact` runs on a copy of the + record in the emitting thread, so neither the queue nor any sink ever holds + an ``Authorization`` header, cookie, password, token or ``secret=`` value. +* **No feedback loop.** Records from this module's own logger and from + ``sqlalchemy.*`` (what a database sink itself triggers) are skipped. +""" + +from __future__ import annotations + +import logging +import logging.handlers +import queue +import re +import threading +import time +from collections.abc import Sequence +from typing import Protocol, runtime_checkable + +from simple_module_core.correlation import correlation_id + +logger = logging.getLogger(__name__) + +DEFAULT_QUEUE_CAPACITY = 10_000 +DEFAULT_BATCH_SIZE = 200 +DEFAULT_FLUSH_INTERVAL_SECONDS = 1.0 +STOP_TIMEOUT_SECONDS = 5.0 +REDACTED = "***" +# Logger names whose records are never forwarded: our own, and the SQL layer a +# database sink writes through. +SKIPPED_LOGGER_PREFIXES: tuple[str, ...] = (__name__, "sqlalchemy") + +_STOP = object() + +_HEADER_RE = re.compile( + r"""(?ix)\b(proxy-authorization|authorization|set-cookie|cookie)(["']?\s*[:=]\s*["']?)[^\r\n"']*""" +) +_SECRET_RE = re.compile( + r"""(?ix)([\w-]*(?:password|passwd|token|secret|api[_-]?key)[\w-]*["']?\s*[:=]\s*["']?)""" + r"""[^\s"',;&}\])]+""" +) +_BEARER_RE = re.compile(r"(?i)\bbearer\s+[\w.~+/=-]+") +_SENSITIVE_KEY_RE = re.compile(r"(?i)password|passwd|token|secret|api[_-]?key|authorization|cookie") +_STANDARD_ATTRS = frozenset(logging.makeLogRecord({}).__dict__) | {"message", "asctime"} + + +@runtime_checkable +class LogSink(Protocol): + """Where batches of log records end up.""" + + def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: + """Persist ``records``. Called from the handler's thread, never a request's.""" + ... + + +def redact(text: str) -> str: + """Mask credentials in ``text``: auth headers, cookies, bearer tokens, ``key=value`` secrets.""" + text = _HEADER_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}{REDACTED}", text) + text = _BEARER_RE.sub(f"Bearer {REDACTED}", text) + return _SECRET_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text) + + +def redact_record(record: logging.LogRecord) -> None: + """Mask ``record`` in place: the message, the traceback text and any extra attributes.""" + record.msg = redact(record.getMessage()) + record.args = None + if record.exc_text: + record.exc_text = redact(record.exc_text) + for key, value in list(record.__dict__.items()): + if key in _STANDARD_ATTRS: + continue + if _SENSITIVE_KEY_RE.search(key): + record.__dict__[key] = REDACTED + elif isinstance(value, str): + record.__dict__[key] = redact(value) + + +class RedactionFilter(logging.Filter): + """Filter form of :func:`redact_record`, for a handler that is not a :class:`QueueSinkHandler`. + + Mutates the shared record, so attach it to a handler that owns its records. + """ + + def filter(self, record: logging.LogRecord) -> bool: + redact_record(record) + return True + + +class QueueSinkHandler(logging.handlers.QueueHandler): + """Forward records to a :class:`LogSink` in batches from a listener thread.""" + + def __init__( + self, + sink: LogSink, + *, + level: int = logging.INFO, + capacity: int = DEFAULT_QUEUE_CAPACITY, + batch_size: int = DEFAULT_BATCH_SIZE, + flush_interval: float = DEFAULT_FLUSH_INTERVAL_SECONDS, + ) -> None: + super().__init__(queue.Queue(maxsize=capacity)) + self.setLevel(level) + self.sink = sink + self.batch_size = batch_size + self.flush_interval = flush_interval + self._dropped = 0 + self._failed_batches = 0 + self._lock = threading.Lock() + self._thread: threading.Thread | None = None + self.addFilter(self._not_own_record) + + @staticmethod + def _not_own_record(record: logging.LogRecord) -> bool: + name = record.name + return not any( + name == prefix or name.startswith(f"{prefix}.") for prefix in SKIPPED_LOGGER_PREFIXES + ) + + @property + def dropped(self) -> int: + """Records discarded because the queue was full.""" + return self._dropped + + @property + def failed_batches(self) -> int: + """Batches the sink raised on; their records are lost.""" + return self._failed_batches + + def prepare(self, record: logging.LogRecord) -> logging.LogRecord: + # QueueHandler.prepare formats the message and traceback into a copy, so + # the caller's record, shared with every other handler, stays untouched. + record = super().prepare(record) + if not getattr(record, "correlation_id", ""): + record.correlation_id = correlation_id.get("") + redact_record(record) + return record + + def enqueue(self, record: logging.LogRecord) -> None: + try: + self.queue.put_nowait(record) + except queue.Full: + with self._lock: + self._dropped += 1 + + # ── lifecycle ─────────────────────────────────────────── + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._thread = threading.Thread(target=self._run, name="log-sink", daemon=True) + self._thread.start() + + def stop(self) -> None: + """Flush what is queued and stop the thread; gives up after a few seconds.""" + thread, self._thread = self._thread, None + if thread is None: + return + try: + self.queue.put(_STOP, timeout=STOP_TIMEOUT_SECONDS) + except queue.Full: + return + thread.join(STOP_TIMEOUT_SECONDS) + + def _run(self) -> None: + stopping = False + while not stopping: + batch: list[logging.LogRecord] = [] + deadline: float | None = None + while len(batch) < self.batch_size: + timeout = self.flush_interval if deadline is None else deadline - time.monotonic() + if timeout <= 0: + break + try: + item = self.queue.get(timeout=timeout) + except queue.Empty: + break + if item is _STOP: + stopping = True + break + batch.append(item) + if deadline is None: + deadline = time.monotonic() + self.flush_interval + if batch: + self._flush(batch) + + def _flush(self, batch: list[logging.LogRecord]) -> None: + try: + self.sink.emit_batch(batch) + except Exception: + with self._lock: + self._failed_batches += 1 + # This module's own logger is skipped by the handler, so a failing + # sink cannot loop back into itself. + logger.warning("Log sink failed; dropped %d record(s)", len(batch), exc_info=True) + + +def install_sink( + sink: LogSink, *, level: int = logging.INFO, target: logging.Logger | None = None, **options +) -> QueueSinkHandler: + """Attach a started :class:`QueueSinkHandler` for ``sink`` to ``target`` (root by default).""" + handler = QueueSinkHandler(sink, level=level, **options) + handler.start() + (target if target is not None else logging.getLogger()).addHandler(handler) + return handler + + +def uninstall_sink(handler: QueueSinkHandler, *, target: logging.Logger | None = None) -> None: + """Detach ``handler`` and flush it.""" + (target if target is not None else logging.getLogger()).removeHandler(handler) + handler.stop() + + +__all__ = [ + "LogSink", + "QueueSinkHandler", + "RedactionFilter", + "install_sink", + "redact", + "redact_record", + "uninstall_sink", +] diff --git a/framework/hosting/tests/test_hosting_log_sinks.py b/framework/hosting/tests/test_hosting_log_sinks.py new file mode 100644 index 000000000..8b1026c5a --- /dev/null +++ b/framework/hosting/tests/test_hosting_log_sinks.py @@ -0,0 +1,154 @@ +"""LogSink / QueueSinkHandler: batching, redaction, and never blocking (GH #537).""" + +from __future__ import annotations + +import logging +import threading +import time +from collections.abc import Iterator, Sequence + +import pytest +from simple_module_hosting.log_sinks import ( + QueueSinkHandler, + install_sink, + redact, + uninstall_sink, +) + + +class Collector: + def __init__(self) -> None: + self.batches: list[list[logging.LogRecord]] = [] + + def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: + self.batches.append(list(records)) + + @property + def messages(self) -> list[str]: + return [str(r.msg) for batch in self.batches for r in batch] + + +class Blocked: + """A sink that never returns until released.""" + + def __init__(self) -> None: + self.release = threading.Event() + self.entered = threading.Event() + + def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: + self.entered.set() + self.release.wait(10) + + +@pytest.fixture +def target() -> Iterator[logging.Logger]: + log = logging.getLogger("hosting_log_sink_test") + log.setLevel(logging.DEBUG) + log.propagate = False + yield log + log.handlers.clear() + + +def test_records_reach_the_sink_in_one_batch(target): + sink = Collector() + handler = install_sink(sink, target=target, flush_interval=0.05) + for i in range(5): + target.info("line %d", i) + uninstall_sink(handler, target=target) + assert sink.messages == [f"line {i}" for i in range(5)] + assert len(sink.batches) == 1 + + +def test_batches_are_capped(target): + sink = Collector() + handler = install_sink(sink, target=target, batch_size=3, flush_interval=0.05) + for i in range(7): + target.info("n%d", i) + uninstall_sink(handler, target=target) + assert [len(b) for b in sink.batches] == [3, 3, 1] + + +def test_level_threshold(target): + sink = Collector() + handler = install_sink(sink, level=logging.WARNING, target=target, flush_interval=0.05) + target.info("quiet") + target.warning("loud") + uninstall_sink(handler, target=target) + assert sink.messages == ["loud"] + + +def test_own_logger_and_sqlalchemy_are_skipped(target): + sink = Collector() + handler = install_sink(sink, target=target, flush_interval=0.05) + for name in ("simple_module_hosting.log_sinks", "sqlalchemy.engine", "sqlalchemyx"): + record = logging.LogRecord(name, logging.ERROR, __file__, 1, f"x-{name}", None, None) + target.handle(record) + uninstall_sink(handler, target=target) + assert sink.messages == ["x-sqlalchemyx"] + + +def test_full_queue_drops_and_counts_without_blocking(target): + sink = Blocked() + handler = install_sink(sink, target=target, capacity=5, batch_size=1, flush_interval=0.01) + target.info("first") + assert sink.entered.wait(5) # the listener is now stuck inside the sink + started = time.monotonic() + for i in range(100): + target.info("burst %d", i) + assert time.monotonic() - started < 2 + assert handler.dropped == 95 # 5 queued, the rest counted + sink.release.set() + uninstall_sink(handler, target=target) + + +def test_failing_sink_is_counted_and_does_not_raise(target): + class Boom: + def emit_batch(self, records): + raise RuntimeError("down") + + handler = install_sink(Boom(), target=target, flush_interval=0.05) + target.info("lost") + uninstall_sink(handler, target=target) + assert handler.failed_batches == 1 + + +def test_secrets_are_masked_before_queueing_and_the_original_record_is_untouched(target): + handler = QueueSinkHandler(Collector()) + target.addHandler(handler) + record = logging.LogRecord( + target.name, + logging.INFO, + __file__, + 1, + "Authorization: Bearer abc.def password=hunter2", + None, + None, + ) + record.session_token = "xyz" + target.handle(record) + queued = handler.queue.get_nowait() + assert "abc.def" not in queued.msg and "hunter2" not in queued.msg + assert queued.session_token == "***" + assert "hunter2" in record.msg + + +@pytest.mark.parametrize( + ("raw", "secret"), + [ + ("GET / Authorization: Bearer abc123", "abc123"), + ("Cookie: sid=s3cret; other=1", "s3cret"), + ("login password=hunter2 ok", "hunter2"), + ('{"access_token": "tok-999"}', "tok-999"), + ("client_secret=shh&x=1", "shh"), + ("sent Bearer eyJhbGciOi.payload.sig", "eyJhbGciOi"), + ("headers={'Authorization': 'Basic Zm9vOmJhcg=='}", "Zm9vOmJhcg"), + ], +) +def test_redact_masks(raw, secret): + out = redact(raw) + assert secret not in out + assert "***" in out + + +def test_redact_keeps_ordinary_text(): + assert redact("user 42 logged in from 10.0.0.1") == "user 42 logged in from 10.0.0.1" diff --git a/host/migrations/versions/1f5985daa10b_background_tasks_retained_log_records.py b/host/migrations/versions/1f5985daa10b_background_tasks_retained_log_records.py new file mode 100644 index 000000000..d63765a68 --- /dev/null +++ b/host/migrations/versions/1f5985daa10b_background_tasks_retained_log_records.py @@ -0,0 +1,50 @@ +"""background_tasks: retained log records (GH #537) + +Adds ``background_tasks_log_record``, the table the opt-in log sink writes to. +Indexed for the three reads that exist: one task's lines in time order, a +time-range purge, and a minimum-level search. + +Revision ID: 1f5985daa10b +Revises: bb906e1a1089 +Create Date: 2026-10-21 18:23:28.568913 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "1f5985daa10b" +down_revision: str | None = "bb906e1a1089" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_TABLE = "background_tasks_log_record" + + +def upgrade() -> None: + op.create_table( + _TABLE, + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("ts", sa.DateTime(timezone=True), nullable=False), + sa.Column("level", sa.Integer(), nullable=False), + sa.Column("logger", sa.String(length=255), nullable=False), + sa.Column("message", sa.String(length=4096), nullable=False), + sa.Column("task_id", sa.String(length=64), nullable=True), + sa.Column("job_id", sa.String(length=64), nullable=True), + sa.Column("tenant_id", sa.String(length=50), nullable=True), + sa.Column("correlation_id", sa.String(length=64), nullable=True), + sa.Column("extra", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_background_tasks_log_record")), + ) + op.create_index("ix_background_tasks_log_record_task_ts", _TABLE, ["task_id", "ts"]) + op.create_index("ix_background_tasks_log_record_ts", _TABLE, ["ts"]) + op.create_index("ix_background_tasks_log_record_level_ts", _TABLE, ["level", "ts"]) + + +def downgrade() -> None: + op.drop_index("ix_background_tasks_log_record_level_ts", table_name=_TABLE) + op.drop_index("ix_background_tasks_log_record_ts", table_name=_TABLE) + op.drop_index("ix_background_tasks_log_record_task_ts", table_name=_TABLE) + op.drop_table(_TABLE) diff --git a/host/tests/test_migration_graph.py b/host/tests/test_migration_graph.py index 2deb94476..8a096e798 100644 --- a/host/tests/test_migration_graph.py +++ b/host/tests/test_migration_graph.py @@ -38,6 +38,7 @@ "b5d3f08a6e17": "background_tasks", "e878882924e7": "background_tasks", "bb906e1a1089": "background_tasks", # task progress + owner (#536) + "1f5985daa10b": "background_tasks", # retained log records (#537) "51b2058d8afd": "api_tokens", "168a2882f443": "keycloak", "f6a3b9d2e8c4": "keycloak", @@ -52,7 +53,7 @@ # head revision -> module owning the tables it creates or alters. _HEADS = { "a7c2e91d4b58": "audit_log", - "bb906e1a1089": "background_tasks", # also carries the inherited ``api_tokens`` + "1f5985daa10b": "background_tasks", # also carries the inherited ``api_tokens`` "f1a9c3d7b2e4": "tenants", "f6a3b9d2e8c4": "keycloak", } @@ -85,6 +86,7 @@ "c8e4f1a2b6d9": "api_tokens", "d7b3a9e5c1f2": "api_tokens", "bb906e1a1089": "api_tokens", # also its own ``background_tasks`` + "1f5985daa10b": "api_tokens", # also inherits ``background_tasks`` "51b2058d8afd": "api_tokens", # ancestors of a labelled revision inherit its label too } diff --git a/modules/background_tasks/background_tasks/constants.py b/modules/background_tasks/background_tasks/constants.py index f116bd01c..aece9b8e7 100644 --- a/modules/background_tasks/background_tasks/constants.py +++ b/modules/background_tasks/background_tasks/constants.py @@ -47,6 +47,8 @@ MENU_LABEL = "Background tasks" MENU_ICON = "activity" MENU_ORDER = 120 +LOGS_MENU_LABEL = "Logs" +LOGS_MENU_ICON = "scroll-text" # ── Page identifiers ──────────────────────────────────────────── # Kept as literals at the call site (see endpoints/views.py) so @@ -178,3 +180,16 @@ def max_outbox_handler_timeout(lease_seconds: float) -> float: OWNER_ID_MAX_LENGTH = 64 PROGRESS_MESSAGE_MAX_LENGTH = 255 PROGRESS_PHASE_MAX_LENGTH = 64 + +# ── Retained logs (GH #537) ───────────────────────────────────── +TABLE_LOG_RECORD = f"{TABLE_PREFIX}log_record" +LOG_MESSAGE_MAX_LENGTH = 4096 +LOG_NAME_MAX_LENGTH = 255 +LOG_ID_MAX_LENGTH = 64 +LOG_EXTRA_MAX_KEYS = 20 +LOG_EXTRA_VALUE_MAX_LENGTH = 1000 +DEFAULT_LOG_SINK_LEVEL = "INFO" +DEFAULT_LOG_RETENTION_DAYS = 7 +LOG_PER_PAGE = 50 +# Level names the admin filter offers, lowest first. +LOG_LEVEL_NAMES = ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") diff --git a/modules/background_tasks/background_tasks/contracts/logs.py b/modules/background_tasks/background_tasks/contracts/logs.py new file mode 100644 index 000000000..f63a3c464 --- /dev/null +++ b/modules/background_tasks/background_tasks/contracts/logs.py @@ -0,0 +1,31 @@ +"""DTOs for the retained-log screens (GH #537).""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from simple_module_db.types import AwareDateTime +from sqlmodel import Field, SQLModel + + +class LogItem(SQLModel): + id: uuid.UUID + ts: datetime = Field(sa_type=AwareDateTime()) + level: int + level_name: str + logger: str + message: str + task_id: str | None = None + job_id: str | None = None + tenant_id: str | None = None + correlation_id: str | None = None + extra: dict[str, Any] = Field(default_factory=dict) + + +class LogListResponse(SQLModel): + items: list[LogItem] + total: int + page: int + per_page: int diff --git a/modules/background_tasks/background_tasks/deps.py b/modules/background_tasks/background_tasks/deps.py index 9a9c68e85..d1abcacdb 100644 --- a/modules/background_tasks/background_tasks/deps.py +++ b/modules/background_tasks/background_tasks/deps.py @@ -8,6 +8,7 @@ from simple_module_db import RequestSession, current_tenant_id from simple_module_db.deps import get_db +from background_tasks.log_service import LogScope, LogService from background_tasks.outbox_service import OutboxScope, OutboxService from background_tasks.outbox_transport import OutboxTransport from background_tasks.service import BackgroundTaskService @@ -49,3 +50,11 @@ def outbox_scope(request: Request) -> OutboxScope: return OutboxScope(tenant_id=tenant_id) multi_tenant = bool(getattr(request.app.state.sm.settings, "multi_tenant", False)) return OutboxScope(everything=not multi_tenant) + + +async def get_log_service( + request: Request, + db: RequestSession = Depends(get_db), +) -> LogService: + scope = outbox_scope(request) + return LogService(db=db, scope=LogScope(everything=scope.everything, tenant_id=scope.tenant_id)) diff --git a/modules/background_tasks/background_tasks/endpoints/logs_api.py b/modules/background_tasks/background_tasks/endpoints/logs_api.py new file mode 100644 index 000000000..993fe4153 --- /dev/null +++ b/modules/background_tasks/background_tasks/endpoints/logs_api.py @@ -0,0 +1,62 @@ +"""Admin REST endpoints for retained logs (GH #537).""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query +from simple_module_db.pagination import PageParams +from simple_module_hosting.pagination import page_params +from simple_module_hosting.permissions import RequiresPermission + +from background_tasks.constants import ADMIN_ROUTER_PREFIX, LOG_PER_PAGE, MODULE_NAME, PERM_VIEW +from background_tasks.contracts.logs import LogListResponse +from background_tasks.deps import get_background_task_service, get_log_service +from background_tasks.log_service import LogService +from background_tasks.service import BackgroundTaskService + +router = APIRouter( + prefix=ADMIN_ROUTER_PREFIX, + dependencies=[Depends(RequiresPermission(PERM_VIEW))], + tags=[f"{MODULE_NAME}-admin"], +) + + +@router.get("/logs", response_model=LogListResponse) +async def search_logs( + q: str | None = Query(default=None), + level: str | None = Query(default=None), + task_id: str | None = Query(default=None), + logger: str | None = Query(default=None), + params: PageParams = Depends(page_params(LOG_PER_PAGE)), + service: LogService = Depends(get_log_service), +) -> LogListResponse: + """Retained log lines, newest first; ``level`` is a minimum.""" + return await service.search( + page=params.page, + per_page=params.per_page, + task_id=task_id, + min_level=level, + q=q, + logger_name=logger, + ) + + +@router.get("/executions/{execution_id}/logs", response_model=LogListResponse) +async def execution_logs( + execution_id: uuid.UUID, + level: str | None = Query(default=None), + params: PageParams = Depends(page_params(LOG_PER_PAGE)), + tasks: BackgroundTaskService = Depends(get_background_task_service), + service: LogService = Depends(get_log_service), +) -> LogListResponse: + """The log lines one execution wrote, found by its Celery task id.""" + execution = await tasks.get(execution_id) + if execution is None: + raise HTTPException(status_code=404, detail="Task execution not found") + celery_id = execution.celery_task_id + if not celery_id: + return LogListResponse(items=[], total=0, page=1, per_page=params.per_page) + return await service.search( + page=params.page, per_page=params.per_page, task_id=celery_id, min_level=level + ) diff --git a/modules/background_tasks/background_tasks/endpoints/logs_views.py b/modules/background_tasks/background_tasks/endpoints/logs_views.py new file mode 100644 index 000000000..2ce35c552 --- /dev/null +++ b/modules/background_tasks/background_tasks/endpoints/logs_views.py @@ -0,0 +1,44 @@ +"""Inertia view for the retained-log search page (GH #537).""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query +from simple_module_db.pagination import PageParams +from simple_module_hosting.inertia_deps import InertiaDep +from simple_module_hosting.pagination import lenient_page_params +from simple_module_hosting.permissions import RequiresPermission +from simple_module_inertia import InertiaResponse + +from background_tasks.constants import LOG_LEVEL_NAMES, LOG_PER_PAGE, PERM_VIEW +from background_tasks.deps import get_log_service +from background_tasks.log_service import LogService + +router = APIRouter(dependencies=[Depends(RequiresPermission(PERM_VIEW))]) + + +@router.get("/logs", response_model=None) +async def logs( + inertia: InertiaDep, + q: str = Query(default=""), + level: str = Query(default=""), + task_id: str = Query(default=""), + params: PageParams = Depends(lenient_page_params(LOG_PER_PAGE)), + service: LogService = Depends(get_log_service), +) -> InertiaResponse: + # Fixed page size: the pager carries only ``page``. + found = await service.search( + page=params.page, + per_page=LOG_PER_PAGE, + task_id=task_id or None, + min_level=level, + q=q or None, + ) + return await inertia.render( + "BackgroundTasks/Logs", + { + "logs": [i.model_dump(mode="json") for i in found.items], + "levels": list(LOG_LEVEL_NAMES), + "pagination": {"page": found.page, "per_page": found.per_page, "total": found.total}, + "filters": {"q": q, "level": level, "task_id": task_id}, + }, + ) diff --git a/modules/background_tasks/background_tasks/locales/en.json b/modules/background_tasks/background_tasks/locales/en.json index a54c57177..2abb5bb7d 100644 --- a/modules/background_tasks/background_tasks/locales/en.json +++ b/modules/background_tasks/background_tasks/locales/en.json @@ -158,7 +158,8 @@ "routes_empty": "No task routes are configured, so every task goes to the default queue. Add routes in a module's task_config.py (TASK_ROUTES)." }, "nav": { - "background_tasks": "Background tasks" + "background_tasks": "Background tasks", + "logs": "Logs" }, "audit": { "task_execution": "Task execution" @@ -192,5 +193,29 @@ "discard_failed": "Failed to discard the event", "conflict": "This event is being delivered right now, so it cannot be changed. The list has been refreshed.", "gone": "This event no longer exists. The list has been refreshed." + }, + "logs": { + "title": "Logs", + "description": "Retained application and task log lines", + "back_button": "Executions", + "tab_details": "Details", + "tab_logs": "Logs", + "tab_list_label": "Task sections", + "search_label": "Search messages", + "search_placeholder": "Search messages", + "task_label": "Task id", + "task_placeholder": "Task id", + "level_label": "Minimum level", + "level_all": "All levels", + "apply": "Search", + "col_time": "Time", + "col_level": "Level", + "col_logger": "Logger", + "col_message": "Message", + "col_task": "Task", + "empty_title": "No log lines", + "empty_description": "Nothing matches. Retained logging is off by default: turn on log_sink_enabled in the Background tasks settings, then restart the web process and workers.", + "load_failed": "Failed to load log lines", + "platform": "Platform" } } diff --git a/modules/background_tasks/background_tasks/log_purge.py b/modules/background_tasks/background_tasks/log_purge.py new file mode 100644 index 000000000..41e926ae5 --- /dev/null +++ b/modules/background_tasks/background_tasks/log_purge.py @@ -0,0 +1,32 @@ +"""Retention for retained log lines (GH #537).""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta + +from sqlalchemy import delete +from sqlalchemy.orm import Session + +from background_tasks.constants import DEFAULT_LOG_RETENTION_DAYS, MODULE_NAME +from background_tasks.models import RetainedLog +from background_tasks.settings import BackgroundTasksSettings +from background_tasks.worker_settings import settings_for + +logger = logging.getLogger(__name__) + + +def log_retention_days() -> int: + """The DB-backed ``log_retention_days``; the default when settings are unreadable.""" + try: + return settings_for(BackgroundTasksSettings, MODULE_NAME).log_retention_days + except Exception: + logger.warning("Could not read log_retention_days; using the default", exc_info=True) + return DEFAULT_LOG_RETENTION_DAYS + + +def purge_log_records(session: Session, days: int | None = None) -> int: + """Delete log lines older than ``days`` (the setting when omitted); returns the count.""" + cutoff = datetime.now(UTC) - timedelta(days=days if days is not None else log_retention_days()) + result = session.execute(delete(RetainedLog).where(RetainedLog.ts < cutoff)) # ty: ignore[invalid-argument-type] + return result.rowcount or 0 diff --git a/modules/background_tasks/background_tasks/log_service.py b/modules/background_tasks/background_tasks/log_service.py new file mode 100644 index 000000000..7ea46a9ab --- /dev/null +++ b/modules/background_tasks/background_tasks/log_service.py @@ -0,0 +1,85 @@ +"""Reading retained log lines for the admin screens (GH #537).""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +from simple_module_db import RequestSession +from simple_module_db.pagination import PageParams, paginate +from sqlalchemy import false, select + +from background_tasks.constants import LOG_LEVEL_NAMES +from background_tasks.contracts.logs import LogItem, LogListResponse +from background_tasks.models import RetainedLog + +_LEVEL_NAMES = {number: name for name, number in logging.getLevelNamesMapping().items()} + + +def level_number(name: str | None) -> int | None: + """``"WARNING"`` → 30; ``None`` for an empty or unknown name (no filter).""" + upper = (name or "").strip().upper() + return logging.getLevelNamesMapping()[upper] if upper in LOG_LEVEL_NAMES else None + + +def _like(text: str) -> str: + escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return f"%{escaped}%" + + +@dataclass(frozen=True) +class LogScope: + """Which tenants' lines the caller may read, as the outbox scope does.""" + + everything: bool = False + tenant_id: str | None = None + + def conditions(self) -> list[Any]: + if self.everything: + return [] + if self.tenant_id: + return [RetainedLog.tenant_id == self.tenant_id] + return [false()] + + +class LogService: + def __init__(self, db: RequestSession, scope: LogScope) -> None: + self.db = db + self.scope = scope + + async def search( + self, + *, + page: int, + per_page: int, + task_id: str | None = None, + min_level: str | None = None, + q: str | None = None, + logger_name: str | None = None, + ) -> LogListResponse: + where = [*self.scope.conditions()] + if task_id is not None: + where.append(RetainedLog.task_id == task_id) + if (level := level_number(min_level)) is not None: + where.append(RetainedLog.level >= level) + if q: + where.append(RetainedLog.message.ilike(_like(q), escape="\\")) + if logger_name: + where.append(RetainedLog.logger.startswith(logger_name)) + found = await paginate( + self.db, + select(RetainedLog).where(*where).order_by(RetainedLog.ts.desc(), RetainedLog.id), # ty: ignore[invalid-argument-type] + PageParams(page=page, per_page=per_page), + ) + return LogListResponse( + items=[ + LogItem.model_validate( + {**r.model_dump(), "level_name": _LEVEL_NAMES.get(r.level, str(r.level))} + ) + for r in found.items + ], + total=found.total, + page=found.page, + per_page=found.per_page, + ) diff --git a/modules/background_tasks/background_tasks/log_sink.py b/modules/background_tasks/background_tasks/log_sink.py new file mode 100644 index 000000000..b456081fc --- /dev/null +++ b/modules/background_tasks/background_tasks/log_sink.py @@ -0,0 +1,127 @@ +"""The database log sink and its installer (GH #537). + +``DbLogSink`` writes batches through the worker-style sync session, so it works +the same in the web process and in a Celery child, and never touches a request +transaction. Installation is idempotent and fork-aware: a prefork child inherits +the parent's handler but not its listener thread, so ``worker_process_init`` +replaces it. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Sequence +from datetime import UTC, datetime +from typing import Any + +from celery.signals import worker_process_init +from simple_module_db import current_tenant_id +from simple_module_hosting.log_sinks import QueueSinkHandler, install_sink, uninstall_sink + +from background_tasks.constants import ( + LOG_EXTRA_MAX_KEYS, + LOG_EXTRA_VALUE_MAX_LENGTH, + LOG_ID_MAX_LENGTH, + LOG_MESSAGE_MAX_LENGTH, + LOG_NAME_MAX_LENGTH, + TENANT_ID_MAX_LENGTH, +) +from background_tasks.log_context import LogContextFilter +from background_tasks.models import RetainedLog +from background_tasks.settings import BackgroundTasksSettings +from background_tasks.sync_db import sync_session + +_log = logging.getLogger(__name__) + +_COLUMN_ATTRS = ("task_id", "job_id", "tenant_id", "correlation_id") +_STANDARD_ATTRS = frozenset(logging.makeLogRecord({}).__dict__) | {"message", "asctime"} +_handler: QueueSinkHandler | None = None + + +def _text(value: object, limit: int) -> str | None: + return None if value in (None, "") else str(value)[:limit] + + +def _extra(record: logging.LogRecord) -> dict[str, Any]: + extra: dict[str, Any] = {} + for key, value in record.__dict__.items(): + if key in _STANDARD_ATTRS or key in _COLUMN_ATTRS: + continue + if len(extra) >= LOG_EXTRA_MAX_KEYS: + break + extra[key[:LOG_NAME_MAX_LENGTH]] = ( + value + if isinstance(value, bool | int | float) + else str(value)[:LOG_EXTRA_VALUE_MAX_LENGTH] + ) + return extra + + +def to_row(record: logging.LogRecord) -> RetainedLog: + """Map an already-redacted ``LogRecord`` onto its table row.""" + return RetainedLog( + ts=datetime.fromtimestamp(record.created, UTC), + level=record.levelno, + logger=record.name[:LOG_NAME_MAX_LENGTH], + message=str(record.msg)[:LOG_MESSAGE_MAX_LENGTH], + task_id=_text(getattr(record, "task_id", None), LOG_ID_MAX_LENGTH), + job_id=_text(getattr(record, "job_id", None), LOG_ID_MAX_LENGTH), + tenant_id=_text(getattr(record, "tenant_id", None), TENANT_ID_MAX_LENGTH), + correlation_id=_text(getattr(record, "correlation_id", None), LOG_ID_MAX_LENGTH), + extra=json.loads(json.dumps(_extra(record), default=str)), + ) + + +class DbLogSink: + """``LogSink`` that inserts one ``RetainedLog`` row per record.""" + + def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: + with sync_session() as session: + session.add_all([to_row(r) for r in records]) + + +class TenantStampFilter(logging.Filter): + """Stamp the emitting context's tenant onto the record, before it is queued.""" + + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "tenant_id", None): + record.tenant_id = current_tenant_id.get() + return True + + +def install_log_sink(settings: BackgroundTasksSettings) -> QueueSinkHandler | None: + """Install the DB sink on the root logger when enabled; replace any earlier one.""" + global _handler + uninstall_log_sink() + if not settings.log_sink_enabled: + return None + level = logging.getLevelNamesMapping()[settings.log_sink_level.upper()] + handler = install_sink(DbLogSink(), level=level) + # Handler filters run on the emitting thread, where the task's log context + # and tenant are bound; the listener thread sees neither. + handler.addFilter(LogContextFilter()) + handler.addFilter(TenantStampFilter()) + _handler = handler + _log.info("Retained log sink installed (level=%s)", settings.log_sink_level) + return handler + + +def uninstall_log_sink() -> None: + global _handler + handler, _handler = _handler, None + if handler is not None: + uninstall_sink(handler) + + +@worker_process_init.connect +def _reinstall_in_child(**_: object) -> None: + """A forked child has the parent's handler but not its thread: build its own.""" + global _handler + handler, _handler = _handler, None + if handler is None: + return + logging.getLogger().removeHandler(handler) + from background_tasks.worker_settings import load_runner_settings + + install_log_sink(load_runner_settings()) diff --git a/modules/background_tasks/background_tasks/models.py b/modules/background_tasks/background_tasks/models.py index 0fd0c8a75..ee24c21cc 100644 --- a/modules/background_tasks/background_tasks/models.py +++ b/modules/background_tasks/background_tasks/models.py @@ -14,11 +14,15 @@ from background_tasks.constants import ( DEFAULT_QUEUE, + LOG_ID_MAX_LENGTH, + LOG_MESSAGE_MAX_LENGTH, + LOG_NAME_MAX_LENGTH, MODULE_NAME, OUTBOX_KEY_MAX_LENGTH, OWNER_ID_MAX_LENGTH, PROGRESS_MESSAGE_MAX_LENGTH, PROGRESS_PHASE_MAX_LENGTH, + TABLE_LOG_RECORD, TABLE_OUTBOX, TABLE_TASK_EXECUTION, TENANT_ID_MAX_LENGTH, @@ -182,3 +186,36 @@ class OutboxMessage(Base, table=True): # ty: ignore[unsupported-base] Index("ix_background_tasks_outbox_status_locked", "status", "locked_until"), Index("ix_background_tasks_outbox_status_delivered", "status", "delivered_at"), ) + + +class RetainedLog(Base, table=True): # ty: ignore[unsupported-base] + """One retained log line from the web process or a worker (GH #537). + + Written in batches by the log sink, never inside a request transaction, and + only while ``log_sink_enabled`` is on. ``tenant_id`` is a plain column, not + ``MultiTenantMixin``, for the same reason as ``TaskExecution``: worker and + platform lines carry no tenant and strict mode would raise on their insert; + the admin views filter on the column instead. ``task_id`` is the Celery task + id, which is what ``TaskExecution.celery_task_id`` holds. + """ + + __tablename__ = TABLE_LOG_RECORD + # Telemetry, written many times a second: auditing it would log the log. + __audit_exclude__ = True + + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + ts: datetime = Field(sa_type=AwareDateTime(), nullable=False) + level: int = Field(nullable=False) + logger: str = Field(max_length=LOG_NAME_MAX_LENGTH) + message: str = Field(max_length=LOG_MESSAGE_MAX_LENGTH) + task_id: str | None = Field(default=None, max_length=LOG_ID_MAX_LENGTH) + job_id: str | None = Field(default=None, max_length=LOG_ID_MAX_LENGTH) + tenant_id: str | None = Field(default=None, max_length=TENANT_ID_MAX_LENGTH) + correlation_id: str | None = Field(default=None, max_length=LOG_ID_MAX_LENGTH) + extra: dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON, nullable=False)) + + __table_args__ = ( + Index("ix_background_tasks_log_record_task_ts", "task_id", "ts"), + Index("ix_background_tasks_log_record_ts", "ts"), + Index("ix_background_tasks_log_record_level_ts", "level", "ts"), + ) diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 195acd828..fe9117234 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -18,6 +18,8 @@ from background_tasks.constants import ( _MODULE_SETTINGS, API_PREFIX, + LOGS_MENU_ICON, + LOGS_MENU_LABEL, MENU_ICON, MENU_LABEL, MENU_ORDER, @@ -120,10 +122,25 @@ def register_menu_items(self, registry: MenuRegistry) -> None: group_key="ui.nav_groups.system", ) ) + registry.add( + MenuItem( + label=LOGS_MENU_LABEL, + label_key="background_tasks.nav.logs", + url=f"{VIEW_PREFIX}/logs", + icon=LOGS_MENU_ICON, + order=MENU_ORDER + 1, + section=MenuSection.ADMIN_SIDEBAR, + roles=["admin"], + group="System", + group_key="ui.nav_groups.system", + ) + ) def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None: from background_tasks.endpoints.api_admin import router as api_admin from background_tasks.endpoints.api_status import router as api_status + from background_tasks.endpoints.logs_api import router as logs_api + from background_tasks.endpoints.logs_views import router as logs_views from background_tasks.endpoints.outbox_api import router as outbox_api from background_tasks.endpoints.outbox_views import router as outbox_views from background_tasks.endpoints.views import router as views @@ -131,8 +148,10 @@ def register_routes(self, api_router: APIRouter, view_router: APIRouter) -> None api_router.include_router(api_admin) api_router.include_router(api_status) api_router.include_router(outbox_api) + api_router.include_router(logs_api) # Before ``views``: its ``/{execution_id}`` would otherwise swallow ``/outbox``. view_router.include_router(outbox_views) + view_router.include_router(logs_views) view_router.include_router(views) def locale_dirs(self) -> dict[str, Path]: @@ -144,6 +163,7 @@ async def on_startup(self, app: FastAPI) -> None: import asyncio from background_tasks.celery_app import build_celery + from background_tasks.log_sink import install_log_sink from background_tasks.signals import bind_event_bus from background_tasks.sync_db import set_database_url from background_tasks.tenant_context import set_default_tenant @@ -166,6 +186,7 @@ async def on_startup(self, app: FastAPI) -> None: # stay a silent no-op — that's the documented cross-process limit. bind_event_bus(app.state.sm.event_bus, asyncio.get_running_loop()) self._install_durable_transport(app) + install_log_sink(services.settings) logger.info( "BackgroundTasks: Celery app ready (broker=%s, queue=%s)", services.settings.broker_url, @@ -253,6 +274,7 @@ async def _start_invalidation_transport(app: FastAPI) -> None: ) async def on_shutdown(self, app: FastAPI) -> None: + from background_tasks.log_sink import uninstall_log_sink from background_tasks.outbox_worker import reset_worker_state from background_tasks.signals import unbind_event_bus from background_tasks.sync_db import dispose_sync_engine @@ -273,4 +295,6 @@ async def on_shutdown(self, app: FastAPI) -> None: if services.celery is not None: services.celery.close() unbind_event_bus() + # Before the engine goes: the final flush still needs it. + uninstall_log_sink() dispose_sync_engine() diff --git a/modules/background_tasks/background_tasks/pages/Detail.tsx b/modules/background_tasks/background_tasks/pages/Detail.tsx index b40f60ec0..e9a2bf479 100644 --- a/modules/background_tasks/background_tasks/pages/Detail.tsx +++ b/modules/background_tasks/background_tasks/pages/Detail.tsx @@ -2,11 +2,13 @@ import { Head, Link, router, usePage } from '@inertiajs/react'; import { keys, useT } from '@simple-module-py/i18n'; import { PageShell } from '@simple-module-py/ui/components/PageShell'; import { Button } from '@simple-module-py/ui/components/ui/button'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@simple-module-py/ui/components/ui/tabs'; import { usePermissions } from '@simple-module-py/ui/hooks/use-permissions'; import { AdminLayout } from '@simple-module-py/ui/layouts/AdminLayout'; import { ArrowLeft, RefreshCcw } from 'lucide-react'; import { useState } from 'react'; import { DetailFacts } from './components/DetailFacts'; +import { LogsPanel } from './components/LogsPanel'; import { PayloadCards } from './components/PayloadCards'; import { ProgressCard } from './components/ProgressCard'; import { RetryConfirmDialog } from './components/RetryConfirmDialog'; @@ -99,13 +101,27 @@ function Detail() { anyone opened this page. On desktop the deck's order returns. */} -
- - - {/* `h-full` so a short payload card does not leave the traceback - floating with 300px of empty card under it. */} - -
+ + + {t(keys.background_tasks.logs.tab_details)} + {t(keys.background_tasks.logs.tab_logs)} + + +
+ + + {/* `h-full` so a short payload card does not leave the traceback + floating with 300px of empty card under it. */} + +
+ + + + + {retryable && ( + } + > +
+ setQ(e.target.value)} + /> + setTaskId(e.target.value)} + /> +
+ +
+ +
+ visit(page, filters)} + /> + + + ); +} + +Logs.layout = [AdminLayout]; +export default Logs; diff --git a/modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx b/modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx new file mode 100644 index 000000000..4b4d79de2 --- /dev/null +++ b/modules/background_tasks/background_tasks/pages/components/LogLevelSelect.tsx @@ -0,0 +1,29 @@ +import { keys, useT } from '@simple-module-py/i18n'; +import { NativeSelect, NativeSelectOption } from '@simple-module-py/ui/components/ui/native-select'; +import { LOG_LEVEL_ALL, LOG_LEVELS } from '../logs'; + +interface Props { + value: string; + onChange: (level: string) => void; +} + +/** Minimum-level picker shared by the task tab and the search page. */ +export function LogLevelSelect({ value, onChange }: Props) { + const { t } = useT(); + return ( + onChange(e.target.value)} + > + + {t(keys.background_tasks.logs.level_all)} + + {LOG_LEVELS.map((level) => ( + + {level} + + ))} + + ); +} diff --git a/modules/background_tasks/background_tasks/pages/components/LogTable.tsx b/modules/background_tasks/background_tasks/pages/components/LogTable.tsx new file mode 100644 index 000000000..f4321bfe5 --- /dev/null +++ b/modules/background_tasks/background_tasks/pages/components/LogTable.tsx @@ -0,0 +1,76 @@ +import { keys, useT } from '@simple-module-py/i18n'; +import { PaginationBar } from '@simple-module-py/ui/components/PaginationBar'; +import { Card } from '@simple-module-py/ui/components/ui/card'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@simple-module-py/ui/components/ui/table'; +import { cn } from '@simple-module-py/ui/lib/utils'; +import { EM_DASH, formatTs } from '../constants'; +import { type LogItem, levelClass } from '../logs'; + +interface Props { + logs: LogItem[]; + page: number; + perPage: number; + total: number; + onPageChange: (page: number) => void; + /** Show the task-id column (the search page); a task's own tab already knows it. */ + showTask?: boolean; +} + +/** Log lines as a table with the shared pager underneath. */ +export function LogTable({ logs, page, perPage, total, onPageChange, showTask = false }: Props) { + const { t } = useT(); + return ( + + + + + {t(keys.background_tasks.logs.col_time)} + {t(keys.background_tasks.logs.col_level)} + {t(keys.background_tasks.logs.col_logger)} + {t(keys.background_tasks.logs.col_message)} + {showTask && {t(keys.background_tasks.logs.col_task)}} + + + + {logs.length === 0 && ( + + +

{t(keys.background_tasks.logs.empty_title)}

+

+ {t(keys.background_tasks.logs.empty_description)} +

+ + + )} + {logs.map((row) => ( + + + {formatTs(row.ts)} + + + {row.level_name} + + + {row.logger} + + + {row.message} + + {showTask && ( + {row.task_id ?? EM_DASH} + )} + + ))} + +
+ + + ); +} diff --git a/modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx b/modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx new file mode 100644 index 000000000..586bc457a --- /dev/null +++ b/modules/background_tasks/background_tasks/pages/components/LogsPanel.tsx @@ -0,0 +1,46 @@ +import { keys, t } from '@simple-module-py/i18n'; +import { useEffect, useState } from 'react'; +import { toast } from 'sonner'; +import { fetchExecutionLogs, LOG_LEVEL_ALL, type LogPage } from '../logs'; +import { LogLevelSelect } from './LogLevelSelect'; +import { LogTable } from './LogTable'; + +const EMPTY: LogPage = { items: [], total: 0, page: 1, per_page: 50 }; + +/** The Logs tab of a task: its retained lines, filtered by minimum level. */ +export function LogsPanel({ executionId }: { executionId: string }) { + const [level, setLevel] = useState(LOG_LEVEL_ALL); + const [page, setPage] = useState(1); + const [data, setData] = useState(EMPTY); + + useEffect(() => { + let live = true; + fetchExecutionLogs(executionId, level, page) + .then((next) => live && setData(next)) + .catch(() => live && toast.error(t(keys.background_tasks.logs.load_failed))); + return () => { + live = false; + }; + }, [executionId, level, page]); + + return ( +
+
+ { + setLevel(next); + setPage(1); + }} + /> +
+ +
+ ); +} diff --git a/modules/background_tasks/background_tasks/pages/logs.ts b/modules/background_tasks/background_tasks/pages/logs.ts new file mode 100644 index 000000000..2e5f7ed4e --- /dev/null +++ b/modules/background_tasks/background_tasks/pages/logs.ts @@ -0,0 +1,50 @@ +// Frontend mirror of the retained-log DTOs (background_tasks/contracts/logs.py). + +import { API_BASE } from './constants'; + +export interface LogItem { + id: string; + ts: string; + level: number; + level_name: string; + logger: string; + message: string; + task_id: string | null; + job_id: string | null; + tenant_id: string | null; + correlation_id: string | null; +} + +export interface LogPage { + items: LogItem[]; + total: number; + page: number; + per_page: number; +} + +/** Minimum-level choices, lowest first; the empty string means no filter. */ +export const LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] as const; + +export const LOG_LEVEL_ALL = ''; + +export const LEVEL_ERROR = 40; +export const LEVEL_WARNING = 30; + +export function levelClass(level: number): string { + if (level >= LEVEL_ERROR) return 'text-destructive'; + if (level >= LEVEL_WARNING) return 'text-amber-600 dark:text-amber-400'; + return 'text-muted-foreground'; +} + +/** One task's log lines, newest first. Throws on a non-2xx so the caller can toast. */ +export async function fetchExecutionLogs( + executionId: string, + level: string, + page: number, +): Promise { + const query = new URLSearchParams({ page: String(page) }); + if (level) query.set('level', level); + const res = await fetch(`${API_BASE}/executions/${executionId}/logs?${query}`); + if (!res.ok) throw new Error(String(res.status)); + return (await res.json()) as LogPage; +} diff --git a/modules/background_tasks/background_tasks/runner.py b/modules/background_tasks/background_tasks/runner.py index 41c145418..0f2d49b44 100644 --- a/modules/background_tasks/background_tasks/runner.py +++ b/modules/background_tasks/background_tasks/runner.py @@ -53,6 +53,7 @@ def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: from simple_module_hosting import merge_host_settings from background_tasks.celery_app import build_celery + from background_tasks.log_sink import install_log_sink from background_tasks.sync_db import set_database_url from background_tasks.tenant_context import set_default_tenant from background_tasks.worker_settings import load_runner_settings @@ -68,4 +69,9 @@ def build_runner_celery(fallback_env_file: Path | None = None) -> Celery: set_default_tenant(default_tenant) # Hydrated from the DB like the web process, so an admin-edited setting # reaches the worker at boot, not only the web app. - return build_celery(load_runner_settings()) + settings = load_runner_settings() + celery = build_celery(settings) + # Task logs: the worker never runs the web lifespan that installs the sink. + # Prefork children rebuild it on ``worker_process_init``. + install_log_sink(settings) + return celery diff --git a/modules/background_tasks/background_tasks/settings.py b/modules/background_tasks/background_tasks/settings.py index 94cba1027..e3bf24316 100644 --- a/modules/background_tasks/background_tasks/settings.py +++ b/modules/background_tasks/background_tasks/settings.py @@ -67,6 +67,7 @@ forbid_localhost_broker_in_production, warn_on_legacy_redis_vars, ) +from background_tasks.settings_logs import LogSinkFields _CELERY_RESTART = {"requires_restart": True, "group": "Celery"} @@ -84,13 +85,12 @@ def _redis_from_env(default: str) -> str: return os.environ.get(ENV_REDIS_URL) or default -class BackgroundTasksSettings(BaseSettings): +class BackgroundTasksSettings(LogSinkFields, BaseSettings): """Configuration for the Celery + Redis task runner.""" - # Without ``env_prefix`` pydantic-settings resolves each field from its - # bare, case-insensitive name — so a container setting ``broker_url`` or - # ``retention_days`` for any other purpose silently reconfigured Celery, - # and the documented ``SM_BG_TASKS_*`` names did nothing (GH #283). + # Without ``env_prefix`` each field resolved from its bare name, so a + # container setting ``broker_url`` or ``retention_days`` for another purpose + # reconfigured Celery and ``SM_BG_TASKS_*`` did nothing (GH #283). model_config = SettingsConfigDict(extra="ignore", env_prefix=ENV_PREFIX) @classmethod diff --git a/modules/background_tasks/background_tasks/settings_logs.py b/modules/background_tasks/background_tasks/settings_logs.py new file mode 100644 index 000000000..a3dccbc0f --- /dev/null +++ b/modules/background_tasks/background_tasks/settings_logs.py @@ -0,0 +1,45 @@ +"""Retained-log settings (GH #537), split out of ``settings.py`` for size.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field, field_validator + +from background_tasks.constants import ( + DEFAULT_LOG_RETENTION_DAYS, + DEFAULT_LOG_SINK_LEVEL, + LOG_LEVEL_NAMES, +) + + +class LogSinkFields(BaseModel): + """Fields mixed into ``BackgroundTasksSettings``.""" + + # ── Retained logs (GH #537) ───────────────────────────── + # Off by default: it writes a row per log line, which a deployment that + # already ships logs elsewhere does not want. Handlers are installed at + # process start, so changing these needs a restart of web and workers. + log_sink_enabled: bool = Field( + default=False, + description="Keep application and task log lines in the database.", + json_schema_extra={"requires_restart": True, "group": "Logs"}, + ) + log_sink_level: str = Field( + default=DEFAULT_LOG_SINK_LEVEL, + description="Lowest level kept (DEBUG, INFO, WARNING, ERROR, CRITICAL).", + json_schema_extra={"requires_restart": True, "group": "Logs"}, + ) + log_retention_days: int = Field( + default=DEFAULT_LOG_RETENTION_DAYS, + ge=1, + le=3650, + description="Days to keep retained log lines (1-3650).", + json_schema_extra={"group": "Logs"}, + ) + + @field_validator("log_sink_level") + @classmethod + def _known_log_level(cls, value: str) -> str: + level = value.strip().upper() + if level not in LOG_LEVEL_NAMES: + raise ValueError(f"log_sink_level must be one of {', '.join(LOG_LEVEL_NAMES)}") + return level diff --git a/modules/background_tasks/background_tasks/tasks.py b/modules/background_tasks/background_tasks/tasks.py index 4a3c7268a..9de1ac7f7 100644 --- a/modules/background_tasks/background_tasks/tasks.py +++ b/modules/background_tasks/background_tasks/tasks.py @@ -36,6 +36,7 @@ OutboxStatus, TaskStatus, ) +from background_tasks.log_purge import purge_log_records from background_tasks.models import OutboxMessage, TaskExecution from background_tasks.settings import BackgroundTasksSettings from background_tasks.sync_db import sync_session @@ -138,7 +139,7 @@ def sweep_stuck_tasks(self: Task) -> int: @shared_task(name=INTERNAL_TASK_PURGE_OLD) def purge_old_executions() -> int: - """Delete terminal rows older than the retention window.""" + """Delete terminal rows, delivered outbox rows and log lines past their retention.""" retention_days = int(os.environ.get("SM_BG_TASKS_RETENTION_DAYS", DEFAULT_RETENTION_DAYS)) cutoff = datetime.now(UTC) - timedelta(days=retention_days) @@ -155,6 +156,7 @@ def purge_old_executions() -> int: .where(OutboxMessage.delivered_at < cutoff) ) count += delivered.rowcount or 0 + count += purge_log_records(session) if count: logger.info( "purge_old_executions removed %d row(s) older than %d days", diff --git a/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx b/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx new file mode 100644 index 000000000..0aad38f3d --- /dev/null +++ b/modules/background_tasks/tests-js/background_tasks_LogsTab.test.tsx @@ -0,0 +1,56 @@ +import '@testing-library/jest-dom/vitest'; +import { configureI18n, keys } from '@simple-module-py/i18n'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +configureI18n({ locale: 'en', messages: {} }); + +vi.mock('sonner', () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +import { LogsPanel } from '../background_tasks/pages/components/LogsPanel'; +import { stubFetch } from './stub-fetch'; + +const line = (message: string, level_name = 'INFO', level = 20) => ({ + id: message, + ts: '2026-01-01T00:00:00Z', + level, + level_name, + logger: 'demo', + message, + task_id: 'c1', + job_id: null, + tenant_id: null, + correlation_id: null, +}); + +describe('LogsPanel', () => { + afterEach(() => vi.unstubAllGlobals()); + + it("loads a task's lines and refetches with the chosen minimum level", async () => { + const fetchMock = stubFetch(200, { + items: [line('copied 3 rows'), line('disk full', 'ERROR', 40)], + total: 2, + page: 1, + per_page: 50, + }); + render(); + const urls = () => (fetchMock.mock.calls as unknown as string[][]).map((c) => c[0]); + + expect(await screen.findByText('copied 3 rows')).toBeInTheDocument(); + expect(screen.getByText('disk full')).toBeInTheDocument(); + expect(urls()[0]).toBe('/api/background_tasks/admin/executions/e1/logs?page=1'); + + fireEvent.change( + screen.getByRole('combobox', { name: keys.background_tasks.logs.level_label }), + { target: { value: 'ERROR' } }, + ); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + expect(urls()[1]).toContain('level=ERROR'); + }); + + it('says why it is empty when nothing was retained', async () => { + stubFetch(200, { items: [], total: 0, page: 1, per_page: 50 }); + render(); + expect(await screen.findByText(keys.background_tasks.logs.empty_title)).toBeInTheDocument(); + }); +}); diff --git a/modules/background_tasks/tests/test_bg_log_api.py b/modules/background_tasks/tests/test_bg_log_api.py new file mode 100644 index 000000000..059031789 --- /dev/null +++ b/modules/background_tasks/tests/test_bg_log_api.py @@ -0,0 +1,118 @@ +"""Retained-log endpoints and pages (GH #537).""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime, timedelta + +import httpx +import pytest +from background_tasks.models import RetainedLog + +BASE = "/api/background_tasks/admin" + +pytestmark = pytest.mark.usefixtures("_stub_celery") + + +@pytest.fixture +def seed_logs(app): + async def _seed(*rows: tuple[str, int, str | None, str | None]) -> None: + """Each row is (message, level, task_id, tenant_id); newest last.""" + base = datetime.now(UTC) - timedelta(minutes=10) + async with app.state.sm.db.session_factory() as session: + for i, (message, level, task_id, tenant_id) in enumerate(rows): + session.add( + RetainedLog( + ts=base + timedelta(seconds=i), + level=level, + logger="demo.worker", + message=message, + task_id=task_id, + tenant_id=tenant_id, + ) + ) + await session.commit() + + return _seed + + +async def test_task_lines_appear_on_the_execution_logs_endpoint( + authenticated_client: httpx.AsyncClient, seed_execution, seed_logs +): + execution = await seed_execution(celery_task_id="task-a") + await seed_logs( + ("step one", logging.INFO, "task-a", None), + ("other task", logging.INFO, "task-b", None), + ("step two", logging.ERROR, "task-a", None), + ) + resp = await authenticated_client.get(f"{BASE}/executions/{execution.id}/logs") + assert resp.status_code == 200 + body = resp.json() + assert [i["message"] for i in body["items"]] == ["step two", "step one"] + assert body["items"][0]["level_name"] == "ERROR" + assert body["total"] == 2 + + +async def test_level_filter_is_a_minimum( + authenticated_client: httpx.AsyncClient, seed_execution, seed_logs +): + execution = await seed_execution(celery_task_id="task-a") + await seed_logs( + ("chatty", logging.DEBUG, "task-a", None), + ("fine", logging.INFO, "task-a", None), + ("careful", logging.WARNING, "task-a", None), + ("bad", logging.ERROR, "task-a", None), + ) + resp = await authenticated_client.get( + f"{BASE}/executions/{execution.id}/logs", params={"level": "WARNING"} + ) + assert [i["message"] for i in resp.json()["items"]] == ["bad", "careful"] + + +async def test_logs_are_paginated( + authenticated_client: httpx.AsyncClient, seed_execution, seed_logs +): + execution = await seed_execution(celery_task_id="task-a") + await seed_logs(*[(f"line {i}", logging.INFO, "task-a", None) for i in range(5)]) + resp = await authenticated_client.get( + f"{BASE}/executions/{execution.id}/logs", params={"per_page": 2, "page": 3} + ) + body = resp.json() + assert (body["total"], body["page"], len(body["items"])) == (5, 3, 1) + + +async def test_unknown_execution_is_404(authenticated_client: httpx.AsyncClient): + resp = await authenticated_client.get( + f"{BASE}/executions/00000000-0000-0000-0000-000000000000/logs" + ) + assert resp.status_code == 404 + + +async def test_search_filters_by_text_and_task(authenticated_client: httpx.AsyncClient, seed_logs): + await seed_logs( + ("disk 100% full", logging.ERROR, "task-a", None), + ("disk fine", logging.INFO, "task-b", None), + ("network up", logging.INFO, None, None), + ) + by_text = await authenticated_client.get(f"{BASE}/logs", params={"q": "disk"}) + assert by_text.json()["total"] == 2 + percent = await authenticated_client.get(f"{BASE}/logs", params={"q": "100%"}) + assert [i["message"] for i in percent.json()["items"]] == ["disk 100% full"] + by_task = await authenticated_client.get(f"{BASE}/logs", params={"task_id": "task-b"}) + assert [i["message"] for i in by_task.json()["items"]] == ["disk fine"] + + +async def test_logs_require_authentication(client: httpx.AsyncClient): + resp = await client.get(f"{BASE}/logs", follow_redirects=False) + assert resp.status_code in {302, 401, 403} + + +async def test_logs_page_renders_for_an_admin(authenticated_client: httpx.AsyncClient, seed_logs): + await seed_logs(("hello", logging.INFO, None, None)) + resp = await authenticated_client.get( + "/admin/background-tasks/logs", headers={"X-Inertia": "true"} + ) + assert resp.status_code == 200 + page = resp.json() + assert page["component"] == "BackgroundTasks/Logs" + assert page["props"]["pagination"]["total"] == 1 diff --git a/modules/background_tasks/tests/test_bg_log_purge.py b/modules/background_tasks/tests/test_bg_log_purge.py new file mode 100644 index 000000000..af4a26d2a --- /dev/null +++ b/modules/background_tasks/tests/test_bg_log_purge.py @@ -0,0 +1,50 @@ +"""Retention for retained log lines (GH #537).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest +from background_tasks import log_purge, sync_db, tasks +from background_tasks.models import RetainedLog +from sqlalchemy import select + +pytestmark = pytest.mark.usefixtures("sync_sqlite") + + +def _seed(*ages_days: int) -> None: + now = datetime.now(UTC) + with sync_db.sync_session() as session: + for i, age in enumerate(ages_days): + session.add( + RetainedLog( + ts=now - timedelta(days=age), level=20, logger="t", message=f"{age}d #{i}" + ) + ) + + +def _messages() -> set[str]: + with sync_db.sync_session() as session: + return set(session.execute(select(RetainedLog.message)).scalars()) + + +def test_purge_removes_rows_older_than_retention(): + _seed(1, 6, 8, 30) + with sync_db.sync_session() as session: + assert log_purge.purge_log_records(session, 7) == 2 + assert _messages() == {"1d #0", "6d #1"} + + +def test_purge_old_executions_applies_the_setting(monkeypatch): + monkeypatch.setattr(log_purge, "log_retention_days", lambda: 3) + _seed(1, 5) + assert tasks.purge_old_executions() == 1 + assert _messages() == {"1d #0"} + + +def test_unreadable_settings_fall_back_to_the_default(monkeypatch): + def boom(*_a, **_k): + raise RuntimeError("no settings table") + + monkeypatch.setattr(log_purge, "settings_for", boom) + assert log_purge.log_retention_days() == 7 diff --git a/modules/background_tasks/tests/test_bg_log_sink.py b/modules/background_tasks/tests/test_bg_log_sink.py new file mode 100644 index 000000000..3ec17bf52 --- /dev/null +++ b/modules/background_tasks/tests/test_bg_log_sink.py @@ -0,0 +1,102 @@ +"""The DB log sink: opt-in, task-correlated, redacted (GH #537).""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator + +import pytest +from background_tasks import log_sink, sync_db +from background_tasks.log_context import bind_task_context +from background_tasks.models import RetainedLog +from background_tasks.settings import BackgroundTasksSettings +from simple_module_db import tenant_context +from sqlalchemy import select + +pytestmark = pytest.mark.usefixtures("sync_sqlite") + +APP_LOGGER = logging.getLogger("bg_log_sink_test") + + +@pytest.fixture(autouse=True) +def _root_at_debug() -> Iterator[None]: + root = logging.getLogger() + before = root.level + root.setLevel(logging.DEBUG) + yield + log_sink.uninstall_log_sink() + root.setLevel(before) + + +def _rows() -> list[RetainedLog]: + with sync_db.get_sync_session_factory()() as session: + return list(session.execute(select(RetainedLog).order_by(RetainedLog.ts)).scalars()) + + +def _enabled(**over: object) -> BackgroundTasksSettings: + return BackgroundTasksSettings(log_sink_enabled=True, **over) + + +def test_disabled_by_default_writes_nothing(): + assert BackgroundTasksSettings().log_sink_enabled is False + assert log_sink.install_log_sink(BackgroundTasksSettings()) is None + APP_LOGGER.error("nobody keeps this") + assert _rows() == [] + + +def test_task_lines_carry_the_task_id(): + handler = log_sink.install_log_sink(_enabled()) + assert handler is not None + with bind_task_context(task_id="celery-1", task_name="demo.task"): + APP_LOGGER.info("inside the task") + APP_LOGGER.info("outside") + log_sink.uninstall_log_sink() + by_message = {r.message: r for r in _rows() if r.logger == APP_LOGGER.name} + assert by_message["inside the task"].task_id == "celery-1" + assert by_message["inside the task"].extra["task_name"] == "demo.task" + assert by_message["outside"].task_id is None + + +def test_level_threshold_and_level_number(): + log_sink.install_log_sink(_enabled(log_sink_level="WARNING")) + APP_LOGGER.info("dropped") + APP_LOGGER.error("kept") + log_sink.uninstall_log_sink() + rows = [r for r in _rows() if r.logger == APP_LOGGER.name] + assert [(r.message, r.level) for r in rows] == [("kept", logging.ERROR)] + + +def test_secrets_never_reach_the_table(): + log_sink.install_log_sink(_enabled()) + APP_LOGGER.warning("calling with Authorization: Bearer abc.def.ghi and password=hunter2") + log_sink.uninstall_log_sink() + (row,) = [r for r in _rows() if r.logger == APP_LOGGER.name] + assert "abc.def.ghi" not in row.message + assert "hunter2" not in row.message + + +def test_tenant_is_stamped_and_long_messages_are_truncated(): + log_sink.install_log_sink(_enabled()) + with tenant_context("acme"): + APP_LOGGER.info("x" * 10_000) + log_sink.uninstall_log_sink() + (row,) = [r for r in _rows() if r.logger == APP_LOGGER.name] + assert row.tenant_id == "acme" + assert len(row.message) == 4096 + + +def test_exception_text_is_kept_and_redacted(): + log_sink.install_log_sink(_enabled()) + try: + raise ValueError("password=hunter2") + except ValueError: + APP_LOGGER.exception("boom") + log_sink.uninstall_log_sink() + (row,) = [r for r in _rows() if r.logger == APP_LOGGER.name] + assert "ValueError" in row.message + assert "hunter2" not in row.message + + +def test_invalid_level_is_rejected(): + with pytest.raises(ValueError, match="log_sink_level"): + BackgroundTasksSettings(log_sink_level="LOUD") diff --git a/packages/i18n/src/generated-resources.ts b/packages/i18n/src/generated-resources.ts index aa35513dc..ff5621f29 100644 --- a/packages/i18n/src/generated-resources.ts +++ b/packages/i18n/src/generated-resources.ts @@ -172,7 +172,30 @@ export default { 'background_tasks.index.search_placeholder': '', 'background_tasks.index.title': '', 'background_tasks.index.workers_button': '', + 'background_tasks.logs.apply': '', + 'background_tasks.logs.back_button': '', + 'background_tasks.logs.col_level': '', + 'background_tasks.logs.col_logger': '', + 'background_tasks.logs.col_message': '', + 'background_tasks.logs.col_task': '', + 'background_tasks.logs.col_time': '', + 'background_tasks.logs.description': '', + 'background_tasks.logs.empty_description': '', + 'background_tasks.logs.empty_title': '', + 'background_tasks.logs.level_all': '', + 'background_tasks.logs.level_label': '', + 'background_tasks.logs.load_failed': '', + 'background_tasks.logs.platform': '', + 'background_tasks.logs.search_label': '', + 'background_tasks.logs.search_placeholder': '', + 'background_tasks.logs.tab_details': '', + 'background_tasks.logs.tab_list_label': '', + 'background_tasks.logs.tab_logs': '', + 'background_tasks.logs.task_label': '', + 'background_tasks.logs.task_placeholder': '', + 'background_tasks.logs.title': '', 'background_tasks.nav.background_tasks': '', + 'background_tasks.nav.logs': '', 'background_tasks.outbox.back_button': '', 'background_tasks.outbox.col_actions': '', 'background_tasks.outbox.col_attempts': '', diff --git a/packages/i18n/src/keys.generated.ts b/packages/i18n/src/keys.generated.ts index 7c3269b58..7756de02f 100644 --- a/packages/i18n/src/keys.generated.ts +++ b/packages/i18n/src/keys.generated.ts @@ -233,8 +233,33 @@ export const keys = { title: 'background_tasks.index.title', workers_button: 'background_tasks.index.workers_button', }, + logs: { + apply: 'background_tasks.logs.apply', + back_button: 'background_tasks.logs.back_button', + col_level: 'background_tasks.logs.col_level', + col_logger: 'background_tasks.logs.col_logger', + col_message: 'background_tasks.logs.col_message', + col_task: 'background_tasks.logs.col_task', + col_time: 'background_tasks.logs.col_time', + description: 'background_tasks.logs.description', + empty_description: 'background_tasks.logs.empty_description', + empty_title: 'background_tasks.logs.empty_title', + level_all: 'background_tasks.logs.level_all', + level_label: 'background_tasks.logs.level_label', + load_failed: 'background_tasks.logs.load_failed', + platform: 'background_tasks.logs.platform', + search_label: 'background_tasks.logs.search_label', + search_placeholder: 'background_tasks.logs.search_placeholder', + tab_details: 'background_tasks.logs.tab_details', + tab_list_label: 'background_tasks.logs.tab_list_label', + tab_logs: 'background_tasks.logs.tab_logs', + task_label: 'background_tasks.logs.task_label', + task_placeholder: 'background_tasks.logs.task_placeholder', + title: 'background_tasks.logs.title', + }, nav: { background_tasks: 'background_tasks.nav.background_tasks', + logs: 'background_tasks.nav.logs', }, outbox: { back_button: 'background_tasks.outbox.back_button', From 884bc51abc4daaf4fd9e50a45e02c4c7d2d2a8ad Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:37:14 +0200 Subject: [PATCH 10/22] refactor(background_tasks): split lifecycle wiring out of module.py; deflake eager outbox test Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../background_tasks/_lifecycle.py | 170 ++++++++++++++++++ .../background_tasks/module.py | 149 +-------------- .../tests/test_bg_outbox_eager_platform.py | 2 +- 3 files changed, 175 insertions(+), 146 deletions(-) create mode 100644 modules/background_tasks/background_tasks/_lifecycle.py diff --git a/modules/background_tasks/background_tasks/_lifecycle.py b/modules/background_tasks/background_tasks/_lifecycle.py new file mode 100644 index 000000000..5a6065a55 --- /dev/null +++ b/modules/background_tasks/background_tasks/_lifecycle.py @@ -0,0 +1,170 @@ +"""Startup/shutdown wiring for the BackgroundTasks module. + +``BackgroundTasksModule`` keeps the ``on_startup`` / ``on_shutdown`` hooks and +delegates here, so the module file stays a declaration of what the module +contributes and this file owns the order things are installed and torn down. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastapi import FastAPI + +logger = logging.getLogger(__name__) + + +async def startup(app: FastAPI, module_name: str) -> None: + """Build the Celery app, install signal handlers, hand-off to worker.""" + import asyncio + + from background_tasks.celery_app import build_celery + from background_tasks.log_sink import install_log_sink + from background_tasks.signals import bind_event_bus + from background_tasks.sync_db import set_database_url + from background_tasks.tenant_context import set_default_tenant + + services = app.state.background_tasks + # Pin the sync engine to the same URL the host's async settings + # resolved — pydantic-settings reads ``.env`` but never propagates + # to ``os.environ``, so signals would otherwise fall back to the + # SQLite default and silently drop ``TaskExecution`` rows. + host = app.state.sm.settings + multi = bool(getattr(host, "multi_tenant", False)) + default_tenant = None if multi else getattr(host, "default_tenant", "") or None + set_database_url(host.database_url, tenant_strict=multi, default_tenant=default_tenant) + set_default_tenant(default_tenant) + # build_celery imports `signals` for side effects and runs + # `autodiscover_tasks` across every installed module. + services.celery = build_celery(services.settings) + # Let signal handlers hop onto the API loop to publish events. + # In a standalone worker process this bind never runs, so signals + # stay a silent no-op — that's the documented cross-process limit. + bind_event_bus(app.state.sm.event_bus, asyncio.get_running_loop()) + install_durable_transport(app) + install_log_sink(services.settings) + logger.info( + "BackgroundTasks: Celery app ready (broker=%s, queue=%s)", + services.settings.broker_url, + services.settings.task_default_queue, + ) + + await start_invalidation_transport(app) + + # Registered here, not in register_health_checks: the Celery app does + # not exist until this hook builds it, and the check must follow later + # settings changes rather than pinning the boot-time instance. + from simple_module_core.health import HealthCheck + + from background_tasks.health import CHECK_REDIS, build_redis_check + + app.state.sm.health_registry.add( + HealthCheck( + name=CHECK_REDIS, + check=build_redis_check(app), + module=module_name, + # On demand only, like the mailer and storage checks. An + # earlier version had this on the readiness probe, reasoning + # that Redis is infrastructure this app owns rather than a + # third party. That conflates two different questions: whether + # this process can serve requests, and whether the system is + # fully functional. Failing readiness pulls the web tier out of + # the load balancer, turning "background jobs are backed up" + # into "the site is down" — and it made /health/ready report + # unhealthy on every deployment without Redis, which CI caught. + # The wizard's connection step and the settings screen's "Test + # connection" both run it explicitly, which is what it is for. + probe=False, + ) + ) + + +def install_durable_transport(app: FastAPI) -> None: + """Make ``bus.publish_durable`` write to this module's outbox (GH #459).""" + from background_tasks.constants import OutboxMode + from background_tasks.outbox_sweep import start_sweep + from background_tasks.outbox_transport import OutboxTransport + from background_tasks.outbox_worker import use_bus + + bus = app.state.sm.event_bus + services = app.state.background_tasks + mode = ( + OutboxMode.INPROCESS + if services.celery.conf.task_always_eager + else OutboxMode(services.settings.outbox_mode) + ) + transport = OutboxTransport(services=services, bus=bus, db_state=app.state.sm.db, mode=mode) + bus.durable_transport = transport + if mode == OutboxMode.INPROCESS: + start_sweep(services, app.state.sm.db.session_factory, bus, transport.policy) + # An eager in-process dispatch delivers with the app's bus, which already + # carries every module's durable subscriptions. + use_bus(bus) + + +async def start_invalidation_transport(app: FastAPI) -> None: + """Hand the framework bus the Redis connection this module configures. + + The transport is installed here rather than in a registration hook + because it needs a running event loop for its listener task, and it is + installed by *this* module because it owns the Redis URL — a cache-owning + module opening its own connection is the duplication GH #318 exists to + avoid. + + A worker process that runs Celery without the FastAPI app never reaches + this, so it publishes nothing and hears nothing. That is the same + documented cross-process limit as ``bind_event_bus`` above: a task that + invalidates a cache should do it through an API call or accept the TTL. + """ + from background_tasks.invalidation import RedisInvalidationTransport + + services = app.state.background_tasks + s = services.settings + if not s.broadcast_invalidations: + logger.info( + "BackgroundTasks: cross-worker cache invalidation disabled " + "(broadcast_invalidations=false) — per-process caches expire on their own TTL" + ) + return + + bus = app.state.sm.invalidation + transport = RedisInvalidationTransport(bus, s.broker_url, s.invalidation_channel) + await transport.start() + bus.set_transport(transport) + services.invalidation_transport = transport + logger.info( + "BackgroundTasks: invalidation transport on %r for channel(s) %s", + s.invalidation_channel, + ", ".join(bus.channels) or "(none subscribed)", + ) + + +async def shutdown(app: FastAPI) -> None: + from background_tasks.log_sink import uninstall_log_sink + from background_tasks.outbox_sweep import stop_sweep + from background_tasks.outbox_worker import reset_worker_state + from background_tasks.signals import unbind_event_bus + from background_tasks.sync_db import dispose_sync_engine + + await stop_sweep(app.state.background_tasks) + bus = getattr(app.state.sm, "event_bus", None) + if bus is not None: + drain = getattr(bus.durable_transport, "drain", None) + if drain is not None: + # Eager deliveries still running on this loop. + await drain() + bus.durable_transport = None + reset_worker_state() + services = app.state.background_tasks + if services.invalidation_transport is not None: + app.state.sm.invalidation.clear_transport() + await services.invalidation_transport.stop() + services.invalidation_transport = None + if services.celery is not None: + services.celery.close() + unbind_event_bus() + # Before the engine goes: the final flush still needs it. + uninstall_log_sink() + dispose_sync_engine() diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 229b66c71..09ceae096 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -160,152 +160,11 @@ def locale_dirs(self) -> dict[str, Path]: async def on_startup(self, app: FastAPI) -> None: """Build the Celery app, install signal handlers, hand-off to worker.""" - import asyncio + from background_tasks._lifecycle import startup - from background_tasks.celery_app import build_celery - from background_tasks.log_sink import install_log_sink - from background_tasks.signals import bind_event_bus - from background_tasks.sync_db import set_database_url - from background_tasks.tenant_context import set_default_tenant - - services = app.state.background_tasks - # Pin the sync engine to the same URL the host's async settings - # resolved — pydantic-settings reads ``.env`` but never propagates - # to ``os.environ``, so signals would otherwise fall back to the - # SQLite default and silently drop ``TaskExecution`` rows. - host = app.state.sm.settings - multi = bool(getattr(host, "multi_tenant", False)) - default_tenant = None if multi else getattr(host, "default_tenant", "") or None - set_database_url(host.database_url, tenant_strict=multi, default_tenant=default_tenant) - set_default_tenant(default_tenant) - # build_celery imports `signals` for side effects and runs - # `autodiscover_tasks` across every installed module. - services.celery = build_celery(services.settings) - # Let signal handlers hop onto the API loop to publish events. - # In a standalone worker process this bind never runs, so signals - # stay a silent no-op — that's the documented cross-process limit. - bind_event_bus(app.state.sm.event_bus, asyncio.get_running_loop()) - self._install_durable_transport(app) - install_log_sink(services.settings) - logger.info( - "BackgroundTasks: Celery app ready (broker=%s, queue=%s)", - services.settings.broker_url, - services.settings.task_default_queue, - ) - - await self._start_invalidation_transport(app) - - # Registered here, not in register_health_checks: the Celery app does - # not exist until this hook builds it, and the check must follow later - # settings changes rather than pinning the boot-time instance. - from simple_module_core.health import HealthCheck - - from background_tasks.health import CHECK_REDIS, build_redis_check - - app.state.sm.health_registry.add( - HealthCheck( - name=CHECK_REDIS, - check=build_redis_check(app), - module=self.meta.name, - # On demand only, like the mailer and storage checks. An - # earlier version had this on the readiness probe, reasoning - # that Redis is infrastructure this app owns rather than a - # third party. That conflates two different questions: whether - # this process can serve requests, and whether the system is - # fully functional. Failing readiness pulls the web tier out of - # the load balancer, turning "background jobs are backed up" - # into "the site is down" — and it made /health/ready report - # unhealthy on every deployment without Redis, which CI caught. - # The wizard's connection step and the settings screen's "Test - # connection" both run it explicitly, which is what it is for. - probe=False, - ) - ) - - @staticmethod - def _install_durable_transport(app: FastAPI) -> None: - """Make ``bus.publish_durable`` write to this module's outbox (GH #459).""" - from background_tasks.constants import OutboxMode - from background_tasks.outbox_sweep import start_sweep - from background_tasks.outbox_transport import OutboxTransport - from background_tasks.outbox_worker import use_bus - - bus = app.state.sm.event_bus - services = app.state.background_tasks - mode = ( - OutboxMode.INPROCESS - if services.celery.conf.task_always_eager - else OutboxMode(services.settings.outbox_mode) - ) - transport = OutboxTransport(services=services, bus=bus, db_state=app.state.sm.db, mode=mode) - bus.durable_transport = transport - if mode == OutboxMode.INPROCESS: - start_sweep(services, app.state.sm.db.session_factory, bus, transport.policy) - # An eager in-process dispatch delivers with the app's bus, which already - # carries every module's durable subscriptions. - use_bus(bus) - - @staticmethod - async def _start_invalidation_transport(app: FastAPI) -> None: - """Hand the framework bus the Redis connection this module configures. - - The transport is installed here rather than in a registration hook - because it needs a running event loop for its listener task, and it is - installed by *this* module because it owns the Redis URL — a cache-owning - module opening its own connection is the duplication GH #318 exists to - avoid. - - A worker process that runs Celery without the FastAPI app never reaches - this, so it publishes nothing and hears nothing. That is the same - documented cross-process limit as ``bind_event_bus`` above: a task that - invalidates a cache should do it through an API call or accept the TTL. - """ - from background_tasks.invalidation import RedisInvalidationTransport - - services = app.state.background_tasks - s = services.settings - if not s.broadcast_invalidations: - logger.info( - "BackgroundTasks: cross-worker cache invalidation disabled " - "(broadcast_invalidations=false) — per-process caches expire on their own TTL" - ) - return - - bus = app.state.sm.invalidation - transport = RedisInvalidationTransport(bus, s.broker_url, s.invalidation_channel) - await transport.start() - bus.set_transport(transport) - services.invalidation_transport = transport - logger.info( - "BackgroundTasks: invalidation transport on %r for channel(s) %s", - s.invalidation_channel, - ", ".join(bus.channels) or "(none subscribed)", - ) + await startup(app, self.meta.name) async def on_shutdown(self, app: FastAPI) -> None: - from background_tasks.log_sink import uninstall_log_sink - from background_tasks.outbox_sweep import stop_sweep - from background_tasks.outbox_worker import reset_worker_state - from background_tasks.signals import unbind_event_bus - from background_tasks.sync_db import dispose_sync_engine + from background_tasks._lifecycle import shutdown - await stop_sweep(app.state.background_tasks) - bus = getattr(app.state.sm, "event_bus", None) - if bus is not None: - drain = getattr(bus.durable_transport, "drain", None) - if drain is not None: - # Eager deliveries still running on this loop. - await drain() - bus.durable_transport = None - reset_worker_state() - services = app.state.background_tasks - if services.invalidation_transport is not None: - app.state.sm.invalidation.clear_transport() - await services.invalidation_transport.stop() - services.invalidation_transport = None - if services.celery is not None: - services.celery.close() - unbind_event_bus() - # Before the engine goes: the final flush still needs it. - uninstall_log_sink() - dispose_sync_engine() + await shutdown(app) diff --git a/modules/background_tasks/tests/test_bg_outbox_eager_platform.py b/modules/background_tasks/tests/test_bg_outbox_eager_platform.py index 87343a4dd..aa6525884 100644 --- a/modules/background_tasks/tests/test_bg_outbox_eager_platform.py +++ b/modules/background_tasks/tests/test_bg_outbox_eager_platform.py @@ -49,7 +49,7 @@ async def delivered() -> bool: rows = await outbox_rows() return bool(rows) and all(r.status == OutboxStatus.DELIVERED for r in rows) - await wait_for(delivered, timeout=3.0) + await wait_for(delivered, timeout=15.0) assert seen == [7] app.state.background_tasks.celery.send_task.assert_not_called() From 2c25d334ee88405cd396507ceb97984f2658cf74 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:38:24 +0200 Subject: [PATCH 11/22] fix(sentry): close remaining scrubber gaps; fail closed on scrub errors - Every event field now goes through the recursive pass (tags, server_name, top-level stacktrace, threads, unknown future fields), not a field list. - Key matching: pwd/passphrase/jwt/bearer/card words, whole-word auth/pass/otp/pin/sid/sig/ssn/cvv/mfa, camelCase-aware (basicAuth). - [key, value] pairs in header/query lists, bytes values, quoted values with spaces, escaped JSON, whole Cookie:/Authorization: lines, userinfo with '@' or '/' in the password, JWTs and bare 32+ char tokens are redacted; db.params and frame vars dropped anywhere; user reduced to id. - The old key=value regex was quadratic (20 KB string: 72 s in before_send); all patterns are now boundary-anchored and bounded, strings capped at 64 KiB. - before_send drops the event if tagging/scrubbing raises, logging only the exception type. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 1 + docs/framework/sentry.md | 40 ++-- .../hosting/simple_module_hosting/sentry.py | 30 ++- .../simple_module_hosting/sentry_scrub.py | 189 ++++++++-------- .../sentry_scrub_text.py | 110 +++++++++ framework/hosting/tests/_sentry_transport.py | 34 +++ .../tests/test_hosting_sentry_scrub.py | 28 +-- .../tests/test_hosting_sentry_scrub_gaps.py | 210 ++++++++++++++++++ 8 files changed, 493 insertions(+), 149 deletions(-) create mode 100644 framework/hosting/simple_module_hosting/sentry_scrub_text.py create mode 100644 framework/hosting/tests/_sentry_transport.py create mode 100644 framework/hosting/tests/test_hosting_sentry_scrub_gaps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4564209bc..2c57a7ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to this project are documented in this file. The format is b ### Added - Optional Sentry error reporting: `pip install simple_module_hosting[sentry]` and set `SM_SENTRY_DSN` (plus `SM_SENTRY_ENVIRONMENT`, `SM_SENTRY_TRACES_SAMPLE_RATE`). Web and Celery worker processes tag events with request id, tenant, user id, module and task, and scrub auth headers, cookies, `token`/`code` query values and request bodies. Without a DSN the SDK is never imported (#538). - Sentry events no longer leak secrets the SDK's exact-name scrubber misses: auth-, session- and token-like headers (`X-Auth-Token`, `Proxy-Authorization`, `X-Forwarded-For`), nested secrets in `extra`/`contexts`, tokens in log and HTTP breadcrumbs, credentials in exception messages, and Celery task args/kwargs are scrubbed for errors and transactions alike (#538). +- Sentry scrubbing now covers every event field (tags, `server_name`, top-level stacktraces), camelCase and short key names (`basicAuth`, `pwd`, `otp`), header/query lists of pairs, bytes, quoted and escaped-JSON values, whole `Cookie:` lines, passwords containing `@`, JWTs and bare long tokens; SQL `db.params` are dropped. Behaviour change: a scrubber failure now drops the event instead of letting the SDK decide, and strings over 64 KiB are truncated (the old patterns were quadratic: a 20 KB string stalled `before_send` for over a minute) (#538). - `make ci-doc-paths` (`scripts/check_doc_paths.py`, part of `make lint` and the `python-lint` job) fails when a backticked repo-rooted path in `docs/`, `CLAUDE.md`, `README.md` or `skills/**/SKILL.md` does not exist; tutorial diff --git a/docs/framework/sentry.md b/docs/framework/sentry.md index 9588b7699..9034daec9 100644 --- a/docs/framework/sentry.md +++ b/docs/framework/sentry.md @@ -42,19 +42,33 @@ is open. The DSN is never logged. only, so `simple_module_hosting.sentry_scrub` runs a stricter pass from both `before_send` and `before_send_transaction`: -- Any key, at any depth of `request`, `extra`, `contexts`, breadcrumbs and - span data, whose name contains `password`, `secret`, `token`, `auth` - (`Authorization`, `X-Auth-*`), `cookie`, `session`, `api_key`, `csrf`, - `credential`, `signature`, `dsn` or a client-address word (`forwarded`, - `real_ip`, `remote_addr`, …) is replaced with `[Filtered]`; such request - headers are removed outright. +- After targeted passes, *every* event field (tags, `server_name`, threads, + a top-level `stacktrace`, fields a future SDK adds) goes through the same + recursive pass: deny by default rather than a list of known fields. +- Any key, at any depth, whose name contains `password`/`passwd`/`pwd`, + `passphrase`, `secret`, `token`, `authorization`, `cookie`, `session`, + `api_key`, `csrf`, `credential`, `private_key`, `signature`, `dsn`, `jwt`, + `bearer`, a card-number word or a client-address word (`forwarded`, + `real_ip`, `remote_addr`, …), or has a whole-word `auth`, `pass`, `otp`, + `pin`, `sid`, `sig`, `ssn`, `cvv`, `mfa` (snake, kebab and camelCase: + `basicAuth`, `DB_PASS`, but not `author`), is replaced with `[Filtered]`. + `["Authorization", "…"]` pairs in header/query lists count as keys. Such + request headers are removed outright. - Request cookies, bodies and `env` (`REMOTE_ADDR`) are dropped; the event user is reduced to `{"id": …}`. -- Free text (exception messages, log messages and params, breadcrumb messages - and URLs, span descriptions, query strings, source context) loses - `key=value` / `key: value` pairs with a sensitive key, the OAuth `code` - query parameter, `Bearer`/`Basic` credentials and URL userinfo. -- Celery task `args`, `kwargs` and return values are dropped (the Celery - integration and Celery's own trace log records both attach them). -- URL-sourced transaction names and request URL paths have id- and +- Every string (bytes too) loses `key=value` / `key: value` / + `"key": "quoted value"` pairs with a sensitive key (escaped JSON included), + whole `Cookie:`/`Authorization:` header lines, the OAuth `code` query + parameter, `Bearer`/`Basic`/`Digest` credentials, URL userinfo, JWTs and + bare 32+ character tokens mixing letters and digits (UUIDs and + snake_case identifiers excepted; Sentry's own event/trace/span ids are + kept). Strings over 64 KiB are truncated before scanning. +- Celery task `args`, `kwargs` and return values, SQL `db.params` and frame + `vars` are dropped wherever they appear. +- URL-sourced transaction names and URL paths have id- and token-shaped segments replaced with `*`. +- If scrubbing raises, the event is dropped (only the exception type is + logged); a half-scrubbed event is never sent. +- Not covered: secrets passed as positional log arguments + (`log.info("%s", token)`) with nothing marking them as secret, and + base64 tokens with `/` shorter than 32 characters between slashes. diff --git a/framework/hosting/simple_module_hosting/sentry.py b/framework/hosting/simple_module_hosting/sentry.py index c8b452358..a8796d299 100644 --- a/framework/hosting/simple_module_hosting/sentry.py +++ b/framework/hosting/simple_module_hosting/sentry.py @@ -102,21 +102,33 @@ def _tags(event: Event, process: Process) -> dict[str, str]: return {k: str(v) for k, v in info.items() if v} -def build_before_send(process: Process) -> Callable[[Event, Mapping[str, Any]], Event]: +def _tag_and_scrub(event: Event, process: Process) -> Event: + tags = _tags(event, process) + event.setdefault("tags", {}).update(tags) + if user_id := tags.get(_TAG_USER_ID): + event["user"] = {"id": user_id} # id only: never email/username/ip + else: + event.pop("user", None) + return scrub_event(event) + + +def build_before_send(process: Process) -> Callable[[Event, Mapping[str, Any]], Event | None]: """Tag and scrub an event; used for errors and transactions alike. Scrubbing is ours, not the SDK's: its ``EventScrubber`` matches exact key names only and skips ``contexts`` and free text (``sentry_scrub``). + Fails closed: if tagging or scrubbing raises, the event is dropped rather + than sent half-scrubbed. Only the exception *type* is logged (its message + may quote the event), at WARNING so the logging integration does not turn + it into another event. """ - def before_send(event: Event, hint: Mapping[str, Any]) -> Event: - tags = _tags(event, process) - event.setdefault("tags", {}).update(tags) - if user_id := tags.get(_TAG_USER_ID): - event["user"] = {"id": user_id} # id only: never email/username/ip - else: - event.pop("user", None) - return scrub_event(event) + def before_send(event: Event, hint: Mapping[str, Any]) -> Event | None: + try: + return _tag_and_scrub(event, process) + except Exception as exc: # any failure must drop the event + logger.warning("Sentry event dropped: scrubbing failed (%s)", type(exc).__name__) + return None return before_send diff --git a/framework/hosting/simple_module_hosting/sentry_scrub.py b/framework/hosting/simple_module_hosting/sentry_scrub.py index bc77d3e9e..5ba1aa52b 100644 --- a/framework/hosting/simple_module_hosting/sentry_scrub.py +++ b/framework/hosting/simple_module_hosting/sentry_scrub.py @@ -4,18 +4,21 @@ never looks at ``contexts`` or free text, so this runs a second, stricter pass from ``before_send``/``before_send_transaction``: -* every mapping key matching :data:`SENSITIVE_KEY` (``password``, ``token``, - ``authorization``, ``cookie``, ``session``, ``api_key``, ``csrf``, …, plus - client-IP headers) has its value replaced, at any depth; -* free text (exception messages, log messages, breadcrumb data, span - descriptions, URLs, query strings) has ``key=value`` pairs with a sensitive - key, ``Bearer``/``Basic`` credentials and URL userinfo redacted; -* request cookies, bodies and the WSGI/ASGI ``env`` (``REMOTE_ADDR``) are - dropped, sensitive headers removed outright; -* Celery task ``args``/``kwargs``/return values are dropped wherever they - appear (the Celery integration's ``extra["celery-job"]`` and the - ``extra["data"]`` the ``celery.app.trace`` log records carry); -* frame locals are dropped even though ``include_local_variables=False``. +* targeted passes first: request cookies, bodies and the WSGI/ASGI ``env`` + (``REMOTE_ADDR``) are dropped, sensitive headers removed outright, URL and + transaction paths lose id-shaped segments, ``user`` keeps only ``id``; +* then *every* top-level field goes through :func:`scrub` (deny by default: + a field the SDK adds later is covered without a code change): mapping keys + matching :data:`SENSITIVE_KEY` and ``[key, value]`` pairs whose key does + have their value replaced at any depth; every string, bytes included, goes + through :func:`scrub_text` (``key=value`` pairs, header lines, + ``Bearer``/``Basic`` credentials, URL userinfo, JWTs, bare long tokens); +* Celery task ``args``/``kwargs``/return values, SQL parameters and frame + locals are dropped wherever they appear. + +Anything beyond :data:`_MAX_DEPTH` is replaced, so a cyclic or absurdly deep +structure ends in ``REDACTED`` instead of a ``RecursionError``. A scrubber +failure must never send the raw event: ``sentry.build_before_send`` drops it. """ from __future__ import annotations @@ -25,32 +28,29 @@ from typing import Any from urllib.parse import urlsplit, urlunsplit -REDACTED = "[Filtered]" +from simple_module_hosting.sentry_scrub_text import ( + REDACTED, + SENSITIVE_KEY, + is_sensitive_key, + scrub_text, +) + +__all__ = ["REDACTED", "SENSITIVE_KEY", "is_sensitive_key", "scrub", "scrub_event", "scrub_text"] + _MAX_DEPTH = 12 _PATH_SEP = "/" _PATH_PLACEHOLDER = "*" _SOURCE_URL = "url" - -_SENSITIVE_WORDS = ( - r"passw(?:or)?d|secret|token|authori[sz]ation|(?["']?[\w.\-]*(?:{_SENSITIVE_WORDS})[\w.\-]*["']?\s*[=:]\s*)""" - r"""(?P["']?)(?P(?:Bearer\s+|Basic\s+)?[^\s&,;"'}\]]+)""", - re.IGNORECASE, -) -_SCHEME_CREDENTIAL = re.compile(r"\b(Bearer|Basic|Token)\s+[A-Za-z0-9._~+/=\-]+", re.IGNORECASE) -_URL_USERINFO = re.compile(r"(?P[a-z][a-z0-9+.\-]*://)[^/@\s]+@", re.IGNORECASE) +_URL_KEY = "url" +_USER_ID = "id" +_PAIR_LEN = 2 +# Celery's integration and its trace log records attach the task payload; SQL +# integrations attach bound parameters; ``vars`` are frame locals. +_DROP_KEYS = frozenset({"args", "kwargs", "return_value", "db.params", "db.sql.params", "vars"}) +_REQUEST_DROP = ("cookies", "data", "env") +# Sentry's own correlation ids look like long hex tokens; keep them intact. +_ID_KEYS = frozenset({"event_id", "trace_id", "span_id", "parent_span_id", "request_id", "release"}) +_ID_VALUE = re.compile(r"[\w.\-@+]{1,128}") _ID_SEGMENT = re.compile( r"^(?:\d+|[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}" r"|(?=[\w\-]*\d)(?=[\w\-]*[A-Za-z])[\w\-]{20,})$", @@ -60,33 +60,35 @@ Event = dict[str, Any] -def is_sensitive_key(key: object) -> bool: - return isinstance(key, str) and (key.lower() in _EXACT_KEYS or bool(SENSITIVE_KEY.search(key))) - - -def scrub_text(text: str) -> str: - """Redact credentials embedded in free text.""" - text = _URL_USERINFO.sub(rf"\g{REDACTED}@", text) - text = _QUERY_CODE.sub(rf"\g<1>{REDACTED}", text) - text = _KEY_VALUE.sub(rf"\g\g{REDACTED}", text) - return _SCHEME_CREDENTIAL.sub(rf"\1 {REDACTED}", text) +def _scrub_entry(key: Any, value: Any, depth: int) -> Any: + if is_sensitive_key(key): + return REDACTED + if key in _ID_KEYS and isinstance(value, str) and _ID_VALUE.fullmatch(value): + return value + if key == _URL_KEY and isinstance(value, str): + return _scrub_url(value) + return scrub(value, depth + 1) def scrub(value: Any, depth: int = 0) -> Any: - """Recursively redact sensitive keys and scrub every string.""" + """Recursively redact sensitive keys and pairs and scrub every string.""" if depth > _MAX_DEPTH: return REDACTED if isinstance(value, Mapping): - return { - k: REDACTED if is_sensitive_key(k) else scrub(v, depth + 1) - for k, v in value.items() - if k not in _TASK_PAYLOAD_KEYS - } - if isinstance(value, list | tuple): - return [scrub(v, depth + 1) for v in value] + return {k: _scrub_entry(k, v, depth) for k, v in value.items() if k not in _DROP_KEYS} + if isinstance(value, list | tuple | set | frozenset): + items = list(value) + # Header and query lists: [["Authorization", "..."], ("Set-Cookie", "...")] + if len(items) == _PAIR_LEN and is_sensitive_key(items[0]): + return [scrub(items[0], depth + 1), REDACTED] + return [scrub(v, depth + 1) for v in items] + if isinstance(value, bytes | bytearray): + return scrub_text(bytes(value).decode("utf-8", "replace")) if isinstance(value, str): return scrub_text(value) - return value + if value is None or isinstance(value, bool | int | float): + return value + return scrub_text(repr(value)) def scrub_path(path: str) -> str: @@ -97,72 +99,55 @@ def scrub_path(path: str) -> str: def _scrub_url(url: str) -> str: - parts = urlsplit(url) + try: + parts = urlsplit(url) + except ValueError: # e.g. an unbalanced IPv6 bracket: still scrub it as text + return scrub_text(url) return scrub_text(urlunsplit(parts._replace(path=scrub_path(parts.path)))) +def _is_sensitive_header(item: Any) -> bool: + return isinstance(item, list | tuple) and len(item) == _PAIR_LEN and is_sensitive_key(item[0]) + + def _scrub_request(request: dict[str, Any]) -> None: - for key in ("cookies", "data", "env"): + for key in _REQUEST_DROP: request.pop(key, None) headers = request.get("headers") if isinstance(headers, Mapping): - request["headers"] = { - k: scrub(v) for k, v in headers.items() if not is_sensitive_key(str(k)) - } - query = request.get("query_string") - if isinstance(query, str): - request["query_string"] = scrub_text(query) - elif query is not None: - request["query_string"] = scrub(query) - if isinstance(request.get("url"), str): - request["url"] = _scrub_url(request["url"]) - - -def _scrub_exceptions(event: Event) -> None: - for exc in (event.get("exception") or {}).get("values") or []: - if isinstance(exc.get("value"), str): - exc["value"] = scrub_text(exc["value"]) - _scrub_frames(exc) - for thread in (event.get("threads") or {}).get("values") or []: - _scrub_frames(thread) - - -def _scrub_frames(holder: Mapping[str, Any]) -> None: - """Drop frame locals; scrub source context (a hard-coded secret is still one).""" - for frame in (holder.get("stacktrace") or {}).get("frames") or []: - frame.pop("vars", None) - for key in ("pre_context", "context_line", "post_context"): - if frame.get(key) is not None: - frame[key] = scrub(frame[key]) - - -def _scrub_spans(event: Event) -> None: - for span in event.get("spans") or []: - if isinstance(span.get("description"), str): - span["description"] = scrub_text(span["description"]) - if isinstance(span.get("data"), Mapping): - span["data"] = scrub(span["data"]) + request["headers"] = {k: v for k, v in headers.items() if not is_sensitive_key(k)} + elif isinstance(headers, list): + request["headers"] = [h for h in headers if not _is_sensitive_header(h)] + if isinstance(request.get(_URL_KEY), str): + request[_URL_KEY] = _scrub_url(request[_URL_KEY]) + + +def _scrub_user(event: Event) -> None: + user = event.get("user") + user_id = user.get(_USER_ID) if isinstance(user, Mapping) else None + if user_id is None: + event.pop("user", None) + else: + event["user"] = {_USER_ID: str(user_id)} # never email/username/ip def _scrub_transaction(event: Event) -> None: transaction = event.get("transaction") source = (event.get("transaction_info") or {}).get("source") if isinstance(transaction, str) and source == _SOURCE_URL: - event["transaction"] = scrub_text(scrub_path(transaction)) + event["transaction"] = scrub_path(transaction) def scrub_event(event: Event) -> Event: - """Scrub ``event`` in place (and return it).""" + """Scrub ``event`` (in place where possible) and return it.""" if isinstance(event.get("request"), dict): _scrub_request(event["request"]) - for key in ("extra", "contexts"): - if isinstance(event.get(key), Mapping): - event[key] = scrub(event[key]) - # Breadcrumbs: log messages and HTTP URLs/queries; logentry: log params. - for key in ("breadcrumbs", "logentry", "message"): - if event.get(key) is not None: - event[key] = scrub(event[key]) - _scrub_exceptions(event) - _scrub_spans(event) + _scrub_user(event) _scrub_transaction(event) + user = event.pop("user", None) + scrubbed = scrub(event) + event.clear() + event.update(scrubbed) + if user is not None: + event["user"] = user return event diff --git a/framework/hosting/simple_module_hosting/sentry_scrub_text.py b/framework/hosting/simple_module_hosting/sentry_scrub_text.py new file mode 100644 index 000000000..8d358ab07 --- /dev/null +++ b/framework/hosting/simple_module_hosting/sentry_scrub_text.py @@ -0,0 +1,110 @@ +"""Sensitive-key matching and free-text credential redaction for Sentry events. + +Every pattern here runs on attacker-influenced text inside ``before_send`` +(which blocks the capturing thread), so each one is linear: matches can only +start at a token boundary and every repetition near a backtracking point is +bounded, and input is capped at :data:`MAX_TEXT` +(``test_hosting_sentry_scrub_gaps`` pins both with megabyte inputs). +""" + +from __future__ import annotations + +import re + +REDACTED = "[Filtered]" +TRUNCATED = "...[truncated]" +# The SDK no longer truncates values (``max_value_length`` is None), and the +# patterns below cost a few microseconds per character, so anything longer +# is cut before scanning: a megabyte string must not stall the request. +MAX_TEXT = 64 * 1024 + +# Substring words: any key containing one is sensitive, case-insensitively. +_SUBSTRING_WORDS = ( + r"passw(?:or)?d|passphrase|pwd|secret|token|authori[sz]ation|cookie|session" + r"|api[-_]?key|csrf|xsrf|credential|private[-_]?key|signature|dsn|jwt|bearer" + r"|credit[-_]?card|card[-_]?number" + # client addresses are PII with send_default_pii=False + r"|forwarded|real[-_]?ip|client[-_]?ip|remote[-_]?addr|ip[-_]?address" +) +# Short words that are only sensitive as a whole word: ``auth`` but not +# ``author``; snake_case, kebab-case and camelCase boundaries all count. +_BOUNDED_WORDS = ("auth", "pass", "otp", "pin", "sid", "sig", "ssn", "cvv", "cvc", "mfa") + + +def _bounded(word: str) -> str: + title, upper = word.capitalize(), word.upper() + return ( + rf"(?-i:(?:(?(?:proxy-)?authori[sz]ation|(?:set-)?cookie)[ \t]*:[^\r\n]*", + re.IGNORECASE, +) +# key=value / key: value / "key": "value" / \"key\": \"value\" (JSON in a string). +_KEY_VALUE = re.compile( + rf"""(?P(?\\?["'])(?:(?!(?P=quote))[^\r\n]){0,4096}""" + r"""|(?:(?:Bearer|Basic|Token|Digest)[ \t]+)?[^\s&,;"'}\]\\]+)""", + re.IGNORECASE, +) +_SCHEME_CREDENTIAL = re.compile( + r"\b(Bearer|Basic|Token|Digest)[ \t]+[A-Za-z0-9._~+/=\-]+", re.IGNORECASE +) +# scheme://user:pass@ (the password may itself hold ``@`` or ``/``) or user@. +_URL_USERINFO = re.compile( + r"(?P\b[a-z][a-z0-9+.\-]{0,30}://)" + r"(?:[^\s:/@?#]{0,256}:[^\s?#]{0,256}|[^\s/?#@]{1,256})@", + re.IGNORECASE, +) +_JWT = re.compile(r"(? bool: + if isinstance(key, bytes | bytearray): + key = key.decode("utf-8", "replace") + return isinstance(key, str) and (key.lower() in _EXACT_KEYS or bool(SENSITIVE_KEY.search(key))) + + +def _redact_key_value(match: re.Match[str]) -> str: + return f"{match['key']}{match['quote'] or ''}{REDACTED}" + + +def _redact_long_token(match: re.Match[str]) -> str: + token = match[0] + if not (_HAS_DIGIT.search(token) and _HAS_LETTER.search(token)): + return token + if _IDENTIFIER.fullmatch(token) or _UUID.fullmatch(token): + return token + return REDACTED + + +def scrub_text(text: str) -> str: + """Redact credentials embedded in free text.""" + if len(text) > MAX_TEXT: + text = text[:MAX_TEXT] + TRUNCATED + text = _URL_USERINFO.sub(rf"\g{REDACTED}@", text) + text = _HEADER_LINE.sub(rf"\g: {REDACTED}", text) + text = _QUERY_CODE.sub(rf"\g<1>{REDACTED}", text) + text = _KEY_VALUE.sub(_redact_key_value, text) + text = _SCHEME_CREDENTIAL.sub(rf"\1 {REDACTED}", text) + text = _JWT.sub(REDACTED, text) + return _LONG_TOKEN.sub(_redact_long_token, text) diff --git a/framework/hosting/tests/_sentry_transport.py b/framework/hosting/tests/_sentry_transport.py new file mode 100644 index 000000000..b8b3a4c2f --- /dev/null +++ b/framework/hosting/tests/_sentry_transport.py @@ -0,0 +1,34 @@ +"""A fake Sentry transport: the real SDK pipeline, envelopes kept in memory.""" + +from __future__ import annotations + +from typing import Any + +import pytest +from sentry_sdk.transport import Transport +from simple_module_hosting import sentry as sm_sentry + +DSN_KEY = "dsnpublickey0123" +DSN = f"https://{DSN_KEY}@o0.ingest.sentry.io/1" + + +class FakeTransport(Transport): + def __init__(self, options: dict[str, Any] | None = None) -> None: + super().__init__(options) + self.payloads: list[dict[str, Any]] = [] + + def capture_envelope(self, envelope: Any) -> None: + for item in envelope.items: + if item.type in ("event", "transaction"): + self.payloads.append(item.payload.json) + + def of(self, kind: str) -> list[dict[str, Any]]: + return [p for p in self.payloads if p.get("type", "event") == kind] + + +def start(monkeypatch: pytest.MonkeyPatch, process: sm_sentry.Process) -> FakeTransport: + monkeypatch.setenv("SM_SENTRY_DSN", DSN) + monkeypatch.setenv("SM_SENTRY_TRACES_SAMPLE_RATE", "1.0") + fake = FakeTransport() + assert sm_sentry.init_sentry(process, transport=fake) + return fake diff --git a/framework/hosting/tests/test_hosting_sentry_scrub.py b/framework/hosting/tests/test_hosting_sentry_scrub.py index 79810706e..86753e485 100644 --- a/framework/hosting/tests/test_hosting_sentry_scrub.py +++ b/framework/hosting/tests/test_hosting_sentry_scrub.py @@ -18,10 +18,10 @@ from simple_module_hosting.sentry_scrub import REDACTED, scrub_event, scrub_text sentry_sdk = pytest.importorskip("sentry_sdk") -from sentry_sdk.transport import Transport # noqa: E402 +from _sentry_transport import DSN_KEY as _DSN_KEY # noqa: E402 +from _sentry_transport import FakeTransport # noqa: E402 +from _sentry_transport import start as _start # noqa: E402 -_DSN_KEY = "dsnpublickey0123" -_DSN = f"https://{_DSN_KEY}@o0.ingest.sentry.io/1" _LEAKS = ( "c00kie-val", "203.0.113.9", @@ -39,28 +39,6 @@ ) -class FakeTransport(Transport): - def __init__(self, options: dict[str, Any] | None = None) -> None: - super().__init__(options) - self.payloads: list[dict[str, Any]] = [] - - def capture_envelope(self, envelope: Any) -> None: - for item in envelope.items: - if item.type in ("event", "transaction"): - self.payloads.append(item.payload.json) - - def of(self, kind: str) -> list[dict[str, Any]]: - return [p for p in self.payloads if p.get("type", "event") == kind] - - -def _start(monkeypatch: pytest.MonkeyPatch, process: sm_sentry.Process) -> FakeTransport: - monkeypatch.setenv("SM_SENTRY_DSN", _DSN) - monkeypatch.setenv("SM_SENTRY_TRACES_SAMPLE_RATE", "1.0") - fake = FakeTransport() - assert sm_sentry.init_sentry(process, transport=fake) - return fake - - @pytest.fixture def web(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeTransport]: fake = _start(monkeypatch, sm_sentry.PROCESS_WEB) diff --git a/framework/hosting/tests/test_hosting_sentry_scrub_gaps.py b/framework/hosting/tests/test_hosting_sentry_scrub_gaps.py new file mode 100644 index 000000000..c8eb4faa8 --- /dev/null +++ b/framework/hosting/tests/test_hosting_sentry_scrub_gaps.py @@ -0,0 +1,210 @@ +"""Scrubber gaps found by an adversarial audit (#538 follow-up). + +Each secret below got past the first scrubber; the SDK-driven tests assert it +is absent from the envelope the fake transport receives. +""" + +from __future__ import annotations + +import json +import logging +import time +from collections.abc import Iterator +from typing import Any + +import pytest +from simple_module_hosting import sentry as sm_sentry +from simple_module_hosting.sentry_scrub import REDACTED, is_sensitive_key, scrub, scrub_event +from simple_module_hosting.sentry_scrub_text import MAX_TEXT, TRUNCATED, scrub_text + +sentry_sdk = pytest.importorskip("sentry_sdk") +from _sentry_transport import FakeTransport, start # noqa: E402 + +_HEX_KEY = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" +_JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJlLWp3dA" + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> Iterator[FakeTransport]: + fake = start(monkeypatch, sm_sentry.PROCESS_WEB) + yield fake + sentry_sdk.get_global_scope().client.close() + + +def _capture(message: str) -> None: + try: + raise RuntimeError(message) + except RuntimeError: + sentry_sdk.capture_exception() + sentry_sdk.flush() + + +def test_tags_contexts_pairs_and_free_text_are_scrubbed(web: FakeTransport) -> None: + sentry_sdk.set_tag("auth_token", "tag-s3cret") + sentry_sdk.set_user({"email": "who@example.com", "username": "who"}) + sentry_sdk.set_context( + "svc", + { + "basicAuth": "camel-s3cret", + "pwd": "pwd-s3cret", + "headers": [["X-Auth-Token", "pair-s3cret"], ("Set-Cookie", "sc-s3cret")], + "raw": b"session=bytes-s3cret", + "region": "eu", + }, + ) + _capture( + f"redis://:p@ss/w0rd@cache:6379 jwt {_JWT} key {_HEX_KEY} " + '{"password": "two words"} {\\"secret\\": \\"esc-s3cret\\"}' + "\nCookie: sid=a; other=ck-s3cret" + ) + (event,) = web.of("event") + dumped = json.dumps(event) + for secret in ( + "tag-s3cret", + "who@example.com", + "camel-s3cret", + "pwd-s3cret", + "pair-s3cret", + "sc-s3cret", + "bytes-s3cret", + "p@ss/w0rd", + "w0rd", + _JWT, + _HEX_KEY, + "two words", + "words", + "esc-s3cret", + "ck-s3cret", + ): + assert secret not in dumped, secret + assert "user" not in event # no id, so nothing about the user at all + assert event["contexts"]["svc"]["region"] == "eu" + assert "cache:6379" in dumped + + +def test_sentry_ids_survive_the_long_token_rule(web: FakeTransport) -> None: + _capture("boom") + (event,) = web.of("event") + trace = event["contexts"]["trace"] + assert len(event["event_id"]) == 32 and REDACTED not in event["event_id"] + assert REDACTED not in trace["trace_id"] and REDACTED not in trace["span_id"] + + +def test_scrubber_failure_drops_the_event( + web: FakeTransport, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + def explode(event: dict[str, Any]) -> dict[str, Any]: + raise ValueError("password=fail-s3cret") + + monkeypatch.setattr(sm_sentry, "scrub_event", explode) + caplog.set_level(logging.WARNING) + _capture("token=raw-s3cret") + assert web.of("event") == [] + assert "fail-s3cret" not in caplog.text and "raw-s3cret" not in caplog.text + assert "ValueError" in caplog.text + + +def test_unlisted_fields_and_top_level_stacktrace_are_scrubbed() -> None: + event: dict[str, Any] = { + "stacktrace": { + "frames": [{"context_line": 'api_key = "src-s3cret"', "vars": {"x": "var-s3cret"}}] + }, + "threads": {"values": [{"stacktrace": {"frames": [{"vars": {"y": "thr-s3cret"}}]}}]}, + "request": {"headers": [["Authorization", "hdr-s3cret"], ["Accept", "json"]]}, + "spans": [{"data": {"db.params": ["sql-s3cret"], "db.system": "postgresql"}}], + "server_name": "web-1 token=srv-s3cret", + "logentry": {"message": "%s", "params": [b"secret=b-s3cret"]}, + "user": {"id": "u-1", "email": "e@x", "ip_address": "198.51.100.4"}, + "future_sdk_field": {"refreshToken": "new-s3cret"}, + } + scrubbed = scrub_event(event) + dumped = json.dumps(scrubbed) + for secret in ("s3cret", "198.51.100.4", "e@x"): + assert secret not in dumped, secret + assert scrubbed["request"]["headers"] == [["Accept", "json"]] + assert scrubbed["user"] == {"id": "u-1"} + assert scrubbed["spans"][0]["data"] == {"db.system": "postgresql"} + + +@pytest.mark.parametrize( + "key", + [ + *("pwd", "passwd", "Passphrase", "access_token", "refresh_token", "client_secret"), + *("private_key", "x-api-key", "apiKey", "authToken", "basicAuth", "authHeader"), + *("X-Amz-Signature", "sig", "jwt", "otp", "userPin", "DB_PASS", "AUTH"), + ], +) +def test_sensitive_key_variants(key: str) -> None: + assert is_sensitive_key(key) + + +@pytest.mark.parametrize("key", ["author", "authority", "spinner", "bypass", "side", "region"]) +def test_bounded_words_do_not_overmatch(key: str) -> None: + assert not is_sensitive_key(key) + + +@pytest.mark.parametrize( + ("raw", "secret"), + [ + ("https://user:p@ss@host/x", "ss"), + ("postgresql://u:a/b@db/x", "a/b"), + ("Authorization: Digest username=bob, response=dg-1", "dg-1"), + ("set-cookie: a=1; Path=/; csrftoken=ck-2", "ck-2"), + ("password='with spaces here'", "spaces"), + (f"Bearer-less {_JWT} trailing", _JWT), + ("token " + "Ab3dE" * 8, "Ab3dE" * 8), + ], +) +def test_scrub_text_gaps(raw: str, secret: str) -> None: + assert secret not in scrub_text(raw) + + +@pytest.mark.parametrize( + "keep", + [ + "3f2b9c1d-8e7a-4b5c-9d2e-1f0a9b8c7d6e", + "test_hosting_sentry_scrub_redacts_inline_credentials2", + "/home/app/simple_module_hosting/sentry_scrub.py", + "http://host:8080/orders?page=2", + ], +) +def test_scrub_text_keeps_debugging_signal(keep: str) -> None: + assert scrub_text(keep) == keep + + +def test_scrub_is_idempotent() -> None: + once = scrub_text("Cookie: a=b; c=d url=http://h/x?session=s1") + assert scrub_text(once) == once + + +def test_scrub_text_is_linear_on_adversarial_input() -> None: + payloads = [ + "a" * 1_000_000, + "password" + " " * 1_000_000, + "a://" * 250_000, + "x" * 500_000 + "=" * 3 + "x" * 500_000, + '"secret": "' + "a" * 1_000_000, + "token_" * 200_000, + ] + started = time.monotonic() + for payload in payloads: + assert len(scrub_text(payload)) <= MAX_TEXT + len(TRUNCATED) + assert time.monotonic() - started < 10 # the quadratic version took minutes + # Below the cap the patterns themselves must stay fast. + started = time.monotonic() + for payload in payloads: + scrub_text(payload[: MAX_TEXT - 1]) + assert time.monotonic() - started < 10 + + +def test_cyclic_and_deep_structures_end_redacted() -> None: + cyclic: dict[str, Any] = {"name": "x"} + cyclic["self"] = cyclic + deep: list[Any] = [] + node = deep + for _ in range(1000): + child: list[Any] = [] + node.append(child) + node = child + assert REDACTED in json.dumps(scrub(cyclic)) + assert REDACTED in json.dumps(scrub(deep)) From 2d83011f97cfcc7d319a40d40b08eeacdb7bed1b Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:39:03 +0200 Subject: [PATCH 12/22] fix(hosting): redact URL userinfo passwords in retained logs; mask broker URL in startup log Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../simple_module_hosting/log_sinks.py | 4 ++- .../hosting/tests/test_hosting_log_sinks.py | 14 ++++++++++ .../background_tasks/_lifecycle.py | 4 ++- .../tests/test_bg_startup_log_masking.py | 26 +++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 modules/background_tasks/tests/test_bg_startup_log_masking.py diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index 9fcb6b8fa..d224bbc00 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -50,6 +50,7 @@ r"""(?ix)([\w-]*(?:password|passwd|token|secret|api[_-]?key)[\w-]*["']?\s*[:=]\s*["']?)""" r"""[^\s"',;&}\])]+""" ) +_URL_CRED_RE = re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://)([^/@\s:]*):([^/@\s]*)@") _BEARER_RE = re.compile(r"(?i)\bbearer\s+[\w.~+/=-]+") _SENSITIVE_KEY_RE = re.compile(r"(?i)password|passwd|token|secret|api[_-]?key|authorization|cookie") _STANDARD_ATTRS = frozenset(logging.makeLogRecord({}).__dict__) | {"message", "asctime"} @@ -65,9 +66,10 @@ def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: def redact(text: str) -> str: - """Mask credentials in ``text``: auth headers, cookies, bearer tokens, ``key=value`` secrets.""" + """Mask credentials in ``text``: auth headers, cookies, bearer tokens, ``key=value`` secrets, URL passwords.""" text = _HEADER_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}{REDACTED}", text) text = _BEARER_RE.sub(f"Bearer {REDACTED}", text) + text = _URL_CRED_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}:{REDACTED}@", text) return _SECRET_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text) diff --git a/framework/hosting/tests/test_hosting_log_sinks.py b/framework/hosting/tests/test_hosting_log_sinks.py index 8b1026c5a..ec35dac97 100644 --- a/framework/hosting/tests/test_hosting_log_sinks.py +++ b/framework/hosting/tests/test_hosting_log_sinks.py @@ -152,3 +152,17 @@ def test_redact_masks(raw, secret): def test_redact_keeps_ordinary_text(): assert redact("user 42 logged in from 10.0.0.1") == "user 42 logged in from 10.0.0.1" + + +@pytest.mark.parametrize( + "url", + ["redis://user:s3cret@host:6379/0", "amqp://u:s3cret@mq/vhost", "postgresql://u:s3cret@db/x"], +) +def test_redact_masks_url_userinfo_password(url): + out = redact(f"connecting to {url} now") + assert "s3cret" not in out + assert "@" in out and url.split("://")[0] in out + + +def test_redact_leaves_credential_free_urls_alone(): + assert redact("redis://localhost:6379/0") == "redis://localhost:6379/0" diff --git a/modules/background_tasks/background_tasks/_lifecycle.py b/modules/background_tasks/background_tasks/_lifecycle.py index 5a6065a55..1824ca8fc 100644 --- a/modules/background_tasks/background_tasks/_lifecycle.py +++ b/modules/background_tasks/background_tasks/_lifecycle.py @@ -20,6 +20,8 @@ async def startup(app: FastAPI, module_name: str) -> None: """Build the Celery app, install signal handlers, hand-off to worker.""" import asyncio + from simple_module_hosting.log_sinks import redact + from background_tasks.celery_app import build_celery from background_tasks.log_sink import install_log_sink from background_tasks.signals import bind_event_bus @@ -47,7 +49,7 @@ async def startup(app: FastAPI, module_name: str) -> None: install_log_sink(services.settings) logger.info( "BackgroundTasks: Celery app ready (broker=%s, queue=%s)", - services.settings.broker_url, + redact(services.settings.broker_url), services.settings.task_default_queue, ) diff --git a/modules/background_tasks/tests/test_bg_startup_log_masking.py b/modules/background_tasks/tests/test_bg_startup_log_masking.py new file mode 100644 index 000000000..d8ee2fbbe --- /dev/null +++ b/modules/background_tasks/tests/test_bg_startup_log_masking.py @@ -0,0 +1,26 @@ +"""The startup line must not leak the broker URL's password.""" + +from __future__ import annotations + +import pytest + +SECRET = "hunter2-broker-pw" + + +@pytest.fixture +def _broker_with_password( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level("INFO") + monkeypatch.setenv("SM_REDIS_URL", f"redis://user:{SECRET}@localhost:6399/0") + + +async def test_startup_log_masks_broker_password( + _broker_with_password, caplog: pytest.LogCaptureFixture, app +) -> None: + # The app fixture boots during setup, so read the setup-phase records. + records = caplog.get_records("setup") + ready = [r for r in records if "Celery app ready" in r.getMessage()] + assert ready, "startup log line not captured" + assert "user:***@localhost" in ready[0].getMessage() + assert all(SECRET not in r.getMessage() for r in records) From ff627482cbc3164355c8c8715fb349f3e5dd2a41 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 18:47:06 +0200 Subject: [PATCH 13/22] fix(background_tasks): keep _start_invalidation_transport entry point; wrap redact docstring Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- framework/hosting/simple_module_hosting/log_sinks.py | 2 +- modules/background_tasks/background_tasks/module.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index d224bbc00..f1238a7a2 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -66,7 +66,7 @@ def emit_batch(self, records: Sequence[logging.LogRecord]) -> None: def redact(text: str) -> str: - """Mask credentials in ``text``: auth headers, cookies, bearer tokens, ``key=value`` secrets, URL passwords.""" + """Mask credentials: auth headers, cookies, bearer tokens, key=value secrets, URL passwords.""" text = _HEADER_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}{REDACTED}", text) text = _BEARER_RE.sub(f"Bearer {REDACTED}", text) text = _URL_CRED_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}:{REDACTED}@", text) diff --git a/modules/background_tasks/background_tasks/module.py b/modules/background_tasks/background_tasks/module.py index 09ceae096..085723be0 100644 --- a/modules/background_tasks/background_tasks/module.py +++ b/modules/background_tasks/background_tasks/module.py @@ -168,3 +168,9 @@ async def on_shutdown(self, app: FastAPI) -> None: from background_tasks._lifecycle import shutdown await shutdown(app) + + @staticmethod + async def _start_invalidation_transport(app: FastAPI) -> None: + from background_tasks._lifecycle import start_invalidation_transport + + await start_invalidation_transport(app) From d2fcd677a2e43932e7c978d6864e3f1c4cdaa121 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 21:22:52 +0200 Subject: [PATCH 14/22] i18n(de): German strings for task progress and retained logs Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../background_tasks/locales/de.json | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/modules/background_tasks/background_tasks/locales/de.json b/modules/background_tasks/background_tasks/locales/de.json index 1e15d01f1..067a21f9d 100644 --- a/modules/background_tasks/background_tasks/locales/de.json +++ b/modules/background_tasks/background_tasks/locales/de.json @@ -73,7 +73,9 @@ "copy": "Kopieren", "copied": "Kopiert", "no_traceback": "Kein Traceback aufgezeichnet.", - "tenant": "Organisation" + "tenant": "Organisation", + "progress": "Fortschritt", + "progress_count": "{current} von {total} ({percent} %)" }, "retry_dialog": { "title": "{name} erneut starten?", @@ -156,7 +158,8 @@ "routes_empty": "Keine Aufgabenrouten konfiguriert. Jede Aufgabe wird an die Standardwarteschlange gesendet. Fügen Sie Routen in task_config.py (TASK_ROUTES) eines Moduls hinzu." }, "nav": { - "background_tasks": "Hintergrundaufgaben" + "background_tasks": "Hintergrundaufgaben", + "logs": "Protokolle" }, "audit": { "task_execution": "Aufgabenausführung" @@ -191,5 +194,29 @@ "conflict": "Dieses Ereignis wird gerade zugestellt und kann daher nicht geändert werden. Die Liste wurde aktualisiert.", "gone": "Dieses Ereignis existiert nicht mehr. Die Liste wurde aktualisiert." }, - "permission_group": "Hintergrundaufgaben" + "permission_group": "Hintergrundaufgaben", + "logs": { + "title": "Protokolle", + "description": "Gespeicherte Anwendungs- und Aufgabenprotokollzeilen", + "back_button": "Ausführungen", + "tab_details": "Details", + "tab_logs": "Protokolle", + "tab_list_label": "Aufgabenbereiche", + "search_label": "Meldungen durchsuchen", + "search_placeholder": "Meldungen durchsuchen", + "task_label": "Aufgaben-ID", + "task_placeholder": "Aufgaben-ID", + "level_label": "Mindeststufe", + "level_all": "Alle Stufen", + "apply": "Suchen", + "col_time": "Zeit", + "col_level": "Stufe", + "col_logger": "Logger", + "col_message": "Meldung", + "col_task": "Aufgabe", + "empty_title": "Keine Protokollzeilen", + "empty_description": "Es gibt keine Treffer. Die Protokollspeicherung ist standardmäßig deaktiviert: Aktivieren Sie log_sink_enabled in den Einstellungen für Hintergrundaufgaben und starten Sie anschließend den Webprozess und die Worker neu.", + "load_failed": "Protokollzeilen konnten nicht geladen werden", + "platform": "Plattform" + } } From f0a33abcbb8358002c2090d3b168c29987e9dbb3 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Wed, 21 Oct 2026 21:26:07 +0200 Subject: [PATCH 15/22] refactor(optimize): one standard-attrs helper, single retention reader for log purge Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../simple_module_hosting/log_sinks.py | 11 ++++++---- .../background_tasks/log_purge.py | 22 +++---------------- .../background_tasks/log_sink.py | 12 ++++++---- .../background_tasks/tasks.py | 9 +++++--- .../tests/test_bg_log_purge.py | 15 +++++-------- 5 files changed, 29 insertions(+), 40 deletions(-) diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index d8f1e8006..cfe22b868 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -25,7 +25,7 @@ import threading import time from collections.abc import Sequence -from typing import Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable from simple_module_core.correlation import correlation_id @@ -60,15 +60,18 @@ def redact(text: str) -> str: return scrub_text(text) +def standard_extra_attrs(record: logging.LogRecord) -> dict[str, Any]: + """The attributes a caller attached via ``extra=``, i.e. everything not a stock record field.""" + return {k: v for k, v in record.__dict__.items() if k not in _STANDARD_ATTRS} + + def redact_record(record: logging.LogRecord) -> None: """Mask ``record`` in place: the message, the traceback text and any extra attributes.""" record.msg = redact(record.getMessage()) record.args = None if record.exc_text: record.exc_text = redact(record.exc_text) - for key, value in list(record.__dict__.items()): - if key in _STANDARD_ATTRS: - continue + for key, value in list(standard_extra_attrs(record).items()): if is_sensitive_key(key): record.__dict__[key] = REDACTED elif isinstance(value, str): diff --git a/modules/background_tasks/background_tasks/log_purge.py b/modules/background_tasks/background_tasks/log_purge.py index 41e926ae5..406b597b5 100644 --- a/modules/background_tasks/background_tasks/log_purge.py +++ b/modules/background_tasks/background_tasks/log_purge.py @@ -2,31 +2,15 @@ from __future__ import annotations -import logging -from datetime import UTC, datetime, timedelta +from datetime import datetime from sqlalchemy import delete from sqlalchemy.orm import Session -from background_tasks.constants import DEFAULT_LOG_RETENTION_DAYS, MODULE_NAME from background_tasks.models import RetainedLog -from background_tasks.settings import BackgroundTasksSettings -from background_tasks.worker_settings import settings_for -logger = logging.getLogger(__name__) - -def log_retention_days() -> int: - """The DB-backed ``log_retention_days``; the default when settings are unreadable.""" - try: - return settings_for(BackgroundTasksSettings, MODULE_NAME).log_retention_days - except Exception: - logger.warning("Could not read log_retention_days; using the default", exc_info=True) - return DEFAULT_LOG_RETENTION_DAYS - - -def purge_log_records(session: Session, days: int | None = None) -> int: - """Delete log lines older than ``days`` (the setting when omitted); returns the count.""" - cutoff = datetime.now(UTC) - timedelta(days=days if days is not None else log_retention_days()) +def purge_log_records(session: Session, cutoff: datetime) -> int: + """Delete log lines older than ``cutoff``; returns the count.""" result = session.execute(delete(RetainedLog).where(RetainedLog.ts < cutoff)) # ty: ignore[invalid-argument-type] return result.rowcount or 0 diff --git a/modules/background_tasks/background_tasks/log_sink.py b/modules/background_tasks/background_tasks/log_sink.py index b456081fc..3e7091388 100644 --- a/modules/background_tasks/background_tasks/log_sink.py +++ b/modules/background_tasks/background_tasks/log_sink.py @@ -17,7 +17,12 @@ from celery.signals import worker_process_init from simple_module_db import current_tenant_id -from simple_module_hosting.log_sinks import QueueSinkHandler, install_sink, uninstall_sink +from simple_module_hosting.log_sinks import ( + QueueSinkHandler, + install_sink, + standard_extra_attrs, + uninstall_sink, +) from background_tasks.constants import ( LOG_EXTRA_MAX_KEYS, @@ -35,7 +40,6 @@ _log = logging.getLogger(__name__) _COLUMN_ATTRS = ("task_id", "job_id", "tenant_id", "correlation_id") -_STANDARD_ATTRS = frozenset(logging.makeLogRecord({}).__dict__) | {"message", "asctime"} _handler: QueueSinkHandler | None = None @@ -45,8 +49,8 @@ def _text(value: object, limit: int) -> str | None: def _extra(record: logging.LogRecord) -> dict[str, Any]: extra: dict[str, Any] = {} - for key, value in record.__dict__.items(): - if key in _STANDARD_ATTRS or key in _COLUMN_ATTRS: + for key, value in standard_extra_attrs(record).items(): + if key in _COLUMN_ATTRS: continue if len(extra) >= LOG_EXTRA_MAX_KEYS: break diff --git a/modules/background_tasks/background_tasks/tasks.py b/modules/background_tasks/background_tasks/tasks.py index 35ceadfae..816fb7e9d 100644 --- a/modules/background_tasks/background_tasks/tasks.py +++ b/modules/background_tasks/background_tasks/tasks.py @@ -134,8 +134,11 @@ def sweep_stuck_tasks(self: Task) -> int: @shared_task(name=INTERNAL_TASK_PURGE_OLD) def purge_old_executions() -> int: """Delete terminal rows, delivered outbox rows and log lines past their retention.""" - retention_days = load_runner_settings().retention_days - cutoff = datetime.now(UTC) - timedelta(days=retention_days) + runner = load_runner_settings() + retention_days = runner.retention_days + now = datetime.now(UTC) + cutoff = now - timedelta(days=retention_days) + log_cutoff = now - timedelta(days=runner.log_retention_days) with sync_session() as session: result = session.execute( @@ -150,7 +153,7 @@ def purge_old_executions() -> int: .where(OutboxMessage.delivered_at < cutoff) ) count += delivered.rowcount or 0 - count += purge_log_records(session) + count += purge_log_records(session, log_cutoff) if count: logger.info( "purge_old_executions removed %d row(s) older than %d days", diff --git a/modules/background_tasks/tests/test_bg_log_purge.py b/modules/background_tasks/tests/test_bg_log_purge.py index af4a26d2a..25a3a4eae 100644 --- a/modules/background_tasks/tests/test_bg_log_purge.py +++ b/modules/background_tasks/tests/test_bg_log_purge.py @@ -7,6 +7,7 @@ import pytest from background_tasks import log_purge, sync_db, tasks from background_tasks.models import RetainedLog +from background_tasks.settings import BackgroundTasksSettings from sqlalchemy import select pytestmark = pytest.mark.usefixtures("sync_sqlite") @@ -31,20 +32,14 @@ def _messages() -> set[str]: def test_purge_removes_rows_older_than_retention(): _seed(1, 6, 8, 30) with sync_db.sync_session() as session: - assert log_purge.purge_log_records(session, 7) == 2 + assert log_purge.purge_log_records(session, datetime.now(UTC) - timedelta(days=7)) == 2 assert _messages() == {"1d #0", "6d #1"} def test_purge_old_executions_applies_the_setting(monkeypatch): - monkeypatch.setattr(log_purge, "log_retention_days", lambda: 3) + monkeypatch.setattr( + tasks, "load_runner_settings", lambda: BackgroundTasksSettings(log_retention_days=3) + ) _seed(1, 5) assert tasks.purge_old_executions() == 1 assert _messages() == {"1d #0"} - - -def test_unreadable_settings_fall_back_to_the_default(monkeypatch): - def boom(*_a, **_k): - raise RuntimeError("no settings table") - - monkeypatch.setattr(log_purge, "settings_for", boom) - assert log_purge.log_retention_days() == 7 From 46355ca25cde248ea3ee26e1695ee6957de208e9 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 09:24:40 +0200 Subject: [PATCH 16/22] fix(hosting): scrub structured log extras recursively; disjoint Redis lease key namespaces Structured extras (dicts/lists/objects) bypassed redaction in the retained-log sink; they now go through the Sentry recursive scrubber. Redis holder and fence keys use separate prefixes so a lease named 'foo:fence' cannot collide with foo's counter. Also makes the Sentry route test independent of the git sha embedded in the release field. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../simple_module_hosting/_lease_redis.py | 15 ++++++------- .../simple_module_hosting/log_sinks.py | 6 +++++- .../tests/test_hosting_lease_backends.py | 13 ++++++++++++ .../hosting/tests/test_hosting_log_sinks.py | 21 +++++++++++++++++++ .../hosting/tests/test_hosting_sentry.py | 4 ++-- 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/framework/hosting/simple_module_hosting/_lease_redis.py b/framework/hosting/simple_module_hosting/_lease_redis.py index 605d36104..7f98a1c9e 100644 --- a/framework/hosting/simple_module_hosting/_lease_redis.py +++ b/framework/hosting/simple_module_hosting/_lease_redis.py @@ -18,9 +18,12 @@ import os from typing import Any -_KEY_PREFIX = "sm:lease:" -_SEM_PREFIX = "sm:sem:" -_FENCE_SUFFIX = ":fence" +# Holder and fence-counter keys live under disjoint prefixes: a name is never +# parsed, so ``foo:fence`` cannot collide with the counter of ``foo``. +_HOLDER_PREFIX = "sm:lease:h:" +_FENCE_PREFIX = "sm:lease:f:" +_SEM_HOLDER_PREFIX = "sm:sem:h:" +_SEM_FENCE_PREFIX = "sm:sem:f:" _ACQUIRE = """ if redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', ARGV[2]) then @@ -120,13 +123,11 @@ async def _eval(self, script: str, keys: list[str], *args: Any) -> int: @staticmethod def _lease_keys(name: str) -> list[str]: - key = _KEY_PREFIX + name - return [key, key + _FENCE_SUFFIX] + return [_HOLDER_PREFIX + name, _FENCE_PREFIX + name] @staticmethod def _sem_keys(name: str) -> list[str]: - key = _SEM_PREFIX + name - return [key, key + _FENCE_SUFFIX] + return [_SEM_HOLDER_PREFIX + name, _SEM_FENCE_PREFIX + name] async def acquire(self, name: str, token: str, ttl: float) -> int | None: return await self._eval(_ACQUIRE, self._lease_keys(name), token, _ms(ttl)) or None diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index cfe22b868..ef29338e1 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -29,7 +29,7 @@ from simple_module_core.correlation import correlation_id -from simple_module_hosting.sentry_scrub_text import REDACTED, is_sensitive_key, scrub_text +from simple_module_hosting.sentry_scrub import REDACTED, is_sensitive_key, scrub, scrub_text logger = logging.getLogger(__name__) @@ -76,6 +76,10 @@ def redact_record(record: logging.LogRecord) -> None: record.__dict__[key] = REDACTED elif isinstance(value, str): record.__dict__[key] = redact(value) + elif not (value is None or isinstance(value, bool | int | float)): + # Mappings, sequences and arbitrary objects: the Sentry scrubber walks them + # (sensitive keys at any depth, repr() of opaque objects) into a copy. + record.__dict__[key] = scrub(value) class RedactionFilter(logging.Filter): diff --git a/framework/hosting/tests/test_hosting_lease_backends.py b/framework/hosting/tests/test_hosting_lease_backends.py index 464fd353a..f260f6964 100644 --- a/framework/hosting/tests/test_hosting_lease_backends.py +++ b/framework/hosting/tests/test_hosting_lease_backends.py @@ -149,3 +149,16 @@ async def test_postgres_lease_race_across_connections(engine) -> None: sem = [semaphore(name + "s", 4, _LONG_TTL, blocking=False, backend=backend) for _ in range(12)] assert (await asyncio.gather(*(h.acquire() for h in sem))).count(True) == 4 + + +async def test_names_that_look_like_internal_keys_do_not_collide(backend: LeaseBackend) -> None: + base = _name() + first = await backend.acquire(base, "a", _LONG_TTL) + second = await backend.acquire(f"{base}:fence", "b", _LONG_TTL) + assert first == 1 + assert second == 1 # its own holder and its own counter + assert await backend.release(base, "a") + third = await backend.acquire(base, "c", _LONG_TTL) + assert third == 2 + sem = await backend.semaphore_acquire(f"{base}:sem:fence", "s", 1, _LONG_TTL) + assert sem == 1 diff --git a/framework/hosting/tests/test_hosting_log_sinks.py b/framework/hosting/tests/test_hosting_log_sinks.py index 2231f91ee..953721b48 100644 --- a/framework/hosting/tests/test_hosting_log_sinks.py +++ b/framework/hosting/tests/test_hosting_log_sinks.py @@ -186,3 +186,24 @@ def test_log_and_sentry_redact_identically(raw): assert redact(raw) == scrub_text(raw) assert not any(s in redact(raw) for s in SECRETS) + + +def test_structured_extras_are_scrubbed_recursively() -> None: + from simple_module_hosting.log_sinks import redact_record + + record = logging.makeLogRecord( + { + "msg": "x", + "payload": {"password": "hunter2", "nested": [{"api_key": "k1"}], "ok": "fine"}, + "pair": ("authorization", "Bearer abc"), + "opaque": object(), + } + ) + original = record.payload + redact_record(record) + dumped = repr((record.payload, record.pair)) + assert "hunter2" not in dumped + assert "k1" not in dumped + assert "abc" not in dumped + assert record.payload["ok"] == "fine" + assert original["password"] == "hunter2" # the caller's object is untouched diff --git a/framework/hosting/tests/test_hosting_sentry.py b/framework/hosting/tests/test_hosting_sentry.py index a07349f30..197321d8e 100644 --- a/framework/hosting/tests/test_hosting_sentry.py +++ b/framework/hosting/tests/test_hosting_sentry.py @@ -64,7 +64,7 @@ async def test_route_exception_is_tagged_and_scrubbed(transport: FakeTransport) base_url="http://test", ) as client: await client.post( - "/boom?token=s3cret&code=abc&keep=1", + "/boom?token=s3cret&code=oauthcode9&keep=1", headers={ "Authorization": "Bearer x", "Cookie": "sm=1", @@ -83,7 +83,7 @@ async def test_route_exception_is_tagged_and_scrubbed(transport: FakeTransport) request = event["request"] assert not {h.lower() for h in request.get("headers", {})} & set(_SECRET_HEADERS) dumped = str(event) - for leaked in ("s3cret", "abc", "hunter2", "Bearer x", "private@example.com"): + for leaked in ("s3cret", "oauthcode9", "hunter2", "Bearer x", "private@example.com"): assert leaked not in dumped assert "keep=1" in dumped From b26da3bb4af208a9ad6a33cacab95b3249dba66f Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 09:42:12 +0200 Subject: [PATCH 17/22] fix(background_tasks): stop the retained log sink re-logging its own writes; tolerate NULL task args Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- framework/db/simple_module_db/listeners.py | 17 +++++- .../simple_module_hosting/log_sinks.py | 12 +++- .../background_tasks/contracts/schemas.py | 22 ++++++- .../tests/test_bg_log_sink_loop.py | 58 +++++++++++++++++++ 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 modules/background_tasks/tests/test_bg_log_sink_loop.py diff --git a/framework/db/simple_module_db/listeners.py b/framework/db/simple_module_db/listeners.py index 569a98d64..8c409da9c 100644 --- a/framework/db/simple_module_db/listeners.py +++ b/framework/db/simple_module_db/listeners.py @@ -131,6 +131,14 @@ def attach_session_listeners(session_class: type[Session]) -> None: setattr(session_class, _LISTENERS_ATTACHED, True) +def _log_entity_change(obj: object, event_name: str, *, extra: dict[str, object]) -> None: + # Models flagged ``__audit_exclude__`` (telemetry such as retained logs) are + # not logged either: the log record would be retained and written again. + if getattr(obj, "__audit_exclude__", False): + return + _db_logger.info(event_name, extra=extra) + + def _before_flush_listener( session: Session, flush_context: object, @@ -147,7 +155,8 @@ def _before_flush_listener( if obj.updated_by is None: obj.updated_by = user_id - _db_logger.info( + _log_entity_change( + obj, _EVENT_ENTITY_CREATED, extra={ "operation": _OP_CREATE, @@ -167,7 +176,8 @@ def _before_flush_listener( if isinstance(obj, VersionedMixin): obj.version += 1 - _db_logger.info( + _log_entity_change( + obj, _EVENT_ENTITY_UPDATED, extra={ "operation": _OP_UPDATE, @@ -189,7 +199,8 @@ def _before_flush_listener( obj.deleted_by = user_id session.add(obj) - _db_logger.info( + _log_entity_change( + obj, _EVENT_ENTITY_SOFT_DELETED, extra={ "operation": _OP_SOFT_DELETE, diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index ef29338e1..334324511 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -13,8 +13,9 @@ * **Secrets are masked before queueing.** :func:`redact` runs on a copy of the record in the emitting thread, so neither the queue nor any sink ever holds an ``Authorization`` header, cookie, password, token or ``secret=`` value. -* **No feedback loop.** Records from this module's own logger and from - ``sqlalchemy.*`` (what a database sink itself triggers) are skipped. +* **No feedback loop.** Records from this module's own logger, from + ``sqlalchemy.*`` and any record emitted on the writer thread while a sink + is writing (what a database sink itself triggers) are skipped. """ from __future__ import annotations @@ -115,6 +116,13 @@ def __init__( self._lock = threading.Lock() self._thread: threading.Thread | None = None self.addFilter(self._not_own_record) + self.addFilter(self._not_from_writer_thread) + + def _not_from_writer_thread(self, record: logging.LogRecord) -> bool: + # Anything logged while the sink writes (entity-change logs from the + # insert itself, session or driver chatter) would be retained, written, + # and logged again: drop it at the source. + return threading.current_thread() is not self._thread @staticmethod def _not_own_record(record: logging.LogRecord) -> bool: diff --git a/modules/background_tasks/background_tasks/contracts/schemas.py b/modules/background_tasks/background_tasks/contracts/schemas.py index e94f6e351..c461a8a64 100644 --- a/modules/background_tasks/background_tasks/contracts/schemas.py +++ b/modules/background_tasks/background_tasks/contracts/schemas.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import Any -from pydantic import ConfigDict +from pydantic import ConfigDict, field_validator from sqlmodel import SQLModel from background_tasks.constants import TaskStatus @@ -39,6 +39,16 @@ class TaskExecutionListItem(SQLModel): exception_type: str | None = None retried_from_id: uuid.UUID | None = None + @field_validator("args", mode="before") + @classmethod + def _args_none(cls, v: Any) -> Any: + return [] if v is None else v + + @field_validator("kwargs", mode="before") + @classmethod + def _kwargs_none(cls, v: Any) -> Any: + return {} if v is None else v + class TaskExecutionDetail(SQLModel): """Full row for the detail page (adds args/kwargs/result/traceback).""" @@ -69,6 +79,16 @@ class TaskExecutionDetail(SQLModel): finished_at: datetime | None = None heartbeat_at: datetime | None = None + @field_validator("args", mode="before") + @classmethod + def _args_none(cls, v: Any) -> Any: + return [] if v is None else v + + @field_validator("kwargs", mode="before") + @classmethod + def _kwargs_none(cls, v: Any) -> Any: + return {} if v is None else v + class TaskStatusDTO(SQLModel): """What the owner of a task may see of it (#536). diff --git a/modules/background_tasks/tests/test_bg_log_sink_loop.py b/modules/background_tasks/tests/test_bg_log_sink_loop.py new file mode 100644 index 000000000..9db393591 --- /dev/null +++ b/modules/background_tasks/tests/test_bg_log_sink_loop.py @@ -0,0 +1,58 @@ +"""The retained log sink must not retain its own writes (BUG-001).""" + +from __future__ import annotations + +import logging +import time +import uuid +from collections.abc import Iterator + +import pytest +from background_tasks import log_sink, sync_db +from background_tasks.contracts.schemas import TaskExecutionDetail, TaskExecutionListItem +from background_tasks.models import RetainedLog +from background_tasks.settings import BackgroundTasksSettings +from sqlalchemy import func, select + +pytestmark = pytest.mark.usefixtures("sync_sqlite") + +APP_LOGGER = logging.getLogger("bg_log_sink_loop_test") + + +@pytest.fixture(autouse=True) +def _root_at_debug() -> Iterator[None]: + root = logging.getLogger() + before = root.level + root.setLevel(logging.DEBUG) + logging.getLogger("simple_module.db").setLevel(logging.INFO) + yield + log_sink.uninstall_log_sink() + root.setLevel(before) + + +def _count() -> int: + with sync_db.get_sync_session_factory()() as session: + return session.execute(select(func.count()).select_from(RetainedLog)).scalar_one() + + +def test_sink_writes_are_not_retained_again(): + handler = log_sink.install_log_sink(BackgroundTasksSettings(log_sink_enabled=True)) + assert handler is not None + for i in range(5): + APP_LOGGER.info("line %d", i) + time.sleep(1.5) # intentional-sleep: absence of feedback growth can only be observed over time + first = _count() + time.sleep(1.5) # intentional-sleep: second observation window + assert _count() == first + log_sink.uninstall_log_sink() + with sync_db.get_sync_session_factory()() as session: + loggers = set(session.execute(select(RetainedLog.logger)).scalars()) + assert APP_LOGGER.name in loggers + assert "simple_module.db" not in loggers + + +def test_null_args_and_kwargs_render(): + row = {"id": uuid.uuid4(), "task_name": "t", "status": "pending", "queue": "q", "retries": 0} + assert TaskExecutionListItem(**row, args=None, kwargs=None).kwargs == {} + detail = TaskExecutionDetail(**row, args=None, kwargs=None) + assert detail.args == [] and detail.kwargs == {} From 9083465f9b5fe424fa60bd3a1777cf017f0c2b9b Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 09:51:47 +0200 Subject: [PATCH 18/22] Log sink: writer-thread filter via thread-local (survives stop); CHANGELOG for audit_exclude log skip Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 4 ++++ framework/hosting/simple_module_hosting/log_sinks.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd1bffda..182634a3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,10 @@ All notable changes to this project are documented in this file. The format is b it, which removes the `users` ↔ `audit_log` import cycle. ### Changed + +- Behaviour change: `simple_module_db` no longer emits the `entity.created` / `entity.updated` / + `entity.soft_deleted` log lines for models flagged `__audit_exclude__ = True` (`AuditEntry`, + `OutboxMessage`, `TaskExecutionLog`-style telemetry), so retained logs cannot re-log their own writes. - `scripts/bump_version.py` also bumps every `modules/*/package.json`; all `package.json` versions are now in lockstep with the Python packages (they were 0.1.0, keycloak 0.0.0) (#510). diff --git a/framework/hosting/simple_module_hosting/log_sinks.py b/framework/hosting/simple_module_hosting/log_sinks.py index 334324511..d6f6ee9fe 100644 --- a/framework/hosting/simple_module_hosting/log_sinks.py +++ b/framework/hosting/simple_module_hosting/log_sinks.py @@ -115,6 +115,7 @@ def __init__( self._failed_batches = 0 self._lock = threading.Lock() self._thread: threading.Thread | None = None + self._writer = threading.local() self.addFilter(self._not_own_record) self.addFilter(self._not_from_writer_thread) @@ -122,7 +123,7 @@ def _not_from_writer_thread(self, record: logging.LogRecord) -> bool: # Anything logged while the sink writes (entity-change logs from the # insert itself, session or driver chatter) would be retained, written, # and logged again: drop it at the source. - return threading.current_thread() is not self._thread + return not getattr(self._writer, "active", False) @staticmethod def _not_own_record(record: logging.LogRecord) -> bool: @@ -177,6 +178,7 @@ def stop(self) -> None: thread.join(STOP_TIMEOUT_SECONDS) def _run(self) -> None: + self._writer.active = True stopping = False while not stopping: batch: list[logging.LogRecord] = [] From 9b959c707f2d8f5b45d7c1be6c991273993a85ae Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 09:56:50 +0200 Subject: [PATCH 19/22] Review fixes: audit_exclude check matches audit.py, restore db logger level in sink test, CHANGELOG model name Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 2 +- framework/db/simple_module_db/listeners.py | 2 +- modules/background_tasks/tests/test_bg_log_sink_loop.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 486acff08..3bc576d34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,7 +131,7 @@ All notable changes to this project are documented in this file. The format is b ### Changed - **Behaviour change:** `simple_module_db` no longer emits the `entity.created` / `entity.updated` / `entity.soft_deleted` log lines for models flagged `__audit_exclude__ = True` (`AuditEntry`, - `OutboxMessage`, `TaskExecutionLog`-style telemetry), so retained logs cannot re-log their own writes (#537). + `OutboxMessage`, `RetainedLog`), so retained logs cannot re-log their own writes (#537). - **Behaviour change:** `rate_limit_authenticated` (off by default) now keeps one budget per signed-in user instead of one per client IP, so users behind one NAT no longer share it (#529). - `scripts/bump_version.py` also bumps every `modules/*/package.json`; all diff --git a/framework/db/simple_module_db/listeners.py b/framework/db/simple_module_db/listeners.py index 8c409da9c..14d0095fe 100644 --- a/framework/db/simple_module_db/listeners.py +++ b/framework/db/simple_module_db/listeners.py @@ -134,7 +134,7 @@ def attach_session_listeners(session_class: type[Session]) -> None: def _log_entity_change(obj: object, event_name: str, *, extra: dict[str, object]) -> None: # Models flagged ``__audit_exclude__`` (telemetry such as retained logs) are # not logged either: the log record would be retained and written again. - if getattr(obj, "__audit_exclude__", False): + if getattr(obj, "__audit_exclude__", False) is True: return _db_logger.info(event_name, extra=extra) diff --git a/modules/background_tasks/tests/test_bg_log_sink_loop.py b/modules/background_tasks/tests/test_bg_log_sink_loop.py index 9db393591..3f09a85de 100644 --- a/modules/background_tasks/tests/test_bg_log_sink_loop.py +++ b/modules/background_tasks/tests/test_bg_log_sink_loop.py @@ -22,12 +22,14 @@ @pytest.fixture(autouse=True) def _root_at_debug() -> Iterator[None]: root = logging.getLogger() - before = root.level + db_logger = logging.getLogger("simple_module.db") + before, db_before = root.level, db_logger.level root.setLevel(logging.DEBUG) - logging.getLogger("simple_module.db").setLevel(logging.INFO) + db_logger.setLevel(logging.INFO) yield log_sink.uninstall_log_sink() root.setLevel(before) + db_logger.setLevel(db_before) def _count() -> int: From 7ab7a382444570434f454421db4263fe858a088f Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 10:05:54 +0200 Subject: [PATCH 20/22] fix(background_tasks): mark owner-scoped task status route Authenticated() for SM031 Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- CHANGELOG.md | 1 + .../background_tasks/endpoints/api_status.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bc576d34..8c66a8a7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project are documented in this file. The format is b (buffered under the existing 30 s throttle, `i == n` writes at once, never audited), and `GET /api/background_tasks/tasks/{celery_task_id}/status` lets the user who enqueued a task read its status and progress without `background_tasks.view` + (marked `Authenticated()`; tokens are refused) (owner in the active tenant, or `view` holders; anyone else gets 404; never returns args, result or traceback). `TaskExecution` gains `progress_*` and an indexed `owner_user_id` stamped from the new `sm_owner_id` message header; diff --git a/modules/background_tasks/background_tasks/endpoints/api_status.py b/modules/background_tasks/background_tasks/endpoints/api_status.py index 909c97fe3..f5a2f27f5 100644 --- a/modules/background_tasks/background_tasks/endpoints/api_status.py +++ b/modules/background_tasks/background_tasks/endpoints/api_status.py @@ -1,9 +1,10 @@ """Owner-scoped task status (#536). Mounted apart from the admin router because it carries no ``PERM_VIEW`` gate: -the person who started a job may follow it without being a task operator. The -check lives in the handler instead, and every refusal is a 404 so the route -does not reveal which task ids exist. +the person who started a job may follow it without being a task operator. It +is marked ``Authenticated()`` (SM031); the ownership check lives in the +handler, and every refusal is a 404 so the route does not reveal which task +ids exist. """ from __future__ import annotations @@ -15,6 +16,7 @@ from simple_module_db import RequestSession, current_tenant_id from simple_module_db.deps import get_db from simple_module_hosting.permissions import ensure_resolved_permissions +from simple_module_hosting.token_guard import Authenticated from sqlalchemy import select from background_tasks.constants import MODULE_NAME, PERM_VIEW, STATUS_ROUTER_PREFIX @@ -38,7 +40,11 @@ async def _visible(request: Request, row: TaskExecution, user_id: str) -> bool: return tenant_id is not None and row.tenant_id == tenant_id -@router.get("/{celery_task_id}/status", response_model=TaskStatusDTO) +@router.get( + "/{celery_task_id}/status", + response_model=TaskStatusDTO, + dependencies=[Depends(Authenticated())], +) async def get_task_status( celery_task_id: str, request: Request, From a3a73d90e113ad41c5975f2cab12905e6e2fc389 Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 10:23:25 +0200 Subject: [PATCH 21/22] test(leases): scope Redis cleanup to this worker's keys CI's xdist workers share one Redis service; the teardown pattern sm:*:t-* deleted another worker's live semaphore, letting all 8 waiters in. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- framework/hosting/tests/test_hosting_lease_backends.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/framework/hosting/tests/test_hosting_lease_backends.py b/framework/hosting/tests/test_hosting_lease_backends.py index f260f6964..7db4aaebe 100644 --- a/framework/hosting/tests/test_hosting_lease_backends.py +++ b/framework/hosting/tests/test_hosting_lease_backends.py @@ -35,14 +35,19 @@ async def backend(request: pytest.FixtureRequest, engine) -> AsyncIterator[Lease redis_backend = RedisLeaseBackend(url) yield redis_backend client = redis_backend._redis() - keys = [k async for k in client.scan_iter(match="sm:*:t-*")] + # Only this process's names: CI's xdist workers share one Redis, and a + # broader pattern deleted another worker's live semaphore mid-test. + keys = [k async for k in client.scan_iter(match=f"sm:*:{_PREFIX}*")] if keys: await client.delete(*keys) await client.aclose() +_PREFIX = f"t-{uuid.uuid4().hex[:12]}-" + + def _name() -> str: - return f"t-{uuid.uuid4().hex}" + return f"{_PREFIX}{uuid.uuid4().hex}" async def test_two_racing_acquirers_one_wins(backend: LeaseBackend) -> None: From 24bef27aab17ddb143e26e25bd5fb689879e581e Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Thu, 22 Oct 2026 11:07:51 +0200 Subject: [PATCH 22/22] fix(background_tasks): exception-type filter rejects a trailing newline re.match with $ accepted "ValueError\n"; use fullmatch. The too-long test case could not be stored on Postgres (varchar(255)); replace it with newline cases that can. Claude-Session: https://claude.ai/code/session_01M9neheZZEe3sVpDi2S3zT4 --- .../background_tasks/endpoints/api_status.py | 8 ++++---- modules/background_tasks/tests/test_bg_task_status_api.py | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/background_tasks/background_tasks/endpoints/api_status.py b/modules/background_tasks/background_tasks/endpoints/api_status.py index f5a2f27f5..c95bc55a0 100644 --- a/modules/background_tasks/background_tasks/endpoints/api_status.py +++ b/modules/background_tasks/background_tasks/endpoints/api_status.py @@ -25,7 +25,7 @@ router = APIRouter(prefix=STATUS_ROUTER_PREFIX, tags=[f"{MODULE_NAME}-status"]) -_TYPE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.]{0,254}$") +_TYPE_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_.]{0,254}") _NOT_FOUND = "Task not found" @@ -63,7 +63,7 @@ async def get_task_status( ).scalar_one_or_none() if row is None or not await _visible(request, row, str(user.id)): raise HTTPException(status_code=404, detail=_NOT_FOUND) - error = ( - row.exception_type if row.exception_type and _TYPE_NAME.match(row.exception_type) else None - ) + error = row.exception_type + if error and not _TYPE_NAME.fullmatch(error): + error = None return TaskStatusDTO.model_validate(row, update={"exception_type": error}) diff --git a/modules/background_tasks/tests/test_bg_task_status_api.py b/modules/background_tasks/tests/test_bg_task_status_api.py index 398b4d0a0..b0ddfc95d 100644 --- a/modules/background_tasks/tests/test_bg_task_status_api.py +++ b/modules/background_tasks/tests/test_bg_task_status_api.py @@ -102,7 +102,10 @@ async def test_the_error_is_a_type_name_never_a_traceback( "bad", [ pytest.param("Boom: leaked message", id="message"), - pytest.param("x" * 300, id="too-long"), + # The column is varchar(255), the same cap as the pattern, so a stored + # value is never too long; a trailing newline is what can leak. + pytest.param("ValueError\nsecret=hunter2", id="multiline"), + pytest.param("ValueError\n", id="trailing-newline"), ], ) async def test_a_malformed_exception_type_is_dropped(