diff --git a/doc/observability/observability.md b/doc/observability/observability.md index a9f8b832..9dcc4d4f 100644 --- a/doc/observability/observability.md +++ b/doc/observability/observability.md @@ -152,6 +152,30 @@ with tracer.span("downstream_step", context=ctx) as span: Call `tracer.shutdown()` (done automatically by `Xcore.shutdown()`) before process exit — otherwise spans still sitting in the `BatchSpanProcessor` buffer are dropped, never exported. +### What each layer costs per call + +Measured on a trivial trusted plugin (best of 7 rounds × 2000 calls, Python 3.14, 8 cores — see +`scripts/profile_runtime.py`): + +| Layer | p50 | Delta | +|---|---|---| +| none | 38.5 µs | — | +| logging at DEBUG | 40.0 µs | +1.5 µs | +| metrics (Prometheus) | 43.4 µs | +5 µs | +| tracing, OTel SDK with a no-op exporter | 95.5 µs | **+57 µs (×2.5)** | +| tracing, OTel console exporter | 181 µs | +143 µs | + +Logging and metrics are cheap. **The OTel span is the only significant cost on the call path** — budget +for it before enabling tracing on a hot service, and prefer sampling at the collector. + +> **The console exporter is for development only.** It uses a synchronous `SimpleSpanProcessor`, so +> every span is written out on the call path (+143 µs per call measured above). In production, set an +> `endpoint`: the OTLP exporter batches. Not measured here: OTLP + `BatchSpanProcessor` under load. + +The in-memory history behind `tracer.export()` is capped by `observability.tracing.max_local_spans` +(default 1000, `0` to keep nothing). It is a debugging aid, not an export pipeline — it keeps the most +recent spans and nothing older. + --- ## 4. Health Checks diff --git a/tests/unit/kernel/test_tracing_local_history.py b/tests/unit/kernel/test_tracing_local_history.py new file mode 100644 index 00000000..7e33af8a --- /dev/null +++ b/tests/unit/kernel/test_tracing_local_history.py @@ -0,0 +1,68 @@ +"""The local span history is bounded (issue #305, memory leak). + +`TracingMiddleware` wraps every plugin call, so `Tracer.span()` is on the hot path even when tracing +is disabled. `Tracer._spans` was a plain list that nothing ever trimmed, so a `Span` leaked per call: +~562 B/call measured, about 4.9 GB/day per worker at 100 req/s. + +What these tests defend: +- the history stops growing past its bound, whatever the number of calls; +- `export()` keeps working (it is the introspection/debug use case the history exists for) and shows + the MOST RECENT spans, not the first ones; +- the bound is configurable, including `0` (keep nothing), and an absurd setting cannot turn the + bound off — `deque(maxlen=None)` would be an unbounded list again, i.e. the leak. +""" + +import pytest + +from xcore.configurations.sections import TracingConfig +from xcore.kernel.observability.tracing import DEFAULT_MAX_LOCAL_SPANS, Tracer + + +def _run(tracer, count, name="op"): + for i in range(count): + with tracer.span(f"{name}-{i}"): + pass + + +class TestLocalHistoryIsBounded: + def test_history_stops_growing_past_the_bound(self): + tracer = Tracer("svc", TracingConfig(max_local_spans=50)) + _run(tracer, 5_000) + assert len(tracer._spans) == 50 + + def test_default_bound_applies_without_any_configuration(self): + tracer = Tracer("svc") + _run(tracer, DEFAULT_MAX_LOCAL_SPANS * 2) + assert len(tracer._spans) == DEFAULT_MAX_LOCAL_SPANS + + def test_export_keeps_working_and_shows_the_most_recent_spans(self): + tracer = Tracer("svc", TracingConfig(max_local_spans=3)) + _run(tracer, 10) + exported = tracer.export() + assert [s["name"] for s in exported] == ["op-7", "op-8", "op-9"] + assert all( + s["duration_ms"] is not None and s["status"] == "ok" for s in exported + ) + + def test_zero_keeps_nothing_at_all(self): + tracer = Tracer("svc", TracingConfig(max_local_spans=0)) + _run(tracer, 100) + assert tracer.export() == [] + + @pytest.mark.parametrize("value", [-1, None, "beaucoup", True, 2.5]) + def test_an_absurd_setting_never_removes_the_bound(self, value): + """`deque(maxlen=None)` is an unbounded list — the leak itself. A bad setting must not reach it.""" + tracer = Tracer("svc", TracingConfig(max_local_spans=value)) + assert tracer._spans.maxlen is not None + _run(tracer, DEFAULT_MAX_LOCAL_SPANS + 100) + assert len(tracer._spans) <= DEFAULT_MAX_LOCAL_SPANS + + def test_a_span_still_carries_what_callers_read_from_it(self): + """The bound must not change the span API: a caller sets attributes and reads ids/duration.""" + tracer = Tracer("svc", TracingConfig(max_local_spans=10)) + with tracer.span("op", tenant="acme") as span: + span.set_attribute("rows", 3) + assert span.attributes == {"tenant": "acme", "rows": 3} + assert len(span.trace_id) == 32 and len(span.span_id) == 16 + assert span.duration_ms is not None + assert tracer.export()[-1]["attributes"] == {"tenant": "acme", "rows": 3} diff --git a/tests/unit/kernel/test_yamlio.py b/tests/unit/kernel/test_yamlio.py new file mode 100644 index 00000000..2b7fd104 --- /dev/null +++ b/tests/unit/kernel/test_yamlio.py @@ -0,0 +1,72 @@ +"""The kernel reads YAML through the C loader when it is available (issue #305, point 4). + +`yaml.safe_load` uses the pure-Python loader even when PyYAML is built against libyaml. The kernel +reads one manifest per plugin at boot, and the sandbox worker re-reads its own at every recycle, so +this is pure startup time: measured 3.5 ms -> 0.37 ms on a real manifest. + +What these tests defend: +- `yamlio.safe_load` parses exactly what `yaml.safe_load` parses (same result, same types); +- it is just as SAFE: an arbitrary-object tag is refused, like `SafeLoader` (the whole point of not + reaching for `yaml.load` with the full loader); +- it works whether or not the C loader is present, so an installation without libyaml is slower and + nothing more. +""" + +import io + +import pytest +import yaml + +from xcore.kernel import yamlio + +DOC = """ +name: te +version: 1.0.0 +enabled: true +retries: 3 +ratio: 1.5 +nothing: null +permissions: + - resource: "db.*" + actions: [read, write] +tags: {a: 1, b: [2, 3]} +""" + + +class TestParity: + def test_it_parses_what_safe_load_parses(self): + assert yamlio.safe_load(DOC) == yaml.safe_load(DOC) + + def test_types_are_the_same_not_just_the_shape(self): + parsed = yamlio.safe_load(DOC) + assert parsed["enabled"] is True + assert isinstance(parsed["retries"], int) and isinstance(parsed["ratio"], float) + assert parsed["nothing"] is None + assert parsed["tags"] == {"a": 1, "b": [2, 3]} + + def test_it_reads_a_stream_as_well_as_a_string(self): + assert yamlio.safe_load(io.StringIO(DOC)) == yaml.safe_load(DOC) + + @pytest.mark.parametrize("empty", ["", "\n", "---\n"]) + def test_an_empty_document_is_none_as_everywhere(self, empty): + assert yamlio.safe_load(empty) is None + + +class TestSafety: + def test_an_arbitrary_object_tag_is_refused(self): + """Same safety as SafeLoader: a manifest must never be able to build a Python object.""" + with pytest.raises(yaml.YAMLError): + yamlio.safe_load("!!python/object/apply:os.system ['echo nope']") + + def test_a_syntax_error_is_a_yaml_error_not_a_crash(self): + with pytest.raises(yaml.YAMLError): + yamlio.safe_load("a: [1, 2\nb: 3") + + +def test_the_fast_flag_says_which_loader_is_in_use(): + """Informational, and it is what a diagnostic reads: it must stay a boolean, never raise.""" + assert isinstance(yamlio.FAST, bool) + if yaml.__with_libyaml__: # pragma: no branch - dépend de l'installation + assert ( + yamlio.FAST + ), "PyYAML est compilé avec libyaml : le loader C devrait être pris" diff --git a/tests/unit/sdk/test_lazy_imports.py b/tests/unit/sdk/test_lazy_imports.py new file mode 100644 index 00000000..d29ea273 --- /dev/null +++ b/tests/unit/sdk/test_lazy_imports.py @@ -0,0 +1,74 @@ +"""`import xcore` does not pay for celery or SQLAlchemy (issues #305 point 2, #306). + +`xcore/sdk/__init__.py` imported `..services.xworker` (celery) eagerly, and the external `xcoresdk` +package (which drags SQLAlchemy through its auth module). Both were paid by every application that +does `import xcore` — and by EVERY sandbox worker, at every recycle, even with no worker service and +no database configured. Measured: `import xcore` 594 ms -> 294 ms once both are lazy. + +What these tests defend, each in a FRESH interpreter (`sys.modules` is global, so an earlier test +importing celery would hide the regression): +- after `import xcore`, neither celery nor sqlalchemy is in `sys.modules`; +- the names are still reachable, and `from xcore.sdk import *` still exposes them; +- the first access does the import, so nothing is silently broken for the plugins that use them. +""" + +import subprocess +import sys + +import pytest + + +def _run(code: str) -> str: + """Run `code` in a fresh interpreter and return its stdout (fails the test on a non-zero exit).""" + done = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) + assert done.returncode == 0, done.stderr + return done.stdout.strip() + + +class TestNothingHeavyAtImportTime: + @pytest.mark.parametrize("heavy", ["celery", "sqlalchemy"]) + def test_import_xcore_does_not_import_it(self, heavy): + loaded = _run(f"import sys, xcore; print({heavy!r} in sys.modules)") + assert loaded == "False" + + @pytest.mark.parametrize("heavy", ["celery", "sqlalchemy"]) + def test_importing_the_sdk_itself_does_not_import_it_either(self, heavy): + loaded = _run(f"import sys, xcore.sdk; print({heavy!r} in sys.modules)") + assert loaded == "False" + + +class TestTheNamesStillWork: + def test_the_worker_service_resolves_and_only_then_pulls_celery(self): + out = _run( + "import sys, xcore.sdk as s\n" + "before = 'celery' in sys.modules\n" + "name = s.WorkerService.__name__\n" + "print(before, name, 'celery' in sys.modules)" + ) + assert out == "False WorkerService True" + + def test_task_and_task_registry_resolve(self): + out = _run( + "from xcore.sdk import task, task_registry\n" + "print(callable(task), task_registry is not None)" + ) + assert out == "True True" + + def test_a_star_import_still_exposes_the_lazy_names(self): + """`__all__` must not shrink: a plugin doing `from xcore.sdk import *` kept working.""" + out = _run( + "import xcore.sdk as s\n" + "names = set(s.__all__)\n" + "print(all(n in names for n in ('WorkerService', 'task', 'task_registry')))" + ) + assert out == "True" + + def test_a_name_that_does_not_exist_still_raises_attribute_error(self): + out = _run( + "import xcore.sdk as s\n" + "try:\n" + " s.PasDuTout\n" + "except AttributeError as e:\n" + " print('AttributeError', 'PasDuTout' in str(e))" + ) + assert out == "AttributeError True" diff --git a/tests/unit/security/test_ast_scanner.py b/tests/unit/security/test_ast_scanner.py index 980d5561..51529a8e 100644 --- a/tests/unit/security/test_ast_scanner.py +++ b/tests/unit/security/test_ast_scanner.py @@ -388,8 +388,13 @@ def test_inject_dotenv_path_traversal(self, validator, tmp_path): assert "traversal" in str(exc_info.value) def test_yaml_import_error(self, validator, tmp_path): - """Test _yaml raises error when pyyaml not installed.""" - with patch.dict("sys.modules", {"yaml": None}): + """Test _yaml raises error when pyyaml not installed. + + `_yaml` passe par `xcore.kernel.yamlio` (loader C quand il est là, #305) : c'est donc + CE module qu'il faut rendre introuvable pour simuler l'absence de PyYAML — rendre + `yaml` seul inutilisable ne change rien quand `yamlio` est déjà importé. + """ + with patch.dict("sys.modules", {"yaml": None, "xcore.kernel.yamlio": None}): (tmp_path / "test.yaml").touch() with pytest.raises(ManifestError) as exc_info: ManifestValidator._yaml(tmp_path / "test.yaml") diff --git a/tests/unit/security/test_scanner_fails_closed.py b/tests/unit/security/test_scanner_fails_closed.py new file mode 100644 index 00000000..2efd8e09 --- /dev/null +++ b/tests/unit/security/test_scanner_fails_closed.py @@ -0,0 +1,61 @@ +"""A security scanner must fail CLOSED: a file it cannot parse is not a file it found clean. + +`ASTScanner.scan` has two import phases: the compiled `scanner_core` extension when it is built, the +Python AST otherwise. Only the Python one reported a syntax error — so with the compiled scanner +installed (the optimised, shipped configuration) a plugin whose source does not parse came back +`passed=True`, and none of its imports, builtins or forbidden attributes had been examined. + +`_check_builtins_and_attrs` parses every file in BOTH phases, so that is where the parse failure is +now reported, and only when the import phase could not do it (no duplicate message). +""" + +from pathlib import Path + +import pytest + +from xcore.kernel.security.validation import ASTScanner + + +def _plugin(tmp_path: Path, source: str) -> Path: + src = tmp_path / "src" + src.mkdir() + (src / "main.py").write_text(source) + return tmp_path + + +class TestUnparsableIsRefused: + def test_a_file_that_does_not_parse_fails_the_scan(self, tmp_path): + result = ASTScanner().scan(_plugin(tmp_path, "def broken(\n pass")) + assert result.passed is False + assert any("syntaxe" in e for e in result.errors) + + def test_it_is_said_once_not_twice(self, tmp_path): + """Both phases parse the file; the operator must read one message, not two.""" + result = ASTScanner().scan(_plugin(tmp_path, "def broken(\n pass")) + assert len([e for e in result.errors if "syntaxe" in e]) == 1 + + def test_the_verdict_does_not_depend_on_the_compiled_scanner( + self, tmp_path, monkeypatch + ): + """The whole point: the C++ path and the Python path must agree on an unparsable file.""" + plugin = _plugin(tmp_path, "def broken(\n pass") + with_cpp = ASTScanner() + without_cpp = ASTScanner() + monkeypatch.setattr(without_cpp, "_cpp", None) + assert with_cpp.scan(plugin).passed is without_cpp.scan(plugin).passed is False + + def test_a_sound_file_still_passes(self, tmp_path): + result = ASTScanner().scan( + _plugin(tmp_path, "import json\n\nVALUE = json.dumps({'a': 1})\n") + ) + assert result.passed is True, result.errors + assert result.errors == [] + + @pytest.mark.parametrize("source", ["eval('1+1')", "exec('x = 1')"]) + def test_a_forbidden_builtin_in_a_sound_file_is_still_caught( + self, tmp_path, source + ): + """The regression to avoid: reporting parse errors must not shadow the real checks.""" + result = ASTScanner().scan(_plugin(tmp_path, source)) + assert result.passed is False + assert any("built-in interdit" in e for e in result.errors) diff --git a/xcore/configurations/loader.py b/xcore/configurations/loader.py index b2a40d4e..d25db942 100644 --- a/xcore/configurations/loader.py +++ b/xcore/configurations/loader.py @@ -74,10 +74,10 @@ def _read(cls, path: str | Path | None) -> dict[str, Any]: suffix = candidate.suffix.lower() try: if suffix in (".yaml", ".yml"): - import yaml + from ..kernel import yamlio with open(candidate, encoding="utf-8") as f: - data = yaml.safe_load(f) or {} + data = yamlio.safe_load(f) or {} else: import json diff --git a/xcore/configurations/sections.py b/xcore/configurations/sections.py index 583b27f1..7410b64a 100644 --- a/xcore/configurations/sections.py +++ b/xcore/configurations/sections.py @@ -315,6 +315,11 @@ class TracingConfig: backend: str = "noop" # noop | opentelemetry | jaeger service_name: str = "xcore" endpoint: str | None = None + # Nombre de spans gardés EN MÉMOIRE pour l'introspection (`Tracer.export()`), les plus + # récents. Ce n'est pas le pipeline d'export OTel : c'est un historique de débogage, et + # il doit être borné — une liste sans limite faisait ~562 octets par appel, soit près de + # 5 Go par jour et par worker à 100 req/s (#305). 0 = ne rien garder. + max_local_spans: int = 1000 @dataclass diff --git a/xcore/kernel/observability/tracing.py b/xcore/kernel/observability/tracing.py index 7f09aa00..8e14d795 100644 --- a/xcore/kernel/observability/tracing.py +++ b/xcore/kernel/observability/tracing.py @@ -12,6 +12,7 @@ import time import uuid +from collections import deque from contextlib import contextmanager from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Generator, Mapping, MutableMapping @@ -40,6 +41,19 @@ _provider_configured = False +DEFAULT_MAX_LOCAL_SPANS = 1000 + + +def _max_local_spans(config: "TracingConfig | None") -> int: + """La taille de l'historique local, prise dans la configuration et bornée à zéro : un réglage + négatif ou aberrant ne doit pas rendre `deque(maxlen=…)` illimité (`maxlen=None`). + """ + value = getattr(config, "max_local_spans", DEFAULT_MAX_LOCAL_SPANS) + if isinstance(value, bool) or not isinstance(value, int): + return DEFAULT_MAX_LOCAL_SPANS + return max(value, 0) + + @dataclass class Span: """ @@ -134,7 +148,11 @@ def __init__( config: "TracingConfig | None" = None, ) -> None: self.service_name = service_name - self._spans: list[Span] = [] + # BORNÉ : `span()` est traversé par chaque appel de plugin (TracingMiddleware enveloppe + # tout, même tracing désactivé). Une liste sans limite gardait donc un Span par appel + # pour toujours — une fuite mesurée à ~562 octets par appel (#305). Un deque garde les + # N plus récents, ce qui est tout ce que `export()` sert à montrer. + self._spans: deque[Span] = deque(maxlen=_max_local_spans(config)) self._otel_tracer = None if ( diff --git a/xcore/kernel/sandbox/worker.py b/xcore/kernel/sandbox/worker.py index 23e443b2..79b29fab 100644 --- a/xcore/kernel/sandbox/worker.py +++ b/xcore/kernel/sandbox/worker.py @@ -801,10 +801,10 @@ def _load_manifest(plugin_dir: Path) -> _PluginManifest: def _parse_manifest_file(fname, manifest_path, manifest: _PluginManifest): if fname.endswith(".yaml"): - import yaml + from xcore.kernel import yamlio with open(manifest_path, encoding="utf-8") as f: - raw = yaml.safe_load(f) or {} + raw = yamlio.safe_load(f) or {} else: import json as _json diff --git a/xcore/kernel/security/validation.py b/xcore/kernel/security/validation.py index 608f0f49..17daaa07 100644 --- a/xcore/kernel/security/validation.py +++ b/xcore/kernel/security/validation.py @@ -156,10 +156,10 @@ def _read_raw(self, plugin_dir: Path) -> dict: @staticmethod def _yaml(path: Path) -> dict: try: - import yaml # type: ignore + from ..yamlio import safe_load # type: ignore with open(path, encoding="utf-8") as f: - return yaml.safe_load(f) or {} + return safe_load(f) or {} except ImportError as e: raise ManifestError("pyyaml non installé") from e @@ -445,8 +445,13 @@ def scan( scanner._scan_imports_python(py_file, result, local_modules) # ── Vérification builtins/attributs (Python AST) ───── + # Le scanner C++ ne rapporte pas les erreurs de syntaxe : dans ce cas c'est cette phase, + # qui parse de toute façon, qui les dit (sans quoi un fichier illisible passait le scan). + used_cpp = scanner._cpp is not None for py_file in py_files: - _check_builtins_and_attrs(py_file, result, scanner._forbidden) + _check_builtins_and_attrs( + py_file, result, scanner._forbidden, report_unparsable=used_cpp + ) result.scanned.append(str(py_file.relative_to(plugin_dir))) return result @@ -548,14 +553,32 @@ def _collect_local_modules(src_dir: Path) -> set[str]: def _check_builtins_and_attrs( - path: Path, result: ScanResult, forbidden: set[str] + path: Path, + result: ScanResult, + forbidden: set[str], + *, + report_unparsable: bool = False, ) -> None: - """Vérifie les builtins dangereux et attributs interdits (nécessite l'AST).""" + """Vérifie les builtins dangereux et attributs interdits (nécessite l'AST). + + `report_unparsable` : signaler ici un fichier qui ne se parse pas. Il le faut quand la phase + d'imports est passée par le scanner C++, qui ne rapporte PAS les erreurs de syntaxe — sinon un + fichier illisible traversait tout le scan sans un mot et le plugin était déclaré conforme, alors + qu'aucun de ses imports ni de ses builtins n'avait pu être examiné. Un scanner de sécurité doit + échouer FERMÉ : ne pas savoir lire un fichier n'est pas la même chose que l'avoir trouvé sain. + Avec le repli Python, `_scan_imports_python` l'a déjà dit : on ne le répète pas. + """ try: source = path.read_text(encoding="utf-8") tree = ast.parse(source, filename=str(path)) - except Exception: - return # déjà signalé par la phase import + except SyntaxError as e: + if report_unparsable: + result.add_error(f"{path}: erreur de syntaxe : {e}") + return + except Exception as e: + if report_unparsable: + result.add_error(f"{path}: lecture impossible : {e}") + return for node in ast.walk(tree): if isinstance(node, ast.Name): diff --git a/xcore/kernel/yamlio.py b/xcore/kernel/yamlio.py new file mode 100644 index 00000000..4f7afe7a --- /dev/null +++ b/xcore/kernel/yamlio.py @@ -0,0 +1,37 @@ +""" +yamlio.py — la lecture YAML du noyau, avec le loader C quand il est là (#305, point 4). + +`yaml.safe_load` passe par le loader écrit en PYTHON, même quand PyYAML est compilé avec libyaml +(`yaml.__with_libyaml__`). Sur un manifeste de plugin réel, mesuré : **3,5 ms → 0,37 ms (×9,5)** +en passant à `CSafeLoader`. Le noyau lit un manifeste par plugin au démarrage, et le worker sandbox +relit le sien à chaque recyclage : c'est du temps de démarrage pur. + +`CSafeLoader` a exactement la sûreté de `SafeLoader` (aucune construction d'objet arbitraire) ; la +seule différence est l'implémentation. Il n'est pas garanti présent (roue sans libyaml, PyYAML +installé depuis les sources) : on retombe alors sur `SafeLoader`, sans rien dire — ce n'est pas une +anomalie, juste une installation plus lente. + +Module FEUILLE, sans aucun import d'xcore : le worker sandbox doit garder son jeu d'imports minimal +(#306), et `xcore/kernel/__init__.py` est vide. +""" + +from __future__ import annotations + +from typing import IO, Any + +import yaml + +try: # pragma: no cover - dépend de la façon dont PyYAML a été installé + from yaml import CSafeLoader as _Loader +except ImportError: # pragma: no cover + from yaml import SafeLoader as _Loader # type: ignore[assignment] + +#: Vrai quand le loader C est utilisé (lisible par un test ou un diagnostic). +FAST = _Loader.__name__ == "CSafeLoader" + +__all__ = ["FAST", "safe_load"] + + +def safe_load(stream: str | bytes | IO[str] | IO[bytes]) -> Any: + """Comme `yaml.safe_load`, mais par le loader C quand il est disponible.""" + return yaml.load(stream, Loader=_Loader) diff --git a/xcore/sdk/__init__.py b/xcore/sdk/__init__.py index 5fdc2e56..f4b621b9 100644 --- a/xcore/sdk/__init__.py +++ b/xcore/sdk/__init__.py @@ -39,7 +39,6 @@ from ..kernel.observability import get_logger from ..kernel.permissions.engine import PermissionDenied from ..kernel.runtime.state_machine import PluginState -from ..services.xworker import WorkerService, task, task_registry from .decorators import ( RoutedPlugin, action, @@ -116,52 +115,57 @@ "task_registry", ] -# ── Adaptateurs SQL (extra `db`) ─────────────────────────────────────────── -# SQLAlchemy est une dépendance OPTIONNELLE depuis la 2.7.0 (XCoreRuntime[db], -# [postgres], [sqlite]). Importer ces deux modules à l'import de `xcore.sdk` -# faisait planter `import xcore` — et donc le boot — sur un simple -# `pip install XCoreRuntime`. Ils sont donc résolus à la demande (PEP 562) : -# `from xcore.sdk import BaseAsyncRepository` fonctionne comme avant quand -# SQLAlchemy est installé, et lève une ImportError explicite sinon. +# ── Ce qui ne s'importe qu'à la demande (PEP 562) ────────────────────────── +# +# Trois familles de noms que ce namespace expose SANS les importer à l'import : +# +# - les adaptateurs SQL : SQLAlchemy est optionnel depuis la 2.7.0 +# (XCoreRuntime[db], [postgres], [sqlite]) et les importer faisait planter +# `import xcore` — donc le boot — sur un simple `pip install XCoreRuntime` ; +# - le service de tâches (`WorkerService`, `task`, `task_registry`) : il tire +# CELERY, mesuré à ~52 ms d'import, payés par toute application qui fait +# `import xcore` et par CHAQUE worker sandbox à chaque recyclage, même sans +# service de tâches configuré (#305, #306) ; +# - les fonctionnalités du package externe `xcoresdk`, qui tire SQLAlchemy par +# son module d'authentification : ~102 ms de plus, dans les mêmes conditions. +# +# Soit ~154 ms sur les ~594 ms d'`import xcore`, et autant par worker sandbox. +# L'accès reste identique pour l'appelant (`from xcore.sdk import WorkerService`) : +# c'est le PREMIER accès qui paie l'import, et lui seul. _SQL_ADAPTERS = { "BaseAsyncRepository": ".adapter.asyncsql", "BaseSyncRepository": ".adapter.syncsql", } +_WORKER = dict.fromkeys( + ("WorkerService", "task", "task_registry"), "..services.xworker" +) +# Pas d'équivalent local : absentes plutôt que simulées (un faux `@cached` qui ne +# cache rien silencieusement serait pire qu'une ImportError explicite). +_XCORESDK = dict.fromkeys( + ( + "AutoMixin", + "BaseMongoRepository", + "BaseRedisRepository", + "EventMixin", + "HookMixin", + "ObservabilityMixin", + "ScheduledMixin", + "cached", + "counted", + "cron", + "health_check", + "interval", + "invalidate", + "on_event", + "on_hook", + "timed", + "traced", + ), + "sdk", +) +_LAZY = {**_SQL_ADAPTERS, **_WORKER, **_XCORESDK} if TYPE_CHECKING: # pragma: no cover - from .adapter.asyncsql import BaseAsyncRepository - from .adapter.syncsql import BaseSyncRepository - -if importlib.util.find_spec("sqlalchemy") is not None: - # Absents de __all__ sans SQLAlchemy : `from xcore.sdk import *` ne doit pas - # lever pour une dépendance dont l'utilisateur n'a pas besoin. - __all__ += list(_SQL_ADAPTERS) - - -def __getattr__(name: str): - module_name = _SQL_ADAPTERS.get(name) - if module_name is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - try: - module = importlib.import_module(module_name, __name__) - except ModuleNotFoundError as e: - if (e.name or "").split(".")[0] == "sqlalchemy": - raise ImportError( - f"xcore.sdk.{name} requires SQLAlchemy, an optional dependency " - "since XCoreRuntime 2.7.0 — install it with: " - "pip install 'XCoreRuntime[db]'" - ) from e - raise - value = getattr(module, name) - globals()[name] = value # mis en cache : __getattr__ n'est appelé qu'une fois - return value - - -# ── Fonctionnalités xcoresdk sans équivalent local ───────────────────────── -# N'existent que si le package externe `xcoresdk` est installé en plus. -# Absentes ici plutôt que simulées : un faux no-op serait pire qu'une -# ImportError explicite (ex: un @cached qui ne cache rien silencieusement). -try: from sdk import ( # type: ignore[import-not-found] AutoMixin, BaseMongoRepository, @@ -182,28 +186,50 @@ def __getattr__(name: str): traced, ) - __all__ += [ - "AutoMixin", - "BaseMongoRepository", - "BaseRedisRepository", - "EventMixin", - "HookMixin", - "ObservabilityMixin", - "ScheduledMixin", - "cached", - "counted", - "cron", - "health_check", - "interval", - "invalidate", - "on_event", - "on_hook", - "timed", - "traced", - ] -except ImportError: + from ..services.xworker import WorkerService, task, task_registry + from .adapter.asyncsql import BaseAsyncRepository + from .adapter.syncsql import BaseSyncRepository + +# `find_spec` RÉSOUT sans exécuter (mesuré : 0,05 ms) : `from xcore.sdk import *` ne doit +# pas lever pour une dépendance que l'utilisateur n'a pas, et ne doit pas non plus importer +# celery ou SQLAlchemy juste pour savoir s'ils sont là. +__all__ += list(_WORKER) +if importlib.util.find_spec("sqlalchemy") is not None: + __all__ += list(_SQL_ADAPTERS) +if importlib.util.find_spec("sdk") is not None: + __all__ += list(_XCORESDK) +else: get_logger("xcore.sdk").debug( "package xcoresdk non installé — EventMixin/HookMixin/ObservabilityMixin/" "ScheduledMixin/cached/invalidate/cron/interval/health_check/AutoMixin/" "BaseMongoRepository/BaseRedisRepository indisponibles dans xcore.sdk" ) + +#: Ce qu'il faut installer pour le nom manquant, quand l'import échoue. +_MISSING = { + "sqlalchemy": "pip install 'XCoreRuntime[db]'", + "celery": "pip install 'XCoreRuntime[worker]'", + "sdk": "le package externe xcoresdk", +} + + +def __getattr__(name: str): + module_name = _LAZY.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + try: + module = importlib.import_module( + module_name, __name__ if module_name[0] == "." else None + ) + except ModuleNotFoundError as e: + missing = (e.name or "").split(".")[0] + if missing in _MISSING: + raise ImportError( + f"xcore.sdk.{name} demande {missing}, une dépendance optionnelle — {_MISSING[missing]}" + ) from e + raise + value = getattr(module, name) + globals()[ + name + ] = value # mis en cache : __getattr__ n'est appelé qu'une fois par nom + return value diff --git a/xcore/services/scheduler/service.py b/xcore/services/scheduler/service.py index d4ab1fbf..defeaf13 100644 --- a/xcore/services/scheduler/service.py +++ b/xcore/services/scheduler/service.py @@ -192,6 +192,7 @@ async def init(self) -> None: return jobstores: dict = {"default": MemoryJobStore()} + degraded = False if self._config.backend == "redis": try: @@ -215,9 +216,26 @@ async def init(self) -> None: ) logger.debug("distributed lock enabled", backend="redis") except ImportError: - logger.warning( - "apscheduler[redis] not installed, falling back to memory" + # PAS un repli discret : l'opérateur a DEMANDÉ le verrou distribué, donc il croit + # ses jobs protégés. Retomber en silence sur « aucun verrou » est le pire des deux + # mondes — chaque worker exécutera chaque job, et rien ne l'aura dit (#310). + logger.error( + "scheduler: verrou distribué DEMANDÉ mais indisponible — chaque processus " + "exécutera chaque job", + backend="redis", + hint="pip install 'apscheduler[redis]' redis", ) + degraded = True + elif self._config.backend != "memory": + # « database » figure dans les valeurs documentées mais n'est pas implémenté : le dire, + # au lieu de donner un backend mémoire à qui a écrit autre chose. + logger.error( + "scheduler: backend inconnu, aucun verrou posé — chaque processus exécutera " + "chaque job", + backend=self._config.backend, + supported="memory | redis", + ) + degraded = True self._scheduler = AsyncIOScheduler( jobstores=jobstores, @@ -233,11 +251,37 @@ async def init(self) -> None: self._add_job_from_config(job_cfg) self._scheduler.start() - self._status = ServiceStatus.READY + self._status = ServiceStatus.DEGRADED if degraded else ServiceStatus.READY logger.info( "scheduler started", timezone=self._config.timezone, backend=self._config.backend, + distributed_lock=_REDIS_LOCK_CLIENT is not None, + ) + self._warn_if_unlocked() + + def _warn_if_unlocked(self) -> None: + """Dire UNE fois, au démarrage, que les jobs ne sont pas protégés contre le multi-processus. + + Sans verrou, chaque processus qui a enregistré un job l'exécute : deux workers uvicorn, deux + paies ; quatre workers, quatre amortissements. Le backend par défaut (`memory`) est dans ce + cas, donc la configuration la plus simple est aussi la seule qui double les écritures. + + Une bibliothèque ne peut PAS savoir combien de processus tournent (uvicorn `--workers` est + décidé dehors, et chaque worker est un process qui se croit seul) : d'où un avertissement + nommant les jobs concernés, et non un refus — refuser en mono-processus empêcherait un + déploiement parfaitement correct de tourner. + """ + if _REDIS_LOCK_CLIENT is not None: + return + jobs = [str(j.get("id") or j.get("func") or "?") for j in self._config.jobs] + if not jobs and not _JOB_REGISTRY: + return # aucun job : rien à dire + logger.warning( + "scheduler sans verrou : en multi-processus, CHAQUE processus exécutera ces jobs", + backend=self._config.backend, + jobs=jobs or sorted(_JOB_REGISTRY), + hint="backend: redis (verrou distribué), ou un seul processus", ) def _add_job_from_config(self, job_cfg: dict) -> None: