From 67b92a72054ebcb8f4f600ebe320efba0e9f08ba Mon Sep 17 00:00:00 2001 From: Masterplanner25 Date: Fri, 9 Oct 2026 21:26:16 -0700 Subject: [PATCH 1/2] fix(registry): run DEBT-COMPAT-1's consumer check once per module set (FR-52); pin PyJWT[crypto] load_plugins() ended unconditionally in check_consumer_requirements, and the registry's getters call load_plugins() lazily -- 26 times in one warm `GET /memory/nodes`. Each check read every installed distribution's metadata (packages_distributions + the shadow scan): ~3 s on every app request since 2.25.0 (app: p50 3,780 ms as shipped, 820 ms with the check stubbed). The check now runs when the plugin module set differs from the last one checked; its answer cannot change while the process lives. The latch is in both registry-reset dicts. Tests count metadata scans: 5 loads -> 1 scan; a different module set is checked again; one warm request through the real route -> 0 scans (26 before). Always-check and never-check mutations each fail them. PyJWT is pinned with its crypto extra: its HMAC guard against asymmetric key material returns early when cryptography is absent (found by nodus-auth, def3ef8). We had cryptography by a separate pin; now the guard does not depend on that, and a test asserts has_crypto. Co-Authored-By: Claude Opus 5.5 --- AINDY/platform_layer/registry.py | 22 +++-- AINDY/requirements.txt | 2 +- pyproject.toml | 2 +- tests/fixtures/client.py | 1 + .../unit/test_debt_compat1_consumer_range.py | 93 +++++++++++++++++++ tests/unit/test_extension_abi.py | 1 + tests/unit/test_jwt_library_migration.py | 10 ++ 7 files changed, 122 insertions(+), 9 deletions(-) diff --git a/AINDY/platform_layer/registry.py b/AINDY/platform_layer/registry.py index ada04fe6..4eec476e 100644 --- a/AINDY/platform_layer/registry.py +++ b/AINDY/platform_layer/registry.py @@ -136,6 +136,10 @@ _required_syscalls: list[str] = [] _symbols: dict[str, Any] = {} _loaded_plugins: set[str] = set() +# FR-52: the plugin module set DEBT-COMPAT-1's consumer check last ran for. The getters call +# load_plugins() lazily, many times per request, and the check reads every installed +# distribution's metadata; its answer cannot change while the process lives. +_consumer_check_modules: frozenset[str] | None = None _registered_apps: list[str] = [] _bootstrap_dependencies: dict[str, list[str]] = {} _loaded_extension_records: dict[str, dict[str, Any]] = {} @@ -2055,7 +2059,7 @@ def load_plugins( manifest_path, profile=profile, ) - global _active_plugin_profile, _active_plugin_profile_source + global _active_plugin_profile, _active_plugin_profile_source, _consumer_check_modules active_profile, plugin_entries, extension_entries, explicitly_selected, selection_source = _resolve_plugin_profile_selection( manifest_path if manifest_path is not None else path, profile=profile, @@ -2202,11 +2206,15 @@ def load_plugins( ", ".join(loaded), ) # DEBT-COMPAT-1 / DEC-080 — compare each plugin distribution's declared aindy-runtime range - # with this runtime; warn, never refuse. - try: - from AINDY.platform_layer.runtime_compatibility import check_consumer_requirements + # with this runtime; warn, never refuse. Once per module set (FR-52): unconditionally, it + # ran on every lazy load_plugins() call and added ~3 s to every request. + module_names = frozenset(entry["module_name"] for entry in plugin_entries) + if module_names != _consumer_check_modules: + _consumer_check_modules = module_names + try: + from AINDY.platform_layer.runtime_compatibility import check_consumer_requirements - check_consumer_requirements(entry["module_name"] for entry in plugin_entries) - except Exception as exc: # noqa: BLE001 — a compatibility probe must never fail a boot - logger.debug("DEBT-COMPAT-1 consumer check skipped: %s", exc) + check_consumer_requirements(sorted(module_names)) + except Exception as exc: # noqa: BLE001 — a compatibility probe must never fail a boot + logger.debug("DEBT-COMPAT-1 consumer check skipped: %s", exc) return loaded diff --git a/AINDY/requirements.txt b/AINDY/requirements.txt index 5cfc2e3a..d7cbf67b 100644 --- a/AINDY/requirements.txt +++ b/AINDY/requirements.txt @@ -52,7 +52,7 @@ pytest==9.1.1 pytest-mock==3.15.1 pytest-asyncio==1.4.0 respx==0.23.1 -PyJWT==2.15.1 +PyJWT[crypto]==2.15.1 cryptography==50.0.1 passlib==1.7.4 bcrypt==4.0.1 diff --git a/pyproject.toml b/pyproject.toml index f03c91a4..fcf2ac5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ "pyphen==0.18.1", "python-dateutil==2.9.0.post0", "python-dotenv==1.2.3", - "PyJWT==2.15.1", + "PyJWT[crypto]==2.15.1", "cryptography==50.0.1", "python-json-logger==4.2.0", "redis==8.1.0", diff --git a/tests/fixtures/client.py b/tests/fixtures/client.py index 6b3b392c..10fdd55b 100644 --- a/tests/fixtures/client.py +++ b/tests/fixtures/client.py @@ -98,6 +98,7 @@ def _patch_session_aliases(monkeypatch, session_factory, engine): "_required_syscalls": [], "_symbols": {}, "_loaded_plugins": set(), + "_consumer_check_modules": None, "_registered_apps": [], "_bootstrap_dependencies": {}, "_loaded_extension_records": {}, diff --git a/tests/unit/test_debt_compat1_consumer_range.py b/tests/unit/test_debt_compat1_consumer_range.py index af8eb8d7..b45b5951 100644 --- a/tests/unit/test_debt_compat1_consumer_range.py +++ b/tests/unit/test_debt_compat1_consumer_range.py @@ -255,3 +255,96 @@ def test_the_unsatisfied_warning_names_the_metadata_path(caplog): ) [warning] = _warnings(caplog) assert "/somewhere/my_app-1.0.0.dist-info" in warning.getMessage() + + +# ── FR-52: once per module set, never once per call ───────────────────────────────────────── +# The registry's getters call load_plugins() lazily — 26 times in one warm `GET /memory/nodes` — +# and the check ended every call reading every installed distribution's metadata: ~3 s on each +# of the app's requests (p50 3,780 ms as shipped, 820 ms with the check stubbed out). + + +@pytest.fixture +def counted(plugin_package, monkeypatch): + """`packages_distributions` (the expensive half) and `load_plugins`, counted.""" + import importlib.metadata as md + + from AINDY.platform_layer import registry + + calls = {"scan": 0, "load_plugins": 0} + owners = md.packages_distributions # the fixture's fake + + def scan(): + calls["scan"] += 1 + return owners() + + real_load = registry.load_plugins + + def load(*args, **kwargs): + calls["load_plugins"] += 1 + return real_load(*args, **kwargs) + + monkeypatch.setattr(md, "packages_distributions", scan) + monkeypatch.setattr(registry, "load_plugins", load) + return calls + + +@pytest.fixture +def empty_registry(): + from AINDY.platform_layer import registry + from tests.unit.test_extension_abi import _REGISTRY_STATE_EMPTY, _copy_registry_value + + snapshot = {name: _copy_registry_value(getattr(registry, name)) for name in _REGISTRY_STATE_EMPTY} + for name, value in _REGISTRY_STATE_EMPTY.items(): + setattr(registry, name, _copy_registry_value(value)) + try: + yield registry + finally: + for name, value in snapshot.items(): + setattr(registry, name, value) + + +def test_the_check_runs_once_across_repeated_loads(counted, plugin_package, empty_registry): + for _ in range(5): + empty_registry.load_plugins(manifest_path=plugin_package) + assert counted["load_plugins"] == 5 + assert counted["scan"] == 1, f"the consumer check ran {counted['scan']} times for one module set" + [record] = rc.consumer_requirement_checks() + assert record["status"] == "unsatisfied", "running it once must still record its answer" + + +def test_a_different_module_set_is_checked(counted, plugin_package, empty_registry, tmp_path): + """Liveness for the latch: it suppresses repeats, not checks.""" + (plugin_package.parent / "apps" / "debtcompat_other.py").write_text("def bootstrap():\n pass\n", encoding="utf-8") + wider = tmp_path / "wider.json" + wider.write_text('{"plugins": ["apps.debtcompat_plugin", "apps.debtcompat_other"]}', encoding="utf-8") + try: + empty_registry.load_plugins(manifest_path=plugin_package) + empty_registry.load_plugins(manifest_path=wider) + empty_registry.load_plugins(manifest_path=wider) + finally: + sys.modules.pop("apps.debtcompat_other", None) + assert counted["scan"] == 2 + + +def test_a_warm_request_scans_no_metadata(runtime_only_client, db_session, counted, plugin_package, monkeypatch): + """The FR's own instrument: count the scans across one warm request, through the real route.""" + import uuid + + from AINDY.db.models.user import User + from AINDY.services.auth_service import create_access_token, hash_password + + user = User(email=f"fr52-{uuid.uuid4().hex[:8]}@aindy.test", username=f"fr52{uuid.uuid4().hex[:8]}", + hashed_password=hash_password("fr52-password"), is_active=True) + db_session.add(user) + db_session.commit() + db_session.refresh(user) + headers = {"Authorization": "Bearer " + create_access_token({"sub": str(user.id)}, token_version=0)} + monkeypatch.setenv("AINDY_PLUGIN_MANIFEST", str(plugin_package)) + try: + assert runtime_only_client.get("/memory/nodes?limit=1", headers=headers).status_code == 200 # warm + counted.update(scan=0, load_plugins=0) + assert runtime_only_client.get("/memory/nodes?limit=1", headers=headers).status_code == 200 + finally: + sys.modules.pop("apps.debtcompat_plugin", None) + assert counted["load_plugins"] > 0, "liveness: the request must go through the lazy getters" + assert counted["scan"] == 0, f"{counted['scan']} metadata scans in one warm request" diff --git a/tests/unit/test_extension_abi.py b/tests/unit/test_extension_abi.py index 9fd5eb71..000217d8 100644 --- a/tests/unit/test_extension_abi.py +++ b/tests/unit/test_extension_abi.py @@ -30,6 +30,7 @@ _REGISTRY_STATE_EMPTY = { "_loaded_plugins": set(), + "_consumer_check_modules": None, "_registered_apps": [], "_bootstrap_dependencies": {}, "_loaded_extension_records": {}, diff --git a/tests/unit/test_jwt_library_migration.py b/tests/unit/test_jwt_library_migration.py index 816e947b..0da90286 100644 --- a/tests/unit/test_jwt_library_migration.py +++ b/tests/unit/test_jwt_library_migration.py @@ -175,3 +175,13 @@ def test_error_log_reads_user_from_an_expired_unverifiable_token(): expired = jwt.encode({**claims, "exp": 1}, "some-other-key-0123456789abcdef0123", algorithm="HS256") assert _extract_user_id_from_request(_request(expired)) == uuid.UUID(LEGACY_SUB) assert _extract_user_id_from_request(_request("not-a-token")) is None + + +def test_pyjwt_runs_with_its_crypto_backend(): + """PyJWT's HMAC guard against asymmetric key material opens with `if not has_crypto: return + False` — on a plain `pip install PyJWT` its DER branch never runs (found by nodus-auth, + def3ef8). The pin is `PyJWT[crypto]`, so the guard does not hang on `cryptography` arriving + by some other route.""" + from jwt.algorithms import has_crypto + + assert has_crypto is True From b47e529be72c2a8cd2c745e1e65aca3b9989a525 Mon Sep 17 00:00:00 2001 From: Masterplanner25 Date: Fri, 9 Oct 2026 21:27:03 -0700 Subject: [PATCH 2/2] docs: close FR-51 (#810) and FR-52 (#814) in the ledger; next app FR is FR-53 Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- TECH_DEBT.md | 32 +++++++++++++++++++++ changelog.d/814-fr52-consumer-check-once.md | 16 +++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 changelog.d/814-fr52-consumer-check-once.md diff --git a/CLAUDE.md b/CLAUDE.md index aceb56de..28e906c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -288,7 +288,7 @@ A soak assertion must not be stricter than the contract. ### Open — programs and multi-item prefixes -- **APP-FR-\*** — app-side feature requests. **Next: FR-51** (the app numbers ahead of this ledger — read its register before numbering). Open: FR-14 recurrence half. Closed: FR-43..50 (FR-46 default ON, DEC-084). +- **APP-FR-\*** — app-side feature requests. **Next: FR-53** (the app numbers ahead of this ledger — read its register before numbering). Open: FR-14 recurrence half. Closed: FR-43..52 (FR-46 default ON, DEC-084; FR-51 #810, FR-52 #814). - **ECOGAP-\*** — ECOGAP-1 ph1–3 and ECOGAP-4 G4b shipped opt-in; G4a built-but-INERT until a policy is registered. ECOGAP-2 is C2/C3, ECOGAP-3 is MEMORY-EMBEDDING-PROVIDER-1 — don't double-track. - **RTR-\*** — 1/5/6 closed; 2/3/4/7 harden-halves done. RTR-4 remaining: soak + flip `AINDY_DELEGATION_PRIVATE_MEMORY`; delegate writes take the deferred path, so `MemoryNodeDAO.save` is the chokepoint. - **DOCS-\*** — check `APP_ROUTERS` + `ROUTE_OWNERSHIP_INVENTORY.md`, never file presence, before calling a route runtime-owned. diff --git a/TECH_DEBT.md b/TECH_DEBT.md index 91dbb39b..e274df18 100644 --- a/TECH_DEBT.md +++ b/TECH_DEBT.md @@ -9233,6 +9233,38 @@ key in each tool dict. --- +## FR-52 — DEBT-COMPAT-1's consumer check ran on every `load_plugins()` call, and the registry's getters call that lazily: ~3 s added to every request 🔴 defect, P1 (app-filed 2026-10-07, runtime 2.25.0) + +**Status: CLOSED (2026-10-09, #814) — UNRELEASED.** `load_plugins()` ended unconditionally in +`check_consumer_requirements`, which calls `packages_distributions()` and scans every distribution +for shadowing copies. The getters (`emit_event`, `get_route_guard`, `get_response_adapter`, +`get_memory_policy`, … — 14 sites) call `load_plugins()` lazily: **26 times in one warm +`GET /memory/nodes`**. The app measured p50 3,780 ms as shipped vs 820 ms with the check stubbed +(their yappi profile: 9.7 s summed in the check across threads, 208,026 `pathlib` parses). +**Fix:** the check runs when the plugin module set differs from the last one checked +(`registry._consumer_check_modules`, in both registry-reset dicts). Its answer cannot change while the +process lives, and a different manifest or profile is still checked. **Why our suites never saw it:** +runtime-only boot's manifest lists no plugins, so `load_plugins()` returns before the check — the +cost exists only for a consumer with plugins (the app has 16). The route test therefore selects a +plugin manifest through `AINDY_PLUGIN_MANIFEST`. Tests: 5 loads → 1 scan; a wider module set → +checked again; one warm request through the real route → 0 scans (26 before). Always-check and +never-check mutations each fail them. Asks 1 and 3 built; ask 2 (cache `packages_distributions`) +unnecessary once the check runs once. + +--- + +## FR-51 — `pymongo==4.18.1` and `python-jose==3.5.0` were exact pins carrying four advisories; pymongo's fix is 4.18.2, python-jose had none 🔴 security (app-filed 2026-10-06, runtime 2.26.0) + +**Status: CLOSED (2026-10-08, #810) — UNRELEASED.** Both asks answered: `pymongo==4.18.2` +(CVE-2026-96747/96748/96749), and python-jose **replaced by PyJWT** (CVE-2026-85394, critical, no +fix). The app's exposure assessment matched ours: HS256 pinned at every verifying decode, the two +unverified reads authorise nothing. jose-minted tokens verify unchanged (pinned fixtures). The app's +four audit ignores come off at the adoption that brings this release. PyJWT carries its `[crypto]` +extra since #814 — its asymmetric-key guard returns early without `cryptography` (nodus-auth +def3ef8). Boot Smoke needed DEC-098 to let a dependency-changing PR boot at all. + +--- + ## FR-50 — the memory routes forwarded their optional fields as None and the syscall schemas refused them: query-only and tags-only recall, and a node created without a node_type, all answered 400 🔴 defect (app-filed 2026-09-30, runtime 2.24.0) **Status: CLOSED (2026-09-30).** VERIFIED LIVE by the app 2026-10-02: a query-only diff --git a/changelog.d/814-fr52-consumer-check-once.md b/changelog.d/814-fr52-consumer-check-once.md new file mode 100644 index 00000000..907d6e46 --- /dev/null +++ b/changelog.d/814-fr52-consumer-check-once.md @@ -0,0 +1,16 @@ +### Fixed — every request with plugins loaded paid ~3 s for a version check (`FR-52`, #814) + +- DEBT-COMPAT-1's consumer check (2.25.0) ran at the end of every `load_plugins()` call, and the + registry's getters call `load_plugins()` lazily, 26 times in one warm `GET /memory/nodes`. Each + run read every installed distribution's metadata. The app measured p50 3,780 ms as shipped vs + 820 ms with the check stubbed out. It now runs once per plugin module set, so a different + manifest or profile is still checked. `/api/version` reports the same records. +- Our own suites never saw it: runtime-only boot loads no plugins, so `load_plugins()` returned + before the check. The regression test boots a plugin manifest and counts metadata scans across a + warm request through the real route (26 before, 0 after). + +### Changed — PyJWT is pinned with its `crypto` extra (#814) + +- PyJWT's guard against asymmetric key material under an HMAC algorithm returns early when + `cryptography` is absent. The runtime had `cryptography` through a separate pin; the pin is now + `PyJWT[crypto]==2.15.1`, so the guard no longer depends on it, and a test asserts it is live.