diff --git a/docs/streaming.md b/docs/streaming.md index 8db5cc2e..768fb77d 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -97,7 +97,7 @@ in `report.to_dict()`) carries: | `batch_id` | 1-based index of this batch | | `rows_in_batch` / `rows_seen_total` | rows in this batch / cumulative across the stream | | `batch_trust_score` | trust score of this cleaned batch (0-100) | -| `rolling_trust_score` | row-weighted trust over the recent window | +| `rolling_trust_score` | mean of per-batch trust scores over the last `rolling_trust_window` batches (unweighted) | | `cumulative_trust_score` | row-weighted trust over the whole stream | | `schema_drift_detected` | whether drift was flagged for this batch | | `warmup_phase` | whether this batch was still in warmup | diff --git a/tests/conftest.py b/tests/conftest.py index 791b7e74..04c42a38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,83 @@ +import re +from pathlib import Path + import pandas as pd import pytest +ROOT = Path(__file__).resolve().parents[1] +TESTS = ROOT / "tests" + +# Repo-only top-level directories that some suites import. The sdist ships +# ``tests/`` but not these, so suites that depend on a missing directory are +# left out of collection instead of failing with ModuleNotFoundError. +# +# Each entry maps a directory to (test directories that need it, top-level +# modules that come from it). ``expectations`` and ``golden_util`` are +# ``tests/`` helpers that import ``dataset_loader`` / ``fixture_download`` +# from ``scripts/``. +_REPO_ONLY_DEPENDENCIES = { + "benchmarks": ( + ("benchmark", "performance", "truthbench"), + ("benchmarks", "bench", "harness_metrics", "public_benchmark", "results_schema"), + ), + "training": ( + ("training",), + ("training",), + ), + "scripts": ( + (), + ( + "dataset_loader", + "fixture_download", + "fetch_online_fixtures", + "expectations", + "golden_util", + ), + ), +} + + +def _missing_repo_only_dependencies(): + dirs = [] + modules = [] + for name, (test_dirs, mods) in _REPO_ONLY_DEPENDENCIES.items(): + if not (ROOT / name).is_dir(): + dirs.extend(TESTS / d for d in test_dirs) + modules.extend(mods) + pattern = None + if modules: + pattern = re.compile( + r"^\s*(from|import)\s+(" + "|".join(map(re.escape, modules)) + r")\b", + re.MULTILINE, + ) + return tuple(dirs), pattern + + +_MISSING_TEST_DIRS, _MISSING_IMPORT_RE = _missing_repo_only_dependencies() + + +def pytest_ignore_collect(collection_path, config): + """Skip suites whose repo-only dependencies are absent (e.g. from the sdist).""" + if not _MISSING_TEST_DIRS and _MISSING_IMPORT_RE is None: + return None + path = Path(collection_path) + for test_dir in _MISSING_TEST_DIRS: + if path == test_dir or test_dir in path.parents: + return True + if ( + _MISSING_IMPORT_RE is not None + and path.suffix == ".py" + and path.name.startswith("test_") + and path.is_file() + ): + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + if _MISSING_IMPORT_RE.search(source): + return True + return None + def pytest_addoption(parser): parser.addoption( diff --git a/tests/test_no_network_in_runtime.py b/tests/test_no_network_in_runtime.py index 66a8dcf4..4ab8b7e3 100644 --- a/tests/test_no_network_in_runtime.py +++ b/tests/test_no_network_in_runtime.py @@ -1,7 +1,12 @@ """FreshData's runtime must never touch the network (offline-by-default, -no runtime LLM, no cloud call). This patches every socket-construction entry -point so any accidental network attempt raises loudly instead of silently -succeeding or silently timing out. +no runtime LLM, no cloud call). This patches every socket connection and +name-resolution entry point so any accidental network attempt raises loudly +instead of silently succeeding or silently timing out. + +``socket.socket`` itself is left a real class: replacing it with a function +breaks any later ``import ssl`` (``class SSLSocket(socket.socket)``), which the +lazily imported model registry pulls in via ``urllib.request``. That only +surfaced when this file ran on its own, before anything else imported ``ssl``. The one documented exception, ``fd.models.pull(...)``, is explicit/opt-in and is not exercised here — these tests cover the default, always-on runtime @@ -24,8 +29,10 @@ def _fail_on_network(monkeypatch): def _blocked(*_args, **_kwargs): raise AssertionError("unexpected network call from the FreshData runtime") - monkeypatch.setattr(socket, "socket", _blocked) + monkeypatch.setattr(socket.socket, "connect", _blocked) + monkeypatch.setattr(socket.socket, "connect_ex", _blocked) monkeypatch.setattr(socket, "create_connection", _blocked) + monkeypatch.setattr(socket, "getaddrinfo", _blocked) def _messy_frame() -> pd.DataFrame: