From dbd841476cb1c174725be8b1d66d59716f17bb52 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:58:51 +0530 Subject: [PATCH] test: make the suite runnable from the sdist and in isolation; fix rolling trust docs tests/test_no_network_in_runtime.py: the autouse fixture replaced socket.socket with a plain function. The clean/learn tests lazily import the model registry, which imports urllib.request and therefore ssl, and ssl subclasses socket.socket, so the import raised TypeError whenever the file ran before anything else had imported ssl (running the file alone, -k selections, sharding, random order). The fixture now patches socket.socket.connect, socket.socket.connect_ex, socket.create_connection and socket.getaddrinfo through monkeypatch, which still fails any network attempt but leaves the socket class intact. tests/conftest.py: the sdist ships tests/ without the repo-only benchmarks/, training/ and scripts/ directories, so collection from an unpacked sdist stopped with ModuleNotFoundError errors. A pytest_ignore_collect hook now checks each directory independently and, only when it is missing, ignores the suites that depend on it: tests/benchmark, tests/performance and tests/truthbench for benchmarks/, tests/training for training/, and any test module importing a module that lives there (benchmarks, bench, harness_metrics, public_benchmark, results_schema; training; dataset_loader, fixture_download, fetch_online_fixtures, plus the expectations and golden_util helpers that import them). In a checkout nothing is ignored. pyproject.toml is unchanged. docs/streaming.md: rolling_trust_score was described as row-weighted, but StreamingState takes an unweighted mean of the last rolling_trust_window batch scores; only cumulative_trust_score is row-weighted. The docs row now says so. Closes #347 Closes #348 Closes #349 --- docs/streaming.md | 2 +- tests/conftest.py | 77 +++++++++++++++++++++++++++++ tests/test_no_network_in_runtime.py | 15 ++++-- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/docs/streaming.md b/docs/streaming.md index 8db5cc2e..768fb77d 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -97,7 +97,7 @@ in `report.to_dict()`) carries: | `batch_id` | 1-based index of this batch | | `rows_in_batch` / `rows_seen_total` | rows in this batch / cumulative across the stream | | `batch_trust_score` | trust score of this cleaned batch (0-100) | -| `rolling_trust_score` | row-weighted trust over the recent window | +| `rolling_trust_score` | mean of per-batch trust scores over the last `rolling_trust_window` batches (unweighted) | | `cumulative_trust_score` | row-weighted trust over the whole stream | | `schema_drift_detected` | whether drift was flagged for this batch | | `warmup_phase` | whether this batch was still in warmup | diff --git a/tests/conftest.py b/tests/conftest.py index 791b7e74..04c42a38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,83 @@ +import re +from pathlib import Path + import pandas as pd import pytest +ROOT = Path(__file__).resolve().parents[1] +TESTS = ROOT / "tests" + +# Repo-only top-level directories that some suites import. The sdist ships +# ``tests/`` but not these, so suites that depend on a missing directory are +# left out of collection instead of failing with ModuleNotFoundError. +# +# Each entry maps a directory to (test directories that need it, top-level +# modules that come from it). ``expectations`` and ``golden_util`` are +# ``tests/`` helpers that import ``dataset_loader`` / ``fixture_download`` +# from ``scripts/``. +_REPO_ONLY_DEPENDENCIES = { + "benchmarks": ( + ("benchmark", "performance", "truthbench"), + ("benchmarks", "bench", "harness_metrics", "public_benchmark", "results_schema"), + ), + "training": ( + ("training",), + ("training",), + ), + "scripts": ( + (), + ( + "dataset_loader", + "fixture_download", + "fetch_online_fixtures", + "expectations", + "golden_util", + ), + ), +} + + +def _missing_repo_only_dependencies(): + dirs = [] + modules = [] + for name, (test_dirs, mods) in _REPO_ONLY_DEPENDENCIES.items(): + if not (ROOT / name).is_dir(): + dirs.extend(TESTS / d for d in test_dirs) + modules.extend(mods) + pattern = None + if modules: + pattern = re.compile( + r"^\s*(from|import)\s+(" + "|".join(map(re.escape, modules)) + r")\b", + re.MULTILINE, + ) + return tuple(dirs), pattern + + +_MISSING_TEST_DIRS, _MISSING_IMPORT_RE = _missing_repo_only_dependencies() + + +def pytest_ignore_collect(collection_path, config): + """Skip suites whose repo-only dependencies are absent (e.g. from the sdist).""" + if not _MISSING_TEST_DIRS and _MISSING_IMPORT_RE is None: + return None + path = Path(collection_path) + for test_dir in _MISSING_TEST_DIRS: + if path == test_dir or test_dir in path.parents: + return True + if ( + _MISSING_IMPORT_RE is not None + and path.suffix == ".py" + and path.name.startswith("test_") + and path.is_file() + ): + try: + source = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + if _MISSING_IMPORT_RE.search(source): + return True + return None + def pytest_addoption(parser): parser.addoption( diff --git a/tests/test_no_network_in_runtime.py b/tests/test_no_network_in_runtime.py index 66a8dcf4..4ab8b7e3 100644 --- a/tests/test_no_network_in_runtime.py +++ b/tests/test_no_network_in_runtime.py @@ -1,7 +1,12 @@ """FreshData's runtime must never touch the network (offline-by-default, -no runtime LLM, no cloud call). This patches every socket-construction entry -point so any accidental network attempt raises loudly instead of silently -succeeding or silently timing out. +no runtime LLM, no cloud call). This patches every socket connection and +name-resolution entry point so any accidental network attempt raises loudly +instead of silently succeeding or silently timing out. + +``socket.socket`` itself is left a real class: replacing it with a function +breaks any later ``import ssl`` (``class SSLSocket(socket.socket)``), which the +lazily imported model registry pulls in via ``urllib.request``. That only +surfaced when this file ran on its own, before anything else imported ``ssl``. The one documented exception, ``fd.models.pull(...)``, is explicit/opt-in and is not exercised here — these tests cover the default, always-on runtime @@ -24,8 +29,10 @@ def _fail_on_network(monkeypatch): def _blocked(*_args, **_kwargs): raise AssertionError("unexpected network call from the FreshData runtime") - monkeypatch.setattr(socket, "socket", _blocked) + monkeypatch.setattr(socket.socket, "connect", _blocked) + monkeypatch.setattr(socket.socket, "connect_ex", _blocked) monkeypatch.setattr(socket, "create_connection", _blocked) + monkeypatch.setattr(socket, "getaddrinfo", _blocked) def _messy_frame() -> pd.DataFrame: