Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/streaming.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ in `report.to_dict()`) carries:
| `batch_id` | 1-based index of this batch |
| `rows_in_batch` / `rows_seen_total` | rows in this batch / cumulative across the stream |
| `batch_trust_score` | trust score of this cleaned batch (0-100) |
| `rolling_trust_score` | row-weighted trust over the recent window |
| `rolling_trust_score` | mean of per-batch trust scores over the last `rolling_trust_window` batches (unweighted) |
| `cumulative_trust_score` | row-weighted trust over the whole stream |
| `schema_drift_detected` | whether drift was flagged for this batch |
| `warmup_phase` | whether this batch was still in warmup |
Expand Down
77 changes: 77 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,83 @@
import re
from pathlib import Path

import pandas as pd
import pytest

ROOT = Path(__file__).resolve().parents[1]
TESTS = ROOT / "tests"

# Repo-only top-level directories that some suites import. The sdist ships
# ``tests/`` but not these, so suites that depend on a missing directory are
# left out of collection instead of failing with ModuleNotFoundError.
#
# Each entry maps a directory to (test directories that need it, top-level
# modules that come from it). ``expectations`` and ``golden_util`` are
# ``tests/`` helpers that import ``dataset_loader`` / ``fixture_download``
# from ``scripts/``.
_REPO_ONLY_DEPENDENCIES = {
"benchmarks": (
("benchmark", "performance", "truthbench"),
("benchmarks", "bench", "harness_metrics", "public_benchmark", "results_schema"),
),
"training": (
("training",),
("training",),
),
"scripts": (
(),
(
"dataset_loader",
"fixture_download",
"fetch_online_fixtures",
"expectations",
"golden_util",
),
),
}


def _missing_repo_only_dependencies():
dirs = []
modules = []
for name, (test_dirs, mods) in _REPO_ONLY_DEPENDENCIES.items():
if not (ROOT / name).is_dir():
dirs.extend(TESTS / d for d in test_dirs)
modules.extend(mods)
pattern = None
if modules:
pattern = re.compile(
r"^\s*(from|import)\s+(" + "|".join(map(re.escape, modules)) + r")\b",
re.MULTILINE,
)
return tuple(dirs), pattern


_MISSING_TEST_DIRS, _MISSING_IMPORT_RE = _missing_repo_only_dependencies()


def pytest_ignore_collect(collection_path, config):
"""Skip suites whose repo-only dependencies are absent (e.g. from the sdist)."""
if not _MISSING_TEST_DIRS and _MISSING_IMPORT_RE is None:
return None
path = Path(collection_path)
for test_dir in _MISSING_TEST_DIRS:
if path == test_dir or test_dir in path.parents:
return True
if (
_MISSING_IMPORT_RE is not None
and path.suffix == ".py"
and path.name.startswith("test_")
and path.is_file()
):
try:
source = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
return None
if _MISSING_IMPORT_RE.search(source):
return True
return None


def pytest_addoption(parser):
parser.addoption(
Expand Down
15 changes: 11 additions & 4 deletions tests/test_no_network_in_runtime.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
"""FreshData's runtime must never touch the network (offline-by-default,
no runtime LLM, no cloud call). This patches every socket-construction entry
point so any accidental network attempt raises loudly instead of silently
succeeding or silently timing out.
no runtime LLM, no cloud call). This patches every socket connection and
name-resolution entry point so any accidental network attempt raises loudly
instead of silently succeeding or silently timing out.

``socket.socket`` itself is left a real class: replacing it with a function
breaks any later ``import ssl`` (``class SSLSocket(socket.socket)``), which the
lazily imported model registry pulls in via ``urllib.request``. That only
surfaced when this file ran on its own, before anything else imported ``ssl``.

The one documented exception, ``fd.models.pull(...)``, is explicit/opt-in and
is not exercised here — these tests cover the default, always-on runtime
Expand All @@ -24,8 +29,10 @@ def _fail_on_network(monkeypatch):
def _blocked(*_args, **_kwargs):
raise AssertionError("unexpected network call from the FreshData runtime")

monkeypatch.setattr(socket, "socket", _blocked)
monkeypatch.setattr(socket.socket, "connect", _blocked)
monkeypatch.setattr(socket.socket, "connect_ex", _blocked)
monkeypatch.setattr(socket, "create_connection", _blocked)
monkeypatch.setattr(socket, "getaddrinfo", _blocked)


def _messy_frame() -> pd.DataFrame:
Expand Down
Loading