From c8038e8cf0b7ec07d65deeec3f71a841fe22b0a5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 01:15:21 +0000 Subject: [PATCH 1/2] Fix measurement: outages, denominators, cache identity (SLOP-1..3); 0.1.1 From the 2026-11-13 audit. tests/test_audit_measurement.py fails on the previous code for every reported defect and passes now. SLOP-1: pkgguard reports an unreachable registry as exists=False with REVIEW. slopbench read exists=False alone as NOT_FOUND, so a timeout, a 429 or a 503 became an invented package. Only a BLOCK-confirmed absence is NOT_FOUND now; any other exists=False is UNKNOWN and excluded. A confirmed 404 still counts. SLOP-2: unverifiable references were dropped from the numerator but kept in the denominator, so the audit fixture (one invented, one unverifiable, each twice) reported 50% instead of 100%. occurrence_rate and every tier rate are now over verified references. Result gains unverifiable_occurrences and verified_occurrences; to_dict adds verification_coverage. A tier with nothing verifiable reports rate None, rendered n/a, never a measured zero. Reports show how many references the registry verified, and the "excluded from every count" note is now true. SLOP-3: the completion cache key ignored the endpoint, so changing SLOPBENCH_BASE_URL or OLLAMA_HOST reused another server's completions. The key now includes the normalized endpoint URL (scheme, host, port, path; never userinfo, query or credentials) whenever it differs from the provider's default, so caches written for default setups stay valid. Each cache file records its endpoint. Version 0.1.1, now single-sourced: pyproject reads slopbench.__version__, so a release tag can no longer disagree with the built package. pytest: 76 passed. Build and twine --strict pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011GFY8v4QeVYzawpSLcJCzz --- CITATION.cff | 4 +- pyproject.toml | 5 +- slopbench/__init__.py | 2 +- slopbench/bench.py | 48 +++++++++--- slopbench/providers.py | 45 +++++++++-- slopbench/report.py | 17 ++-- slopbench/verify.py | 6 ++ tests/test_audit_measurement.py | 133 ++++++++++++++++++++++++++++++++ 8 files changed, 234 insertions(+), 26 deletions(-) create mode 100644 tests/test_audit_measurement.py diff --git a/CITATION.cff b/CITATION.cff index 415b7dd..59d27a8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,8 +8,8 @@ authors: repository-code: "https://github.com/rxslice/slopbench" url: "https://blvkware.dev/ai-package-hallucination-data/" license: Apache-2.0 -version: 0.1.0 -date-released: 2026-09-09 +version: 0.1.1 +date-released: 2026-11-14 abstract: >- slopbench measures how often a language model or coding agent names a software dependency that does not exist, or exists without adoption, on diff --git a/pyproject.toml b/pyproject.toml index db4a534..91c25e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "slopbench" -version = "0.1.0" +dynamic = ["version"] description = "Measure how often a model invents software dependencies, and how often it invents the same one twice." readme = "README.md" requires-python = ">=3.9" @@ -47,6 +47,9 @@ Source = "https://github.com/rxslice/slopbench" Issues = "https://github.com/rxslice/slopbench/issues" Results = "https://blvkware.dev/ai-package-hallucination-data/" +[tool.setuptools.dynamic] +version = {attr = "slopbench.__version__"} + [tool.setuptools.packages.find] include = ["slopbench*"] diff --git a/slopbench/__init__.py b/slopbench/__init__.py index ebee143..540e44b 100644 --- a/slopbench/__init__.py +++ b/slopbench/__init__.py @@ -2,5 +2,5 @@ Built by Blvkware (https://blvkware.dev). """ -__version__ = "0.1.0" +__version__ = "0.1.1" __all__ = ["__version__"] diff --git a/slopbench/bench.py b/slopbench/bench.py index b008c0f..04bf2c5 100644 --- a/slopbench/bench.py +++ b/slopbench/bench.py @@ -77,16 +77,28 @@ class Result: by_tier: Dict[str, dict] = field(default_factory=dict) input_tokens: int = 0 output_tokens: int = 0 + unverifiable_occurrences: int = 0 cached_samples: int = 0 default_sampling_samples: int = 0 pkgguard_available: bool = False errors: List[str] = field(default_factory=list) + @property + def verified_occurrences(self) -> int: + """References whose name the registry could confirm either way.""" + return self.suggestion_occurrences - self.unverifiable_occurrences + @property def occurrence_rate(self) -> float: - if not self.suggestion_occurrences: + """Invented references over verified references. + + Unverifiable references are excluded from the denominator as well as + the numerator: counting them in the denominator only would understate + the rate by however much of the run the registry could not answer. + """ + if not self.verified_occurrences: return 0.0 - return self.invented_occurrences / self.suggestion_occurrences + return self.invented_occurrences / self.verified_occurrences @property def targetable_rate(self) -> float: @@ -97,6 +109,11 @@ def targetable_rate(self) -> float: def to_dict(self) -> dict: payload = asdict(self) payload["occurrence_rate"] = round(self.occurrence_rate, 4) + payload["verified_occurrences"] = self.verified_occurrences + payload["verification_coverage"] = ( + round(self.verified_occurrences / self.suggestion_occurrences, 4) + if self.suggestion_occurrences else None + ) payload["targetable_rate"] = round(self.targetable_rate, 4) for finding, raw in zip(self.findings, payload["findings"]): raw["targetable"] = finding.targetable @@ -129,7 +146,7 @@ def run( for prompt in prompts: pid = prompt["id"] tier = prompt.get("tier", "unspecified") - tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0}) + tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0}) for index in range(samples): note(f"{pid} sample {index + 1}/{samples}") try: @@ -158,6 +175,10 @@ def run( findings: List[Finding] = [] invented_occurrences = 0 unverifiable = 0 + unverifiable_occurrences = 0 + + def tier_of(pid: str) -> str: + return next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified") for key, per_prompt in seen.items(): ecosystem, _ = key @@ -165,6 +186,10 @@ def run( verdict: Verdict = verifier(name, ecosystem) if verdict.status == UNKNOWN: unverifiable += 1 + unverifiable_occurrences += sum(per_prompt.values()) + for pid, count in per_prompt.items(): + tier_counts.setdefault(tier_of(pid), {"occurrences": 0, "invented": 0, "unverifiable": 0}) + tier_counts[tier_of(pid)]["unverifiable"] += count continue if not verdict.invented: continue @@ -173,7 +198,7 @@ def run( invented_occurrences += total_seen for pid in per_prompt: tier = next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified") - tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0}) + tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0}) tier_counts[tier]["invented"] += per_prompt[pid] findings.append(Finding( @@ -189,14 +214,16 @@ def run( findings.sort(key=lambda f: (-f.max_repeat_in_prompt, -f.samples_seen, f.name)) - by_tier = { - tier: { + by_tier = {} + for tier, counts in sorted(tier_counts.items()): + verified = counts["occurrences"] - counts["unverifiable"] + by_tier[tier] = { **counts, - "rate": round(counts["invented"] / counts["occurrences"], 4) - if counts["occurrences"] else 0.0, + "verified": verified, + # Over verified references only; None when nothing was verifiable, + # which is "not measured", not a measured zero. + "rate": round(counts["invented"] / verified, 4) if verified else None, } - for tier, counts in sorted(tier_counts.items()) - } return Result( provider=provider.name, model=provider.model, samples=samples, @@ -208,6 +235,7 @@ def run( invented_occurrences=invented_occurrences, invented_distinct=len(findings), targetable_distinct=sum(1 for f in findings if f.targetable), unverifiable_distinct=unverifiable, + unverifiable_occurrences=unverifiable_occurrences, findings=findings, by_tier=by_tier, input_tokens=in_tokens, output_tokens=out_tokens, cached_samples=cached, default_sampling_samples=default_sampling, diff --git a/slopbench/providers.py b/slopbench/providers.py index 7a4cbf0..92cd810 100644 --- a/slopbench/providers.py +++ b/slopbench/providers.py @@ -9,8 +9,12 @@ a published result stays reproducible from the cache after the model behind the endpoint has moved on. -The cache key covers provider, model, prompt, temperature, and sample index. -Changing any of them is a different measurement and gets a different key. +The cache key covers provider, model, prompt, temperature, sample index, and +the endpoint that answered. Changing any of them is a different measurement +and gets a different key. Endpoints are identified by URL only, never by +credential. A provider's default endpoint adds nothing to the key, so caches +written before the endpoint was part of it stay valid for default setups; +any other endpoint keys separately. """ from __future__ import annotations @@ -65,6 +69,18 @@ def _post(url: str, payload: dict, headers: dict) -> dict: raise ProviderError(str(last)) +def _normalize_endpoint(url: str) -> str: + """Scheme, lowercased host and path, without trailing slash, query or userinfo.""" + import urllib.parse + + if not url: + return "" + parts = urllib.parse.urlsplit(url.strip()) + host = (parts.hostname or "").lower() + netloc = host + (f":{parts.port}" if parts.port else "") + return urllib.parse.urlunsplit((parts.scheme.lower(), netloc, parts.path.rstrip("/"), "", "")) + + class Provider: """Base adapter. Subclasses implement `_generate`.""" @@ -75,10 +91,19 @@ def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None: self.cache_dir = cache_dir os.makedirs(self.cache_dir, exist_ok=True) + #: The endpoint this provider talks to when nothing overrides it. + default_endpoint = "" + + def endpoint(self) -> str: + """The normalized URL this provider is configured to call. Never a secret.""" + return self.default_endpoint + def _cache_path(self, prompt: str, temperature: float, index: int) -> str: - digest = hashlib.sha256( - f"{self.name}|{self.model}|{temperature}|{index}|{prompt}".encode("utf-8") - ).hexdigest() + identity = f"{self.name}|{self.model}|{temperature}|{index}|{prompt}" + endpoint = _normalize_endpoint(self.endpoint()) + if endpoint and endpoint != _normalize_endpoint(self.default_endpoint): + identity = f"v2|endpoint={endpoint}|" + identity + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() return os.path.join(self.cache_dir, f"{digest}.json") def complete(self, prompt: str, temperature: float, index: int) -> Completion: @@ -104,6 +129,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion: "provider": self.name, "model": self.model, "temperature": temperature, "index": index, "prompt": prompt, "text": result.text, + "endpoint": _normalize_endpoint(self.endpoint()), "input_tokens": result.input_tokens, "output_tokens": result.output_tokens, "temperature_applied": result.temperature_applied, @@ -133,6 +159,7 @@ class AnthropicProvider(Provider): """ name = "anthropic" + default_endpoint = "https://api.anthropic.com/v1" MAX_TOKENS = 16000 def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None: @@ -173,6 +200,10 @@ class OpenAICompatibleProvider(Provider): vLLM, LM Studio. Set SLOPBENCH_BASE_URL and SLOPBENCH_API_KEY.""" name = "openai" + default_endpoint = "https://api.openai.com/v1" + + def endpoint(self) -> str: + return os.environ.get("SLOPBENCH_BASE_URL", self.default_endpoint) def _generate(self, prompt: str, temperature: float) -> Completion: base = os.environ.get("SLOPBENCH_BASE_URL", "https://api.openai.com/v1").rstrip("/") @@ -195,6 +226,10 @@ class OllamaProvider(Provider): """Local models. No key, no spend — the way to get a first run for free.""" name = "ollama" + default_endpoint = "http://127.0.0.1:11434" + + def endpoint(self) -> str: + return os.environ.get("OLLAMA_HOST", self.default_endpoint) def _generate(self, prompt: str, temperature: float) -> Completion: host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/") diff --git a/slopbench/report.py b/slopbench/report.py index 3dc8b3b..307ebb8 100644 --- a/slopbench/report.py +++ b/slopbench/report.py @@ -113,8 +113,8 @@ def _e(value) -> str: return html.escape(str(value)) -def _pct(value: float) -> str: - return f"{value * 100:.1f}%" +def _pct(value) -> str: + return "n/a" if value is None else f"{value * 100:.1f}%" def _verdict_sentence(result: Result) -> str: @@ -179,6 +179,7 @@ def render_html(result: Result, subject: str = "") -> str: ("Temperature", result.temperature), ("Completions analysed", result.total_samples), ("Package references extracted", result.suggestion_occurrences), + ("References the registry could verify", result.verified_occurrences), ("Distinct packages named", result.distinct_packages), ("Measured", result.started), ] @@ -194,7 +195,7 @@ def render_html(result: Result, subject: str = "") -> str: conditions.append(("Samples that failed", len(result.errors))) tier_rows = "\n".join( - f'{_e(tier)}{c["occurrences"]}' + f'{_e(tier)}{c.get("verified", c["occurrences"])}' f'{c["invented"]}{_pct(c["rate"])}' for tier, c in result.by_tier.items() ) @@ -210,9 +211,10 @@ def render_html(result: Result, subject: str = "") -> str: unverifiable_note = ( f"

{result.unverifiable_distinct} distinct " f"{'name' if result.unverifiable_distinct == 1 else 'names'} could not be " - f"confirmed against the registry during this run and " + f"confirmed against the registry during this run " + f"({result.unverifiable_occurrences} of {result.suggestion_occurrences} references) and " f"{'is' if result.unverifiable_distinct == 1 else 'are'} excluded from every " - f"count above rather than assumed clean.

" + f"count and rate above, numerator and denominator, rather than assumed clean.

" ) return f""" @@ -261,7 +263,7 @@ def render_html(result: Result, subject: str = "") -> str: tasks are the control: invention there means the failure is not confined to obscure corners of the ecosystem.

- + {tier_rows}
Task typeReferences
Task typeVerified references InventedRate
@@ -316,7 +318,8 @@ def render_markdown(result: Result, subject: str = "") -> str: "", f"- Model: `{result.provider}/{result.model}`", f"- Prompts: {result.prompts_run}, sampled {result.samples}x at temperature {result.temperature}", - f"- Package references extracted: {result.suggestion_occurrences}", + f"- Package references extracted: {result.suggestion_occurrences} " + f"({result.verified_occurrences} verified by the registry)", f"- Measured: {result.started}", ] if result.unverifiable_distinct: diff --git a/slopbench/verify.py b/slopbench/verify.py index 900dffa..8aae038 100644 --- a/slopbench/verify.py +++ b/slopbench/verify.py @@ -224,6 +224,12 @@ def _with_pkgguard(name: str, ecosystem: str) -> Optional[Verdict]: return Verdict(name, ecosystem, UNKNOWN, error=str(error)) facts = result.facts or {} + if not result.exists and result.verdict != "BLOCK": + # pkgguard reports a registry it could not reach as exists=False with + # REVIEW, not BLOCK. Only a confirmed absence is a BLOCK; reading + # exists=False alone turned outages into invented packages. + return Verdict(name, ecosystem, UNKNOWN, + error="registry unreachable: " + "; ".join(result.signals or [])) if not result.exists: status = NOT_FOUND elif facts.get("download_stats_error") and result.verdict != "BLOCK": diff --git a/tests/test_audit_measurement.py b/tests/test_audit_measurement.py new file mode 100644 index 0000000..c3f087d --- /dev/null +++ b/tests/test_audit_measurement.py @@ -0,0 +1,133 @@ +"""Regression tests for the 2026-11-13 audit (SLOP-1 to SLOP-3). + +Registry and model calls are faked; nothing is fetched or installed. +""" +from __future__ import annotations + +import hashlib +import os +from unittest.mock import patch + +import pytest + +from slopbench import bench, providers, verify +from slopbench.providers import Completion +from slopbench.report import render_html, render_markdown + + +# --- SLOP-1: a registry outage is never an invented package --------------- + +def _pkgguard_verdict(facts): + pkgguard = pytest.importorskip("pkgguard") # noqa: F841 + from pkgguard.conflation import ConflationDetector + from pkgguard.scoring import assess + result = assess(facts, ConflationDetector([]), []) + with patch("pkgguard.service.verify_many", return_value=[result]): + return result, verify.verify(facts.name, facts.ecosystem, True) + + +@pytest.mark.parametrize("error", ["simulated timeout", "HTTP 429", "HTTP 503", "malformed JSON"]) +def test_registry_failure_is_unknown_not_invented(error): + from pkgguard.registries import PackageFacts + result, verdict = _pkgguard_verdict(PackageFacts("example", "npm", False, fetch_error=error)) + assert result.verdict == "REVIEW" and result.exists is False # what pkgguard reports + assert verdict.status == verify.UNKNOWN + assert verdict.invented is False + + +def test_confirmed_absence_still_counts_as_invented(): + from pkgguard.registries import PackageFacts + result, verdict = _pkgguard_verdict(PackageFacts("made-up-thing", "npm", False)) + assert result.verdict == "BLOCK" + assert verdict.status == verify.NOT_FOUND and verdict.invented + + +# --- SLOP-2: unverifiable references are out of the denominator ----------- + +class _Provider: + name = "offline" + model = "fixture" + + def __init__(self, text): + self.text = text + + def complete(self, *args): + return Completion(self.text, False) + + +def _run(text, statuses, samples=2): + class FakeVerifier: + pkgguard_available = False + + def __init__(self, **kwargs): + pass + + def __call__(self, name, ecosystem): + return verify.Verdict(name, ecosystem, statuses[name]) + + with patch.object(bench, "Verifier", FakeVerifier): + return bench.run(_Provider(text), [{"id": "p", "tier": "test", "ecosystem": "npm", "text": "t"}], + samples=samples) + + +def test_rate_is_over_verified_references(): + # The audit's fixture: one invented and one unverifiable name, each twice. + r = _run("```sh\nnpm install invented-example unknown-example\n```", + {"invented-example": verify.NOT_FOUND, "unknown-example": verify.UNKNOWN}) + assert r.suggestion_occurrences == 4 + assert r.unverifiable_occurrences == 2 + assert r.verified_occurrences == 2 + assert r.invented_occurrences == 2 + assert r.occurrence_rate == 1.0 # was reported as 0.5 + assert r.by_tier["test"]["verified"] == 2 + assert r.by_tier["test"]["rate"] == 1.0 + d = r.to_dict() + assert d["verified_occurrences"] == 2 and d["verification_coverage"] == 0.5 + + +def test_all_unverifiable_run_is_not_a_measured_zero(): + r = _run("```sh\nnpm install unknown-example\n```", {"unknown-example": verify.UNKNOWN}) + assert r.verified_occurrences == 0 + assert r.by_tier["test"]["rate"] is None + assert "n/a" in render_html(r) + assert "0 verified by the registry" in render_markdown(r) + + +# --- SLOP-3: different endpoints never share a cache entry ----------------- + +def test_custom_endpoints_get_separate_cache_entries(tmp_path): + p = providers.OpenAICompatibleProvider("same-model", str(tmp_path)) + with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://provider-one.invalid/v1"}): + one = p._cache_path("fixture", 0.7, 0) + one_again = p._cache_path("fixture", 0.7, 0) + with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://provider-two.invalid/v1/"}): + two = p._cache_path("fixture", 0.7, 0) + assert one == one_again + assert one != two + + +def test_ollama_host_is_part_of_the_key(tmp_path): + p = providers.OllamaProvider("m", str(tmp_path)) + with patch.dict(os.environ, {"OLLAMA_HOST": "http://gpu-box:11434"}): + a = p._cache_path("x", 0.7, 0) + with patch.dict(os.environ, {"OLLAMA_HOST": "http://other-box:11434"}): + b = p._cache_path("x", 0.7, 0) + assert a != b + + +def test_default_endpoint_keeps_existing_caches_valid(tmp_path): + p = providers.OpenAICompatibleProvider("m", str(tmp_path)) + legacy = hashlib.sha256("openai|m|0.7|0|x".encode()).hexdigest() + ".json" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("SLOPBENCH_BASE_URL", None) + assert os.path.basename(p._cache_path("x", 0.7, 0)) == legacy + with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://API.openai.com/v1/"}): + assert os.path.basename(p._cache_path("x", 0.7, 0)) == legacy # same endpoint, normalized + + +def test_credentials_never_enter_the_key(tmp_path): + p = providers.OpenAICompatibleProvider("m", str(tmp_path)) + with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://user:secret@host.invalid/v1?token=abc"}): + path_with_secret = p._cache_path("x", 0.7, 0) + with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://host.invalid/v1"}): + assert p._cache_path("x", 0.7, 0) == path_with_secret From 5531e8ef62dc84bb1d9677c0f8f863d1f35a6c2f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 01:16:34 +0000 Subject: [PATCH 2/2] Tests: exercise the SLOP-1 adapter without requiring pkgguard CI installs slopbench without the optional pkgguard extra, so the SLOP-1 tests failed on import. They now drive the adapter with a stand-in shaped like pkgguard's Assessment (REVIEW/exists=False for an outage, BLOCK for a confirmed absence), as the existing pkgguard tests do. A separate test runs against the real pkgguard when it is installed and skips otherwise. Without pkgguard: 75 passed, 2 skipped; the new tests still fail on the previous code for every reported defect (8 of 8). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011GFY8v4QeVYzawpSLcJCzz --- tests/test_audit_measurement.py | 54 ++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_audit_measurement.py b/tests/test_audit_measurement.py index c3f087d..a4bda51 100644 --- a/tests/test_audit_measurement.py +++ b/tests/test_audit_measurement.py @@ -17,31 +17,55 @@ # --- SLOP-1: a registry outage is never an invented package --------------- -def _pkgguard_verdict(facts): - pkgguard = pytest.importorskip("pkgguard") # noqa: F841 - from pkgguard.conflation import ConflationDetector - from pkgguard.scoring import assess - result = assess(facts, ConflationDetector([]), []) - with patch("pkgguard.service.verify_many", return_value=[result]): - return result, verify.verify(facts.name, facts.ecosystem, True) +class _PkgguardResult: + """Shaped like pkgguard's Assessment, so the adapter is tested without pkgguard.""" + + def __init__(self, verdict, exists, signals=(), facts=None): + self.verdict, self.exists, self.signals, self.facts = verdict, exists, list(signals), facts + self.risk_score = 50 if verdict == "REVIEW" else 90 + + +def _stub_pkgguard(monkeypatch, result): + import sys + import types + + service = types.ModuleType("pkgguard.service") + service.verify_many = lambda names, ecosystem: [result] + monkeypatch.setitem(sys.modules, "pkgguard", types.ModuleType("pkgguard")) + monkeypatch.setitem(sys.modules, "pkgguard.service", service) @pytest.mark.parametrize("error", ["simulated timeout", "HTTP 429", "HTTP 503", "malformed JSON"]) -def test_registry_failure_is_unknown_not_invented(error): - from pkgguard.registries import PackageFacts - result, verdict = _pkgguard_verdict(PackageFacts("example", "npm", False, fetch_error=error)) - assert result.verdict == "REVIEW" and result.exists is False # what pkgguard reports +def test_registry_failure_is_unknown_not_invented(monkeypatch, error): + # What pkgguard returns when it cannot reach the registry. + _stub_pkgguard(monkeypatch, _PkgguardResult( + "REVIEW", False, [f"Could not reach the npm registry: {error}"])) + verdict = verify._with_pkgguard("example", "npm") assert verdict.status == verify.UNKNOWN assert verdict.invented is False -def test_confirmed_absence_still_counts_as_invented(): - from pkgguard.registries import PackageFacts - result, verdict = _pkgguard_verdict(PackageFacts("made-up-thing", "npm", False)) - assert result.verdict == "BLOCK" +def test_confirmed_absence_still_counts_as_invented(monkeypatch): + _stub_pkgguard(monkeypatch, _PkgguardResult("BLOCK", False, ["No package named 'x' exists"])) + verdict = verify._with_pkgguard("made-up-thing", "npm") assert verdict.status == verify.NOT_FOUND and verdict.invented +def test_against_real_pkgguard_when_installed(): + pytest.importorskip("pkgguard.scoring") + from pkgguard.conflation import ConflationDetector + from pkgguard.registries import PackageFacts + from pkgguard.scoring import assess + + for facts, expected in [ + (PackageFacts("example", "npm", False, fetch_error="simulated timeout"), verify.UNKNOWN), + (PackageFacts("made-up-thing", "npm", False), verify.NOT_FOUND), + ]: + result = assess(facts, ConflationDetector([]), []) + with patch("pkgguard.service.verify_many", return_value=[result]): + assert verify.verify(facts.name, facts.ecosystem, True).status == expected + + # --- SLOP-2: unverifiable references are out of the denominator ----------- class _Provider: