From c8038e8cf0b7ec07d65deeec3f71a841fe22b0a5 Mon Sep 17 00:00:00 2001
From: Claude
Date: Sat, 14 Nov 2026 01:15:21 +0000
Subject: [PATCH 1/2] Fix measurement: outages, denominators, cache identity
(SLOP-1..3); 0.1.1
From the 2026-11-13 audit. tests/test_audit_measurement.py fails on the
previous code for every reported defect and passes now.
SLOP-1: pkgguard reports an unreachable registry as exists=False with
REVIEW. slopbench read exists=False alone as NOT_FOUND, so a timeout, a
429 or a 503 became an invented package. Only a BLOCK-confirmed absence is
NOT_FOUND now; any other exists=False is UNKNOWN and excluded. A confirmed
404 still counts.
SLOP-2: unverifiable references were dropped from the numerator but kept
in the denominator, so the audit fixture (one invented, one unverifiable,
each twice) reported 50% instead of 100%. occurrence_rate and every tier
rate are now over verified references. Result gains
unverifiable_occurrences and verified_occurrences; to_dict adds
verification_coverage. A tier with nothing verifiable reports rate None,
rendered n/a, never a measured zero. Reports show how many references the
registry verified, and the "excluded from every count" note is now true.
SLOP-3: the completion cache key ignored the endpoint, so changing
SLOPBENCH_BASE_URL or OLLAMA_HOST reused another server's completions.
The key now includes the normalized endpoint URL (scheme, host, port, path;
never userinfo, query or credentials) whenever it differs from the
provider's default, so caches written for default setups stay valid. Each
cache file records its endpoint.
Version 0.1.1, now single-sourced: pyproject reads slopbench.__version__,
so a release tag can no longer disagree with the built package.
pytest: 76 passed. Build and twine --strict pass.
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_011GFY8v4QeVYzawpSLcJCzz
---
CITATION.cff | 4 +-
pyproject.toml | 5 +-
slopbench/__init__.py | 2 +-
slopbench/bench.py | 48 +++++++++---
slopbench/providers.py | 45 +++++++++--
slopbench/report.py | 17 ++--
slopbench/verify.py | 6 ++
tests/test_audit_measurement.py | 133 ++++++++++++++++++++++++++++++++
8 files changed, 234 insertions(+), 26 deletions(-)
create mode 100644 tests/test_audit_measurement.py
diff --git a/CITATION.cff b/CITATION.cff
index 415b7dd..59d27a8 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -8,8 +8,8 @@ authors:
repository-code: "https://github.com/rxslice/slopbench"
url: "https://blvkware.dev/ai-package-hallucination-data/"
license: Apache-2.0
-version: 0.1.0
-date-released: 2026-09-09
+version: 0.1.1
+date-released: 2026-11-14
abstract: >-
slopbench measures how often a language model or coding agent names a
software dependency that does not exist, or exists without adoption, on
diff --git a/pyproject.toml b/pyproject.toml
index db4a534..91c25e3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "slopbench"
-version = "0.1.0"
+dynamic = ["version"]
description = "Measure how often a model invents software dependencies, and how often it invents the same one twice."
readme = "README.md"
requires-python = ">=3.9"
@@ -47,6 +47,9 @@ Source = "https://github.com/rxslice/slopbench"
Issues = "https://github.com/rxslice/slopbench/issues"
Results = "https://blvkware.dev/ai-package-hallucination-data/"
+[tool.setuptools.dynamic]
+version = {attr = "slopbench.__version__"}
+
[tool.setuptools.packages.find]
include = ["slopbench*"]
diff --git a/slopbench/__init__.py b/slopbench/__init__.py
index ebee143..540e44b 100644
--- a/slopbench/__init__.py
+++ b/slopbench/__init__.py
@@ -2,5 +2,5 @@
Built by Blvkware (https://blvkware.dev).
"""
-__version__ = "0.1.0"
+__version__ = "0.1.1"
__all__ = ["__version__"]
diff --git a/slopbench/bench.py b/slopbench/bench.py
index b008c0f..04bf2c5 100644
--- a/slopbench/bench.py
+++ b/slopbench/bench.py
@@ -77,16 +77,28 @@ class Result:
by_tier: Dict[str, dict] = field(default_factory=dict)
input_tokens: int = 0
output_tokens: int = 0
+ unverifiable_occurrences: int = 0
cached_samples: int = 0
default_sampling_samples: int = 0
pkgguard_available: bool = False
errors: List[str] = field(default_factory=list)
+ @property
+ def verified_occurrences(self) -> int:
+ """References whose name the registry could confirm either way."""
+ return self.suggestion_occurrences - self.unverifiable_occurrences
+
@property
def occurrence_rate(self) -> float:
- if not self.suggestion_occurrences:
+ """Invented references over verified references.
+
+ Unverifiable references are excluded from the denominator as well as
+ the numerator: counting them in the denominator only would understate
+ the rate by however much of the run the registry could not answer.
+ """
+ if not self.verified_occurrences:
return 0.0
- return self.invented_occurrences / self.suggestion_occurrences
+ return self.invented_occurrences / self.verified_occurrences
@property
def targetable_rate(self) -> float:
@@ -97,6 +109,11 @@ def targetable_rate(self) -> float:
def to_dict(self) -> dict:
payload = asdict(self)
payload["occurrence_rate"] = round(self.occurrence_rate, 4)
+ payload["verified_occurrences"] = self.verified_occurrences
+ payload["verification_coverage"] = (
+ round(self.verified_occurrences / self.suggestion_occurrences, 4)
+ if self.suggestion_occurrences else None
+ )
payload["targetable_rate"] = round(self.targetable_rate, 4)
for finding, raw in zip(self.findings, payload["findings"]):
raw["targetable"] = finding.targetable
@@ -129,7 +146,7 @@ def run(
for prompt in prompts:
pid = prompt["id"]
tier = prompt.get("tier", "unspecified")
- tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0})
+ tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0})
for index in range(samples):
note(f"{pid} sample {index + 1}/{samples}")
try:
@@ -158,6 +175,10 @@ def run(
findings: List[Finding] = []
invented_occurrences = 0
unverifiable = 0
+ unverifiable_occurrences = 0
+
+ def tier_of(pid: str) -> str:
+ return next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified")
for key, per_prompt in seen.items():
ecosystem, _ = key
@@ -165,6 +186,10 @@ def run(
verdict: Verdict = verifier(name, ecosystem)
if verdict.status == UNKNOWN:
unverifiable += 1
+ unverifiable_occurrences += sum(per_prompt.values())
+ for pid, count in per_prompt.items():
+ tier_counts.setdefault(tier_of(pid), {"occurrences": 0, "invented": 0, "unverifiable": 0})
+ tier_counts[tier_of(pid)]["unverifiable"] += count
continue
if not verdict.invented:
continue
@@ -173,7 +198,7 @@ def run(
invented_occurrences += total_seen
for pid in per_prompt:
tier = next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified")
- tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0})
+ tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0})
tier_counts[tier]["invented"] += per_prompt[pid]
findings.append(Finding(
@@ -189,14 +214,16 @@ def run(
findings.sort(key=lambda f: (-f.max_repeat_in_prompt, -f.samples_seen, f.name))
- by_tier = {
- tier: {
+ by_tier = {}
+ for tier, counts in sorted(tier_counts.items()):
+ verified = counts["occurrences"] - counts["unverifiable"]
+ by_tier[tier] = {
**counts,
- "rate": round(counts["invented"] / counts["occurrences"], 4)
- if counts["occurrences"] else 0.0,
+ "verified": verified,
+ # Over verified references only; None when nothing was verifiable,
+ # which is "not measured", not a measured zero.
+ "rate": round(counts["invented"] / verified, 4) if verified else None,
}
- for tier, counts in sorted(tier_counts.items())
- }
return Result(
provider=provider.name, model=provider.model, samples=samples,
@@ -208,6 +235,7 @@ def run(
invented_occurrences=invented_occurrences, invented_distinct=len(findings),
targetable_distinct=sum(1 for f in findings if f.targetable),
unverifiable_distinct=unverifiable,
+ unverifiable_occurrences=unverifiable_occurrences,
findings=findings, by_tier=by_tier,
input_tokens=in_tokens, output_tokens=out_tokens, cached_samples=cached,
default_sampling_samples=default_sampling,
diff --git a/slopbench/providers.py b/slopbench/providers.py
index 7a4cbf0..92cd810 100644
--- a/slopbench/providers.py
+++ b/slopbench/providers.py
@@ -9,8 +9,12 @@
a published result stays reproducible from the cache after the model behind the
endpoint has moved on.
-The cache key covers provider, model, prompt, temperature, and sample index.
-Changing any of them is a different measurement and gets a different key.
+The cache key covers provider, model, prompt, temperature, sample index, and
+the endpoint that answered. Changing any of them is a different measurement
+and gets a different key. Endpoints are identified by URL only, never by
+credential. A provider's default endpoint adds nothing to the key, so caches
+written before the endpoint was part of it stay valid for default setups;
+any other endpoint keys separately.
"""
from __future__ import annotations
@@ -65,6 +69,18 @@ def _post(url: str, payload: dict, headers: dict) -> dict:
raise ProviderError(str(last))
+def _normalize_endpoint(url: str) -> str:
+ """Scheme, lowercased host and path, without trailing slash, query or userinfo."""
+ import urllib.parse
+
+ if not url:
+ return ""
+ parts = urllib.parse.urlsplit(url.strip())
+ host = (parts.hostname or "").lower()
+ netloc = host + (f":{parts.port}" if parts.port else "")
+ return urllib.parse.urlunsplit((parts.scheme.lower(), netloc, parts.path.rstrip("/"), "", ""))
+
+
class Provider:
"""Base adapter. Subclasses implement `_generate`."""
@@ -75,10 +91,19 @@ def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None:
self.cache_dir = cache_dir
os.makedirs(self.cache_dir, exist_ok=True)
+ #: The endpoint this provider talks to when nothing overrides it.
+ default_endpoint = ""
+
+ def endpoint(self) -> str:
+ """The normalized URL this provider is configured to call. Never a secret."""
+ return self.default_endpoint
+
def _cache_path(self, prompt: str, temperature: float, index: int) -> str:
- digest = hashlib.sha256(
- f"{self.name}|{self.model}|{temperature}|{index}|{prompt}".encode("utf-8")
- ).hexdigest()
+ identity = f"{self.name}|{self.model}|{temperature}|{index}|{prompt}"
+ endpoint = _normalize_endpoint(self.endpoint())
+ if endpoint and endpoint != _normalize_endpoint(self.default_endpoint):
+ identity = f"v2|endpoint={endpoint}|" + identity
+ digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()
return os.path.join(self.cache_dir, f"{digest}.json")
def complete(self, prompt: str, temperature: float, index: int) -> Completion:
@@ -104,6 +129,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion:
"provider": self.name, "model": self.model,
"temperature": temperature, "index": index,
"prompt": prompt, "text": result.text,
+ "endpoint": _normalize_endpoint(self.endpoint()),
"input_tokens": result.input_tokens,
"output_tokens": result.output_tokens,
"temperature_applied": result.temperature_applied,
@@ -133,6 +159,7 @@ class AnthropicProvider(Provider):
"""
name = "anthropic"
+ default_endpoint = "https://api.anthropic.com/v1"
MAX_TOKENS = 16000
def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None:
@@ -173,6 +200,10 @@ class OpenAICompatibleProvider(Provider):
vLLM, LM Studio. Set SLOPBENCH_BASE_URL and SLOPBENCH_API_KEY."""
name = "openai"
+ default_endpoint = "https://api.openai.com/v1"
+
+ def endpoint(self) -> str:
+ return os.environ.get("SLOPBENCH_BASE_URL", self.default_endpoint)
def _generate(self, prompt: str, temperature: float) -> Completion:
base = os.environ.get("SLOPBENCH_BASE_URL", "https://api.openai.com/v1").rstrip("/")
@@ -195,6 +226,10 @@ class OllamaProvider(Provider):
"""Local models. No key, no spend — the way to get a first run for free."""
name = "ollama"
+ default_endpoint = "http://127.0.0.1:11434"
+
+ def endpoint(self) -> str:
+ return os.environ.get("OLLAMA_HOST", self.default_endpoint)
def _generate(self, prompt: str, temperature: float) -> Completion:
host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
diff --git a/slopbench/report.py b/slopbench/report.py
index 3dc8b3b..307ebb8 100644
--- a/slopbench/report.py
+++ b/slopbench/report.py
@@ -113,8 +113,8 @@ def _e(value) -> str:
return html.escape(str(value))
-def _pct(value: float) -> str:
- return f"{value * 100:.1f}%"
+def _pct(value) -> str:
+ return "n/a" if value is None else f"{value * 100:.1f}%"
def _verdict_sentence(result: Result) -> str:
@@ -179,6 +179,7 @@ def render_html(result: Result, subject: str = "") -> str:
("Temperature", result.temperature),
("Completions analysed", result.total_samples),
("Package references extracted", result.suggestion_occurrences),
+ ("References the registry could verify", result.verified_occurrences),
("Distinct packages named", result.distinct_packages),
("Measured", result.started),
]
@@ -194,7 +195,7 @@ def render_html(result: Result, subject: str = "") -> str:
conditions.append(("Samples that failed", len(result.errors)))
tier_rows = "\n".join(
- f'| {_e(tier)} | {c["occurrences"]} | '
+ f'
| {_e(tier)} | {c.get("verified", c["occurrences"])} | '
f'{c["invented"]} | {_pct(c["rate"])} |
'
for tier, c in result.by_tier.items()
)
@@ -210,9 +211,10 @@ def render_html(result: Result, subject: str = "") -> str:
unverifiable_note = (
f"{result.unverifiable_distinct} distinct "
f"{'name' if result.unverifiable_distinct == 1 else 'names'} could not be "
- f"confirmed against the registry during this run and "
+ f"confirmed against the registry during this run "
+ f"({result.unverifiable_occurrences} of {result.suggestion_occurrences} references) and "
f"{'is' if result.unverifiable_distinct == 1 else 'are'} excluded from every "
- f"count above rather than assumed clean.
"
+ f"count and rate above, numerator and denominator, rather than assumed clean.
"
)
return f"""
@@ -261,7 +263,7 @@ def render_html(result: Result, subject: str = "") -> str:
tasks are the control: invention there means the failure is not confined to
obscure corners of the ecosystem.
- | Task type | References |
+ | Task type | Verified references |
Invented | Rate |
{tier_rows}
@@ -316,7 +318,8 @@ def render_markdown(result: Result, subject: str = "") -> str:
"",
f"- Model: `{result.provider}/{result.model}`",
f"- Prompts: {result.prompts_run}, sampled {result.samples}x at temperature {result.temperature}",
- f"- Package references extracted: {result.suggestion_occurrences}",
+ f"- Package references extracted: {result.suggestion_occurrences} "
+ f"({result.verified_occurrences} verified by the registry)",
f"- Measured: {result.started}",
]
if result.unverifiable_distinct:
diff --git a/slopbench/verify.py b/slopbench/verify.py
index 900dffa..8aae038 100644
--- a/slopbench/verify.py
+++ b/slopbench/verify.py
@@ -224,6 +224,12 @@ def _with_pkgguard(name: str, ecosystem: str) -> Optional[Verdict]:
return Verdict(name, ecosystem, UNKNOWN, error=str(error))
facts = result.facts or {}
+ if not result.exists and result.verdict != "BLOCK":
+ # pkgguard reports a registry it could not reach as exists=False with
+ # REVIEW, not BLOCK. Only a confirmed absence is a BLOCK; reading
+ # exists=False alone turned outages into invented packages.
+ return Verdict(name, ecosystem, UNKNOWN,
+ error="registry unreachable: " + "; ".join(result.signals or []))
if not result.exists:
status = NOT_FOUND
elif facts.get("download_stats_error") and result.verdict != "BLOCK":
diff --git a/tests/test_audit_measurement.py b/tests/test_audit_measurement.py
new file mode 100644
index 0000000..c3f087d
--- /dev/null
+++ b/tests/test_audit_measurement.py
@@ -0,0 +1,133 @@
+"""Regression tests for the 2026-11-13 audit (SLOP-1 to SLOP-3).
+
+Registry and model calls are faked; nothing is fetched or installed.
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+from unittest.mock import patch
+
+import pytest
+
+from slopbench import bench, providers, verify
+from slopbench.providers import Completion
+from slopbench.report import render_html, render_markdown
+
+
+# --- SLOP-1: a registry outage is never an invented package ---------------
+
+def _pkgguard_verdict(facts):
+ pkgguard = pytest.importorskip("pkgguard") # noqa: F841
+ from pkgguard.conflation import ConflationDetector
+ from pkgguard.scoring import assess
+ result = assess(facts, ConflationDetector([]), [])
+ with patch("pkgguard.service.verify_many", return_value=[result]):
+ return result, verify.verify(facts.name, facts.ecosystem, True)
+
+
+@pytest.mark.parametrize("error", ["simulated timeout", "HTTP 429", "HTTP 503", "malformed JSON"])
+def test_registry_failure_is_unknown_not_invented(error):
+ from pkgguard.registries import PackageFacts
+ result, verdict = _pkgguard_verdict(PackageFacts("example", "npm", False, fetch_error=error))
+ assert result.verdict == "REVIEW" and result.exists is False # what pkgguard reports
+ assert verdict.status == verify.UNKNOWN
+ assert verdict.invented is False
+
+
+def test_confirmed_absence_still_counts_as_invented():
+ from pkgguard.registries import PackageFacts
+ result, verdict = _pkgguard_verdict(PackageFacts("made-up-thing", "npm", False))
+ assert result.verdict == "BLOCK"
+ assert verdict.status == verify.NOT_FOUND and verdict.invented
+
+
+# --- SLOP-2: unverifiable references are out of the denominator -----------
+
+class _Provider:
+ name = "offline"
+ model = "fixture"
+
+ def __init__(self, text):
+ self.text = text
+
+ def complete(self, *args):
+ return Completion(self.text, False)
+
+
+def _run(text, statuses, samples=2):
+ class FakeVerifier:
+ pkgguard_available = False
+
+ def __init__(self, **kwargs):
+ pass
+
+ def __call__(self, name, ecosystem):
+ return verify.Verdict(name, ecosystem, statuses[name])
+
+ with patch.object(bench, "Verifier", FakeVerifier):
+ return bench.run(_Provider(text), [{"id": "p", "tier": "test", "ecosystem": "npm", "text": "t"}],
+ samples=samples)
+
+
+def test_rate_is_over_verified_references():
+ # The audit's fixture: one invented and one unverifiable name, each twice.
+ r = _run("```sh\nnpm install invented-example unknown-example\n```",
+ {"invented-example": verify.NOT_FOUND, "unknown-example": verify.UNKNOWN})
+ assert r.suggestion_occurrences == 4
+ assert r.unverifiable_occurrences == 2
+ assert r.verified_occurrences == 2
+ assert r.invented_occurrences == 2
+ assert r.occurrence_rate == 1.0 # was reported as 0.5
+ assert r.by_tier["test"]["verified"] == 2
+ assert r.by_tier["test"]["rate"] == 1.0
+ d = r.to_dict()
+ assert d["verified_occurrences"] == 2 and d["verification_coverage"] == 0.5
+
+
+def test_all_unverifiable_run_is_not_a_measured_zero():
+ r = _run("```sh\nnpm install unknown-example\n```", {"unknown-example": verify.UNKNOWN})
+ assert r.verified_occurrences == 0
+ assert r.by_tier["test"]["rate"] is None
+ assert "n/a" in render_html(r)
+ assert "0 verified by the registry" in render_markdown(r)
+
+
+# --- SLOP-3: different endpoints never share a cache entry -----------------
+
+def test_custom_endpoints_get_separate_cache_entries(tmp_path):
+ p = providers.OpenAICompatibleProvider("same-model", str(tmp_path))
+ with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://provider-one.invalid/v1"}):
+ one = p._cache_path("fixture", 0.7, 0)
+ one_again = p._cache_path("fixture", 0.7, 0)
+ with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://provider-two.invalid/v1/"}):
+ two = p._cache_path("fixture", 0.7, 0)
+ assert one == one_again
+ assert one != two
+
+
+def test_ollama_host_is_part_of_the_key(tmp_path):
+ p = providers.OllamaProvider("m", str(tmp_path))
+ with patch.dict(os.environ, {"OLLAMA_HOST": "http://gpu-box:11434"}):
+ a = p._cache_path("x", 0.7, 0)
+ with patch.dict(os.environ, {"OLLAMA_HOST": "http://other-box:11434"}):
+ b = p._cache_path("x", 0.7, 0)
+ assert a != b
+
+
+def test_default_endpoint_keeps_existing_caches_valid(tmp_path):
+ p = providers.OpenAICompatibleProvider("m", str(tmp_path))
+ legacy = hashlib.sha256("openai|m|0.7|0|x".encode()).hexdigest() + ".json"
+ with patch.dict(os.environ, {}, clear=False):
+ os.environ.pop("SLOPBENCH_BASE_URL", None)
+ assert os.path.basename(p._cache_path("x", 0.7, 0)) == legacy
+ with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://API.openai.com/v1/"}):
+ assert os.path.basename(p._cache_path("x", 0.7, 0)) == legacy # same endpoint, normalized
+
+
+def test_credentials_never_enter_the_key(tmp_path):
+ p = providers.OpenAICompatibleProvider("m", str(tmp_path))
+ with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://user:secret@host.invalid/v1?token=abc"}):
+ path_with_secret = p._cache_path("x", 0.7, 0)
+ with patch.dict(os.environ, {"SLOPBENCH_BASE_URL": "https://host.invalid/v1"}):
+ assert p._cache_path("x", 0.7, 0) == path_with_secret
From 5531e8ef62dc84bb1d9677c0f8f863d1f35a6c2f Mon Sep 17 00:00:00 2001
From: Claude
Date: Sat, 14 Nov 2026 01:16:34 +0000
Subject: [PATCH 2/2] Tests: exercise the SLOP-1 adapter without requiring
pkgguard
CI installs slopbench without the optional pkgguard extra, so the SLOP-1
tests failed on import. They now drive the adapter with a stand-in shaped
like pkgguard's Assessment (REVIEW/exists=False for an outage, BLOCK for a
confirmed absence), as the existing pkgguard tests do. A separate test runs
against the real pkgguard when it is installed and skips otherwise.
Without pkgguard: 75 passed, 2 skipped; the new tests still fail on the
previous code for every reported defect (8 of 8).
Co-Authored-By: Claude Opus 5.5
Claude-Session: https://claude.ai/code/session_011GFY8v4QeVYzawpSLcJCzz
---
tests/test_audit_measurement.py | 54 ++++++++++++++++++++++++---------
1 file changed, 39 insertions(+), 15 deletions(-)
diff --git a/tests/test_audit_measurement.py b/tests/test_audit_measurement.py
index c3f087d..a4bda51 100644
--- a/tests/test_audit_measurement.py
+++ b/tests/test_audit_measurement.py
@@ -17,31 +17,55 @@
# --- SLOP-1: a registry outage is never an invented package ---------------
-def _pkgguard_verdict(facts):
- pkgguard = pytest.importorskip("pkgguard") # noqa: F841
- from pkgguard.conflation import ConflationDetector
- from pkgguard.scoring import assess
- result = assess(facts, ConflationDetector([]), [])
- with patch("pkgguard.service.verify_many", return_value=[result]):
- return result, verify.verify(facts.name, facts.ecosystem, True)
+class _PkgguardResult:
+ """Shaped like pkgguard's Assessment, so the adapter is tested without pkgguard."""
+
+ def __init__(self, verdict, exists, signals=(), facts=None):
+ self.verdict, self.exists, self.signals, self.facts = verdict, exists, list(signals), facts
+ self.risk_score = 50 if verdict == "REVIEW" else 90
+
+
+def _stub_pkgguard(monkeypatch, result):
+ import sys
+ import types
+
+ service = types.ModuleType("pkgguard.service")
+ service.verify_many = lambda names, ecosystem: [result]
+ monkeypatch.setitem(sys.modules, "pkgguard", types.ModuleType("pkgguard"))
+ monkeypatch.setitem(sys.modules, "pkgguard.service", service)
@pytest.mark.parametrize("error", ["simulated timeout", "HTTP 429", "HTTP 503", "malformed JSON"])
-def test_registry_failure_is_unknown_not_invented(error):
- from pkgguard.registries import PackageFacts
- result, verdict = _pkgguard_verdict(PackageFacts("example", "npm", False, fetch_error=error))
- assert result.verdict == "REVIEW" and result.exists is False # what pkgguard reports
+def test_registry_failure_is_unknown_not_invented(monkeypatch, error):
+ # What pkgguard returns when it cannot reach the registry.
+ _stub_pkgguard(monkeypatch, _PkgguardResult(
+ "REVIEW", False, [f"Could not reach the npm registry: {error}"]))
+ verdict = verify._with_pkgguard("example", "npm")
assert verdict.status == verify.UNKNOWN
assert verdict.invented is False
-def test_confirmed_absence_still_counts_as_invented():
- from pkgguard.registries import PackageFacts
- result, verdict = _pkgguard_verdict(PackageFacts("made-up-thing", "npm", False))
- assert result.verdict == "BLOCK"
+def test_confirmed_absence_still_counts_as_invented(monkeypatch):
+ _stub_pkgguard(monkeypatch, _PkgguardResult("BLOCK", False, ["No package named 'x' exists"]))
+ verdict = verify._with_pkgguard("made-up-thing", "npm")
assert verdict.status == verify.NOT_FOUND and verdict.invented
+def test_against_real_pkgguard_when_installed():
+ pytest.importorskip("pkgguard.scoring")
+ from pkgguard.conflation import ConflationDetector
+ from pkgguard.registries import PackageFacts
+ from pkgguard.scoring import assess
+
+ for facts, expected in [
+ (PackageFacts("example", "npm", False, fetch_error="simulated timeout"), verify.UNKNOWN),
+ (PackageFacts("made-up-thing", "npm", False), verify.NOT_FOUND),
+ ]:
+ result = assess(facts, ConflationDetector([]), [])
+ with patch("pkgguard.service.verify_many", return_value=[result]):
+ assert verify.verify(facts.name, facts.ecosystem, True).status == expected
+
+
# --- SLOP-2: unverifiable references are out of the denominator -----------
class _Provider: