Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ authors:
repository-code: "https://github.com/rxslice/slopbench"
url: "https://blvkware.dev/ai-package-hallucination-data/"
license: Apache-2.0
version: 0.1.0
date-released: 2026-09-09
version: 0.1.1
date-released: 2026-11-14
abstract: >-
slopbench measures how often a language model or coding agent names a
software dependency that does not exist, or exists without adoption, on
Expand Down
5 changes: 4 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "slopbench"
version = "0.1.0"
dynamic = ["version"]
description = "Measure how often a model invents software dependencies, and how often it invents the same one twice."
readme = "README.md"
requires-python = ">=3.9"
Expand Down Expand Up @@ -47,6 +47,9 @@ Source = "https://github.com/rxslice/slopbench"
Issues = "https://github.com/rxslice/slopbench/issues"
Results = "https://blvkware.dev/ai-package-hallucination-data/"

[tool.setuptools.dynamic]
version = {attr = "slopbench.__version__"}

[tool.setuptools.packages.find]
include = ["slopbench*"]

Expand Down
2 changes: 1 addition & 1 deletion slopbench/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@

Built by Blvkware (https://blvkware.dev).
"""
__version__ = "0.1.0"
__version__ = "0.1.1"
__all__ = ["__version__"]
48 changes: 38 additions & 10 deletions slopbench/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,28 @@ class Result:
by_tier: Dict[str, dict] = field(default_factory=dict)
input_tokens: int = 0
output_tokens: int = 0
unverifiable_occurrences: int = 0
cached_samples: int = 0
default_sampling_samples: int = 0
pkgguard_available: bool = False
errors: List[str] = field(default_factory=list)

@property
def verified_occurrences(self) -> int:
"""References whose name the registry could confirm either way."""
return self.suggestion_occurrences - self.unverifiable_occurrences

@property
def occurrence_rate(self) -> float:
if not self.suggestion_occurrences:
"""Invented references over verified references.

Unverifiable references are excluded from the denominator as well as
the numerator: counting them in the denominator only would understate
the rate by however much of the run the registry could not answer.
"""
if not self.verified_occurrences:
return 0.0
return self.invented_occurrences / self.suggestion_occurrences
return self.invented_occurrences / self.verified_occurrences

@property
def targetable_rate(self) -> float:
Expand All @@ -97,6 +109,11 @@ def targetable_rate(self) -> float:
def to_dict(self) -> dict:
payload = asdict(self)
payload["occurrence_rate"] = round(self.occurrence_rate, 4)
payload["verified_occurrences"] = self.verified_occurrences
payload["verification_coverage"] = (
round(self.verified_occurrences / self.suggestion_occurrences, 4)
if self.suggestion_occurrences else None
)
payload["targetable_rate"] = round(self.targetable_rate, 4)
for finding, raw in zip(self.findings, payload["findings"]):
raw["targetable"] = finding.targetable
Expand Down Expand Up @@ -129,7 +146,7 @@ def run(
for prompt in prompts:
pid = prompt["id"]
tier = prompt.get("tier", "unspecified")
tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0})
tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0})
for index in range(samples):
note(f"{pid} sample {index + 1}/{samples}")
try:
Expand Down Expand Up @@ -158,13 +175,21 @@ def run(
findings: List[Finding] = []
invented_occurrences = 0
unverifiable = 0
unverifiable_occurrences = 0

def tier_of(pid: str) -> str:
return next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified")

for key, per_prompt in seen.items():
ecosystem, _ = key
name = display[key]
verdict: Verdict = verifier(name, ecosystem)
if verdict.status == UNKNOWN:
unverifiable += 1
unverifiable_occurrences += sum(per_prompt.values())
for pid, count in per_prompt.items():
tier_counts.setdefault(tier_of(pid), {"occurrences": 0, "invented": 0, "unverifiable": 0})
tier_counts[tier_of(pid)]["unverifiable"] += count
continue
if not verdict.invented:
continue
Expand All @@ -173,7 +198,7 @@ def run(
invented_occurrences += total_seen
for pid in per_prompt:
tier = next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified")
tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0})
tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0})
tier_counts[tier]["invented"] += per_prompt[pid]

findings.append(Finding(
Expand All @@ -189,14 +214,16 @@ def run(

findings.sort(key=lambda f: (-f.max_repeat_in_prompt, -f.samples_seen, f.name))

by_tier = {
tier: {
by_tier = {}
for tier, counts in sorted(tier_counts.items()):
verified = counts["occurrences"] - counts["unverifiable"]
by_tier[tier] = {
**counts,
"rate": round(counts["invented"] / counts["occurrences"], 4)
if counts["occurrences"] else 0.0,
"verified": verified,
# Over verified references only; None when nothing was verifiable,
# which is "not measured", not a measured zero.
"rate": round(counts["invented"] / verified, 4) if verified else None,
}
for tier, counts in sorted(tier_counts.items())
}

return Result(
provider=provider.name, model=provider.model, samples=samples,
Expand All @@ -208,6 +235,7 @@ def run(
invented_occurrences=invented_occurrences, invented_distinct=len(findings),
targetable_distinct=sum(1 for f in findings if f.targetable),
unverifiable_distinct=unverifiable,
unverifiable_occurrences=unverifiable_occurrences,
findings=findings, by_tier=by_tier,
input_tokens=in_tokens, output_tokens=out_tokens, cached_samples=cached,
default_sampling_samples=default_sampling,
Expand Down
45 changes: 40 additions & 5 deletions slopbench/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@
a published result stays reproducible from the cache after the model behind the
endpoint has moved on.

The cache key covers provider, model, prompt, temperature, and sample index.
Changing any of them is a different measurement and gets a different key.
The cache key covers provider, model, prompt, temperature, sample index, and
the endpoint that answered. Changing any of them is a different measurement
and gets a different key. Endpoints are identified by URL only, never by
credential. A provider's default endpoint adds nothing to the key, so caches
written before the endpoint was part of it stay valid for default setups;
any other endpoint keys separately.
"""
from __future__ import annotations

Expand Down Expand Up @@ -65,6 +69,18 @@ def _post(url: str, payload: dict, headers: dict) -> dict:
raise ProviderError(str(last))


def _normalize_endpoint(url: str) -> str:
"""Scheme, lowercased host and path, without trailing slash, query or userinfo."""
import urllib.parse

if not url:
return ""
parts = urllib.parse.urlsplit(url.strip())
host = (parts.hostname or "").lower()
netloc = host + (f":{parts.port}" if parts.port else "")
return urllib.parse.urlunsplit((parts.scheme.lower(), netloc, parts.path.rstrip("/"), "", ""))


class Provider:
"""Base adapter. Subclasses implement `_generate`."""

Expand All @@ -75,10 +91,19 @@ def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None:
self.cache_dir = cache_dir
os.makedirs(self.cache_dir, exist_ok=True)

#: The endpoint this provider talks to when nothing overrides it.
default_endpoint = ""

def endpoint(self) -> str:
"""The normalized URL this provider is configured to call. Never a secret."""
return self.default_endpoint

def _cache_path(self, prompt: str, temperature: float, index: int) -> str:
digest = hashlib.sha256(
f"{self.name}|{self.model}|{temperature}|{index}|{prompt}".encode("utf-8")
).hexdigest()
identity = f"{self.name}|{self.model}|{temperature}|{index}|{prompt}"
endpoint = _normalize_endpoint(self.endpoint())
if endpoint and endpoint != _normalize_endpoint(self.default_endpoint):
identity = f"v2|endpoint={endpoint}|" + identity
digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()
return os.path.join(self.cache_dir, f"{digest}.json")

def complete(self, prompt: str, temperature: float, index: int) -> Completion:
Expand All @@ -104,6 +129,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion:
"provider": self.name, "model": self.model,
"temperature": temperature, "index": index,
"prompt": prompt, "text": result.text,
"endpoint": _normalize_endpoint(self.endpoint()),
"input_tokens": result.input_tokens,
"output_tokens": result.output_tokens,
"temperature_applied": result.temperature_applied,
Expand Down Expand Up @@ -133,6 +159,7 @@ class AnthropicProvider(Provider):
"""

name = "anthropic"
default_endpoint = "https://api.anthropic.com/v1"
MAX_TOKENS = 16000

def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None:
Expand Down Expand Up @@ -173,6 +200,10 @@ class OpenAICompatibleProvider(Provider):
vLLM, LM Studio. Set SLOPBENCH_BASE_URL and SLOPBENCH_API_KEY."""

name = "openai"
default_endpoint = "https://api.openai.com/v1"

def endpoint(self) -> str:
return os.environ.get("SLOPBENCH_BASE_URL", self.default_endpoint)

def _generate(self, prompt: str, temperature: float) -> Completion:
base = os.environ.get("SLOPBENCH_BASE_URL", "https://api.openai.com/v1").rstrip("/")
Expand All @@ -195,6 +226,10 @@ class OllamaProvider(Provider):
"""Local models. No key, no spend — the way to get a first run for free."""

name = "ollama"
default_endpoint = "http://127.0.0.1:11434"

def endpoint(self) -> str:
return os.environ.get("OLLAMA_HOST", self.default_endpoint)

def _generate(self, prompt: str, temperature: float) -> Completion:
host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
Expand Down
17 changes: 10 additions & 7 deletions slopbench/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ def _e(value) -> str:
return html.escape(str(value))


def _pct(value: float) -> str:
return f"{value * 100:.1f}%"
def _pct(value) -> str:
return "n/a" if value is None else f"{value * 100:.1f}%"


def _verdict_sentence(result: Result) -> str:
Expand Down Expand Up @@ -179,6 +179,7 @@ def render_html(result: Result, subject: str = "") -> str:
("Temperature", result.temperature),
("Completions analysed", result.total_samples),
("Package references extracted", result.suggestion_occurrences),
("References the registry could verify", result.verified_occurrences),
("Distinct packages named", result.distinct_packages),
("Measured", result.started),
]
Expand All @@ -194,7 +195,7 @@ def render_html(result: Result, subject: str = "") -> str:
conditions.append(("Samples that failed", len(result.errors)))

tier_rows = "\n".join(
f'<tr><td>{_e(tier)}</td><td class="num">{c["occurrences"]}</td>'
f'<tr><td>{_e(tier)}</td><td class="num">{c.get("verified", c["occurrences"])}</td>'
f'<td class="num">{c["invented"]}</td><td class="num">{_pct(c["rate"])}</td></tr>'
for tier, c in result.by_tier.items()
)
Expand All @@ -210,9 +211,10 @@ def render_html(result: Result, subject: str = "") -> str:
unverifiable_note = (
f"<p>{result.unverifiable_distinct} distinct "
f"{'name' if result.unverifiable_distinct == 1 else 'names'} could not be "
f"confirmed against the registry during this run and "
f"confirmed against the registry during this run "
f"({result.unverifiable_occurrences} of {result.suggestion_occurrences} references) and "
f"{'is' if result.unverifiable_distinct == 1 else 'are'} excluded from every "
f"count above rather than assumed clean.</p>"
f"count and rate above, numerator and denominator, rather than assumed clean.</p>"
)

return f"""<!doctype html>
Expand Down Expand Up @@ -261,7 +263,7 @@ def render_html(result: Result, subject: str = "") -> str:
tasks are the control: invention there means the failure is not confined to
obscure corners of the ecosystem.</p>
<table>
<thead><tr><th>Task type</th><th class="num">References</th>
<thead><tr><th>Task type</th><th class="num">Verified references</th>
<th class="num">Invented</th><th class="num">Rate</th></tr></thead>
<tbody>{tier_rows}</tbody>
</table>
Expand Down Expand Up @@ -316,7 +318,8 @@ def render_markdown(result: Result, subject: str = "") -> str:
"",
f"- Model: `{result.provider}/{result.model}`",
f"- Prompts: {result.prompts_run}, sampled {result.samples}x at temperature {result.temperature}",
f"- Package references extracted: {result.suggestion_occurrences}",
f"- Package references extracted: {result.suggestion_occurrences} "
f"({result.verified_occurrences} verified by the registry)",
f"- Measured: {result.started}",
]
if result.unverifiable_distinct:
Expand Down
6 changes: 6 additions & 0 deletions slopbench/verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,12 @@ def _with_pkgguard(name: str, ecosystem: str) -> Optional[Verdict]:
return Verdict(name, ecosystem, UNKNOWN, error=str(error))

facts = result.facts or {}
if not result.exists and result.verdict != "BLOCK":
# pkgguard reports a registry it could not reach as exists=False with
# REVIEW, not BLOCK. Only a confirmed absence is a BLOCK; reading
# exists=False alone turned outages into invented packages.
return Verdict(name, ecosystem, UNKNOWN,
error="registry unreachable: " + "; ".join(result.signals or []))
if not result.exists:
status = NOT_FOUND
elif facts.get("download_stats_error") and result.verdict != "BLOCK":
Expand Down
Loading
Loading