diff --git a/CITATION.cff b/CITATION.cff index 415b7dd..59d27a8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -8,8 +8,8 @@ authors: repository-code: "https://github.com/rxslice/slopbench" url: "https://blvkware.dev/ai-package-hallucination-data/" license: Apache-2.0 -version: 0.1.0 -date-released: 2026-09-09 +version: 0.1.1 +date-released: 2026-11-14 abstract: >- slopbench measures how often a language model or coding agent names a software dependency that does not exist, or exists without adoption, on diff --git a/pyproject.toml b/pyproject.toml index db4a534..91c25e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "slopbench" -version = "0.1.0" +dynamic = ["version"] description = "Measure how often a model invents software dependencies, and how often it invents the same one twice." readme = "README.md" requires-python = ">=3.9" @@ -47,6 +47,9 @@ Source = "https://github.com/rxslice/slopbench" Issues = "https://github.com/rxslice/slopbench/issues" Results = "https://blvkware.dev/ai-package-hallucination-data/" +[tool.setuptools.dynamic] +version = {attr = "slopbench.__version__"} + [tool.setuptools.packages.find] include = ["slopbench*"] diff --git a/slopbench/__init__.py b/slopbench/__init__.py index ebee143..540e44b 100644 --- a/slopbench/__init__.py +++ b/slopbench/__init__.py @@ -2,5 +2,5 @@ Built by Blvkware (https://blvkware.dev). """ -__version__ = "0.1.0" +__version__ = "0.1.1" __all__ = ["__version__"] diff --git a/slopbench/bench.py b/slopbench/bench.py index b008c0f..04bf2c5 100644 --- a/slopbench/bench.py +++ b/slopbench/bench.py @@ -77,16 +77,28 @@ class Result: by_tier: Dict[str, dict] = field(default_factory=dict) input_tokens: int = 0 output_tokens: int = 0 + unverifiable_occurrences: int = 0 cached_samples: int = 0 default_sampling_samples: int = 0 pkgguard_available: bool = False errors: List[str] = field(default_factory=list) + @property + def verified_occurrences(self) -> int: + """References whose name the registry could confirm either way.""" + return self.suggestion_occurrences - self.unverifiable_occurrences + @property def occurrence_rate(self) -> float: - if not self.suggestion_occurrences: + """Invented references over verified references. + + Unverifiable references are excluded from the denominator as well as + the numerator: counting them in the denominator only would understate + the rate by however much of the run the registry could not answer. + """ + if not self.verified_occurrences: return 0.0 - return self.invented_occurrences / self.suggestion_occurrences + return self.invented_occurrences / self.verified_occurrences @property def targetable_rate(self) -> float: @@ -97,6 +109,11 @@ def targetable_rate(self) -> float: def to_dict(self) -> dict: payload = asdict(self) payload["occurrence_rate"] = round(self.occurrence_rate, 4) + payload["verified_occurrences"] = self.verified_occurrences + payload["verification_coverage"] = ( + round(self.verified_occurrences / self.suggestion_occurrences, 4) + if self.suggestion_occurrences else None + ) payload["targetable_rate"] = round(self.targetable_rate, 4) for finding, raw in zip(self.findings, payload["findings"]): raw["targetable"] = finding.targetable @@ -129,7 +146,7 @@ def run( for prompt in prompts: pid = prompt["id"] tier = prompt.get("tier", "unspecified") - tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0}) + tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0}) for index in range(samples): note(f"{pid} sample {index + 1}/{samples}") try: @@ -158,6 +175,10 @@ def run( findings: List[Finding] = [] invented_occurrences = 0 unverifiable = 0 + unverifiable_occurrences = 0 + + def tier_of(pid: str) -> str: + return next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified") for key, per_prompt in seen.items(): ecosystem, _ = key @@ -165,6 +186,10 @@ def run( verdict: Verdict = verifier(name, ecosystem) if verdict.status == UNKNOWN: unverifiable += 1 + unverifiable_occurrences += sum(per_prompt.values()) + for pid, count in per_prompt.items(): + tier_counts.setdefault(tier_of(pid), {"occurrences": 0, "invented": 0, "unverifiable": 0}) + tier_counts[tier_of(pid)]["unverifiable"] += count continue if not verdict.invented: continue @@ -173,7 +198,7 @@ def run( invented_occurrences += total_seen for pid in per_prompt: tier = next((p.get("tier", "unspecified") for p in prompts if p["id"] == pid), "unspecified") - tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0}) + tier_counts.setdefault(tier, {"occurrences": 0, "invented": 0, "unverifiable": 0}) tier_counts[tier]["invented"] += per_prompt[pid] findings.append(Finding( @@ -189,14 +214,16 @@ def run( findings.sort(key=lambda f: (-f.max_repeat_in_prompt, -f.samples_seen, f.name)) - by_tier = { - tier: { + by_tier = {} + for tier, counts in sorted(tier_counts.items()): + verified = counts["occurrences"] - counts["unverifiable"] + by_tier[tier] = { **counts, - "rate": round(counts["invented"] / counts["occurrences"], 4) - if counts["occurrences"] else 0.0, + "verified": verified, + # Over verified references only; None when nothing was verifiable, + # which is "not measured", not a measured zero. + "rate": round(counts["invented"] / verified, 4) if verified else None, } - for tier, counts in sorted(tier_counts.items()) - } return Result( provider=provider.name, model=provider.model, samples=samples, @@ -208,6 +235,7 @@ def run( invented_occurrences=invented_occurrences, invented_distinct=len(findings), targetable_distinct=sum(1 for f in findings if f.targetable), unverifiable_distinct=unverifiable, + unverifiable_occurrences=unverifiable_occurrences, findings=findings, by_tier=by_tier, input_tokens=in_tokens, output_tokens=out_tokens, cached_samples=cached, default_sampling_samples=default_sampling, diff --git a/slopbench/providers.py b/slopbench/providers.py index 7a4cbf0..92cd810 100644 --- a/slopbench/providers.py +++ b/slopbench/providers.py @@ -9,8 +9,12 @@ a published result stays reproducible from the cache after the model behind the endpoint has moved on. -The cache key covers provider, model, prompt, temperature, and sample index. -Changing any of them is a different measurement and gets a different key. +The cache key covers provider, model, prompt, temperature, sample index, and +the endpoint that answered. Changing any of them is a different measurement +and gets a different key. Endpoints are identified by URL only, never by +credential. A provider's default endpoint adds nothing to the key, so caches +written before the endpoint was part of it stay valid for default setups; +any other endpoint keys separately. """ from __future__ import annotations @@ -65,6 +69,18 @@ def _post(url: str, payload: dict, headers: dict) -> dict: raise ProviderError(str(last)) +def _normalize_endpoint(url: str) -> str: + """Scheme, lowercased host and path, without trailing slash, query or userinfo.""" + import urllib.parse + + if not url: + return "" + parts = urllib.parse.urlsplit(url.strip()) + host = (parts.hostname or "").lower() + netloc = host + (f":{parts.port}" if parts.port else "") + return urllib.parse.urlunsplit((parts.scheme.lower(), netloc, parts.path.rstrip("/"), "", "")) + + class Provider: """Base adapter. Subclasses implement `_generate`.""" @@ -75,10 +91,19 @@ def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None: self.cache_dir = cache_dir os.makedirs(self.cache_dir, exist_ok=True) + #: The endpoint this provider talks to when nothing overrides it. + default_endpoint = "" + + def endpoint(self) -> str: + """The normalized URL this provider is configured to call. Never a secret.""" + return self.default_endpoint + def _cache_path(self, prompt: str, temperature: float, index: int) -> str: - digest = hashlib.sha256( - f"{self.name}|{self.model}|{temperature}|{index}|{prompt}".encode("utf-8") - ).hexdigest() + identity = f"{self.name}|{self.model}|{temperature}|{index}|{prompt}" + endpoint = _normalize_endpoint(self.endpoint()) + if endpoint and endpoint != _normalize_endpoint(self.default_endpoint): + identity = f"v2|endpoint={endpoint}|" + identity + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest() return os.path.join(self.cache_dir, f"{digest}.json") def complete(self, prompt: str, temperature: float, index: int) -> Completion: @@ -104,6 +129,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion: "provider": self.name, "model": self.model, "temperature": temperature, "index": index, "prompt": prompt, "text": result.text, + "endpoint": _normalize_endpoint(self.endpoint()), "input_tokens": result.input_tokens, "output_tokens": result.output_tokens, "temperature_applied": result.temperature_applied, @@ -133,6 +159,7 @@ class AnthropicProvider(Provider): """ name = "anthropic" + default_endpoint = "https://api.anthropic.com/v1" MAX_TOKENS = 16000 def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None: @@ -173,6 +200,10 @@ class OpenAICompatibleProvider(Provider): vLLM, LM Studio. Set SLOPBENCH_BASE_URL and SLOPBENCH_API_KEY.""" name = "openai" + default_endpoint = "https://api.openai.com/v1" + + def endpoint(self) -> str: + return os.environ.get("SLOPBENCH_BASE_URL", self.default_endpoint) def _generate(self, prompt: str, temperature: float) -> Completion: base = os.environ.get("SLOPBENCH_BASE_URL", "https://api.openai.com/v1").rstrip("/") @@ -195,6 +226,10 @@ class OllamaProvider(Provider): """Local models. No key, no spend — the way to get a first run for free.""" name = "ollama" + default_endpoint = "http://127.0.0.1:11434" + + def endpoint(self) -> str: + return os.environ.get("OLLAMA_HOST", self.default_endpoint) def _generate(self, prompt: str, temperature: float) -> Completion: host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/") diff --git a/slopbench/report.py b/slopbench/report.py index 3dc8b3b..307ebb8 100644 --- a/slopbench/report.py +++ b/slopbench/report.py @@ -113,8 +113,8 @@ def _e(value) -> str: return html.escape(str(value)) -def _pct(value: float) -> str: - return f"{value * 100:.1f}%" +def _pct(value) -> str: + return "n/a" if value is None else f"{value * 100:.1f}%" def _verdict_sentence(result: Result) -> str: @@ -179,6 +179,7 @@ def render_html(result: Result, subject: str = "") -> str: ("Temperature", result.temperature), ("Completions analysed", result.total_samples), ("Package references extracted", result.suggestion_occurrences), + ("References the registry could verify", result.verified_occurrences), ("Distinct packages named", result.distinct_packages), ("Measured", result.started), ] @@ -194,7 +195,7 @@ def render_html(result: Result, subject: str = "") -> str: conditions.append(("Samples that failed", len(result.errors))) tier_rows = "\n".join( - f'
{result.unverifiable_distinct} distinct " f"{'name' if result.unverifiable_distinct == 1 else 'names'} could not be " - f"confirmed against the registry during this run and " + f"confirmed against the registry during this run " + f"({result.unverifiable_occurrences} of {result.suggestion_occurrences} references) and " f"{'is' if result.unverifiable_distinct == 1 else 'are'} excluded from every " - f"count above rather than assumed clean.
" + f"count and rate above, numerator and denominator, rather than assumed clean." ) return f""" @@ -261,7 +263,7 @@ def render_html(result: Result, subject: str = "") -> str: tasks are the control: invention there means the failure is not confined to obscure corners of the ecosystem.| Task type | References | +||||
|---|---|---|---|---|---|
| Task type | Verified references | Invented | Rate |