diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..912fa8f --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,17 @@ +{ + "permissions": { + "allow": [ + "Bash(pip install *)", + "Bash(pip3 install *)", + "Bash(pip download *)", + "Bash(pip show *)", + "Bash(python -m pip install *)", + "Bash(python3 -m pip install *)", + "Bash(python -m venv *)", + "Bash(python3 -m venv *)", + "Bash(.venv/bin/pip install *)", + "Bash(.venv/bin/python *)", + "Bash(.venv/bin/slopbench *)" + ] + } +} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a301328 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Release + +# Publishes to PyPI when a version tag (v0.1.0, v0.2.0, ...) is pushed. +# Uses PyPI trusted publishing: no API token is stored anywhere. PyPI trusts +# this workflow, in this repository, running in the `pypi` environment. + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip build twine + - run: python -m pip install -e ".[dev]" + - run: python -m pytest -q + - name: Tag must match the package version + run: | + version=$(python -c "import slopbench; print(slopbench.__version__)") + test "v${version}" = "${GITHUB_REF_NAME}" || { + echo "Tag ${GITHUB_REF_NAME} does not match package version ${version}"; exit 1; } + - run: python -m build + - run: python -m twine check --strict dist/* + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/slopbench/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..49ff635 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,46 @@ +# AGENTS.md + +Notes for AI coding agents working in this repository. + +## What this is + +slopbench measures package hallucination in model output: how often a model +names a PyPI, npm or crates.io package that does not exist, and how often the +same invented name recurs across samples of one prompt. A name that recurs in at +least two samples is "targetable". The `README.md` explains why that is the +measurement. + +## Setup and checks + +```bash +pip install -e ".[dev]" +python -m pytest -q +``` + +The tests make no network calls and need no API keys. CI runs them on Python +3.9 and 3.12 (`.github/workflows/ci.yml`). The package has no runtime +dependencies. Keep it that way: it uses only the standard library. + +## Layout + +| Path | Role | +|---|---| +| `slopbench/cli.py` | `slopbench run` and `slopbench prompts`. Exit 1 means a targetable name was found; exit 2 means nothing was measured. | +| `slopbench/providers.py` | Model adapters (`ollama`, `anthropic`, `openai`-compatible) and the on-disk completion cache. | +| `slopbench/extract.py` | Reads package names from model output. It is conservative on purpose. | +| `slopbench/verify.py` | Registry checks. pkgguard is used when installed, with a direct-API fallback. | +| `slopbench/bench.py` | Sampling, recurrence counting, and pkgguard corpus output and merging. | +| `slopbench/report.py` | HTML and Markdown reports. | +| `slopbench/data/prompts.json` | The 30-prompt corpus. No prompt may name a package; a test enforces this. | +| `reports/` | Published aggregate results (CC BY 4.0). | + +## Rules that protect the numbers + +- A registry failure is never a pass. Unverifiable names are excluded and + reported, never scored as clean. +- Do not widen extraction to running prose. Install commands are read only from + fenced blocks, inline code and lines that begin with a package manager. +- Do not publish invented names that anyone could register. `reports/` holds + aggregates only. +- Any change to what counts as invented or targetable needs a test showing the + input that motivated it. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..415b7dd --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,25 @@ +cff-version: 1.2.0 +message: "If you use slopbench or its panel data, please cite it as below." +title: "slopbench: measuring package hallucination and its recurrence in AI model output" +type: software +authors: + - name: Blvkware + website: "https://blvkware.dev" +repository-code: "https://github.com/rxslice/slopbench" +url: "https://blvkware.dev/ai-package-hallucination-data/" +license: Apache-2.0 +version: 0.1.0 +date-released: 2026-09-09 +abstract: >- + slopbench measures how often a language model or coding agent names a + software dependency that does not exist, or exists without adoption, on + PyPI, npm or crates.io, and how often it names the same invented package + in independent samples of the same prompt. Only recurring names can be + pre-registered by an attacker (slopsquatting), so recurrence is the headline. +keywords: + - slopsquatting + - package hallucination + - LLM security + - software supply chain + - AI coding agents + - benchmark diff --git a/README.md b/README.md index c0b3e34..00101c3 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,25 @@ # slopbench — measure package hallucination in model output -> **Latest results:** [Where AI coding models invent packages](reports/2026-10-hallux-panel/) (HALLUX panel, Oct 2026): 3.83% of 2,532 package suggestions named packages that do not exist; 0.17% on routine prompts, 5.83% on fast-moving topics. +> **Latest results:** [Where AI coding models invent packages](https://github.com/rxslice/slopbench/tree/main/reports/2026-10-hallux-panel) (HALLUX panel, 22 Sep to 2 Oct 2026): 3.83% of 2,532 package suggestions from four model families named packages that do not exist; 0.17% on routine prompts, 5.83% on prompts about emerging tools. [Full write-up](https://blvkware.dev/ai-package-hallucination-data/). **Built by [Blvkware](https://blvkware.dev)** [![CI](https://github.com/rxslice/slopbench/actions/workflows/ci.yml/badge.svg)](https://github.com/rxslice/slopbench/actions/workflows/ci.yml) -[![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](./LICENSE) +[![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](https://github.com/rxslice/slopbench/blob/main/LICENSE) [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/) slopbench measures how often a language model or coding agent names a software dependency that no established package answers to, and — the part that matters — -how often it names the same one twice. +how often it names the same one twice. It covers PyPI, npm and crates.io, needs +nothing beyond the Python standard library, and runs free against a local model. ```bash -pip install -e . +pip install slopbench slopbench run --provider ollama --model qwen2.5-coder:7b --samples 5 --report out.html ``` +To work on slopbench itself, clone this repository and run `pip install -e ".[dev]"`. + ## Why recurrence is the measurement A hallucination rate on its own overstates the threat. A package name a model @@ -82,8 +85,8 @@ running prose is missed, and that is the trade being made deliberately. | Provider | Configure with | Notes | |---|---|---| | `ollama` | `OLLAMA_HOST` | Local, free. Start here. | -| `anthropic` | `ANTHROPIC_API_KEY` | | -| `openai` | `SLOPBENCH_BASE_URL`, `SLOPBENCH_API_KEY` | Any `/v1/chat/completions` endpoint: OpenAI, Groq, Together, OpenRouter, vLLM, LM Studio. | +| `anthropic` | `ANTHROPIC_API_KEY` | Current Claude models sample only at their own defaults and refuse a `temperature`. slopbench then drops it for the rest of the run and states in the report how many samples were drawn that way. | +| `openai` | `SLOPBENCH_BASE_URL`, `SLOPBENCH_API_KEY` (or `OPENAI_API_KEY`) | Any `/v1/chat/completions` endpoint: OpenAI, Groq, Together, OpenRouter, vLLM, LM Studio. The base URL defaults to `https://api.openai.com/v1`. | Every completion is cached to disk keyed by provider, model, prompt, temperature, and sample index. Generation is the only step that costs money; @@ -102,7 +105,7 @@ slopbench run --provider ollama --model qwen2.5-coder:7b \ --ecosystem npm --limit 6 --samples 5 --report out.html # full run with every output -slopbench run --provider anthropic --model claude-sonnet-4-6 \ +slopbench run --provider anthropic --model claude-sonnet-5-5 \ --samples 5 --report out.html --markdown out.md \ --json out.json --corpus corpus.json --subject "Acme Agents" ``` @@ -126,7 +129,8 @@ pkgguard loads a malformed corpus without error and simply finds no names in it, which is the kind of failure nobody notices. ```bash -slopbench run --provider ollama --model qwen2.5-coder:7b --samples 5 --corpus path/to/pkgguard/data/known_hallucinations.json +slopbench run --provider ollama --model qwen2.5-coder:7b --samples 5 \ + --corpus path/to/pkgguard/data/known_hallucinations.json # corpus: 2 new name(s) written to path/to/pkgguard/data/known_hallucinations.json ``` @@ -139,17 +143,22 @@ classifier. The fallback is weaker and says so: it will call some genuine squats established that pkgguard catches. ```bash -pip install -e ".[pkgguard]" +pip install "slopbench[pkgguard]" ``` +The extra installs pkgguard's published distribution, +[`blvkware-pkgguard`](https://pypi.org/project/blvkware-pkgguard/), which +provides the `pkgguard` module. Install it under that name. The bare name +`pkgguard` is not registered on PyPI. + ## The prompt corpus -30 prompts across PyPI, npm, and crates.io, in three tiers: +30 prompts (12 PyPI, 11 npm, 7 crates.io), in three tiers: -- **routine** — well-covered tasks. This is the control. Invention here means +- **routine** (8 prompts) — well-covered tasks. This is the control. Invention here means the problem is not confined to obscure corners. -- **niche** — specific integrations with real but thinner public coverage. -- **emerging** — areas where training data is thinnest, which is where invention +- **niche** (12) — specific integrations with real but thinner public coverage. +- **emerging** (10) — areas where training data is thinnest, which is where invention concentrates. No prompt names a package. A prompt that names a library measures @@ -170,6 +179,42 @@ The corpus is the part worth customizing. - Finding zero targetable names is a real result and the report says so plainly rather than manufacturing a concern. +## FAQ + +**What is slopsquatting?** +Registering a package name that AI models invent, so that code or an agent which +installs the invented name gets the attacker's package. It works only when the +invented name is predictable, which is why slopbench measures recurrence. + +**How often do AI coding models invent packages?** +In the [October 2026 HALLUX panel](https://github.com/rxslice/slopbench/tree/main/reports/2026-10-hallux-panel), 3.83% of 2,532 +package suggestions from four model families named packages that did not exist +(95% interval 3.15% to 4.65%). The rate was 0.17% on routine prompts and 5.83% on +prompts about emerging tools. The four families were statistically +indistinguishable. + +**What makes an invented name "targetable"?** +It appeared in at least two independent samples of the same prompt. A name +invented once cannot be predicted, so nobody can register it in advance. + +**Can I test my own agent or prompts?** +Yes. Point `--prompts` at a JSON file in the same shape as +[`slopbench/data/prompts.json`](https://github.com/rxslice/slopbench/blob/main/slopbench/data/prompts.json), and use the +`openai` provider for any OpenAI-compatible endpoint. The CLI exits `1` when it +finds a targetable name, so it can gate CI. + +**Does it cost anything?** +Only the model calls. Ollama runs locally for free, and every completion is +cached, so re-scoring a run costs nothing. + +## Citing + +If you use slopbench or the panel data, cite it with +[`CITATION.cff`](https://github.com/rxslice/slopbench/blob/main/CITATION.cff) (GitHub's "Cite this repository" button reads +it). The panel data is CC BY 4.0. Cite +https://blvkware.dev/ai-package-hallucination-data/. + ## License -Apache 2.0. Built by [Blvkware](https://blvkware.dev). +Code: Apache 2.0. Panel data in [`reports/`](https://github.com/rxslice/slopbench/tree/main/reports): CC BY 4.0. +Built by [Blvkware](https://blvkware.dev). diff --git a/pyproject.toml b/pyproject.toml index 0507285..db4a534 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -8,18 +8,34 @@ version = "0.1.0" description = "Measure how often a model invents software dependencies, and how often it invents the same one twice." readme = "README.md" requires-python = ">=3.9" -license = {text = "Apache-2.0"} +license = "Apache-2.0" +license-files = ["LICENSE"] authors = [{name = "Blvkware"}] -keywords = ["llm", "hallucination", "supply-chain", "slopsquatting", "benchmark", "security"] +keywords = [ + "slopsquatting", "package-hallucination", "llm", "llm-security", "ai-coding-agents", + "supply-chain-security", "hallucination", "benchmark", "pypi", "npm", "crates-io", +] classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.12", "Topic :: Security", - "License :: OSI Approved :: Apache Software License", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Quality Assurance", ] dependencies = [] [project.optional-dependencies] -pkgguard = ["pkgguard"] +# The distribution is blvkware-pkgguard; it installs the `pkgguard` module. +# The bare PyPI name "pkgguard" is unregistered: depending on it would let +# whoever registers it first run code on every machine that installs this extra. +pkgguard = ["blvkware-pkgguard>=0.1.2"] dev = ["pytest>=7"] [project.scripts] @@ -27,6 +43,9 @@ slopbench = "slopbench.cli:main" [project.urls] Homepage = "https://blvkware.dev" +Source = "https://github.com/rxslice/slopbench" +Issues = "https://github.com/rxslice/slopbench/issues" +Results = "https://blvkware.dev/ai-package-hallucination-data/" [tool.setuptools.packages.find] include = ["slopbench*"] diff --git a/reports/2026-10-hallux-panel/README.md b/reports/2026-10-hallux-panel/README.md index 541d84d..89a029f 100644 --- a/reports/2026-10-hallux-panel/README.md +++ b/reports/2026-10-hallux-panel/README.md @@ -32,4 +32,30 @@ know whether models invent names around their project: russ@blvkware.dev. - `hallux-panel-2026-10.csv`: suggestions, inventions and rates by tier and by model. - `hallux-panel-2026-10-registrability.csv`: invented names by registry and what protected them. +Both files start with `#` comment lines that give the source, dates and licence. Skip them +when parsing (for example `pandas.read_csv(path, comment="#")`). + +### Columns: `hallux-panel-2026-10.csv` + +| Column | Meaning | +|---|---| +| `dimension` | `tier`, `model` or `all` | +| `group` | Tier name, model identifier, or `all` | +| `suggestions` | Package references extracted from usable completions | +| `invented` | References to packages that do not exist | +| `rate_pct` | `invented / suggestions`, as a percentage | +| `ci95_low_pct`, `ci95_high_pct` | 95% Wilson score interval for `rate_pct` | + +### Columns: `hallux-panel-2026-10-registrability.csv` + +| Column | Meaning | +|---|---| +| `registry` | `pypi`, `crates.io`, `npm`, or `all` | +| `invented_names` | Distinct invented names on that registry | +| `registrable_by_anyone` | Names a stranger could register on 2 Oct 2026 | +| `protected_org_scope` | npm names under an organisation scope that already exists | +| `protected_types_scope` | npm names under `@types` | +| `protected_npm_punctuation_rule` | npm names blocked because they collide with an existing name under npm's punctuation rule | +| `unclear` | Names whose status could not be settled | + Data licence: CC BY 4.0. Cite https://blvkware.dev/ai-package-hallucination-data/ diff --git a/slopbench/bench.py b/slopbench/bench.py index d9ebe73..b008c0f 100644 --- a/slopbench/bench.py +++ b/slopbench/bench.py @@ -78,6 +78,7 @@ class Result: input_tokens: int = 0 output_tokens: int = 0 cached_samples: int = 0 + default_sampling_samples: int = 0 pkgguard_available: bool = False errors: List[str] = field(default_factory=list) @@ -121,7 +122,7 @@ def run( occurrences = 0 empty = 0 completed = 0 - in_tokens = out_tokens = cached = 0 + in_tokens = out_tokens = cached = default_sampling = 0 errors: List[str] = [] tier_counts: Dict[str, Dict[str, int]] = {} @@ -138,6 +139,7 @@ def run( continue completed += 1 cached += 1 if completion.cached else 0 + default_sampling += 0 if completion.temperature_applied else 1 in_tokens += completion.input_tokens or 0 out_tokens += completion.output_tokens or 0 if not completion.text.strip(): @@ -208,6 +210,7 @@ def run( unverifiable_distinct=unverifiable, findings=findings, by_tier=by_tier, input_tokens=in_tokens, output_tokens=out_tokens, cached_samples=cached, + default_sampling_samples=default_sampling, pkgguard_available=verifier.pkgguard_available, errors=errors, ) diff --git a/slopbench/cli.py b/slopbench/cli.py index e43541a..e36dd4e 100644 --- a/slopbench/cli.py +++ b/slopbench/cli.py @@ -149,6 +149,9 @@ def progress(message: str) -> None: f"{result.total_samples} completions.") if result.unverifiable_distinct: print(f"{result.unverifiable_distinct} name(s) unverifiable and excluded.") + if result.default_sampling_samples: + print(f"{result.default_sampling_samples} completion(s) drawn at the model's " + f"default sampling: the endpoint refused temperature {result.temperature}.") for error in result.errors[:5]: print(f"error: {error}", file=sys.stderr) diff --git a/slopbench/providers.py b/slopbench/providers.py index e73b6e0..7a4cbf0 100644 --- a/slopbench/providers.py +++ b/slopbench/providers.py @@ -38,6 +38,10 @@ class Completion: cached: bool input_tokens: Optional[int] = None output_tokens: Optional[int] = None + # False when the endpoint refused the requested temperature and the sample + # was drawn at the model's own default sampling instead. Recorded, never + # hidden: it changes what the recurrence figure means. + temperature_applied: bool = True def _post(url: str, payload: dict, headers: dict) -> dict: @@ -87,6 +91,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion: text=data["text"], cached=True, input_tokens=data.get("input_tokens"), output_tokens=data.get("output_tokens"), + temperature_applied=data.get("temperature_applied", True), ) except (OSError, ValueError, KeyError): pass @@ -101,6 +106,7 @@ def complete(self, prompt: str, temperature: float, index: int) -> Completion: "prompt": prompt, "text": result.text, "input_tokens": result.input_tokens, "output_tokens": result.output_tokens, + "temperature_applied": result.temperature_applied, }, fh) os.replace(temp, path) except OSError: @@ -112,23 +118,54 @@ def _generate(self, prompt: str, temperature: float) -> Completion: class AnthropicProvider(Provider): + """Claude via the Messages API. + + Current Claude models sample only at their own defaults: Opus 4.7 and + later, Sonnet 5 and the Fable models reject any `temperature` with a 400, + and Sonnet 5.5 and Haiku 5.5 reject non-default values. The first such + rejection switches this provider to default sampling for the rest of the + run, and every affected sample is marked so the report can say so, rather + than the run failing every sample and reporting nothing. + + `max_tokens` is higher than the other providers' because on current models + thinking is always on and its tokens count against the same limit; at 2048 + the visible answer could be cut off before its install command. + """ + name = "anthropic" + MAX_TOKENS = 16000 + + def __init__(self, model: str, cache_dir: str = DEFAULT_CACHE) -> None: + super().__init__(model, cache_dir) + self._send_temperature = True def _generate(self, prompt: str, temperature: float) -> Completion: key = os.environ.get("ANTHROPIC_API_KEY") if not key: raise ProviderError("ANTHROPIC_API_KEY is not set.") - data = _post( - "https://api.anthropic.com/v1/messages", - {"model": self.model, "max_tokens": 2048, "temperature": temperature, - "messages": [{"role": "user", "content": prompt}]}, - {"content-type": "application/json", "x-api-key": key, - "anthropic-version": "2023-06-01"}, - ) + headers = {"content-type": "application/json", "x-api-key": key, + "anthropic-version": "2023-06-01"} + payload = {"model": self.model, "max_tokens": self.MAX_TOKENS, + "messages": [{"role": "user", "content": prompt}]} + + data = None + if self._send_temperature: + try: + data = _post("https://api.anthropic.com/v1/messages", + {**payload, "temperature": temperature}, headers) + except ProviderError as error: + message = str(error) + if not (message.startswith("HTTP 400") and "temperature" in message): + raise + self._send_temperature = False + if data is None: + data = _post("https://api.anthropic.com/v1/messages", payload, headers) + text = "".join(b.get("text", "") for b in data.get("content", []) if b.get("type") == "text") usage = data.get("usage") or {} - return Completion(text, False, usage.get("input_tokens"), usage.get("output_tokens")) + return Completion(text, False, usage.get("input_tokens"), usage.get("output_tokens"), + temperature_applied=self._send_temperature) class OpenAICompatibleProvider(Provider): diff --git a/slopbench/report.py b/slopbench/report.py index 3652287..3dc8b3b 100644 --- a/slopbench/report.py +++ b/slopbench/report.py @@ -184,6 +184,10 @@ def render_html(result: Result, subject: str = "") -> str: ] if result.unverifiable_distinct: conditions.append(("Names registry could not confirm", result.unverifiable_distinct)) + if result.default_sampling_samples: + conditions.append(("Samples at the model's default sampling " + "(endpoint refused the temperature)", + result.default_sampling_samples)) if result.errors: # A run that lost samples measured less than it looks like it did, and # the reader is entitled to that before the number rather than after. @@ -320,6 +324,11 @@ def render_markdown(result: Result, subject: str = "") -> str: f"- Excluded as unverifiable: {result.unverifiable_distinct} " f"(registry unreachable; not counted as clean)" ) + if result.default_sampling_samples: + lines.append( + f"- Samples drawn at the model's default sampling because the endpoint " + f"refused temperature {result.temperature}: {result.default_sampling_samples}" + ) if result.errors: lines.append(f"- Samples that failed and were not measured: {len(result.errors)}") lines += ["", "## Findings", "", diff --git a/tests/test_slopbench.py b/tests/test_slopbench.py index 544f726..8d1b639 100644 --- a/tests/test_slopbench.py +++ b/tests/test_slopbench.py @@ -668,3 +668,67 @@ def test_a_closing_backtick_ends_the_command(): # code block, followed by prose. text = '```python\nprint("Install with `pip install weasyprint` and ensure Cairo is present.")\n```' assert names(text) == {"weasyprint"} + + +# -------------------------------------------------------------------------- +# Anthropic provider: current models refuse a sampling temperature +# -------------------------------------------------------------------------- + +def _anthropic_reply(text="```bash\npip install requests\n```"): + return {"content": [{"type": "text", "text": text}], + "usage": {"input_tokens": 10, "output_tokens": 20}} + + +def test_refused_temperature_falls_back_to_default_sampling(monkeypatch, tmp_path): + from slopbench import providers + + sent = [] + + def fake_post(url, payload, headers): + sent.append(payload) + if "temperature" in payload: + raise providers.ProviderError( + 'HTTP 400: {"error":{"message":"temperature is deprecated for this model."}}') + return _anthropic_reply() + + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + monkeypatch.setattr(providers, "_post", fake_post) + provider = providers.AnthropicProvider("claude-opus-5-5", cache_dir=str(tmp_path)) + + first = provider.complete("task", 0.7, 0) + second = provider.complete("task", 0.7, 1) + + assert first.text and not first.temperature_applied + assert not second.temperature_applied + # One refused request, then no temperature is sent again in this run. + assert ["temperature" in p for p in sent] == [True, False, False] + # The cache records the truth, so a re-run from cache reports it too. + assert not provider.complete("task", 0.7, 0).temperature_applied + + +def test_other_bad_requests_are_not_retried(monkeypatch, tmp_path): + from slopbench import providers + + def fake_post(url, payload, headers): + raise providers.ProviderError("HTTP 400: model not found") + + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + monkeypatch.setattr(providers, "_post", fake_post) + provider = providers.AnthropicProvider("claude-nope", cache_dir=str(tmp_path)) + with pytest.raises(providers.ProviderError): + provider.complete("task", 0.7, 0) + + +def test_default_sampling_is_reported(monkeypatch, tmp_path): + from slopbench.report import render_html, render_markdown + + class DefaultSampling(ScriptedProvider): + def complete(self, prompt, temperature, index): + return Completion("```bash\npip install requests\n```", cached=False, + temperature_applied=False) + + _patch_verify(monkeypatch, {}) + result = run(DefaultSampling([""], tmp_path), _prompts(), samples=2) + assert result.default_sampling_samples == 2 + assert "default sampling" in render_markdown(result) + assert "default sampling" in render_html(result)