fix(http): bound the on-disk response cache

src/pyscrappy/core/async_http.py

@@ -284,7 +284,9 @@ def _cache_put(self, key: str, resp: httpx.Response) -> None:
         if self.config.cache_ttl > 0:
             _SHARED_CACHE.put(key, resp, self.config.cache_max_size)
             if self.config.cache_dir:
-                _disk_cache_for(self.config.cache_dir).put(key, resp)
+                _disk_cache_for(self.config.cache_dir).put(
+                    key, resp, self.config.cache_ttl, self.config.cache_dir_max_size
+                )
 
     async def _rate_limit(self, url: str, min_delay: float | None = None) -> None:
         domain = urlparse(url).netloc

src/pyscrappy/core/config.py

@@ -71,6 +71,11 @@ class ScraperConfig:
             hits survive across process restarts and separate runs — the in-memory
             cache still fronts it for speed. ``None`` (the default) keeps caching
             in memory only.
+        cache_dir_max_size: Maximum number of live entries in the on-disk cache,
+            mirroring ``cache_max_size`` for the in-memory tier. A long-running
+            process (or a large crawl reusing one ``cache_dir``) stays within
+            this cap rather than growing one file per distinct URL forever.
+            Defaults to ``512``. Only takes effect when ``cache_dir`` is set.
         impersonate: Impersonate a real browser's TLS/JA3 fingerprint, to get
             past anti-bot filters that block plain clients (e.g. ``"chrome"``,
             ``"chrome124"``, ``"safari"``, ``"firefox"``). Works on both the sync
@@ -109,6 +114,7 @@ class ScraperConfig:
     cache_ttl: float = 0.0
     cache_max_size: int = 512
     cache_dir: str | None = None
+    cache_dir_max_size: int = 512
     impersonate: str | None = None
     retry_jitter: bool = True
     # Observability hooks (best-effort; a raising callback never breaks a scrape).

src/pyscrappy/core/http.py

@@ -143,11 +143,21 @@ class _DiskCache:
     read against the caller's ``ttl``; an expired file is deleted lazily. All ops
     are best-effort — a caching layer must never break a scrape, so any OS/JSON
     error just behaves as a miss.
+
+    Bounded like :class:`_ResponseCache`: ``put()`` prunes afterward, sweeping
+    any expired entry (so a key fetched once and never again does not linger
+    forever) and then, if the directory still exceeds ``max_size``, deleting
+    the oldest files by mtime until it does not. Unlike the in-memory LRU, a
+    read does not refresh recency here — mtime is set-on-write, not set-on-hit
+    — so a cold key that keeps getting read but never rewritten ages out under
+    a full cache; that trade is what keeps pruning an O(entries) directory
+    scan instead of a second per-entry access-time write on every hit.
     """
 
-    def __init__(self, cache_dir: str) -> None:
+    def __init__(self, cache_dir: str, max_size: int = _DEFAULT_CACHE_MAX_SIZE) -> None:
         self._dir = Path(cache_dir)
         self._lock = threading.Lock()
+        self._max_size = max_size
 
     def _path(self, key: str) -> Path:
         return self._dir / (hashlib.sha256(key.encode("utf-8")).hexdigest() + ".json")
@@ -181,7 +191,9 @@ def get(self, key: str, ttl: float) -> httpx.Response | None:
         except Exception:  # noqa: BLE001 - a bad cache entry is a miss, not an error
             return None
 
-    def put(self, key: str, resp: httpx.Response) -> None:
+    def put(
+        self, key: str, resp: httpx.Response, ttl: float | None = None, max_size: int | None = None
+    ) -> None:
         tmp = None
         try:
             self._dir.mkdir(parents=True, exist_ok=True)
@@ -219,6 +231,35 @@ def put(self, key: str, resp: httpx.Response) -> None:
                     tmp.unlink(missing_ok=True)
                 except OSError:
                     pass
+        self._prune(ttl, self._max_size if max_size is None else max_size)
+
+    def _prune(self, ttl: float | None, max_size: int) -> None:
+        """Sweep expired entries, then trim to ``max_size`` (oldest mtime first).
+
+        Called after every write so growth stays bounded even for a key that is
+        never re-requested — ``get()``'s lazy expiry only reaps the ones that
+        are. Best-effort like the rest of this class: never raises.
+        """
+        try:
+            with self._lock:
+                now = time.time()
+                live: list[tuple[float, Path]] = []
+                for f in self._dir.glob("*.json"):
+                    try:
+                        if ttl is not None and ttl > 0:
+                            data = json.loads(f.read_text(encoding="utf-8"))
+                            if now - data["ts"] > ttl:
+                                f.unlink(missing_ok=True)
+                                continue
+                        live.append((f.stat().st_mtime, f))
+                    except (OSError, ValueError, KeyError):
+                        continue  # unreadable/malformed entry: not this pass's job
+                if len(live) > max_size:
+                    live.sort(key=lambda entry: entry[0])
+                    for _, f in live[: len(live) - max_size]:
+                        f.unlink(missing_ok=True)
+        except OSError:
+            pass
 
     def clear(self) -> None:
         try:
@@ -529,7 +570,9 @@ def _cache_put(self, key: str, resp: httpx.Response) -> None:
         if self.config.cache_ttl > 0:
             _SHARED_CACHE.put(key, resp, self.config.cache_max_size)
             if self.config.cache_dir:
-                _disk_cache_for(self.config.cache_dir).put(key, resp)
+                _disk_cache_for(self.config.cache_dir).put(
+                    key, resp, self.config.cache_ttl, self.config.cache_dir_max_size
+                )
 
     @staticmethod
     def clear_cache() -> None:

tests/test_core/test_http.py

@@ -2,6 +2,8 @@
 
 import os
 import random
+import shutil
+import time
 from datetime import datetime, timezone
 from unittest.mock import MagicMock, patch
 
@@ -728,6 +730,59 @@ def test_disk_cache_preserves_duplicate_headers(self, tmp_path):
         assert hit.headers.get_list("Set-Cookie") == ["a=1", "b=2"]
         assert hit.headers.get("Content-Type") == "text/html"
 
+    def _disk_resp(self, url):
+        return httpx.Response(200, content=b"x", request=httpx.Request("GET", url))
+
+    def test_disk_cache_evicts_oldest_past_max_size(self, tmp_path):
+        # #166: an unbounded on-disk cache grows one file per distinct URL
+        # forever. put() must cap it, keeping the most recently written entries.
+        import pyscrappy.core.http as http_mod
+
+        dc = http_mod._DiskCache(str(tmp_path / "capped"), max_size=5)
+        for i in range(20):
+            dc.put(f"http://x/{i}", self._disk_resp(f"http://x/{i}"), ttl=100)
+
+        remaining = list((tmp_path / "capped").glob("*.json"))
+        assert len(remaining) == 5
+        # The 5 most recently written keys are the last 5 (0..19 in order).
+        for i in range(15, 20):
+            assert dc.get(f"http://x/{i}", 100) is not None
+        for i in range(0, 15):
+            assert dc.get(f"http://x/{i}", 100) is None
+
+    def test_disk_cache_put_max_size_overrides_the_instance_default(self, tmp_path):
+        import pyscrappy.core.http as http_mod
+
+        dc = http_mod._DiskCache(str(tmp_path / "override"))  # default max_size
+        for i in range(10):
+            dc.put(f"http://x/{i}", self._disk_resp(f"http://x/{i}"), ttl=100, max_size=3)
+        assert len(list((tmp_path / "override").glob("*.json"))) == 3
+
+    def test_disk_cache_sweeps_an_expired_entry_even_if_never_reread(self, tmp_path):
+        # get()'s lazy expiry only catches a key that is re-requested. put()'s
+        # prune must also reap a key that expires and is never asked for again.
+        import pyscrappy.core.http as http_mod
+
+        dc = http_mod._DiskCache(str(tmp_path / "sweep"), max_size=100)
+        dc.put("http://x/expired", self._disk_resp("http://x/expired"), ttl=0.01)
+        time.sleep(0.05)
+        dc.put("http://x/other", self._disk_resp("http://x/other"), ttl=0.01)
+
+        remaining = list((tmp_path / "sweep").glob("*.json"))
+        assert len(remaining) == 1
+        assert dc.get("http://x/other", 0.01) is not None
+
+    def test_disk_cache_prune_failure_does_not_raise(self, tmp_path):
+        # A prune that cannot even list the directory (e.g. removed out from
+        # under the cache) must behave as a no-op, not break the write it follows.
+        import pyscrappy.core.http as http_mod
+
+        cache_dir = tmp_path / "gone"
+        dc = http_mod._DiskCache(str(cache_dir), max_size=1)
+        dc.put("http://x/a", self._disk_resp("http://x/a"), ttl=100)
+        shutil.rmtree(cache_dir)
+        dc.put("http://x/b", self._disk_resp("http://x/b"), ttl=100)  # must not raise
+
 
 class TestObservabilityHooks:
     @pytest.fixture(autouse=True)