Feat/optional max pages

db/init/02_sources_config.sql

@@ -25,13 +25,11 @@
 -- fresh-volume schema looks like" and "what the live db was migrated to" —
 -- exactly the bug class this file exists to avoid.
 --
--- max_pages is intentionally left NULLABLE here: the pydantic SourceConfig
--- model (ingestion/app/config.py) is what enforces `max_pages` is required
--- and > 0 on every write path (sync/propose). Enforcing NOT NULL at the SQL
--- layer would break the 9 existing rows that predate this column with no
--- backfill value to give them. This split (permissive schema + strict
--- application-layer validation) is deliberate — do not "fix" it by adding a
--- NOT NULL constraint here.
+-- max_pages is NULLABLE and NULL is meaningful: it means "no page limit"
+-- (crawl all in-scope pages). The pydantic SourceConfig model
+-- (ingestion/app/config.py) makes `max_pages` OPTIONAL and, when provided,
+-- enforces it is > 0 on every write path (sync/propose). Do not add a
+-- NOT NULL constraint here — a NULL is a valid, intentional "unlimited".
 
 ALTER TABLE doc_sources
     ADD COLUMN IF NOT EXISTS sitemap            TEXT,

ingestion/app/admin.py

@@ -519,7 +519,7 @@ def _build_source_config(
             sitemap=sitemap.strip() or None,
             include_prefixes=_split_prefixes(include_prefixes),
             exclude_prefixes=_split_prefixes(exclude_prefixes),
-            max_pages=max_pages.strip(),
+            max_pages=(max_pages.strip() or None),
             language=language.strip() or "english",
             rate_limit_rps=rate_limit_rps.strip(),
             llms_txt=(llms_txt.strip() or "auto"),
@@ -543,7 +543,7 @@ def _record_to_config(record: SourceRecord) -> SourceConfig:
         sitemap=record.sitemap,
         include_prefixes=record.include_prefixes,
         exclude_prefixes=record.exclude_prefixes,
-        max_pages=record.max_pages if (record.max_pages is not None and record.max_pages > 0) else 100,
+        max_pages=record.max_pages,
         language=record.language or "english",
         rate_limit_rps=record.rate_limit_rps if (record.rate_limit_rps is not None and record.rate_limit_rps > 0) else 1.0,
         llms_txt=record.llms_txt or "auto",
@@ -637,7 +637,7 @@ def create_source_submit(
     sitemap: str = Form(default=""),
     include_prefixes: str = Form(default=""),
     exclude_prefixes: str = Form(default=""),
-    max_pages: str = Form(...),
+    max_pages: str = Form(default=""),
     language: str = Form(default="english"),
     rate_limit_rps: str = Form(default="1.0"),
     llms_txt: str = Form(default="auto"),
@@ -720,7 +720,7 @@ def update_source_submit(
     sitemap: str = Form(default=""),
     include_prefixes: str = Form(default=""),
     exclude_prefixes: str = Form(default=""),
-    max_pages: str = Form(...),
+    max_pages: str = Form(default=""),
     language: str = Form(default="english"),
     rate_limit_rps: str = Form(default="1.0"),
     llms_txt: str = Form(default="auto"),

ingestion/app/config.py

@@ -8,7 +8,7 @@
         sitemap: https://fastapi.tiangolo.com/sitemap.xml   # optional
         include_prefixes: ["/tutorial/", "/reference/"]      # optional allowlist
         exclude_prefixes: ["/blog/", "/release-notes/"]      # optional denylist (wins)
-        max_pages: 500              # required
+        max_pages: 500              # optional — omit for no page limit (crawl all in-scope pages)
         language: english           # optional, default english
         rate_limit_rps: 1.0         # optional, default 1.0
 
@@ -99,7 +99,7 @@ class SourceConfig(BaseModel):
     sitemap: HttpUrl | None = None
     include_prefixes: list[str] = Field(default_factory=list)
     exclude_prefixes: list[str] = Field(default_factory=list)
-    max_pages: int = Field(gt=0)
+    max_pages: int | None = Field(default=None, gt=0)
     language: str = "english"
     rate_limit_rps: float = Field(default=1.0, gt=0)
     llms_txt: Literal["auto", "off", "only"] = "auto"

ingestion/app/crawler.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import ipaddress
+import sys
 import time
 import urllib.robotparser
 import warnings
@@ -397,7 +398,8 @@ def crawl(
     client: httpx.Client | None = None,
     conditional: dict[str, tuple[str | None, str | None]] | None = None,
 ) -> Iterator[dict]:
-    """Discover and fetch up to `source.max_pages` pages for `source`.
+    """Discover and fetch up to `source.max_pages` pages for `source` (or ALL
+    in-scope pages when `source.max_pages` is None — no page limit).
 
     YIELDS `{"url": str, "html": str | None, "fetch_ok": bool}` dicts one at a
     time as pages are visited, rather than materializing the whole crawl in
@@ -462,6 +464,11 @@ def crawl(
         rp = load_robots(client, base_url)
         limiter = RateLimiter(source.rate_limit_rps)
 
+        # `max_pages` is optional: None means "no page limit". Internally we use
+        # a very large int so the existing cap comparisons/slicing need no
+        # special-casing — same-host + prefix scoping still bounds the crawl.
+        page_cap = source.max_pages if source.max_pages is not None else sys.maxsize
+
         pages_fetched = 0
         visited: set[str] = set()
 
@@ -525,7 +532,7 @@ def crawl(
                     sections = llms_txt.split_llms_full(llms_text, index_url)
                     llms_count = 0
                     for section in sections:
-                        if llms_count >= source.max_pages:
+                        if llms_count >= page_cap:
                             break
                         yield {
                             "url": section["url"],
@@ -560,7 +567,7 @@ def crawl(
                     continue
                 if u not in filtered:
                     filtered.append(u)
-            candidate_urls = filtered[: source.max_pages]
+            candidate_urls = filtered[:page_cap]
             log.info("llms_index_candidates", count=len(candidate_urls))
         elif source.sitemap and not _same_host(str(source.sitemap), base_url):
             # Defence in depth: SourceConfig already rejects an off-host
@@ -572,7 +579,7 @@ def crawl(
                 candidate_urls = discover_sitemap_urls(
                     client,
                     str(source.sitemap),
-                    source.max_pages,
+                    page_cap,
                     limiter,
                     log,
                     base_url,
@@ -672,7 +679,7 @@ def _visit(url: str) -> dict | None:
 
         if candidate_urls is not None:
             for url in candidate_urls:
-                if pages_fetched >= source.max_pages:
+                if pages_fetched >= page_cap:
                     break
                 url = _strip_fragment(url)
                 if url in visited:
@@ -685,7 +692,7 @@ def _visit(url: str) -> dict | None:
                     yield item
         else:
             queue = [base_url]
-            while queue and pages_fetched < source.max_pages:
+            while queue and pages_fetched < page_cap:
                 url = _strip_fragment(queue.pop(0))
                 if url in visited:
                     continue

ingestion/app/llms_txt.py

@@ -41,22 +41,18 @@ def discover(
     base_url: str,
     *,
     prefer_full: bool = True,
-    max_bytes: int = 10_000_000,
 ) -> tuple[str, str] | None:
     """Try to fetch `/llms-full.txt` then `/llms.txt` (or the reverse order
     when `prefer_full=False`) at `base_url`'s origin.
 
-    Returns `(fetched_url, text)` for the first response that is status 200,
-    has a non-empty body, and is within `max_bytes`. Returns `None` if
-    neither candidate qualifies.
+    Returns `(fetched_url, text)` for the first response that is status 200
+    with a non-empty body. There is NO size limit — the full file is fetched
+    however large it is (llms-full.txt files are legitimately large). Returns
+    `None` if neither candidate qualifies.
 
-    The body is streamed and the download is ABORTED as soon as it exceeds
-    `max_bytes`, so an oversized file (e.g. a 24MB `/llms-full.txt`) is not
-    fully downloaded only to be discarded.
-
-    NEVER raises: any httpx error, oversize body, empty body, or non-200
-    status is treated as "skip this candidate" (and, if both candidates are
-    exhausted, "skip this source" — return None) rather than propagating.
+    NEVER raises: any httpx error, empty body, or non-200 status is treated as
+    "skip this candidate" (and, if both candidates are exhausted, "skip this
+    source" — return None) rather than propagating.
     """
     parsed = urlparse(base_url)
     origin = f"{parsed.scheme}://{parsed.netloc}"
@@ -68,7 +64,7 @@ def discover(
     log = logger.bind(base_url=base_url)
 
     for url in candidates:
-        text = _fetch_capped(client, url, max_bytes, log)
+        text = _fetch_body(client, url, log)
         if text is None:
             continue
         log.info("llms_txt_discovered", url=url, size=len(text))
@@ -77,43 +73,18 @@ def discover(
     return None
 
 
-def _fetch_capped(client, url: str, max_bytes: int, log) -> str | None:
-    """GET `url`, streaming the body and aborting the download once it exceeds
-    `max_bytes`. Returns the decoded text, or None to skip this candidate
-    (non-200, empty, oversize, or any error — discovery is best-effort and
-    never raises).
-
-    Streaming (rather than `client.get(...).content`) is what makes the
-    `max_bytes` guard cheap: a huge `/llms-full.txt` is stopped mid-transfer
-    instead of being pulled into memory in full and then thrown away.
-    """
-    chunks: list[bytes] = []
-    encoding = "utf-8"
+def _fetch_body(client, url: str, log) -> str | None:
+    """GET `url` and return its decoded text, or None to skip this candidate
+    (non-200, empty, or any error — discovery is best-effort and never raises)."""
     try:
-        with client.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=15) as resp:
-            if resp.status_code != 200:
-                log.info("llms_txt_non_200", url=url, status=resp.status_code)
-                return None
-            total = 0
-            for chunk in resp.iter_bytes():
-                total += len(chunk)
-                if total > max_bytes:
-                    log.info("llms_txt_too_large", url=url, size_at_least=total, max_bytes=max_bytes)
-                    return None
-                chunks.append(chunk)
-            encoding = resp.encoding or "utf-8"
+        resp = client.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
     except Exception as e:  # noqa: BLE001 - discovery is best-effort, never raises
         log.info("llms_txt_fetch_failed", url=url, error=str(e))
         return None
-
-    body = b"".join(chunks)
-    if not body:
-        log.info("llms_txt_empty", url=url)
+    if resp.status_code != 200:
+        log.info("llms_txt_non_200", url=url, status=resp.status_code)
         return None
-    try:
-        text = body.decode(encoding, errors="replace")
-    except LookupError:
-        text = body.decode("utf-8", errors="replace")
+    text = resp.text
     if not text.strip():
         log.info("llms_txt_empty", url=url)
         return None

ingestion/app/sources_repo.py

@@ -189,7 +189,7 @@ def _row_to_record(row: tuple) -> SourceRecord:
         sitemap=sitemap,
         include_prefixes=list(include_prefixes) if include_prefixes is not None else [],
         exclude_prefixes=list(exclude_prefixes) if exclude_prefixes is not None else [],
-        max_pages=max_pages if (max_pages is not None and max_pages > 0) else 100,
+        max_pages=max_pages,  # None means "no page limit" (optional)
         language=language or "english",
         rate_limit_rps=rate_limit_rps if (rate_limit_rps is not None and rate_limit_rps > 0) else 1.0,
         llms_txt=llms_txt if llms_txt else "auto",

ingestion/app/templates/admin/form.html

@@ -35,8 +35,8 @@ <h2>{% if record %}Edit {{ record.name }}{% else %}Add a source{% endif %}</h2>
       <label for="exclude_prefixes">exclude_prefixes (one per line, optional &mdash; always wins over include)</label>
       <textarea id="exclude_prefixes" name="exclude_prefixes" rows="3">{{ values.exclude_prefixes or '' }}</textarea>
 
-      <label for="max_pages">max_pages</label>
-      <input type="number" id="max_pages" name="max_pages" min="1" required value="{{ values.max_pages or '' }}">
+      <label for="max_pages">max_pages <span class="muted">(optional — leave blank for no limit)</span></label>
+      <input type="number" id="max_pages" name="max_pages" min="1" placeholder="no limit" value="{{ values.max_pages or '' }}">
 
       <label for="language">language</label>
       <select id="language" name="language">

ingestion/app/templates/admin/index.html

@@ -68,7 +68,7 @@ <h2>&#9888; Pending review (agent-proposed)</h2>
         </td>
         <td>{% if s.include_prefixes %}{{ s.include_prefixes | join(", ") }}{% else %}<span class="muted">(none)</span>{% endif %}</td>
         <td>{% if s.exclude_prefixes %}{{ s.exclude_prefixes | join(", ") }}{% else %}<span class="muted">(none)</span>{% endif %}</td>
-        <td>{{ s.max_pages if s.max_pages is not none else "(none)" }}</td>
+        <td>{% if s.max_pages is not none %}{{ s.max_pages }}{% else %}<span class="muted">unlimited</span>{% endif %}</td>
         <td><span class="proposed-by">agent: {{ s.proposed_by or "unknown" }}</span></td>
         <td class="actions">
           <div class="btn-group">

ingestion/tests/test_config.py

@@ -98,7 +98,8 @@ def test_invalid_name_pattern_raises(tmp_path):
         load_sources(p)
 
 
-def test_missing_max_pages_raises(tmp_path):
+def test_missing_max_pages_is_allowed_and_means_unlimited(tmp_path):
+    # max_pages is optional: omitting it is valid and means "no page limit".
     p = write_yaml(
         tmp_path,
         """
@@ -107,6 +108,22 @@ def test_missing_max_pages_raises(tmp_path):
             base_url: https://example.com
         """,
     )
+    sources = load_sources(p)
+    assert len(sources) == 1
+    assert sources[0].max_pages is None
+
+
+def test_zero_or_negative_max_pages_still_raises(tmp_path):
+    # When provided, max_pages must be positive (gt=0).
+    p = write_yaml(
+        tmp_path,
+        """
+        sources:
+          - name: bad-max-pages
+            base_url: https://example.com
+            max_pages: 0
+        """,
+    )
     with pytest.raises(ConfigError):
         load_sources(p)
 

ingestion/tests/test_crawler.py

@@ -85,6 +85,31 @@ def test_sitemap_discovery_bounded_by_max_pages():
     assert len(pages) == 1
 
 
+def test_sitemap_discovery_unlimited_when_max_pages_none():
+    # max_pages=None (omitted) => no page limit: all in-scope pages are crawled.
+    source = SourceConfig(
+        name="example",
+        base_url="https://example.com/",
+        sitemap="https://example.com/sitemap.xml",
+        include_prefixes=["/docs/"],
+        rate_limit_rps=1000,
+    )
+    assert source.max_pages is None
+    handler = _handler_factory(
+        sitemap_body=SITEMAP_XML,
+        page_bodies={
+            "https://example.com/docs/a": PAGE_HTML,
+            "https://example.com/docs/b": PAGE_HTML,
+        },
+    )
+    client = make_client(handler)
+    pages = list(crawl(source, client=client))
+    assert {p["url"] for p in pages} == {
+        "https://example.com/docs/a",
+        "https://example.com/docs/b",
+    }
+
+
 def test_bfs_fallback_when_no_sitemap():
     def handler(request: httpx.Request) -> httpx.Response:
         url = str(request.url)

ingestion/tests/test_llms_txt.py

@@ -27,52 +27,23 @@ def handler(request: httpx.Request) -> httpx.Response:
     assert discover(client, "https://example.com/") is None
 
 
-def test_discover_returns_none_when_over_max_bytes():
-    def handler(request: httpx.Request) -> httpx.Response:
-        url = str(request.url)
-        if url == "https://example.com/llms-full.txt":
-            return httpx.Response(200, text="x" * 100)
-        return httpx.Response(404)
-
-    client = make_client(handler)
-    assert discover(client, "https://example.com/", max_bytes=10) is None
-
-
-def test_discover_aborts_download_once_over_cap():
-    """The oversized body must be abandoned mid-stream, not fully downloaded and
-    then discarded — otherwise a 24MB llms-full.txt costs a 24MB transfer just
-    to be rejected."""
-    consumed = {"chunks": 0}
-
-    def body_gen():
-        for _ in range(1000):  # up to 1000 * 1KB = 1MB if fully read
-            consumed["chunks"] += 1
-            yield b"x" * 1024
+def test_discover_has_no_size_limit():
+    """There is no size cap: a large llms-full.txt is fetched in full (real
+    files are legitimately large, e.g. anthropic's ~24MB)."""
+    big = "# Full\n" + ("word " * 500_000)  # ~2.5MB
 
     def handler(request: httpx.Request) -> httpx.Response:
         url = str(request.url)
         if url.endswith("/llms-full.txt"):
-            return httpx.Response(200, content=body_gen())
+            return httpx.Response(200, text=big)
         return httpx.Response(404)
 
     client = make_client(handler)
-    result = discover(client, "https://example.com/", max_bytes=5_000)  # 5KB cap
-    assert result is None
-    # Stopped shortly after crossing 5KB (~6 chunks), not the whole 1000.
-    assert consumed["chunks"] < 20
-
-
-def test_discover_accepts_body_at_the_cap_boundary():
-    def handler(request: httpx.Request) -> httpx.Response:
-        url = str(request.url)
-        if url.endswith("/llms-full.txt"):
-            return httpx.Response(200, text="# T\n" + "y" * 96)  # exactly 100 bytes
-        return httpx.Response(404)
-
-    client = make_client(handler)
-    result = discover(client, "https://example.com/", max_bytes=100)
+    result = discover(client, "https://example.com/")
     assert result is not None
-    assert result[0].endswith("/llms-full.txt")
+    url, text = result
+    assert url.endswith("/llms-full.txt")
+    assert len(text) == len(big)  # whole body returned, nothing truncated
 
 
 def test_discover_prefers_llms_full_over_llms_txt():