From ccf51270b05dba93b2d7dd99d126772c40ac1486 Mon Sep 17 00:00:00 2001
From: ItayUliel <ulielitay@gmail.com>
Date: Wed, 22 Jul 2026 20:02:38 +0300
Subject: [PATCH 1/2] feat: make max_pages optional (omit for no page limit)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A source had to specify max_pages. Make it optional across both services:
omitting it (or passing null) means "no page limit" — crawl all in-scope pages,
still bounded by same-host + include/exclude prefix scoping.

- config.SourceConfig / retrieval.ProposedSourceConfig: max_pages is now
  `int | None = Field(default=None, gt=0)` — optional, and positive only when
  provided. The DB column was already NULLABLE; NULL now means unlimited (schema
  comment updated).
- crawler.crawl(): when max_pages is None, use sys.maxsize internally as the cap
  (page_cap) so the existing sitemap/BFS/llms bound checks and slicing need no
  special-casing.
- propose_doc_source MCP tool + retrieval.propose_source: max_pages optional
  (default None); docstring updated (omit for no limit; positive if given).
- admin: form field no longer required (blank = no limit); dropped the
  None -> 100 fallbacks in _record_to_config and sources_repo._row_to_record so
  an unset limit stays unlimited instead of silently becoming 100; index view
  shows "unlimited".

Tests: omitting max_pages is accepted and yields None (config + ProposedSource);
a None-limit sitemap crawl fetches all in-scope pages; propose with max_pages=None
writes NULL; zero/negative still rejected. Updated the two tests that asserted
max_pages was required.
---
 db/init/02_sources_config.sql            | 12 +++++-------
 ingestion/app/admin.py                   |  8 ++++----
 ingestion/app/config.py                  |  4 ++--
 ingestion/app/crawler.py                 | 19 ++++++++++++------
 ingestion/app/sources_repo.py            |  2 +-
 ingestion/app/templates/admin/form.html  |  4 ++--
 ingestion/app/templates/admin/index.html |  2 +-
 ingestion/tests/test_config.py           | 19 +++++++++++++++++-
 ingestion/tests/test_crawler.py          | 25 ++++++++++++++++++++++++
 ingestion/tests/test_sources_repo.py     |  9 ++++-----
 mcp-server/app/retrieval.py              |  4 ++--
 mcp-server/app/server.py                 | 10 ++++++----
 mcp-server/tests/test_server.py          | 18 +++++++++++++++++
 13 files changed, 101 insertions(+), 35 deletions(-)

diff --git a/db/init/02_sources_config.sql b/db/init/02_sources_config.sql
index 3b19d0d..5cf005c 100644
--- a/db/init/02_sources_config.sql
+++ b/db/init/02_sources_config.sql
@@ -25,13 +25,11 @@
 -- fresh-volume schema looks like" and "what the live db was migrated to" —
 -- exactly the bug class this file exists to avoid.
 --
--- max_pages is intentionally left NULLABLE here: the pydantic SourceConfig
--- model (ingestion/app/config.py) is what enforces `max_pages` is required
--- and > 0 on every write path (sync/propose). Enforcing NOT NULL at the SQL
--- layer would break the 9 existing rows that predate this column with no
--- backfill value to give them. This split (permissive schema + strict
--- application-layer validation) is deliberate — do not "fix" it by adding a
--- NOT NULL constraint here.
+-- max_pages is NULLABLE and NULL is meaningful: it means "no page limit"
+-- (crawl all in-scope pages). The pydantic SourceConfig model
+-- (ingestion/app/config.py) makes `max_pages` OPTIONAL and, when provided,
+-- enforces it is > 0 on every write path (sync/propose). Do not add a
+-- NOT NULL constraint here — a NULL is a valid, intentional "unlimited".
 
 ALTER TABLE doc_sources
     ADD COLUMN IF NOT EXISTS sitemap            TEXT,
diff --git a/ingestion/app/admin.py b/ingestion/app/admin.py
index c1401a7..1e764ec 100644
--- a/ingestion/app/admin.py
+++ b/ingestion/app/admin.py
@@ -519,7 +519,7 @@ def _build_source_config(
             sitemap=sitemap.strip() or None,
             include_prefixes=_split_prefixes(include_prefixes),
             exclude_prefixes=_split_prefixes(exclude_prefixes),
-            max_pages=max_pages.strip(),
+            max_pages=(max_pages.strip() or None),
             language=language.strip() or "english",
             rate_limit_rps=rate_limit_rps.strip(),
             llms_txt=(llms_txt.strip() or "auto"),
@@ -543,7 +543,7 @@ def _record_to_config(record: SourceRecord) -> SourceConfig:
         sitemap=record.sitemap,
         include_prefixes=record.include_prefixes,
         exclude_prefixes=record.exclude_prefixes,
-        max_pages=record.max_pages if (record.max_pages is not None and record.max_pages > 0) else 100,
+        max_pages=record.max_pages,
         language=record.language or "english",
         rate_limit_rps=record.rate_limit_rps if (record.rate_limit_rps is not None and record.rate_limit_rps > 0) else 1.0,
         llms_txt=record.llms_txt or "auto",
@@ -637,7 +637,7 @@ def create_source_submit(
     sitemap: str = Form(default=""),
     include_prefixes: str = Form(default=""),
     exclude_prefixes: str = Form(default=""),
-    max_pages: str = Form(...),
+    max_pages: str = Form(default=""),
     language: str = Form(default="english"),
     rate_limit_rps: str = Form(default="1.0"),
     llms_txt: str = Form(default="auto"),
@@ -720,7 +720,7 @@ def update_source_submit(
     sitemap: str = Form(default=""),
     include_prefixes: str = Form(default=""),
     exclude_prefixes: str = Form(default=""),
-    max_pages: str = Form(...),
+    max_pages: str = Form(default=""),
     language: str = Form(default="english"),
     rate_limit_rps: str = Form(default="1.0"),
     llms_txt: str = Form(default="auto"),
diff --git a/ingestion/app/config.py b/ingestion/app/config.py
index 801c101..5da2cc5 100644
--- a/ingestion/app/config.py
+++ b/ingestion/app/config.py
@@ -8,7 +8,7 @@
         sitemap: https://fastapi.tiangolo.com/sitemap.xml   # optional
         include_prefixes: ["/tutorial/", "/reference/"]      # optional allowlist
         exclude_prefixes: ["/blog/", "/release-notes/"]      # optional denylist (wins)
-        max_pages: 500              # required
+        max_pages: 500              # optional — omit for no page limit (crawl all in-scope pages)
         language: english           # optional, default english
         rate_limit_rps: 1.0         # optional, default 1.0
 
@@ -99,7 +99,7 @@ class SourceConfig(BaseModel):
     sitemap: HttpUrl | None = None
     include_prefixes: list[str] = Field(default_factory=list)
     exclude_prefixes: list[str] = Field(default_factory=list)
-    max_pages: int = Field(gt=0)
+    max_pages: int | None = Field(default=None, gt=0)
     language: str = "english"
     rate_limit_rps: float = Field(default=1.0, gt=0)
     llms_txt: Literal["auto", "off", "only"] = "auto"
diff --git a/ingestion/app/crawler.py b/ingestion/app/crawler.py
index 038a701..16008b2 100644
--- a/ingestion/app/crawler.py
+++ b/ingestion/app/crawler.py
@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import ipaddress
+import sys
 import time
 import urllib.robotparser
 import warnings
@@ -397,7 +398,8 @@ def crawl(
     client: httpx.Client | None = None,
     conditional: dict[str, tuple[str | None, str | None]] | None = None,
 ) -> Iterator[dict]:
-    """Discover and fetch up to `source.max_pages` pages for `source`.
+    """Discover and fetch up to `source.max_pages` pages for `source` (or ALL
+    in-scope pages when `source.max_pages` is None — no page limit).
 
     YIELDS `{"url": str, "html": str | None, "fetch_ok": bool}` dicts one at a
     time as pages are visited, rather than materializing the whole crawl in
@@ -462,6 +464,11 @@ def crawl(
         rp = load_robots(client, base_url)
         limiter = RateLimiter(source.rate_limit_rps)
 
+        # `max_pages` is optional: None means "no page limit". Internally we use
+        # a very large int so the existing cap comparisons/slicing need no
+        # special-casing — same-host + prefix scoping still bounds the crawl.
+        page_cap = source.max_pages if source.max_pages is not None else sys.maxsize
+
         pages_fetched = 0
         visited: set[str] = set()
 
@@ -525,7 +532,7 @@ def crawl(
                     sections = llms_txt.split_llms_full(llms_text, index_url)
                     llms_count = 0
                     for section in sections:
-                        if llms_count >= source.max_pages:
+                        if llms_count >= page_cap:
                             break
                         yield {
                             "url": section["url"],
@@ -560,7 +567,7 @@ def crawl(
                     continue
                 if u not in filtered:
                     filtered.append(u)
-            candidate_urls = filtered[: source.max_pages]
+            candidate_urls = filtered[:page_cap]
             log.info("llms_index_candidates", count=len(candidate_urls))
         elif source.sitemap and not _same_host(str(source.sitemap), base_url):
             # Defence in depth: SourceConfig already rejects an off-host
@@ -572,7 +579,7 @@ def crawl(
                 candidate_urls = discover_sitemap_urls(
                     client,
                     str(source.sitemap),
-                    source.max_pages,
+                    page_cap,
                     limiter,
                     log,
                     base_url,
@@ -672,7 +679,7 @@ def _visit(url: str) -> dict | None:
 
         if candidate_urls is not None:
             for url in candidate_urls:
-                if pages_fetched >= source.max_pages:
+                if pages_fetched >= page_cap:
                     break
                 url = _strip_fragment(url)
                 if url in visited:
@@ -685,7 +692,7 @@ def _visit(url: str) -> dict | None:
                     yield item
         else:
             queue = [base_url]
-            while queue and pages_fetched < source.max_pages:
+            while queue and pages_fetched < page_cap:
                 url = _strip_fragment(queue.pop(0))
                 if url in visited:
                     continue
diff --git a/ingestion/app/sources_repo.py b/ingestion/app/sources_repo.py
index 28fd91e..bdb5b2a 100644
--- a/ingestion/app/sources_repo.py
+++ b/ingestion/app/sources_repo.py
@@ -189,7 +189,7 @@ def _row_to_record(row: tuple) -> SourceRecord:
         sitemap=sitemap,
         include_prefixes=list(include_prefixes) if include_prefixes is not None else [],
         exclude_prefixes=list(exclude_prefixes) if exclude_prefixes is not None else [],
-        max_pages=max_pages if (max_pages is not None and max_pages > 0) else 100,
+        max_pages=max_pages,  # None means "no page limit" (optional)
         language=language or "english",
         rate_limit_rps=rate_limit_rps if (rate_limit_rps is not None and rate_limit_rps > 0) else 1.0,
         llms_txt=llms_txt if llms_txt else "auto",
diff --git a/ingestion/app/templates/admin/form.html b/ingestion/app/templates/admin/form.html
index 4af3d71..cdbb3d3 100644
--- a/ingestion/app/templates/admin/form.html
+++ b/ingestion/app/templates/admin/form.html
@@ -35,8 +35,8 @@ <h2>{% if record %}Edit {{ record.name }}{% else %}Add a source{% endif %}</h2>
       <label for="exclude_prefixes">exclude_prefixes (one per line, optional &mdash; always wins over include)</label>
       <textarea id="exclude_prefixes" name="exclude_prefixes" rows="3">{{ values.exclude_prefixes or '' }}</textarea>
 
-      <label for="max_pages">max_pages</label>
-      <input type="number" id="max_pages" name="max_pages" min="1" required value="{{ values.max_pages or '' }}">
+      <label for="max_pages">max_pages <span class="muted">(optional — leave blank for no limit)</span></label>
+      <input type="number" id="max_pages" name="max_pages" min="1" placeholder="no limit" value="{{ values.max_pages or '' }}">
 
       <label for="language">language</label>
       <select id="language" name="language">
diff --git a/ingestion/app/templates/admin/index.html b/ingestion/app/templates/admin/index.html
index 8cb8d48..3cbd933 100644
--- a/ingestion/app/templates/admin/index.html
+++ b/ingestion/app/templates/admin/index.html
@@ -68,7 +68,7 @@ <h2>&#9888; Pending review (agent-proposed)</h2>
         </td>
         <td>{% if s.include_prefixes %}{{ s.include_prefixes | join(", ") }}{% else %}<span class="muted">(none)</span>{% endif %}</td>
         <td>{% if s.exclude_prefixes %}{{ s.exclude_prefixes | join(", ") }}{% else %}<span class="muted">(none)</span>{% endif %}</td>
-        <td>{{ s.max_pages if s.max_pages is not none else "(none)" }}</td>
+        <td>{% if s.max_pages is not none %}{{ s.max_pages }}{% else %}<span class="muted">unlimited</span>{% endif %}</td>
         <td><span class="proposed-by">agent: {{ s.proposed_by or "unknown" }}</span></td>
         <td class="actions">
           <div class="btn-group">
diff --git a/ingestion/tests/test_config.py b/ingestion/tests/test_config.py
index 52331d9..727a88b 100644
--- a/ingestion/tests/test_config.py
+++ b/ingestion/tests/test_config.py
@@ -98,7 +98,8 @@ def test_invalid_name_pattern_raises(tmp_path):
         load_sources(p)
 
 
-def test_missing_max_pages_raises(tmp_path):
+def test_missing_max_pages_is_allowed_and_means_unlimited(tmp_path):
+    # max_pages is optional: omitting it is valid and means "no page limit".
     p = write_yaml(
         tmp_path,
         """
@@ -107,6 +108,22 @@ def test_missing_max_pages_raises(tmp_path):
             base_url: https://example.com
         """,
     )
+    sources = load_sources(p)
+    assert len(sources) == 1
+    assert sources[0].max_pages is None
+
+
+def test_zero_or_negative_max_pages_still_raises(tmp_path):
+    # When provided, max_pages must be positive (gt=0).
+    p = write_yaml(
+        tmp_path,
+        """
+        sources:
+          - name: bad-max-pages
+            base_url: https://example.com
+            max_pages: 0
+        """,
+    )
     with pytest.raises(ConfigError):
         load_sources(p)
 
diff --git a/ingestion/tests/test_crawler.py b/ingestion/tests/test_crawler.py
index 3cddaf3..e7a3334 100644
--- a/ingestion/tests/test_crawler.py
+++ b/ingestion/tests/test_crawler.py
@@ -85,6 +85,31 @@ def test_sitemap_discovery_bounded_by_max_pages():
     assert len(pages) == 1
 
 
+def test_sitemap_discovery_unlimited_when_max_pages_none():
+    # max_pages=None (omitted) => no page limit: all in-scope pages are crawled.
+    source = SourceConfig(
+        name="example",
+        base_url="https://example.com/",
+        sitemap="https://example.com/sitemap.xml",
+        include_prefixes=["/docs/"],
+        rate_limit_rps=1000,
+    )
+    assert source.max_pages is None
+    handler = _handler_factory(
+        sitemap_body=SITEMAP_XML,
+        page_bodies={
+            "https://example.com/docs/a": PAGE_HTML,
+            "https://example.com/docs/b": PAGE_HTML,
+        },
+    )
+    client = make_client(handler)
+    pages = list(crawl(source, client=client))
+    assert {p["url"] for p in pages} == {
+        "https://example.com/docs/a",
+        "https://example.com/docs/b",
+    }
+
+
 def test_bfs_fallback_when_no_sitemap():
     def handler(request: httpx.Request) -> httpx.Response:
         url = str(request.url)
diff --git a/ingestion/tests/test_sources_repo.py b/ingestion/tests/test_sources_repo.py
index 6898a82..f28cb83 100644
--- a/ingestion/tests/test_sources_repo.py
+++ b/ingestion/tests/test_sources_repo.py
@@ -199,11 +199,10 @@ def test_cfg_matches_record_true_when_llms_txt_also_matches() -> None:
 # --- SourceConfig rejects invalid input on both create and update paths ----
 
 
-def test_invalid_config_rejected_missing_max_pages() -> None:
-    with pytest.raises(Exception):
-        SourceConfig.model_validate(
-            {"name": "bad", "base_url": "https://example.com/"}
-        )  # max_pages is required (gt=0, no default)
+def test_missing_max_pages_is_accepted_and_none() -> None:
+    # max_pages is optional (None => no page limit).
+    cfg = SourceConfig.model_validate({"name": "ok", "base_url": "https://example.com/"})
+    assert cfg.max_pages is None
 
 
 def test_invalid_config_rejected_bad_name_pattern() -> None:
diff --git a/mcp-server/app/retrieval.py b/mcp-server/app/retrieval.py
index 2ebef14..5c681ae 100644
--- a/mcp-server/app/retrieval.py
+++ b/mcp-server/app/retrieval.py
@@ -397,7 +397,7 @@ class ProposedSourceConfig(BaseModel):
     sitemap: HttpUrl | None = None
     include_prefixes: list[str] = Field(default_factory=list)
     exclude_prefixes: list[str] = Field(default_factory=list)
-    max_pages: int = Field(gt=0)
+    max_pages: int | None = Field(default=None, gt=0)
     language: str = "english"
     rate_limit_rps: float = Field(default=1.0, gt=0)
     llms_txt: Literal["auto", "off", "only"] = "auto"
@@ -507,7 +507,7 @@ def propose_source(
     *,
     name: str,
     base_url: str,
-    max_pages: int,
+    max_pages: int | None = None,
     sitemap: str | None = None,
     include_prefixes: list[str] | None = None,
     exclude_prefixes: list[str] | None = None,
diff --git a/mcp-server/app/server.py b/mcp-server/app/server.py
index 8742f89..1db56cc 100644
--- a/mcp-server/app/server.py
+++ b/mcp-server/app/server.py
@@ -88,7 +88,7 @@ def list_doc_sources() -> str:
 def propose_doc_source(
     name: str,
     base_url: str,
-    max_pages: int,
+    max_pages: int | None = None,
     sitemap: str | None = None,
     include_prefixes: list[str] | None = None,
     exclude_prefixes: list[str] | None = None,
@@ -100,9 +100,11 @@ def propose_doc_source(
     and approve in the admin UI before anything is ever crawled. Do not tell
     the user their docs are "being indexed" — tell them the proposal is
     queued for human approval. `name` must match ^[a-z0-9-]+$ and be unique;
-    `base_url`/`sitemap` must be valid http(s) URLs; `max_pages` and
-    `rate_limit_rps` must be positive. Rejected (with a clear reason) on
-    invalid input or a name that is already taken, pending or not."""
+    `base_url`/`sitemap` must be valid http(s) URLs; `rate_limit_rps` must be
+    positive. `max_pages` is OPTIONAL — omit it (or pass null) to crawl all
+    in-scope pages with no page limit; if given it must be positive. Rejected
+    (with a clear reason) on invalid input or a name that is already taken,
+    pending or not."""
     token = get_access_token()
     if token is None:
         # Should not happen in practice: every /mcp request is already
diff --git a/mcp-server/tests/test_server.py b/mcp-server/tests/test_server.py
index cb425ad..d34a7dc 100644
--- a/mcp-server/tests/test_server.py
+++ b/mcp-server/tests/test_server.py
@@ -320,6 +320,24 @@ def on_execute(sql, params):
     assert params["name"] == "my-new-source"
 
 
+def test_propose_source_accepts_optional_max_pages_none(monkeypatch):
+    # max_pages is optional (None => no page limit): a proposal without it is
+    # valid and writes NULL for max_pages.
+    calls = []
+
+    def on_execute(sql, params):
+        calls.append((sql, params))
+        return {"id": 7}
+
+    _install_fake_pool(monkeypatch, on_execute)
+
+    source_id = retrieval.propose_source(**_valid_kwargs(max_pages=None))
+    assert source_id == 7
+    assert len(calls) == 1
+    _, params = calls[0]
+    assert params["max_pages"] is None
+
+
 def test_propose_source_invalid_config_inserts_nothing(monkeypatch):
     calls = []
     _install_fake_pool(monkeypatch, lambda sql, params: calls.append((sql, params)))

From 9c6aa83de0a10a6c0246cc7b62e267a3be081e55 Mon Sep 17 00:00:00 2001
From: ItayUliel <ulielitay@gmail.com>
Date: Wed, 22 Jul 2026 20:14:03 +0300
Subject: [PATCH 2/2] feat(crawler): remove the llms.txt download size limit

Drop the max_bytes cap on llms-full.txt / llms.txt discovery entirely. The cap
rejected legitimately large full-content files (docs.anthropic.com's
/llms-full.txt is ~24MB), forcing a fallback to the thin index and, when that
index links off-host, an empty crawl. Now the full file is fetched however large
it is.

- discover(): no max_bytes parameter; no size check.
- _fetch_capped (streaming/abort) replaced by a plain _fetch_body GET.
- Removed the corresponding cap tests; added one asserting a large body is
  returned in full (no truncation).

Sources are human-approved before any crawl, so an unbounded fetch here is
bounded by that approval gate.
---
 ingestion/app/llms_txt.py        | 59 ++++++++------------------------
 ingestion/tests/test_llms_txt.py | 47 +++++--------------------
 2 files changed, 24 insertions(+), 82 deletions(-)

diff --git a/ingestion/app/llms_txt.py b/ingestion/app/llms_txt.py
index 837efb5..16fc2bc 100644
--- a/ingestion/app/llms_txt.py
+++ b/ingestion/app/llms_txt.py
@@ -41,22 +41,18 @@ def discover(
     base_url: str,
     *,
     prefer_full: bool = True,
-    max_bytes: int = 10_000_000,
 ) -> tuple[str, str] | None:
     """Try to fetch `/llms-full.txt` then `/llms.txt` (or the reverse order
     when `prefer_full=False`) at `base_url`'s origin.
 
-    Returns `(fetched_url, text)` for the first response that is status 200,
-    has a non-empty body, and is within `max_bytes`. Returns `None` if
-    neither candidate qualifies.
+    Returns `(fetched_url, text)` for the first response that is status 200
+    with a non-empty body. There is NO size limit — the full file is fetched
+    however large it is (llms-full.txt files are legitimately large). Returns
+    `None` if neither candidate qualifies.
 
-    The body is streamed and the download is ABORTED as soon as it exceeds
-    `max_bytes`, so an oversized file (e.g. a 24MB `/llms-full.txt`) is not
-    fully downloaded only to be discarded.
-
-    NEVER raises: any httpx error, oversize body, empty body, or non-200
-    status is treated as "skip this candidate" (and, if both candidates are
-    exhausted, "skip this source" — return None) rather than propagating.
+    NEVER raises: any httpx error, empty body, or non-200 status is treated as
+    "skip this candidate" (and, if both candidates are exhausted, "skip this
+    source" — return None) rather than propagating.
     """
     parsed = urlparse(base_url)
     origin = f"{parsed.scheme}://{parsed.netloc}"
@@ -68,7 +64,7 @@ def discover(
     log = logger.bind(base_url=base_url)
 
     for url in candidates:
-        text = _fetch_capped(client, url, max_bytes, log)
+        text = _fetch_body(client, url, log)
         if text is None:
             continue
         log.info("llms_txt_discovered", url=url, size=len(text))
@@ -77,43 +73,18 @@ def discover(
     return None
 
 
-def _fetch_capped(client, url: str, max_bytes: int, log) -> str | None:
-    """GET `url`, streaming the body and aborting the download once it exceeds
-    `max_bytes`. Returns the decoded text, or None to skip this candidate
-    (non-200, empty, oversize, or any error — discovery is best-effort and
-    never raises).
-
-    Streaming (rather than `client.get(...).content`) is what makes the
-    `max_bytes` guard cheap: a huge `/llms-full.txt` is stopped mid-transfer
-    instead of being pulled into memory in full and then thrown away.
-    """
-    chunks: list[bytes] = []
-    encoding = "utf-8"
+def _fetch_body(client, url: str, log) -> str | None:
+    """GET `url` and return its decoded text, or None to skip this candidate
+    (non-200, empty, or any error — discovery is best-effort and never raises)."""
     try:
-        with client.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=15) as resp:
-            if resp.status_code != 200:
-                log.info("llms_txt_non_200", url=url, status=resp.status_code)
-                return None
-            total = 0
-            for chunk in resp.iter_bytes():
-                total += len(chunk)
-                if total > max_bytes:
-                    log.info("llms_txt_too_large", url=url, size_at_least=total, max_bytes=max_bytes)
-                    return None
-                chunks.append(chunk)
-            encoding = resp.encoding or "utf-8"
+        resp = client.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
     except Exception as e:  # noqa: BLE001 - discovery is best-effort, never raises
         log.info("llms_txt_fetch_failed", url=url, error=str(e))
         return None
-
-    body = b"".join(chunks)
-    if not body:
-        log.info("llms_txt_empty", url=url)
+    if resp.status_code != 200:
+        log.info("llms_txt_non_200", url=url, status=resp.status_code)
         return None
-    try:
-        text = body.decode(encoding, errors="replace")
-    except LookupError:
-        text = body.decode("utf-8", errors="replace")
+    text = resp.text
     if not text.strip():
         log.info("llms_txt_empty", url=url)
         return None
diff --git a/ingestion/tests/test_llms_txt.py b/ingestion/tests/test_llms_txt.py
index cd93ce0..399fa53 100644
--- a/ingestion/tests/test_llms_txt.py
+++ b/ingestion/tests/test_llms_txt.py
@@ -27,52 +27,23 @@ def handler(request: httpx.Request) -> httpx.Response:
     assert discover(client, "https://example.com/") is None
 
 
-def test_discover_returns_none_when_over_max_bytes():
-    def handler(request: httpx.Request) -> httpx.Response:
-        url = str(request.url)
-        if url == "https://example.com/llms-full.txt":
-            return httpx.Response(200, text="x" * 100)
-        return httpx.Response(404)
-
-    client = make_client(handler)
-    assert discover(client, "https://example.com/", max_bytes=10) is None
-
-
-def test_discover_aborts_download_once_over_cap():
-    """The oversized body must be abandoned mid-stream, not fully downloaded and
-    then discarded — otherwise a 24MB llms-full.txt costs a 24MB transfer just
-    to be rejected."""
-    consumed = {"chunks": 0}
-
-    def body_gen():
-        for _ in range(1000):  # up to 1000 * 1KB = 1MB if fully read
-            consumed["chunks"] += 1
-            yield b"x" * 1024
+def test_discover_has_no_size_limit():
+    """There is no size cap: a large llms-full.txt is fetched in full (real
+    files are legitimately large, e.g. anthropic's ~24MB)."""
+    big = "# Full\n" + ("word " * 500_000)  # ~2.5MB
 
     def handler(request: httpx.Request) -> httpx.Response:
         url = str(request.url)
         if url.endswith("/llms-full.txt"):
-            return httpx.Response(200, content=body_gen())
+            return httpx.Response(200, text=big)
         return httpx.Response(404)
 
     client = make_client(handler)
-    result = discover(client, "https://example.com/", max_bytes=5_000)  # 5KB cap
-    assert result is None
-    # Stopped shortly after crossing 5KB (~6 chunks), not the whole 1000.
-    assert consumed["chunks"] < 20
-
-
-def test_discover_accepts_body_at_the_cap_boundary():
-    def handler(request: httpx.Request) -> httpx.Response:
-        url = str(request.url)
-        if url.endswith("/llms-full.txt"):
-            return httpx.Response(200, text="# T\n" + "y" * 96)  # exactly 100 bytes
-        return httpx.Response(404)
-
-    client = make_client(handler)
-    result = discover(client, "https://example.com/", max_bytes=100)
+    result = discover(client, "https://example.com/")
     assert result is not None
-    assert result[0].endswith("/llms-full.txt")
+    url, text = result
+    assert url.endswith("/llms-full.txt")
+    assert len(text) == len(big)  # whole body returned, nothing truncated
 
 
 def test_discover_prefers_llms_full_over_llms_txt():