From 3fa79c380bc0b673413d24cbeb03e1293bb33602 Mon Sep 17 00:00:00 2001 From: ItayUliel Date: Wed, 22 Jul 2026 14:30:02 +0300 Subject: [PATCH 1/2] fix(crawler): crawl llms.txt index links instead of ingesting them as content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An llms.txt INDEX (`/llms.txt`) is a curated list of links to the real docs, whereas `/llms-full.txt` is the concatenated documentation prose. The crawler treated both identically: `discover()` falls back from the full file to the index, and `split_llms_full()` was run on whatever it got. The only guard (`_looks_like_llms_txt`) just checks for a leading `# `, which an index also has. Result: for any source whose site serves only `/llms.txt` — or whose `/llms-full.txt` exceeds discover()'s 10MB cap and falls back to the index — the corpus became a list of links with one-line descriptions instead of actual documentation content ("the crawler only returns metadata"). Reproduced against docs.anthropic.com: its /llms-full.txt is 24MB (rejected by the 10MB cap), so discovery fell back to the 56KB /llms.txt index and ingested the link list as content. Fix — distinguish the two file types and handle the index as a discovery source: - llms_txt.looks_like_index(): an index is dominated by markdown link bullets (>=3 bullets AND a majority of content lines), a full file is prose/code. - llms_txt.parse_llms_index(): extract the linked page URLs (absolute + relative resolved, deduped, code-fenced links ignored). - crawler.crawl(): when discovery returns an index, extract its URLs and crawl each as a normal HTML page (filtered same-host + include/exclude prefixes, capped to max_pages, still re-validated per-URL in _visit), so pages run through extract.extract and yield real body text. Full-content files keep the existing section-splitting fast path. "only" mode crawls an index's links rather than yielding nothing. Also gives index sources proper per-page ETag change detection on re-sync (each page fetched via _visit), instead of the all-or-nothing index conditional. Tests: unit coverage for looks_like_index/parse_llms_index, and crawl-level tests proving an index is fetched as HTML (not ingested as metadata), links are scope-filtered, "only"+index crawls the links without BFS, and a real /llms-full.txt still yields markdown sections. Full ingestion suite: 258 passed, 44 skipped (DB-gated), 1 pre-existing WSL2-only failure unrelated to this change. --- ingestion/app/crawler.py | 101 ++++++++++++++----- ingestion/app/llms_txt.py | 84 +++++++++++++++- ingestion/tests/test_crawler.py | 167 +++++++++++++++++++++++++++++++ ingestion/tests/test_llms_txt.py | 87 +++++++++++++++- 4 files changed, 412 insertions(+), 27 deletions(-) diff --git a/ingestion/app/crawler.py b/ingestion/app/crawler.py index db61d4a..038a701 100644 --- a/ingestion/app/crawler.py +++ b/ingestion/app/crawler.py @@ -417,12 +417,23 @@ def crawl( yielded instead of re-fetching/re-extracting. When `source.llms_txt` is `"auto"` or `"only"`, `crawl` first tries the - llmstxt.org convention (`llms-full.txt`/`llms.txt`) via `llms_txt`. If a - file is discovered, its sections are yielded as - `{"url", "markdown", "heading_path", "fetch_ok": True}` items (capped at - `source.max_pages`) and the generator returns — the sitemap/BFS HTML - crawl is skipped entirely. If no file is found: `"auto"` falls through to - the normal HTML crawl; `"only"` yields nothing. If `conditional` carries a + llmstxt.org convention (`llms-full.txt`/`llms.txt`) via `llms_txt`. The two + file types are handled differently: + + - FULL CONTENT (`llms-full.txt`, or an `llms.txt` that is actually full + prose): split into sections yielded as + `{"url", "markdown", "heading_path", "fetch_ok": True}` items (capped at + `source.max_pages`); the generator then returns. + - INDEX (an `llms.txt` that is a list of links to the docs — detected via + `llms_txt.looks_like_index`): the linked page URLs are extracted and + crawled as normal HTML pages below (each yielded as an + `{"url", "html", "fetch_ok": True}` item), NOT ingested as content. + Ingesting the link list verbatim would make the corpus link-list + metadata instead of documentation. + + If no file is found: `"auto"` falls through to the normal sitemap/BFS HTML + crawl; `"only"` yields nothing (but `"only"` DOES crawl an index's links, + since those are what llms.txt points at). If `conditional` carries a validator for the llms-index URL and it 304s, a single sentinel `{"kind": "llms_index_unchanged", "url", "not_modified": True, "fetch_ok": True}` is yielded and the generator returns. @@ -454,6 +465,11 @@ def crawl( pages_fetched = 0 visited: set[str] = set() + # When llms.txt discovery finds an INDEX (a list of links rather than + # full content), these are the page URLs it points at — crawled below + # via the normal HTML path instead of being ingested as content. + llms_index_urls: list[str] | None = None + if source.llms_txt in ("auto", "only"): parsed_base = urlparse(base_url) origin = f"{parsed_base.scheme}://{parsed_base.netloc}" @@ -489,29 +505,64 @@ def crawl( if discovered is not None: index_url, llms_text = discovered - limiter.wait() - sections = llms_txt.split_llms_full(llms_text, index_url) - llms_count = 0 - for section in sections: - if llms_count >= source.max_pages: - break - yield { - "url": section["url"], - "markdown": section["markdown"], - "heading_path": section["heading_path"], - "fetch_ok": True, - } - llms_count += 1 - log.info("crawl_complete", pages_fetched=llms_count, mode="llms_txt") - return - if source.llms_txt == "only": + if llms_txt.looks_like_index(llms_text): + # `/llms.txt` INDEX: a list of links to the real docs, NOT + # content. Extract the linked page URLs and crawl each as a + # normal HTML page below (so it runs through extract.extract + # and yields real body text) — ingesting the index verbatim + # is the "corpus is only link-list metadata" bug this fixes. + parsed = llms_txt.parse_llms_index(llms_text, base_url) + if parsed: + llms_index_urls = parsed + log.info("llms_index_discovered", url=index_url, links=len(parsed)) + else: + log.info("llms_index_empty", url=index_url) + else: + # `/llms-full.txt` (or an llms.txt that is actually full + # content): split into per-section markdown and yield as + # already-extracted content. + limiter.wait() + sections = llms_txt.split_llms_full(llms_text, index_url) + llms_count = 0 + for section in sections: + if llms_count >= source.max_pages: + break + yield { + "url": section["url"], + "markdown": section["markdown"], + "heading_path": section["heading_path"], + "fetch_ok": True, + } + llms_count += 1 + log.info("crawl_complete", pages_fetched=llms_count, mode="llms_txt") + return + + if llms_index_urls is None and source.llms_txt == "only": + # "only" means don't fall back to a full HTML BFS crawl. With no + # full-content file and no usable index, there is nothing to do. log.info("crawl_complete", pages_fetched=0, mode="llms_txt") return - # source.llms_txt == "auto" and nothing discovered: fall through - # to the normal sitemap/BFS HTML crawl below. + # Otherwise fall through to the HTML crawl below: seeded with the + # index's URLs when we have them (both "auto" and "only"), or — for + # "auto" with nothing discovered — the normal sitemap/BFS path. candidate_urls: list[str] | None = None - if source.sitemap and not _same_host(str(source.sitemap), base_url): + if llms_index_urls is not None: + # Crawl exactly the pages the llms.txt index lists — filtered to + # same-host + include/exclude prefixes and capped to max_pages, + # mirroring sitemap discovery — instead of a sitemap/BFS crawl. Each + # URL is still re-validated (host/scope/private/robots) in `_visit`. + filtered: list[str] = [] + for u in llms_index_urls: + if not _same_host(u, base_url): + continue + if not _allowed(urlparse(u).path, source.include_prefixes, source.exclude_prefixes): + continue + if u not in filtered: + filtered.append(u) + candidate_urls = filtered[: source.max_pages] + log.info("llms_index_candidates", count=len(candidate_urls)) + elif source.sitemap and not _same_host(str(source.sitemap), base_url): # Defence in depth: SourceConfig already rejects an off-host # sitemap (H1). This catches a SourceConfig built via # `model_construct` or mutated after validation. diff --git a/ingestion/app/llms_txt.py b/ingestion/app/llms_txt.py index fffc062..6340d26 100644 --- a/ingestion/app/llms_txt.py +++ b/ingestion/app/llms_txt.py @@ -14,7 +14,7 @@ from __future__ import annotations import re -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse from .logging_config import get_logger @@ -28,6 +28,12 @@ _MD_LINK_RE = re.compile(r"^\[([^\]]*)\]\(([^)]+)\)$") _SOURCE_LINE_RE = re.compile(r"^Source:\s*(https?://\S+)", re.IGNORECASE) _SLUG_NONALNUM_RE = re.compile(r"[^a-z0-9]+") +# An llms.txt INDEX lists documentation pages as markdown link bullets, e.g. +# - [Quickstart](https://x.io/docs/quickstart): get started fast +# Match such a bullet and capture its URL. Unlike `_MD_LINK_RE` this is NOT +# whole-line anchored: index bullets carry a leading `-`/`*` marker and an +# optional trailing `: description`. +_INDEX_LINK_ITEM_RE = re.compile(r"^\s*[-*]\s*\[[^\]]*\]\(([^)]+)\)") def discover( @@ -92,6 +98,82 @@ def discover( return None +def _content_lines(text: str) -> list[str]: + """Non-blank, non-heading lines outside fenced code blocks. Used to judge + whether a file is an index (mostly link bullets) or full prose/code.""" + out: list[str] = [] + in_fence = False + for line in text.splitlines(): + stripped = line.strip() + if _FENCE_RE.match(stripped): + in_fence = not in_fence + continue + if in_fence or not stripped or stripped.startswith("#"): + continue + out.append(stripped) + return out + + +def looks_like_index(text: str) -> bool: + """True if `text` is an llms.txt INDEX (a list of links to the real docs) + rather than `/llms-full.txt` full content. + + Per the llmstxt.org convention, `/llms.txt` is a curated index: an H1/H2 + outline whose body is markdown link bullets (`- [Title](url): note`), while + `/llms-full.txt` is the concatenated documentation prose. This distinction + matters because an index must be CRAWLED (fetch each linked page), never + ingested verbatim — otherwise the corpus becomes a list of links with no + actual documentation content. + + Heuristic: an index is dominated by link bullets. If at least half of the + content lines (excluding headings/blank/code) are markdown link bullets AND + there are several of them, it is an index. A full-content file has prose and + code between its headings, so its link-bullet ratio is low. + """ + content = _content_lines(text) + if not content: + return False + link_bullets = sum(1 for line in content if _INDEX_LINK_ITEM_RE.match(line)) + # Require a real list (>=3 bullets) that is the majority of the content, so + # a full-content page that merely happens to contain a couple of bullet + # links is not misclassified as an index. + return link_bullets >= 3 and link_bullets / len(content) >= 0.5 + + +def parse_llms_index(text: str, base_url: str) -> list[str]: + """Extract the documentation page URLs an llms.txt index links to. + + Returns absolute, order-preserving, de-duplicated URLs (relative links are + resolved against `base_url`). Fragment-only or non-http(s) targets are + dropped. Host/scope/private-address filtering is deliberately left to the + crawler's per-URL `_visit` guards, mirroring how sitemap discovery hands + raw candidate URLs to the same gate. + """ + urls: list[str] = [] + seen: set[str] = set() + in_fence = False + for line in text.splitlines(): + stripped = line.strip() + if _FENCE_RE.match(stripped): + in_fence = not in_fence + continue + if in_fence: + continue + m = _INDEX_LINK_ITEM_RE.match(line) + if not m: + continue + target = m.group(1).strip() + if not target or target.startswith("#"): + continue + absolute = urljoin(base_url, target) + if not absolute.lower().startswith(("http://", "https://")): + continue + if absolute not in seen: + seen.add(absolute) + urls.append(absolute) + return urls + + def _slugify(title: str) -> str: """Deterministic slug: lowercase, runs of non-alphanumeric collapsed to a single '-', leading/trailing '-' stripped.""" diff --git a/ingestion/tests/test_crawler.py b/ingestion/tests/test_crawler.py index af874d6..3cddaf3 100644 --- a/ingestion/tests/test_crawler.py +++ b/ingestion/tests/test_crawler.py @@ -372,6 +372,173 @@ def handler(request: httpx.Request) -> httpx.Response: assert not any("169.254.169.254" in u for u in requested) +# --- llms.txt INDEX vs full-content routing ----------------------------- + +LLMS_INDEX = """# Example Docs + +> Summary of the project. + +## Guides +- [Quickstart](https://example.com/docs/quickstart): get started +- [Configuration](https://example.com/docs/config) +- [API](https://example.com/docs/api) +""" + +LLMS_FULL = """# Quickstart + +Real documentation prose describing how to install and use the package across +several sentences of genuine content that is clearly not a list of links. + +## Configuration + +More prose about configuration options, again long enough to read as content. +""" + + +def test_llms_index_is_crawled_as_html_not_ingested_as_content(): + """Regression: a site serving `/llms.txt` (a link INDEX) but no + `/llms-full.txt` must have its linked pages FETCHED as HTML, not have the + index link-list ingested verbatim as content.""" + requested: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + requested.append(url) + if url.endswith("/robots.txt"): + return httpx.Response(200, text=ROBOTS_ALLOW_ALL) + if url.endswith("/llms-full.txt"): + return httpx.Response(404, text="not found") + if url.endswith("/llms.txt"): + return httpx.Response(200, text=LLMS_INDEX) + return httpx.Response(200, text=PAGE_HTML) + + source = SourceConfig( + name="example", + base_url="https://example.com/", + max_pages=10, + rate_limit_rps=1000, + llms_txt="auto", + ) + client = make_client(handler) + pages = list(crawl(source, client=client)) + + # Each linked doc page is yielded as an HTML item (html set, markdown absent), + # so downstream extraction runs on the real page — NOT the index link-list. + assert [p["url"] for p in pages] == [ + "https://example.com/docs/quickstart", + "https://example.com/docs/config", + "https://example.com/docs/api", + ] + for p in pages: + assert p["fetch_ok"] is True + assert "html" in p and p["html"] == PAGE_HTML + assert "markdown" not in p + # The pages were actually fetched over HTTP. + assert "https://example.com/docs/quickstart" in requested + + +def test_llms_index_links_filtered_by_include_prefixes(): + """Index-derived URLs are scoped by include/exclude prefixes, like sitemap + discovery — an out-of-scope link is not crawled.""" + index = ( + "# Docs\n\n## S\n" + "- [in](https://example.com/docs/in)\n" + "- [out](https://example.com/blog/out)\n" + "- [also](https://example.com/docs/also)\n" + ) + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/robots.txt"): + return httpx.Response(200, text=ROBOTS_ALLOW_ALL) + if url.endswith("/llms-full.txt"): + return httpx.Response(404, text="nope") + if url.endswith("/llms.txt"): + return httpx.Response(200, text=index) + return httpx.Response(200, text=PAGE_HTML) + + source = SourceConfig( + name="example", + base_url="https://example.com/docs/", + include_prefixes=["/docs/"], + max_pages=10, + rate_limit_rps=1000, + llms_txt="auto", + ) + client = make_client(handler) + pages = list(crawl(source, client=client)) + assert [p["url"] for p in pages] == [ + "https://example.com/docs/in", + "https://example.com/docs/also", + ] + + +def test_llms_full_content_still_yields_markdown_sections(): + """A real `/llms-full.txt` is still split into already-extracted markdown + sections (the fast path), not crawled page-by-page.""" + fetched_pages: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/robots.txt"): + return httpx.Response(200, text=ROBOTS_ALLOW_ALL) + if url.endswith("/llms-full.txt"): + return httpx.Response(200, text=LLMS_FULL) + fetched_pages.append(url) + return httpx.Response(200, text=PAGE_HTML) + + source = SourceConfig( + name="example", + base_url="https://example.com/", + max_pages=10, + rate_limit_rps=1000, + llms_txt="auto", + ) + client = make_client(handler) + pages = list(crawl(source, client=client)) + assert pages, "expected llms-full sections" + for p in pages: + assert "markdown" in p and p["markdown"] + assert "html" not in p + # No individual doc pages were HTTP-fetched — the full file was the source. + assert fetched_pages == [] + + +def test_llms_only_with_index_crawls_links_not_bfs(): + """`llms_txt="only"` + an index: crawl exactly the index's links, and do + NOT fall back to a BFS crawl of the site.""" + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/robots.txt"): + return httpx.Response(200, text=ROBOTS_ALLOW_ALL) + if url.endswith("/llms-full.txt"): + return httpx.Response(404, text="nope") + if url.endswith("/llms.txt"): + return httpx.Response(200, text=LLMS_INDEX) + # A page whose HTML links elsewhere — BFS would follow these. + return httpx.Response( + 200, + text='xcontent', + ) + + source = SourceConfig( + name="example", + base_url="https://example.com/", + max_pages=10, + rate_limit_rps=1000, + llms_txt="only", + ) + client = make_client(handler) + pages = list(crawl(source, client=client)) + urls = [p["url"] for p in pages] + # Only the three index links, no BFS-discovered /other/x. + assert urls == [ + "https://example.com/docs/quickstart", + "https://example.com/docs/config", + "https://example.com/docs/api", + ] + + def test_crawl_is_generator_and_preserves_fetch_ok_false_contract(): """Both load-bearing invariants asserted together: crawl() stays a generator (memory bounding / per-page commit), and an attempted-but- diff --git a/ingestion/tests/test_llms_txt.py b/ingestion/tests/test_llms_txt.py index ad1fde8..c63195c 100644 --- a/ingestion/tests/test_llms_txt.py +++ b/ingestion/tests/test_llms_txt.py @@ -8,7 +8,8 @@ from __future__ import annotations import httpx -from app.llms_txt import discover, split_llms_full + +from app.llms_txt import discover, looks_like_index, parse_llms_index, split_llms_full def make_client(handler) -> httpx.Client: @@ -146,3 +147,87 @@ def test_split_is_deterministic_across_runs(): urls1 = [s["url"] for s in split_llms_full(text, source_url)] urls2 = [s["url"] for s in split_llms_full(text, source_url)] assert urls1 == urls2 + + +# --- looks_like_index() / parse_llms_index() ---------------------------- + +INDEX_TXT = """# Example Docs + +> A short summary of the project. + +## Guides +- [Quickstart](https://example.com/docs/quickstart): get started +- [Configuration](https://example.com/docs/config) +- [Deployment](/docs/deploy): how to deploy + +## Reference +- [API](https://example.com/docs/api): the API reference +""" + +FULL_TXT = """# Quickstart + +Install the package and import it. This paragraph is real documentation prose, +not a list of links, so the file is full content rather than an index. + +``` +pip install example +``` + +## Configuration + +Configuration lives in a YAML file. Again this is prose describing behavior in +enough detail that it clearly is not a bullet list of links. +""" + + +def test_looks_like_index_true_for_link_list(): + assert looks_like_index(INDEX_TXT) is True + + +def test_looks_like_index_false_for_full_content(): + assert looks_like_index(FULL_TXT) is False + + +def test_looks_like_index_false_for_prose_with_a_couple_of_links(): + # A full page that merely contains one or two inline link bullets must not + # be misclassified as an index (needs >=3 bullets AND a majority). + text = ( + "# Overview\n\n" + "This is a real page with lots of prose describing the system in detail " + "across multiple sentences of genuine content.\n\n" + "- [See also](https://example.com/x)\n" + "More prose after the single link, continuing the explanation.\n" + ) + assert looks_like_index(text) is False + + +def test_parse_llms_index_extracts_absolute_and_relative_urls_in_order(): + urls = parse_llms_index(INDEX_TXT, "https://example.com/") + assert urls == [ + "https://example.com/docs/quickstart", + "https://example.com/docs/config", + "https://example.com/docs/deploy", + "https://example.com/docs/api", + ] + + +def test_parse_llms_index_dedupes_and_skips_fragments_and_non_http(): + text = ( + "## S\n" + "- [A](https://example.com/a)\n" + "- [A again](https://example.com/a)\n" + "- [frag](#section)\n" + "- [mail](mailto:x@example.com)\n" + ) + assert parse_llms_index(text, "https://example.com/") == ["https://example.com/a"] + + +def test_parse_llms_index_ignores_links_inside_code_fences(): + text = ( + "## S\n" + "- [real](https://example.com/real)\n" + "```\n" + "- [fenced](https://example.com/fenced)\n" + "```\n" + ) + assert parse_llms_index(text, "https://example.com/") == ["https://example.com/real"] From aa5580a016d635f5ade50cd0e353d2739f6426b4 Mon Sep 17 00:00:00 2001 From: ItayUliel Date: Wed, 22 Jul 2026 14:37:43 +0300 Subject: [PATCH 2/2] perf(crawler): stream llms.txt download and abort once over the size cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit discover()'s max_bytes guard previously downloaded the entire file with client.get(...).content and only then checked its length — so an oversized /llms-full.txt (docs.anthropic.com serves a 24MB one) cost a full 24MB transfer just to be rejected before falling back to the index. Stream the body instead and stop reading as soon as it crosses max_bytes, so an oversized file is abandoned mid-transfer. Behavior is otherwise unchanged: non-200, empty, oversize, and error cases still skip the candidate, and a valid small file is returned as before. Extracted into a _fetch_capped() helper. Tests: assert the download aborts after a few KB (not the whole body) once the cap is exceeded, and that a body exactly at the cap boundary is still accepted. --- ingestion/app/llms_txt.py | 76 ++++++++++++++++++++------------ ingestion/tests/test_llms_txt.py | 38 +++++++++++++++- 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/ingestion/app/llms_txt.py b/ingestion/app/llms_txt.py index 6340d26..837efb5 100644 --- a/ingestion/app/llms_txt.py +++ b/ingestion/app/llms_txt.py @@ -50,6 +50,10 @@ def discover( has a non-empty body, and is within `max_bytes`. Returns `None` if neither candidate qualifies. + The body is streamed and the download is ABORTED as soon as it exceeds + `max_bytes`, so an oversized file (e.g. a 24MB `/llms-full.txt`) is not + fully downloaded only to be discarded. + NEVER raises: any httpx error, oversize body, empty body, or non-200 status is treated as "skip this candidate" (and, if both candidates are exhausted, "skip this source" — return None) rather than propagating. @@ -64,38 +68,56 @@ def discover( log = logger.bind(base_url=base_url) for url in candidates: - try: - resp = client.get(url, headers={"User-Agent": USER_AGENT}, timeout=15) - except Exception as e: # noqa: BLE001 - discovery is best-effort, never raises - log.info("llms_txt_fetch_failed", url=url, error=str(e)) - continue - - if resp.status_code != 200: - log.info("llms_txt_non_200", url=url, status=resp.status_code) - continue - - try: - body = resp.content - except Exception: # noqa: BLE001 - fall back to text-derived length - body = resp.text.encode("utf-8", errors="replace") - - if not body: - log.info("llms_txt_empty", url=url) + text = _fetch_capped(client, url, max_bytes, log) + if text is None: continue + log.info("llms_txt_discovered", url=url, size=len(text)) + return url, text - if len(body) > max_bytes: - log.info("llms_txt_too_large", url=url, size=len(body), max_bytes=max_bytes) - continue + return None - text = resp.text - if not text.strip(): - log.info("llms_txt_empty", url=url) - continue - log.info("llms_txt_discovered", url=url, size=len(body)) - return url, text +def _fetch_capped(client, url: str, max_bytes: int, log) -> str | None: + """GET `url`, streaming the body and aborting the download once it exceeds + `max_bytes`. Returns the decoded text, or None to skip this candidate + (non-200, empty, oversize, or any error — discovery is best-effort and + never raises). - return None + Streaming (rather than `client.get(...).content`) is what makes the + `max_bytes` guard cheap: a huge `/llms-full.txt` is stopped mid-transfer + instead of being pulled into memory in full and then thrown away. + """ + chunks: list[bytes] = [] + encoding = "utf-8" + try: + with client.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=15) as resp: + if resp.status_code != 200: + log.info("llms_txt_non_200", url=url, status=resp.status_code) + return None + total = 0 + for chunk in resp.iter_bytes(): + total += len(chunk) + if total > max_bytes: + log.info("llms_txt_too_large", url=url, size_at_least=total, max_bytes=max_bytes) + return None + chunks.append(chunk) + encoding = resp.encoding or "utf-8" + except Exception as e: # noqa: BLE001 - discovery is best-effort, never raises + log.info("llms_txt_fetch_failed", url=url, error=str(e)) + return None + + body = b"".join(chunks) + if not body: + log.info("llms_txt_empty", url=url) + return None + try: + text = body.decode(encoding, errors="replace") + except LookupError: + text = body.decode("utf-8", errors="replace") + if not text.strip(): + log.info("llms_txt_empty", url=url) + return None + return text def _content_lines(text: str) -> list[str]: diff --git a/ingestion/tests/test_llms_txt.py b/ingestion/tests/test_llms_txt.py index c63195c..cd93ce0 100644 --- a/ingestion/tests/test_llms_txt.py +++ b/ingestion/tests/test_llms_txt.py @@ -8,7 +8,6 @@ from __future__ import annotations import httpx - from app.llms_txt import discover, looks_like_index, parse_llms_index, split_llms_full @@ -39,6 +38,43 @@ def handler(request: httpx.Request) -> httpx.Response: assert discover(client, "https://example.com/", max_bytes=10) is None +def test_discover_aborts_download_once_over_cap(): + """The oversized body must be abandoned mid-stream, not fully downloaded and + then discarded — otherwise a 24MB llms-full.txt costs a 24MB transfer just + to be rejected.""" + consumed = {"chunks": 0} + + def body_gen(): + for _ in range(1000): # up to 1000 * 1KB = 1MB if fully read + consumed["chunks"] += 1 + yield b"x" * 1024 + + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/llms-full.txt"): + return httpx.Response(200, content=body_gen()) + return httpx.Response(404) + + client = make_client(handler) + result = discover(client, "https://example.com/", max_bytes=5_000) # 5KB cap + assert result is None + # Stopped shortly after crossing 5KB (~6 chunks), not the whole 1000. + assert consumed["chunks"] < 20 + + +def test_discover_accepts_body_at_the_cap_boundary(): + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("/llms-full.txt"): + return httpx.Response(200, text="# T\n" + "y" * 96) # exactly 100 bytes + return httpx.Response(404) + + client = make_client(handler) + result = discover(client, "https://example.com/", max_bytes=100) + assert result is not None + assert result[0].endswith("/llms-full.txt") + + def test_discover_prefers_llms_full_over_llms_txt(): def handler(request: httpx.Request) -> httpx.Response: url = str(request.url)