Skip to content

fix(crawler): crawl llms.txt index links instead of ingesting them as content #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AdamRussak merged 2 commits into from
Jul 22, 2026
+493 −50
Merged

fix(crawler): crawl llms.txt index links instead of ingesting them as content #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fa79c3
fix(crawler): crawl llms.txt index links instead of ingesting them as…
ulielitay Jul 22, 2026
aa5580a
perf(crawler): stream llms.txt download and abort once over the size cap
ulielitay Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 76 additions & 25 deletions ingestion/app/crawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -417,12 +417,23 @@ def crawl(
yielded instead of re-fetching/re-extracting.

When `source.llms_txt` is `"auto"` or `"only"`, `crawl` first tries the
llmstxt.org convention (`llms-full.txt`/`llms.txt`) via `llms_txt`. If a
file is discovered, its sections are yielded as
`{"url", "markdown", "heading_path", "fetch_ok": True}` items (capped at
`source.max_pages`) and the generator returns — the sitemap/BFS HTML
crawl is skipped entirely. If no file is found: `"auto"` falls through to
the normal HTML crawl; `"only"` yields nothing. If `conditional` carries a
llmstxt.org convention (`llms-full.txt`/`llms.txt`) via `llms_txt`. The two
file types are handled differently:

- FULL CONTENT (`llms-full.txt`, or an `llms.txt` that is actually full
prose): split into sections yielded as
`{"url", "markdown", "heading_path", "fetch_ok": True}` items (capped at
`source.max_pages`); the generator then returns.
- INDEX (an `llms.txt` that is a list of links to the docs — detected via
`llms_txt.looks_like_index`): the linked page URLs are extracted and
crawled as normal HTML pages below (each yielded as an
`{"url", "html", "fetch_ok": True}` item), NOT ingested as content.
Ingesting the link list verbatim would make the corpus link-list
metadata instead of documentation.

If no file is found: `"auto"` falls through to the normal sitemap/BFS HTML
crawl; `"only"` yields nothing (but `"only"` DOES crawl an index's links,
since those are what llms.txt points at). If `conditional` carries a
validator for the llms-index URL and it 304s, a single sentinel
`{"kind": "llms_index_unchanged", "url", "not_modified": True,
"fetch_ok": True}` is yielded and the generator returns.
Expand Down Expand Up @@ -454,6 +465,11 @@ def crawl(
pages_fetched = 0
visited: set[str] = set()

# When llms.txt discovery finds an INDEX (a list of links rather than
# full content), these are the page URLs it points at — crawled below
# via the normal HTML path instead of being ingested as content.
llms_index_urls: list[str] | None = None

if source.llms_txt in ("auto", "only"):
parsed_base = urlparse(base_url)
origin = f"{parsed_base.scheme}://{parsed_base.netloc}"
Expand Down Expand Up @@ -489,29 +505,64 @@ def crawl(

if discovered is not None:
index_url, llms_text = discovered
limiter.wait()
sections = llms_txt.split_llms_full(llms_text, index_url)
llms_count = 0
for section in sections:
if llms_count >= source.max_pages:
break
yield {
"url": section["url"],
"markdown": section["markdown"],
"heading_path": section["heading_path"],
"fetch_ok": True,
}
llms_count += 1
log.info("crawl_complete", pages_fetched=llms_count, mode="llms_txt")
return
if source.llms_txt == "only":
if llms_txt.looks_like_index(llms_text):
# `/llms.txt` INDEX: a list of links to the real docs, NOT
# content. Extract the linked page URLs and crawl each as a
# normal HTML page below (so it runs through extract.extract
# and yields real body text) — ingesting the index verbatim
# is the "corpus is only link-list metadata" bug this fixes.
parsed = llms_txt.parse_llms_index(llms_text, base_url)
if parsed:
llms_index_urls = parsed
log.info("llms_index_discovered", url=index_url, links=len(parsed))
else:
log.info("llms_index_empty", url=index_url)
else:
# `/llms-full.txt` (or an llms.txt that is actually full
# content): split into per-section markdown and yield as
# already-extracted content.
limiter.wait()
sections = llms_txt.split_llms_full(llms_text, index_url)
llms_count = 0
for section in sections:
if llms_count >= source.max_pages:
break
yield {
"url": section["url"],
"markdown": section["markdown"],
"heading_path": section["heading_path"],
"fetch_ok": True,
}
llms_count += 1
log.info("crawl_complete", pages_fetched=llms_count, mode="llms_txt")
return

if llms_index_urls is None and source.llms_txt == "only":
# "only" means don't fall back to a full HTML BFS crawl. With no
# full-content file and no usable index, there is nothing to do.
log.info("crawl_complete", pages_fetched=0, mode="llms_txt")
return
# source.llms_txt == "auto" and nothing discovered: fall through
# to the normal sitemap/BFS HTML crawl below.
# Otherwise fall through to the HTML crawl below: seeded with the
# index's URLs when we have them (both "auto" and "only"), or — for
# "auto" with nothing discovered — the normal sitemap/BFS path.

candidate_urls: list[str] | None = None
if source.sitemap and not _same_host(str(source.sitemap), base_url):
if llms_index_urls is not None:
# Crawl exactly the pages the llms.txt index lists — filtered to
# same-host + include/exclude prefixes and capped to max_pages,
# mirroring sitemap discovery — instead of a sitemap/BFS crawl. Each
# URL is still re-validated (host/scope/private/robots) in `_visit`.
filtered: list[str] = []
for u in llms_index_urls:
if not _same_host(u, base_url):
continue
if not _allowed(urlparse(u).path, source.include_prefixes, source.exclude_prefixes):
continue
if u not in filtered:
filtered.append(u)
candidate_urls = filtered[: source.max_pages]
log.info("llms_index_candidates", count=len(candidate_urls))
elif source.sitemap and not _same_host(str(source.sitemap), base_url):
# Defence in depth: SourceConfig already rejects an off-host
# sitemap (H1). This catches a SourceConfig built via
# `model_construct` or mutated after validation.
Expand Down
152 changes: 128 additions & 24 deletions ingestion/app/llms_txt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from __future__ import annotations

import re
from urllib.parse import urlparse
from urllib.parse import urljoin, urlparse

from .logging_config import get_logger

Expand All @@ -28,6 +28,12 @@
_MD_LINK_RE = re.compile(r"^\[([^\]]*)\]\(([^)]+)\)$")
_SOURCE_LINE_RE = re.compile(r"^Source:\s*(https?://\S+)", re.IGNORECASE)
_SLUG_NONALNUM_RE = re.compile(r"[^a-z0-9]+")
# An llms.txt INDEX lists documentation pages as markdown link bullets, e.g.
# - [Quickstart](https://x.io/docs/quickstart): get started fast
# Match such a bullet and capture its URL. Unlike `_MD_LINK_RE` this is NOT
# whole-line anchored: index bullets carry a leading `-`/`*` marker and an
# optional trailing `: description`.
_INDEX_LINK_ITEM_RE = re.compile(r"^\s*[-*]\s*\[[^\]]*\]\(([^)]+)\)")


def discover(
Expand All @@ -44,6 +50,10 @@ def discover(
has a non-empty body, and is within `max_bytes`. Returns `None` if
neither candidate qualifies.

The body is streamed and the download is ABORTED as soon as it exceeds
`max_bytes`, so an oversized file (e.g. a 24MB `/llms-full.txt`) is not
fully downloaded only to be discarded.

NEVER raises: any httpx error, oversize body, empty body, or non-200
status is treated as "skip this candidate" (and, if both candidates are
exhausted, "skip this source" — return None) rather than propagating.
Expand All @@ -58,38 +68,132 @@ def discover(
log = logger.bind(base_url=base_url)

for url in candidates:
try:
resp = client.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
except Exception as e: # noqa: BLE001 - discovery is best-effort, never raises
log.info("llms_txt_fetch_failed", url=url, error=str(e))
text = _fetch_capped(client, url, max_bytes, log)
if text is None:
continue
log.info("llms_txt_discovered", url=url, size=len(text))
return url, text

if resp.status_code != 200:
log.info("llms_txt_non_200", url=url, status=resp.status_code)
continue
return None

try:
body = resp.content
except Exception: # noqa: BLE001 - fall back to text-derived length
body = resp.text.encode("utf-8", errors="replace")

if not body:
log.info("llms_txt_empty", url=url)
continue
def _fetch_capped(client, url: str, max_bytes: int, log) -> str | None:
"""GET `url`, streaming the body and aborting the download once it exceeds
`max_bytes`. Returns the decoded text, or None to skip this candidate
(non-200, empty, oversize, or any error — discovery is best-effort and
never raises).

if len(body) > max_bytes:
log.info("llms_txt_too_large", url=url, size=len(body), max_bytes=max_bytes)
Streaming (rather than `client.get(...).content`) is what makes the
`max_bytes` guard cheap: a huge `/llms-full.txt` is stopped mid-transfer
instead of being pulled into memory in full and then thrown away.
"""
chunks: list[bytes] = []
encoding = "utf-8"
try:
with client.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=15) as resp:
if resp.status_code != 200:
log.info("llms_txt_non_200", url=url, status=resp.status_code)
return None
total = 0
for chunk in resp.iter_bytes():
total += len(chunk)
if total > max_bytes:
log.info("llms_txt_too_large", url=url, size_at_least=total, max_bytes=max_bytes)
return None
chunks.append(chunk)
encoding = resp.encoding or "utf-8"
except Exception as e: # noqa: BLE001 - discovery is best-effort, never raises
log.info("llms_txt_fetch_failed", url=url, error=str(e))
return None

body = b"".join(chunks)
if not body:
log.info("llms_txt_empty", url=url)
return None
try:
text = body.decode(encoding, errors="replace")
except LookupError:
text = body.decode("utf-8", errors="replace")
if not text.strip():
log.info("llms_txt_empty", url=url)
return None
return text


def _content_lines(text: str) -> list[str]:
"""Non-blank, non-heading lines outside fenced code blocks. Used to judge
whether a file is an index (mostly link bullets) or full prose/code."""
out: list[str] = []
in_fence = False
for line in text.splitlines():
stripped = line.strip()
if _FENCE_RE.match(stripped):
in_fence = not in_fence
continue

text = resp.text
if not text.strip():
log.info("llms_txt_empty", url=url)
if in_fence or not stripped or stripped.startswith("#"):
continue
out.append(stripped)
return out

log.info("llms_txt_discovered", url=url, size=len(body))
return url, text

return None
def looks_like_index(text: str) -> bool:
"""True if `text` is an llms.txt INDEX (a list of links to the real docs)
rather than `/llms-full.txt` full content.

Per the llmstxt.org convention, `/llms.txt` is a curated index: an H1/H2
outline whose body is markdown link bullets (`- [Title](url): note`), while
`/llms-full.txt` is the concatenated documentation prose. This distinction
matters because an index must be CRAWLED (fetch each linked page), never
ingested verbatim — otherwise the corpus becomes a list of links with no
actual documentation content.

Heuristic: an index is dominated by link bullets. If at least half of the
content lines (excluding headings/blank/code) are markdown link bullets AND
there are several of them, it is an index. A full-content file has prose and
code between its headings, so its link-bullet ratio is low.
"""
content = _content_lines(text)
if not content:
return False
link_bullets = sum(1 for line in content if _INDEX_LINK_ITEM_RE.match(line))
# Require a real list (>=3 bullets) that is the majority of the content, so
# a full-content page that merely happens to contain a couple of bullet
# links is not misclassified as an index.
return link_bullets >= 3 and link_bullets / len(content) >= 0.5


def parse_llms_index(text: str, base_url: str) -> list[str]:
"""Extract the documentation page URLs an llms.txt index links to.

Returns absolute, order-preserving, de-duplicated URLs (relative links are
resolved against `base_url`). Fragment-only or non-http(s) targets are
dropped. Host/scope/private-address filtering is deliberately left to the
crawler's per-URL `_visit` guards, mirroring how sitemap discovery hands
raw candidate URLs to the same gate.
"""
urls: list[str] = []
seen: set[str] = set()
in_fence = False
for line in text.splitlines():
stripped = line.strip()
if _FENCE_RE.match(stripped):
in_fence = not in_fence
continue
if in_fence:
continue
m = _INDEX_LINK_ITEM_RE.match(line)
if not m:
continue
target = m.group(1).strip()
if not target or target.startswith("#"):
continue
absolute = urljoin(base_url, target)
if not absolute.lower().startswith(("http://", "https://")):
continue
if absolute not in seen:
seen.add(absolute)
urls.append(absolute)
return urls


def _slugify(title: str) -> str:
Expand Down
Loading
Loading