diff --git a/db/init/02_sources_config.sql b/db/init/02_sources_config.sql index 3b19d0d..5cf005c 100644 --- a/db/init/02_sources_config.sql +++ b/db/init/02_sources_config.sql @@ -25,13 +25,11 @@ -- fresh-volume schema looks like" and "what the live db was migrated to" — -- exactly the bug class this file exists to avoid. -- --- max_pages is intentionally left NULLABLE here: the pydantic SourceConfig --- model (ingestion/app/config.py) is what enforces `max_pages` is required --- and > 0 on every write path (sync/propose). Enforcing NOT NULL at the SQL --- layer would break the 9 existing rows that predate this column with no --- backfill value to give them. This split (permissive schema + strict --- application-layer validation) is deliberate — do not "fix" it by adding a --- NOT NULL constraint here. +-- max_pages is NULLABLE and NULL is meaningful: it means "no page limit" +-- (crawl all in-scope pages). The pydantic SourceConfig model +-- (ingestion/app/config.py) makes `max_pages` OPTIONAL and, when provided, +-- enforces it is > 0 on every write path (sync/propose). Do not add a +-- NOT NULL constraint here — a NULL is a valid, intentional "unlimited". ALTER TABLE doc_sources ADD COLUMN IF NOT EXISTS sitemap TEXT, diff --git a/ingestion/app/admin.py b/ingestion/app/admin.py index c1401a7..1e764ec 100644 --- a/ingestion/app/admin.py +++ b/ingestion/app/admin.py @@ -519,7 +519,7 @@ def _build_source_config( sitemap=sitemap.strip() or None, include_prefixes=_split_prefixes(include_prefixes), exclude_prefixes=_split_prefixes(exclude_prefixes), - max_pages=max_pages.strip(), + max_pages=(max_pages.strip() or None), language=language.strip() or "english", rate_limit_rps=rate_limit_rps.strip(), llms_txt=(llms_txt.strip() or "auto"), @@ -543,7 +543,7 @@ def _record_to_config(record: SourceRecord) -> SourceConfig: sitemap=record.sitemap, include_prefixes=record.include_prefixes, exclude_prefixes=record.exclude_prefixes, - max_pages=record.max_pages if (record.max_pages is not None and record.max_pages > 0) else 100, + max_pages=record.max_pages, language=record.language or "english", rate_limit_rps=record.rate_limit_rps if (record.rate_limit_rps is not None and record.rate_limit_rps > 0) else 1.0, llms_txt=record.llms_txt or "auto", @@ -637,7 +637,7 @@ def create_source_submit( sitemap: str = Form(default=""), include_prefixes: str = Form(default=""), exclude_prefixes: str = Form(default=""), - max_pages: str = Form(...), + max_pages: str = Form(default=""), language: str = Form(default="english"), rate_limit_rps: str = Form(default="1.0"), llms_txt: str = Form(default="auto"), @@ -720,7 +720,7 @@ def update_source_submit( sitemap: str = Form(default=""), include_prefixes: str = Form(default=""), exclude_prefixes: str = Form(default=""), - max_pages: str = Form(...), + max_pages: str = Form(default=""), language: str = Form(default="english"), rate_limit_rps: str = Form(default="1.0"), llms_txt: str = Form(default="auto"), diff --git a/ingestion/app/config.py b/ingestion/app/config.py index 801c101..5da2cc5 100644 --- a/ingestion/app/config.py +++ b/ingestion/app/config.py @@ -8,7 +8,7 @@ sitemap: https://fastapi.tiangolo.com/sitemap.xml # optional include_prefixes: ["/tutorial/", "/reference/"] # optional allowlist exclude_prefixes: ["/blog/", "/release-notes/"] # optional denylist (wins) - max_pages: 500 # required + max_pages: 500 # optional — omit for no page limit (crawl all in-scope pages) language: english # optional, default english rate_limit_rps: 1.0 # optional, default 1.0 @@ -99,7 +99,7 @@ class SourceConfig(BaseModel): sitemap: HttpUrl | None = None include_prefixes: list[str] = Field(default_factory=list) exclude_prefixes: list[str] = Field(default_factory=list) - max_pages: int = Field(gt=0) + max_pages: int | None = Field(default=None, gt=0) language: str = "english" rate_limit_rps: float = Field(default=1.0, gt=0) llms_txt: Literal["auto", "off", "only"] = "auto" diff --git a/ingestion/app/crawler.py b/ingestion/app/crawler.py index 038a701..16008b2 100644 --- a/ingestion/app/crawler.py +++ b/ingestion/app/crawler.py @@ -15,6 +15,7 @@ from __future__ import annotations import ipaddress +import sys import time import urllib.robotparser import warnings @@ -397,7 +398,8 @@ def crawl( client: httpx.Client | None = None, conditional: dict[str, tuple[str | None, str | None]] | None = None, ) -> Iterator[dict]: - """Discover and fetch up to `source.max_pages` pages for `source`. + """Discover and fetch up to `source.max_pages` pages for `source` (or ALL + in-scope pages when `source.max_pages` is None — no page limit). YIELDS `{"url": str, "html": str | None, "fetch_ok": bool}` dicts one at a time as pages are visited, rather than materializing the whole crawl in @@ -462,6 +464,11 @@ def crawl( rp = load_robots(client, base_url) limiter = RateLimiter(source.rate_limit_rps) + # `max_pages` is optional: None means "no page limit". Internally we use + # a very large int so the existing cap comparisons/slicing need no + # special-casing — same-host + prefix scoping still bounds the crawl. + page_cap = source.max_pages if source.max_pages is not None else sys.maxsize + pages_fetched = 0 visited: set[str] = set() @@ -525,7 +532,7 @@ def crawl( sections = llms_txt.split_llms_full(llms_text, index_url) llms_count = 0 for section in sections: - if llms_count >= source.max_pages: + if llms_count >= page_cap: break yield { "url": section["url"], @@ -560,7 +567,7 @@ def crawl( continue if u not in filtered: filtered.append(u) - candidate_urls = filtered[: source.max_pages] + candidate_urls = filtered[:page_cap] log.info("llms_index_candidates", count=len(candidate_urls)) elif source.sitemap and not _same_host(str(source.sitemap), base_url): # Defence in depth: SourceConfig already rejects an off-host @@ -572,7 +579,7 @@ def crawl( candidate_urls = discover_sitemap_urls( client, str(source.sitemap), - source.max_pages, + page_cap, limiter, log, base_url, @@ -672,7 +679,7 @@ def _visit(url: str) -> dict | None: if candidate_urls is not None: for url in candidate_urls: - if pages_fetched >= source.max_pages: + if pages_fetched >= page_cap: break url = _strip_fragment(url) if url in visited: @@ -685,7 +692,7 @@ def _visit(url: str) -> dict | None: yield item else: queue = [base_url] - while queue and pages_fetched < source.max_pages: + while queue and pages_fetched < page_cap: url = _strip_fragment(queue.pop(0)) if url in visited: continue diff --git a/ingestion/app/llms_txt.py b/ingestion/app/llms_txt.py index 837efb5..16fc2bc 100644 --- a/ingestion/app/llms_txt.py +++ b/ingestion/app/llms_txt.py @@ -41,22 +41,18 @@ def discover( base_url: str, *, prefer_full: bool = True, - max_bytes: int = 10_000_000, ) -> tuple[str, str] | None: """Try to fetch `/llms-full.txt` then `/llms.txt` (or the reverse order when `prefer_full=False`) at `base_url`'s origin. - Returns `(fetched_url, text)` for the first response that is status 200, - has a non-empty body, and is within `max_bytes`. Returns `None` if - neither candidate qualifies. + Returns `(fetched_url, text)` for the first response that is status 200 + with a non-empty body. There is NO size limit — the full file is fetched + however large it is (llms-full.txt files are legitimately large). Returns + `None` if neither candidate qualifies. - The body is streamed and the download is ABORTED as soon as it exceeds - `max_bytes`, so an oversized file (e.g. a 24MB `/llms-full.txt`) is not - fully downloaded only to be discarded. - - NEVER raises: any httpx error, oversize body, empty body, or non-200 - status is treated as "skip this candidate" (and, if both candidates are - exhausted, "skip this source" — return None) rather than propagating. + NEVER raises: any httpx error, empty body, or non-200 status is treated as + "skip this candidate" (and, if both candidates are exhausted, "skip this + source" — return None) rather than propagating. """ parsed = urlparse(base_url) origin = f"{parsed.scheme}://{parsed.netloc}" @@ -68,7 +64,7 @@ def discover( log = logger.bind(base_url=base_url) for url in candidates: - text = _fetch_capped(client, url, max_bytes, log) + text = _fetch_body(client, url, log) if text is None: continue log.info("llms_txt_discovered", url=url, size=len(text)) @@ -77,43 +73,18 @@ def discover( return None -def _fetch_capped(client, url: str, max_bytes: int, log) -> str | None: - """GET `url`, streaming the body and aborting the download once it exceeds - `max_bytes`. Returns the decoded text, or None to skip this candidate - (non-200, empty, oversize, or any error — discovery is best-effort and - never raises). - - Streaming (rather than `client.get(...).content`) is what makes the - `max_bytes` guard cheap: a huge `/llms-full.txt` is stopped mid-transfer - instead of being pulled into memory in full and then thrown away. - """ - chunks: list[bytes] = [] - encoding = "utf-8" +def _fetch_body(client, url: str, log) -> str | None: + """GET `url` and return its decoded text, or None to skip this candidate + (non-200, empty, or any error — discovery is best-effort and never raises).""" try: - with client.stream("GET", url, headers={"User-Agent": USER_AGENT}, timeout=15) as resp: - if resp.status_code != 200: - log.info("llms_txt_non_200", url=url, status=resp.status_code) - return None - total = 0 - for chunk in resp.iter_bytes(): - total += len(chunk) - if total > max_bytes: - log.info("llms_txt_too_large", url=url, size_at_least=total, max_bytes=max_bytes) - return None - chunks.append(chunk) - encoding = resp.encoding or "utf-8" + resp = client.get(url, headers={"User-Agent": USER_AGENT}, timeout=15) except Exception as e: # noqa: BLE001 - discovery is best-effort, never raises log.info("llms_txt_fetch_failed", url=url, error=str(e)) return None - - body = b"".join(chunks) - if not body: - log.info("llms_txt_empty", url=url) + if resp.status_code != 200: + log.info("llms_txt_non_200", url=url, status=resp.status_code) return None - try: - text = body.decode(encoding, errors="replace") - except LookupError: - text = body.decode("utf-8", errors="replace") + text = resp.text if not text.strip(): log.info("llms_txt_empty", url=url) return None diff --git a/ingestion/app/sources_repo.py b/ingestion/app/sources_repo.py index 28fd91e..bdb5b2a 100644 --- a/ingestion/app/sources_repo.py +++ b/ingestion/app/sources_repo.py @@ -189,7 +189,7 @@ def _row_to_record(row: tuple) -> SourceRecord: sitemap=sitemap, include_prefixes=list(include_prefixes) if include_prefixes is not None else [], exclude_prefixes=list(exclude_prefixes) if exclude_prefixes is not None else [], - max_pages=max_pages if (max_pages is not None and max_pages > 0) else 100, + max_pages=max_pages, # None means "no page limit" (optional) language=language or "english", rate_limit_rps=rate_limit_rps if (rate_limit_rps is not None and rate_limit_rps > 0) else 1.0, llms_txt=llms_txt if llms_txt else "auto", diff --git a/ingestion/app/templates/admin/form.html b/ingestion/app/templates/admin/form.html index 4af3d71..cdbb3d3 100644 --- a/ingestion/app/templates/admin/form.html +++ b/ingestion/app/templates/admin/form.html @@ -35,8 +35,8 @@