From ff436d92b3d54971151813a35bbd70ad9e321976 Mon Sep 17 00:00:00 2001 From: ItayUliel Date: Tue, 21 Jul 2026 01:37:30 +0300 Subject: [PATCH 1/3] feat: selectable embedding models + quality/security hardening Make the embedding model selectable from a registry where choosing a model auto-derives its vector dimension and the services' Docker memory limits, and upgrade the default to a stronger model. Also add lint/type tooling, harden sitemap parsing, and add parity guards. Embedding model registry - config/models.yaml is the single source of truth (model -> dim, memory, query/passage prompts). Default is mixedbread-ai/mxbai-embed-large-v1 (1024d), a quality upgrade over bge-small (384d). - `make configure MODEL=` resolves the selection into .env (EMBEDDING_*, {INGESTION,MCP}_MEM_LIMIT) and renders db/init/01_schema.sql's vector(N) from a template. `make reindex` re-embeds after a switch. - Services read the model/dim/prompts from env with defaults matching the registry default; docker-compose derives memory limits and build-bakes the model. FastEmbed applies no query/passage prefix for these models, so the per-model prompt is applied manually around embed() (mxbai: query-only; e5: both sides). The chunker's tokenizer now follows the embedding model. Tooling + CI - ruff (lint) and mypy configured; `make lint` / `make typecheck` and new CI jobs. mypy quarantines a documented pre-existing typing backlog so the gate enforces types on new/changed code. Coverage reporting added to `make test`. - CI configures the small 384d model so the suite doesn't pull the 1.2GB default on every run (and exercises `make configure`). Security / robustness - Sitemap XML now parsed with defusedxml (forbids DTD/entity expansion/XXE) against untrusted upstream sources. Parity guards - tests assert the committed schema matches the rendered template, the service defaults match the registry, and the duplicated SSRF _addr_is_private helper stays byte-identical across the two services. Housekeeping - Add LICENSE; small B904/B905 fixes surfaced by the new lint gate. --- .env.example | 13 ++ .github/workflows/test.yml | 23 +++ .gitignore | 4 + LICENSE | 18 ++ Makefile | 51 +++++- config/models.yaml | 49 ++++++ db/init/01_schema.sql | 5 +- db/init/01_schema.sql.template | 46 +++++ docker-compose.yml | 23 ++- docs/runbook.md | 46 +++++ ingestion/Dockerfile | 14 +- ingestion/app/chunker.py | 17 +- ingestion/app/config.py | 6 +- ingestion/app/crawler.py | 3 +- ingestion/app/embedder.py | 45 +++-- ingestion/app/main.py | 6 +- ingestion/app/scheduler.py | 12 +- ingestion/app/sources_repo.py | 2 +- ingestion/app/store.py | 3 +- ingestion/app/urlscope.py | 15 +- ingestion/pyproject.toml | 1 + ingestion/tests/conftest.py | 1 - ingestion/tests/test_admin.py | 13 +- ingestion/tests/test_chunker.py | 1 - ingestion/tests/test_config.py | 1 - ingestion/tests/test_crawler.py | 3 +- ingestion/tests/test_embedder.py | 36 ++-- ingestion/tests/test_llms_txt.py | 1 - ingestion/tests/test_main.py | 6 +- ingestion/tests/test_scheduler.py | 1 - ingestion/tests/test_sources_repo.py | 5 +- ingestion/tests/test_store.py | 5 +- ingestion/tests/test_urlscope.py | 1 - ingestion/uv.lock | 99 +++++++++++ mcp-server/Dockerfile | 9 +- mcp-server/app/retrieval.py | 37 ++-- mcp-server/pyproject.toml | 4 + mcp-server/tests/test_registry_defaults.py | 35 ++++ mcp-server/tests/test_retrieval.py | 3 +- .../tests/test_retrieval_integration.py | 7 +- mcp-server/tests/test_server.py | 5 +- mcp-server/uv.lock | 17 +- mypy.ini | 31 ++++ ruff.toml | 32 ++++ scripts/configure_model.py | 161 ++++++++++++++++++ scripts/push_sources.py | 6 +- tests/test_e2e.py | 1 - tests/test_model_registry.py | 100 +++++++++++ tests/test_push_sources.py | 2 - 49 files changed, 912 insertions(+), 113 deletions(-) create mode 100644 LICENSE create mode 100644 config/models.yaml create mode 100644 db/init/01_schema.sql.template create mode 100644 mcp-server/tests/test_registry_defaults.py create mode 100644 mypy.ini create mode 100644 ruff.toml create mode 100644 scripts/configure_model.py create mode 100644 tests/test_model_registry.py diff --git a/.env.example b/.env.example index c476023..d23ad68 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,16 @@ SYNC_TOKEN=change-me # Required — the mcp-server refuses tool calls without this and requests # must send `Authorization: Bearer $MCP_TOKEN`. MCP_TOKEN=change-me + +# --- Embedding model (managed by `make configure`) --- +# These are written/overwritten by `make configure MODEL=`, which resolves +# them from config/models.yaml (the single source of truth). You normally do NOT +# edit them by hand — run `make configure` to switch models, then `make reindex`. +# The values below are the registry default (mixedbread-ai/mxbai-embed-large-v1); +# both services fall back to these defaults if the vars are unset. +# EMBEDDING_MODEL_NAME=mixedbread-ai/mxbai-embed-large-v1 +# EMBEDDING_DIM=1024 # MUST match db/init/01_schema.sql's vector(N) +# EMBEDDING_QUERY_PROMPT=Represent this sentence for searching relevant passages: +# EMBEDDING_PASSAGE_PROMPT= # empty for mxbai/bge; "passage: " for e5 +# INGESTION_MEM_LIMIT=2g # docker-compose memory limit, sized per model +# MCP_MEM_LIMIT=2g diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88731e9..85eaa6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,19 @@ on: branches: [main] jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Lint (ruff) + run: make lint + - name: Type-check (mypy) + run: make typecheck + test: runs-on: ubuntu-latest # or: self-hosted @@ -48,6 +61,16 @@ jobs: sleep 1 done + - name: Configure embedding model (small/fast for CI) + run: | + # Use the small 384-dim model in CI so the suite doesn't download the + # 1.2GB production default (mxbai-large) on every run. This also + # exercises `make configure`: it writes EMBEDDING_* into .env (which + # the Makefile exports to the test venvs) and re-renders 01_schema.sql + # to vector(384) so the seeded test vectors match the column width. + python -m pip install --quiet pyyaml + make configure MODEL=BAAI/bge-small-en-v1.5 + - name: Init database schema run: | # Apply both init files, matching scripts/migrate.sh's production diff --git a/.gitignore b/.gitignore index b7ef692..eecd27a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,11 @@ __pycache__/ *.pyc *.egg-info/ .venv/ +.tooling-venv/ .pytest_cache/ +.mypy_cache/ +.coverage +.ruff_cache/ # OS cruft .DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6b06350 --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2026 Itay Uliel + +All rights reserved. + +This software and its source code are proprietary and confidential. No license, +express or implied, is granted to use, copy, modify, merge, publish, distribute, +sublicense, or sell copies of this software, in whole or in part, without the +prior written permission of the copyright holder. + +Unauthorized copying, distribution, or use of this software, via any medium, is +strictly prohibited. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile index 52715dd..43c128a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,25 @@ -include .env export -.PHONY: up down up-prod down-prod sync test eval backup backup-prune backup-auto restore +.PHONY: up down up-prod down-prod sync test eval lint typecheck configure reindex backup backup-prune backup-auto restore + +# Select the embedding model from config/models.yaml. Resolves the model's +# vector dimension and per-service memory limits, writes them into .env, and +# renders db/init/01_schema.sql. No MODEL => the registry default. +# Usage: make configure (default model) +# make configure MODEL=BAAI/bge-base-en-v1.5 (a specific model) +configure: + python3 scripts/configure_model.py "$(MODEL)" + +# Re-embed the entire corpus with the currently-configured model. Required +# after `make configure` changes the model (content-hash change-detection would +# otherwise skip unchanged pages and leave stale/mismatched vectors). Truncates +# the crawled pages/chunks (NOT doc_sources) then triggers a fresh sync. +reindex: + @echo "Truncating doc_pages/doc_chunks (sources preserved) then re-syncing..." + docker compose exec -T db psql -U $${POSTGRES_USER} -d $${POSTGRES_DB} \ + -c "TRUNCATE doc_pages, doc_chunks RESTART IDENTITY CASCADE;" + $(MAKE) sync # Bring up the full stack locally (db + ingestion + mcp-server) using loopback ports. up: @@ -54,11 +72,13 @@ test: @test -d mcp-server/.venv || python3 -m venv mcp-server/.venv @mcp-server/.venv/bin/pip install -q -U pip @mcp-server/.venv/bin/pip install -q -e mcp-server - @mcp-server/.venv/bin/pip install -q pytest + @mcp-server/.venv/bin/pip install -q pytest pyyaml defusedxml + @ingestion/.venv/bin/pip install -q pytest-cov + @mcp-server/.venv/bin/pip install -q pytest-cov @echo "=== ingestion test suite ===" - cd ingestion && ../ingestion/.venv/bin/pytest -q + cd ingestion && ../ingestion/.venv/bin/pytest -q --cov=app --cov-report=term-missing:skip-covered @echo "=== mcp-server test suite ===" - cd mcp-server && ../mcp-server/.venv/bin/pytest -q + cd mcp-server && ../mcp-server/.venv/bin/pytest -q --cov=app --cov-report=term-missing:skip-covered @echo "=== e2e (cross-package) test suite ===" cd tests && ../ingestion/.venv/bin/python -m pytest -q @echo "make test: all suites green (DB-dependent tests skip cleanly if 'docker compose up -d db' wasn't run first)." @@ -75,6 +95,29 @@ eval: @echo "=== retrieval quality eval ===" cd tests/eval && ../../mcp-server/.venv/bin/python -m pytest -q -m eval +# Tooling venv for lint/typecheck (ruff + mypy). Kept separate from the two +# package venvs; ruff is a standalone binary, mypy runs with +# --ignore-missing-imports so it needn't install every runtime dependency. +TOOLS_VENV = .tooling-venv +$(TOOLS_VENV): + python3 -m venv $(TOOLS_VENV) + @$(TOOLS_VENV)/bin/pip install -q -U pip ruff mypy + +# Lint across both packages, scripts, and tests. (Formatting is available via +# `ruff format` but intentionally NOT gated — this codebase uses deliberate +# hand-alignment in its long explanatory comments/tables.) +lint: $(TOOLS_VENV) + $(TOOLS_VENV)/bin/ruff check . + +# Static type-check the application code and scripts. Each package is checked +# separately (both use a top-level `app` package, so a single invocation would +# see two modules named `app`). mypy.ini quarantines the pre-existing typing +# backlog so the gate enforces types on new/changed code. +typecheck: $(TOOLS_VENV) + cd ingestion && MYPYPATH=. ../$(TOOLS_VENV)/bin/mypy --config-file ../mypy.ini app + cd mcp-server && MYPYPATH=. ../$(TOOLS_VENV)/bin/mypy --config-file ../mypy.ini app + $(TOOLS_VENV)/bin/mypy --config-file mypy.ini scripts + # Dump the docs database to a timestamped custom-format archive under ./backups. backup: mkdir -p backups diff --git a/config/models.yaml b/config/models.yaml new file mode 100644 index 0000000..41dca9a --- /dev/null +++ b/config/models.yaml @@ -0,0 +1,49 @@ +# Embedding model registry — SINGLE SOURCE OF TRUTH. +# +# Consumed by scripts/configure_model.py (`make configure MODEL=`), which +# resolves the selected row into .env vars and renders db/init/01_schema.sql's +# vector(N) dimension. The two services (ingestion, mcp-server) never read this +# file at runtime — they consume the resolved env vars, defaulting to the row +# marked `default: true` below so a fresh clone / CI works without configuring. +# +# Every model here is verified FastEmbed-supported (ONNX, CPU, no torch/GPU). +# `dim` MUST equal FastEmbed's reported dimension for the model (it drives the +# pgvector column width). `mem_ingestion` / `mem_mcp` are docker-compose memory +# limits sized for the model's ONNX weights plus runtime headroom. +# +# Prompting: FastEmbed's query_embed/passage_embed apply NO prefix for these +# (non-multitask) models, so the services apply these prompts MANUALLY around +# plain embed(). Leave a prompt empty ("") when the model wants no prefix on +# that side. bge/mxbai want an instruction on the QUERY only; e5 wants a prefix +# on BOTH sides. + +default: mixedbread-ai/mxbai-embed-large-v1 + +models: + mixedbread-ai/mxbai-embed-large-v1: + dim: 1024 + mem_ingestion: "2g" + mem_mcp: "2g" + query_prompt: "Represent this sentence for searching relevant passages: " + passage_prompt: "" + + intfloat/multilingual-e5-large: + dim: 1024 + mem_ingestion: "2g" + mem_mcp: "2g" + query_prompt: "query: " + passage_prompt: "passage: " + + BAAI/bge-base-en-v1.5: + dim: 768 + mem_ingestion: "1500m" + mem_mcp: "1g" + query_prompt: "Represent this sentence for searching relevant passages: " + passage_prompt: "" + + BAAI/bge-small-en-v1.5: + dim: 384 + mem_ingestion: "1500m" + mem_mcp: "1g" + query_prompt: "Represent this sentence for searching relevant passages: " + passage_prompt: "" diff --git a/db/init/01_schema.sql b/db/init/01_schema.sql index 8360057..5090d76 100644 --- a/db/init/01_schema.sql +++ b/db/init/01_schema.sql @@ -1,3 +1,6 @@ +-- GENERATED from db/init/01_schema.sql.template by scripts/configure_model.py. +-- Do not edit by hand — run `make configure MODEL=` to change the vector +-- dimension. Rendered for embedding dimension 1024. CREATE EXTENSION IF NOT EXISTS vector; CREATE TABLE doc_sources ( @@ -28,7 +31,7 @@ CREATE TABLE doc_chunks ( heading_path TEXT, -- "Guide > Routing > Dynamic Routes" chunk_index INT NOT NULL, content TEXT NOT NULL, -- markdown - embedding vector(384) NOT NULL, + embedding vector(1024) NOT NULL, fts_config regconfig NOT NULL DEFAULT 'english', fts tsvector GENERATED ALWAYS AS (to_tsvector(fts_config, content)) STORED ); diff --git a/db/init/01_schema.sql.template b/db/init/01_schema.sql.template new file mode 100644 index 0000000..477f665 --- /dev/null +++ b/db/init/01_schema.sql.template @@ -0,0 +1,46 @@ +-- TEMPLATE — do not edit db/init/01_schema.sql by hand. +-- +-- scripts/configure_model.py renders this file into db/init/01_schema.sql, +-- substituting __EMBEDDING_DIM__ with the selected model's vector dimension +-- (see config/models.yaml). The committed 01_schema.sql is the rendering for +-- the registry's default model; `make configure MODEL=` re-renders it. +-- A parity test (tests/test_model_registry.py) fails CI if the two drift. +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE doc_sources ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE, -- e.g. "nextjs", matches sources.yaml key + base_url TEXT NOT NULL, + last_synced TIMESTAMPTZ, + last_status TEXT, -- ok | partial | failed + llms_txt TEXT NOT NULL DEFAULT 'auto' + CHECK (llms_txt IN ('auto', 'off', 'only')), + llms_etag TEXT, + llms_last_modified TEXT +); + +CREATE TABLE doc_pages ( + id SERIAL PRIMARY KEY, + source_id INT NOT NULL REFERENCES doc_sources(id) ON DELETE CASCADE, + url TEXT NOT NULL UNIQUE, + content_hash CHAR(64) NOT NULL, -- SHA-256 of extracted markdown + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + etag TEXT, + last_modified TEXT +); + +CREATE TABLE doc_chunks ( + id BIGSERIAL PRIMARY KEY, + page_id INT NOT NULL REFERENCES doc_pages(id) ON DELETE CASCADE, + heading_path TEXT, -- "Guide > Routing > Dynamic Routes" + chunk_index INT NOT NULL, + content TEXT NOT NULL, -- markdown + embedding vector(__EMBEDDING_DIM__) NOT NULL, + fts_config regconfig NOT NULL DEFAULT 'english', + fts tsvector GENERATED ALWAYS AS (to_tsvector(fts_config, content)) STORED +); + +CREATE INDEX doc_chunks_embedding_idx ON doc_chunks + USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64); +CREATE INDEX doc_chunks_fts_idx ON doc_chunks USING gin (fts); +CREATE INDEX doc_chunks_page_idx ON doc_chunks (page_id); diff --git a/docker-compose.yml b/docker-compose.yml index 0d0f347..83103de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,8 @@ services: ingestion: build: context: ./ingestion + args: + EMBEDDING_MODEL_NAME: ${EMBEDDING_MODEL_NAME:-mixedbread-ai/mxbai-embed-large-v1} container_name: self-docs-ingestion restart: unless-stopped profiles: @@ -51,6 +53,12 @@ services: OMP_NUM_THREADS: 1 ORT_NUM_THREADS: 2 SOURCES_YAML: /config/sources.yaml + # Embedding model config — resolved by `make configure` into .env from + # config/models.yaml. The defaults below mirror the registry default + # (mixedbread-ai/mxbai-embed-large-v1) so `make up` works unconfigured. + EMBEDDING_MODEL_NAME: ${EMBEDDING_MODEL_NAME:-mixedbread-ai/mxbai-embed-large-v1} + EMBEDDING_DIM: ${EMBEDDING_DIM:-1024} + EMBEDDING_PASSAGE_PROMPT: "${EMBEDDING_PASSAGE_PROMPT:-}" ports: - "127.0.0.1:8080:8080" volumes: @@ -65,12 +73,16 @@ services: deploy: resources: limits: - memory: 1.5G + # Sized per embedding model by `make configure` (config/models.yaml); + # default fits the 1024-dim mxbai-large ONNX weights + headroom. + memory: ${INGESTION_MEM_LIMIT:-2g} # No Traefik exposure — reachable inside the compose network. mcp-server: build: context: ./mcp-server + args: + EMBEDDING_MODEL_NAME: ${EMBEDDING_MODEL_NAME:-mixedbread-ai/mxbai-embed-large-v1} container_name: self-docs-mcp-server restart: unless-stopped profiles: @@ -88,6 +100,11 @@ services: POSTGRES_PORT: 5432 OMP_NUM_THREADS: 1 ORT_NUM_THREADS: 2 + # Must match the ingestion service's model for vectors to be comparable; + # `make configure` keeps both in sync via .env. Defaults mirror the + # registry default (config/models.yaml). + EMBEDDING_MODEL_NAME: ${EMBEDDING_MODEL_NAME:-mixedbread-ai/mxbai-embed-large-v1} + EMBEDDING_QUERY_PROMPT: "${EMBEDDING_QUERY_PROMPT:-Represent this sentence for searching relevant passages: }" ports: - "127.0.0.1:${DOCS_MCP_HOST_PORT:-8081}:8000" networks: @@ -95,7 +112,9 @@ services: deploy: resources: limits: - memory: 1G + # Sized per embedding model by `make configure` (config/models.yaml); + # default fits the 1024-dim mxbai-large ONNX weights + headroom. + memory: ${MCP_MEM_LIMIT:-2g} healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/metrics', timeout=3)"] interval: 15s diff --git a/docs/runbook.md b/docs/runbook.md index 7cbcaf2..3775878 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -415,6 +415,52 @@ with `\dx` (pgvector extension) and `\dt` (three tables) via --- +## Switch the embedding model + +The embedding model is selected from a registry (`config/models.yaml`, the +single source of truth). Selecting a model auto-derives its vector dimension and +the two services' Docker memory limits. The default is +`mixedbread-ai/mxbai-embed-large-v1` (1024-dim). To see the options: + +```bash +grep -E '^ [A-Za-z]' config/models.yaml # the model keys under `models:` +``` + +Switching models changes the vectors AND (usually) the `vector(N)` column width, +so it requires re-rendering the schema, rebuilding the images (the model is +baked in at build time), and a full re-embed. Because change-detection +(content-hash) skips unchanged pages, an in-place re-sync is not enough — the +corpus must be truncated and re-embedded. + +```bash +# 1. Select the model: writes EMBEDDING_* + *_MEM_LIMIT into .env and renders +# db/init/01_schema.sql to the new vector(N). No MODEL => the registry default. +make configure MODEL=intfloat/multilingual-e5-large + +# 2. Rebuild images so the new model is pre-baked, and recreate the DB schema. +# If the vector dimension changed, the column must be recreated — the +# simplest correct path is the nuke-and-rebuild above: +docker compose build ingestion mcp-server +docker compose down -v db && docker compose up -d db # re-runs the rendered schema +docker compose ps db # wait for healthy +docker compose up -d # (make up) + +# 3. Re-embed the corpus. If you kept the DB (same dimension), use `make reindex` +# instead of the nuke above; it truncates doc_pages/doc_chunks and re-syncs: +make reindex + +# 4. Verify quality held/improved against your eval set: +make eval +``` + +`make configure` requires PyYAML on the machine running it (`pip install pyyaml`, +or use the ingestion venv: `ingestion/.venv/bin/python scripts/configure_model.py `). +The ingestion and mcp-server services MUST run the same model — `make configure` +keeps both in sync via `.env`, and a startup dimension mismatch surfaces as a +pgvector error on the first embed/search. + +--- + ## Backup ### Manual diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index e947054..7e2a5ab 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -23,11 +23,15 @@ RUN pip install --no-cache-dir uv \ && uv export --no-dev --no-editable --format requirements-txt -o /tmp/requirements.txt \ && uv pip install --system --no-cache -r /tmp/requirements.txt -# Pre-download the FastEmbed embedding model (BAAI/bge-small-en-v1.5) and the -# BGE tokenizer at build time so the first real sync doesn't pay a cold-start -# download. Cached under /home/ingest/.cache, owned by the runtime user below. -RUN python -c "from fastembed import TextEmbedding; TextEmbedding(model_name='BAAI/bge-small-en-v1.5')" \ - && python -c "from tokenizers import Tokenizer; Tokenizer.from_pretrained('BAAI/bge-small-en-v1.5')" +# Pre-download the FastEmbed embedding model and its tokenizer at build time so +# the first real sync doesn't pay a cold-start download. Cached under +# /home/ingest/.cache, owned by the runtime user below. The model is a build +# arg so `make configure` / compose can bake the selected model (default = +# config/models.yaml registry default); the runtime env var must match it. +ARG EMBEDDING_MODEL_NAME=mixedbread-ai/mxbai-embed-large-v1 +ENV EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME} +RUN python -c "import os; from fastembed import TextEmbedding; TextEmbedding(model_name=os.environ['EMBEDDING_MODEL_NAME'])" \ + && python -c "import os; from tokenizers import Tokenizer; Tokenizer.from_pretrained(os.environ['EMBEDDING_MODEL_NAME'])" # Non-root runtime user. RUN useradd --create-home --uid 1000 --shell /usr/sbin/nologin ingest \ diff --git a/ingestion/app/chunker.py b/ingestion/app/chunker.py index a2104fc..6b773a3 100644 --- a/ingestion/app/chunker.py +++ b/ingestion/app/chunker.py @@ -19,6 +19,7 @@ from __future__ import annotations +import os import re import threading @@ -29,7 +30,11 @@ MIN_TOKENS = 400 MAX_TOKENS = 600 OVERLAP_RATIO = 0.15 -BGE_MODEL_ID = "BAAI/bge-small-en-v1.5" +# Token counting follows the embedding model so chunk sizing matches the model +# that will actually embed the text (mismatched tokenizers can push a chunk +# past the model's context and silently truncate). Env-driven, same default as +# the embedder (config/models.yaml registry default). +TOKENIZER_MODEL_ID = os.environ.get("EMBEDDING_MODEL_NAME", "mixedbread-ai/mxbai-embed-large-v1") logger = get_logger(component="chunker") @@ -41,16 +46,16 @@ def get_tokenizer() -> Tokenizer: - """Lazily load (and cache) the BGE tokenizer. + """Lazily load (and cache) the embedding model's tokenizer. - Downloads/loads BAAI/bge-small-en-v1.5's tokenizer.json (via the - tokenizers `from_pretrained` HF Hub path, which shares the local HF cache - with FastEmbed's model download). + Downloads/loads TOKENIZER_MODEL_ID's tokenizer.json (via the tokenizers + `from_pretrained` HF Hub path, which shares the local HF cache with + FastEmbed's model download). """ global _tokenizer with _tokenizer_lock: if _tokenizer is None: - _tokenizer = Tokenizer.from_pretrained(BGE_MODEL_ID) + _tokenizer = Tokenizer.from_pretrained(TOKENIZER_MODEL_ID) return _tokenizer diff --git a/ingestion/app/config.py b/ingestion/app/config.py index 2e2320c..801c101 100644 --- a/ingestion/app/config.py +++ b/ingestion/app/config.py @@ -121,7 +121,7 @@ def _language_must_be_supported(cls, v: str) -> str: return normalized @model_validator(mode="after") - def _sitemap_shares_base_url_host(self) -> "SourceConfig": + def _sitemap_shares_base_url_host(self) -> SourceConfig: # SSRF guard (security review H1): `sitemap` is fetched BEFORE any of # its `` entries are host-filtered, and a `` fans # out to its children equally unvalidated. Constraining the sitemap to @@ -142,7 +142,7 @@ def _sitemap_shares_base_url_host(self) -> "SourceConfig": return self @model_validator(mode="after") - def _hosts_must_not_be_private(self) -> "SourceConfig": + def _hosts_must_not_be_private(self) -> SourceConfig: # SSRF guard (security review H2): source URLs are untrusted input # (admin web form + an MCP tool callable by an AI agent), so reject a # host that IS or RESOLVES TO private/loopback/link-local/reserved @@ -166,7 +166,7 @@ def _hosts_must_not_be_private(self) -> "SourceConfig": return self @model_validator(mode="after") - def _base_url_passes_own_prefix_filters(self) -> "SourceConfig": + def _base_url_passes_own_prefix_filters(self) -> SourceConfig: # Without a sitemap, the crawler seeds its BFS queue with base_url # itself; if base_url's path is excluded (or not included) by this # source's own include/exclude_prefixes, the seed is filtered out diff --git a/ingestion/app/crawler.py b/ingestion/app/crawler.py index 07e0f40..db61d4a 100644 --- a/ingestion/app/crawler.py +++ b/ingestion/app/crawler.py @@ -30,9 +30,8 @@ from . import llms_txt from .config import SourceConfig from .logging_config import get_logger -from .urlscope import parse_sitemap +from .urlscope import parse_sitemap, url_host_is_private from .urlscope import path_allowed as _path_allowed -from .urlscope import url_host_is_private USER_AGENT = "self-docs-crawler/0.1" diff --git a/ingestion/app/embedder.py b/ingestion/app/embedder.py index 9f7e719..4b8092f 100644 --- a/ingestion/app/embedder.py +++ b/ingestion/app/embedder.py @@ -1,21 +1,41 @@ """FastEmbed passage-embedding wrapper. -BAAI/bge-small-en-v1.5 is asymmetric: documents MUST be embedded with -`passage_embed()` (FastEmbed applies the BGE `passage:` prefix), while -queries use `query_embed()` — that half is the mcp-server's job, not -ingestion's. Skipping the passage prefix measurably hurts recall. +The embedding model, its dimension, and the per-model PASSAGE prompt are all +env-driven (set by `make configure` from config/models.yaml), defaulting to the +registry default so a fresh checkout / CI works with no configuration: + + EMBEDDING_MODEL_NAME default mixedbread-ai/mxbai-embed-large-v1 + EMBEDDING_DIM default 1024 + EMBEDDING_PASSAGE_PROMPT default "" (mxbai wants no passage prefix) + +Asymmetric retrieval: many models want an instruction/prefix that DIFFERS +between documents and queries. FastEmbed's `passage_embed`/`query_embed` do NOT +apply any prefix for the models we support (bge, mxbai, e5 — they are not +"multitask" models), so we apply the prompt ourselves around plain `embed()`. +Documents get EMBEDDING_PASSAGE_PROMPT here; queries get EMBEDDING_QUERY_PROMPT +in the mcp-server (`retrieval._embed_query`). For mxbai/bge that means no +document prefix and an instruction on the query; for e5 both sides get a prefix. +Keep the two sides consistent — they must use the SAME model. """ from __future__ import annotations +import os import threading from fastembed import TextEmbedding from .logging_config import get_logger -MODEL_NAME = "BAAI/bge-small-en-v1.5" -EMBEDDING_DIM = 384 +# Fallbacks used when EMBEDDING_* env is unset. These MUST equal the registry +# default row in config/models.yaml — tests/test_model_registry.py enforces it. +DEFAULT_MODEL_NAME = "mixedbread-ai/mxbai-embed-large-v1" +DEFAULT_EMBEDDING_DIM = 1024 +DEFAULT_PASSAGE_PROMPT = "" + +MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", DEFAULT_MODEL_NAME) +EMBEDDING_DIM = int(os.environ.get("EMBEDDING_DIM", str(DEFAULT_EMBEDDING_DIM))) +PASSAGE_PROMPT = os.environ.get("EMBEDDING_PASSAGE_PROMPT", DEFAULT_PASSAGE_PROMPT) BATCH_SIZE = 32 logger = get_logger(component="embedder") @@ -35,20 +55,21 @@ def get_model() -> TextEmbedding: def embed_chunks(chunks: list[dict], model: TextEmbedding | None = None, batch_size: int = BATCH_SIZE) -> list[dict]: - """Add an `embedding: list[float]` (len 384) field to each chunk dict via - passage_embed, in batches of `batch_size`. Returns the same list of dicts - (mutated in place) for convenience. + """Add an `embedding: list[float]` (len EMBEDDING_DIM) field to each chunk + dict, in batches of `batch_size`. The document text is prefixed with + EMBEDDING_PASSAGE_PROMPT (empty for mxbai/bge) before embedding. Returns the + same list of dicts (mutated in place) for convenience. """ if not chunks: return chunks m = model or get_model() - texts = [c["content"] for c in chunks] + texts = [PASSAGE_PROMPT + c["content"] for c in chunks] embeddings: list[list[float]] = [] for start in range(0, len(texts), batch_size): batch = texts[start : start + batch_size] - for vec in m.passage_embed(batch): + for vec in m.embed(batch): embeddings.append(vec.tolist() if hasattr(vec, "tolist") else list(vec)) if len(embeddings) != len(chunks): @@ -56,7 +77,7 @@ def embed_chunks(chunks: list[dict], model: TextEmbedding | None = None, batch_s f"embedding count mismatch: got {len(embeddings)} vectors for {len(chunks)} chunks" ) - for chunk, vec in zip(chunks, embeddings): + for chunk, vec in zip(chunks, embeddings, strict=True): if len(vec) != EMBEDDING_DIM: raise RuntimeError(f"unexpected embedding dim {len(vec)} (expected {EMBEDDING_DIM})") chunk["embedding"] = vec diff --git a/ingestion/app/main.py b/ingestion/app/main.py index e0161f1..ef386aa 100644 --- a/ingestion/app/main.py +++ b/ingestion/app/main.py @@ -103,7 +103,7 @@ def _load_initial_sources() -> list[SourceRecord]: _INITIAL_SOURCES: list[SourceRecord] = _load_initial_sources() except Exception as e: # noqa: BLE001 - any DB/connectivity failure is fatal at boot print(f"FATAL: could not load sources from the database: {e}", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(1) from e # --- Opt-in YAML -> DB import (NEVER automatic) ----------------------------------------- # `sources.yaml` -> `doc_sources` import (sources_repo.import_from_yaml) is a @@ -141,7 +141,7 @@ def _maybe_import_sources_yaml_on_boot() -> None: _maybe_import_sources_yaml_on_boot() except Exception as e: # noqa: BLE001 - an explicitly opted-in import failing is fatal too print(f"FATAL: IMPORT_SOURCES_YAML_ON_BOOT import failed: {e}", file=sys.stderr) - raise SystemExit(1) + raise SystemExit(1) from e # --- Dynamic re-read with a last-known-good cache for test observability ---------------- # Startup (above) is fail-fast: an unreachable database at boot aborts the @@ -616,7 +616,7 @@ async def sync(req: SyncRequest | None = None, authorization: str | None = Heade try: all_sources = get_sources() except SourcesUnavailable as e: - raise HTTPException(status_code=503, detail=f"sources unavailable: {e}") + raise HTTPException(status_code=503, detail=f"sources unavailable: {e}") from e sources_by_name = {s.name: s for s in all_sources} sources_by_id = {s.id: s for s in all_sources} diff --git a/ingestion/app/scheduler.py b/ingestion/app/scheduler.py index 6a70200..4c4b4c6 100644 --- a/ingestion/app/scheduler.py +++ b/ingestion/app/scheduler.py @@ -44,7 +44,7 @@ import asyncio from collections.abc import Awaitable, Callable -from datetime import datetime, timezone +from datetime import UTC, datetime from .logging_config import get_logger from .sources_repo import SourceRecord, _select_due_records @@ -90,7 +90,7 @@ def next_due(records: list[SourceRecord], now: datetime) -> list[SourceRecord]: def _default_now() -> datetime: - return datetime.now(timezone.utc) + return datetime.now(UTC) def _default_fetch_candidates() -> list[SourceRecord]: @@ -99,8 +99,10 @@ def _default_fetch_candidates() -> list[SourceRecord]: seam only needs to hand it a full, unfiltered batch. Never executed by this test suite (pytest cannot reach Postgres from the host); tests replace `fetch_candidates_fn` directly.""" - from . import store # local import: keep a hard DB dependency out of the pure test path - from . import sources_repo + from . import ( + sources_repo, + store, # local import: keep a hard DB dependency out of the pure test path + ) conn = store.get_connection() try: @@ -223,5 +225,5 @@ async def run_scheduler(stop: asyncio.Event) -> None: try: await asyncio.wait_for(stop.wait(), timeout=poll_interval_seconds) - except asyncio.TimeoutError: + except TimeoutError: pass diff --git a/ingestion/app/sources_repo.py b/ingestion/app/sources_repo.py index 38b4d5f..28fd91e 100644 --- a/ingestion/app/sources_repo.py +++ b/ingestion/app/sources_repo.py @@ -301,7 +301,7 @@ def parse_cron(expr: str) -> tuple[set[int], set[int], set[int], set[int], set[i ) return tuple( # type: ignore[return-value] _parse_cron_field(token, name, lo, hi) - for token, name, (lo, hi) in zip(fields, _CRON_FIELD_NAMES, _CRON_FIELD_RANGES) + for token, name, (lo, hi) in zip(fields, _CRON_FIELD_NAMES, _CRON_FIELD_RANGES, strict=True) ) diff --git a/ingestion/app/store.py b/ingestion/app/store.py index 7b522ad..71a2f4f 100644 --- a/ingestion/app/store.py +++ b/ingestion/app/store.py @@ -28,9 +28,10 @@ import hashlib import os +from collections.abc import Callable from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable +from typing import Any from urllib.parse import urlparse import psycopg diff --git a/ingestion/app/urlscope.py b/ingestion/app/urlscope.py index 7bf03db..de5f021 100644 --- a/ingestion/app/urlscope.py +++ b/ingestion/app/urlscope.py @@ -53,7 +53,10 @@ import os import socket from urllib.parse import urlparse -from xml.etree import ElementTree +from xml.etree import ElementTree # kept for ElementTree.ParseError below + +from defusedxml import ElementTree as DefusedET +from defusedxml.common import DefusedXmlException _SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} @@ -275,9 +278,13 @@ def parse_sitemap(xml_bytes: bytes) -> tuple[list[str], list[str]]: if not xml_bytes: raise ValueError("parse_sitemap: empty input") try: - root = ElementTree.fromstring(xml_bytes) - except ElementTree.ParseError as e: - raise ValueError(f"parse_sitemap: malformed XML: {e}") from e + # defusedxml forbids DTDs / entity expansion / external entities by + # default — sitemap XML is fetched from untrusted upstream sources, so + # this closes billion-laughs / XXE against the crawler. Both malformed + # XML and a rejected entity/DTD attack surface as ValueError here. + root = DefusedET.fromstring(xml_bytes) + except (ElementTree.ParseError, DefusedXmlException) as e: + raise ValueError(f"parse_sitemap: malformed or unsafe XML: {e}") from e root_tag = _strip_ns(root.tag) diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml index b7b14a4..7b39d2f 100644 --- a/ingestion/pyproject.toml +++ b/ingestion/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "httpx>=0.27", "beautifulsoup4>=4.12", "trafilatura>=1.12", + "defusedxml>=0.7", "fastembed>=0.3", "psycopg[binary]>=3.1", "pyyaml>=6.0", diff --git a/ingestion/tests/conftest.py b/ingestion/tests/conftest.py index 5d5b651..0779a32 100644 --- a/ingestion/tests/conftest.py +++ b/ingestion/tests/conftest.py @@ -55,7 +55,6 @@ import socket import pytest - from app.urlscope import _resolve_host_addrs # Any public, non-private/non-link-local/non-loopback/non-reserved address diff --git a/ingestion/tests/test_admin.py b/ingestion/tests/test_admin.py index be8df3c..d9f0f8c 100644 --- a/ingestion/tests/test_admin.py +++ b/ingestion/tests/test_admin.py @@ -16,17 +16,16 @@ from __future__ import annotations import time +from datetime import UTC, datetime from unittest.mock import MagicMock import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from datetime import datetime, timezone from app import admin -from app.config import ConfigError, SourceConfig +from app.config import SourceConfig from app.sources_repo import SourceRecord from app.store import ChunkRecord, PageRecord, SourceOutcome +from fastapi import FastAPI +from fastapi.testclient import TestClient SYNC_TOKEN = "test-admin-token-xyz" @@ -739,7 +738,7 @@ def test_list_docs_view(client, csrf_token, monkeypatch): source_name="widget", url="https://widget.example.com/docs/guide", content_hash="abc123hash", - fetched_at=datetime.now(timezone.utc), + fetched_at=datetime.now(UTC), chunk_count=4, ) monkeypatch.setattr(admin.store, "list_doc_pages", MagicMock(return_value=[page_rec])) @@ -799,7 +798,7 @@ def test_sync_target_submit(client, csrf_token, monkeypatch): def test_store_row_records(): - now = datetime.now(timezone.utc) + now = datetime.now(UTC) page = admin.store._row_to_page_record((1, 5, "widget", "https://example.com", "hash123", now, 3)) assert page.id == 1 assert page.source_name == "widget" diff --git a/ingestion/tests/test_chunker.py b/ingestion/tests/test_chunker.py index d59f108..c59d1de 100644 --- a/ingestion/tests/test_chunker.py +++ b/ingestion/tests/test_chunker.py @@ -1,4 +1,3 @@ -import pytest from app.chunker import chunk_markdown, count_tokens, get_tokenizer diff --git a/ingestion/tests/test_config.py b/ingestion/tests/test_config.py index a2c1778..45651af 100644 --- a/ingestion/tests/test_config.py +++ b/ingestion/tests/test_config.py @@ -2,7 +2,6 @@ from pathlib import Path import pytest - from app.config import ConfigError, load_sources from app.urlscope import _resolve_host_addrs diff --git a/ingestion/tests/test_crawler.py b/ingestion/tests/test_crawler.py index a256e3a..af874d6 100644 --- a/ingestion/tests/test_crawler.py +++ b/ingestion/tests/test_crawler.py @@ -1,7 +1,6 @@ import inspect import httpx - from app.config import SourceConfig from app.crawler import RateLimiter, _is_private_ip_host, _validate_final_url, crawl, discover_sitemap_urls from app.logging_config import get_logger @@ -916,8 +915,8 @@ def handler(request: httpx.Request) -> httpx.Response: def test_extract_links_suppresses_xml_warning(recwarn): """extract_links() should not emit XMLParsedAsHTMLWarning when processing XML content.""" - from bs4 import XMLParsedAsHTMLWarning from app.crawler import extract_links + from bs4 import XMLParsedAsHTMLWarning xml_content = 'https://example.com/page1' extract_links(xml_content, "https://example.com/sitemap.xml") diff --git a/ingestion/tests/test_embedder.py b/ingestion/tests/test_embedder.py index 3140c2e..6c6e800 100644 --- a/ingestion/tests/test_embedder.py +++ b/ingestion/tests/test_embedder.py @@ -1,26 +1,24 @@ -import pytest +import app.embedder as embedder from app.embedder import EMBEDDING_DIM, embed_chunks class _FakeModel: - """Stand-in for fastembed.TextEmbedding — asserts passage_embed (not - query_embed / embed) is the method used, per the asymmetric-embedding - contract.""" + """Stand-in for fastembed.TextEmbedding — records the exact texts passed to + `embed`, so tests can assert the passage prompt is applied and that plain + `embed` (not query_embed) is the method used.""" def __init__(self): self.calls = [] - def passage_embed(self, texts): - self.calls.append(list(texts)) + def embed(self, texts): + texts = list(texts) + self.calls.append(texts) for _ in texts: yield [0.1] * EMBEDDING_DIM - def query_embed(self, texts): # pragma: no cover - must never be called here - raise AssertionError("embedder must use passage_embed, not query_embed") - -def test_embed_chunks_uses_passage_embed_and_sets_384_dim_vectors(): +def test_embed_chunks_sets_dim_vectors(): fake = _FakeModel() chunks = [ {"url": "u", "heading_path": "H", "chunk_index": 0, "content": "hello world"}, @@ -32,10 +30,24 @@ def test_embed_chunks_uses_passage_embed_and_sets_384_dim_vectors(): assert "embedding" in c assert len(c["embedding"]) == EMBEDDING_DIM assert all(isinstance(v, float) for v in c["embedding"]) - assert fake.calls # passage_embed was invoked + assert fake.calls # embed was invoked + + +def test_embed_chunks_applies_passage_prompt(monkeypatch): + monkeypatch.setattr(embedder, "PASSAGE_PROMPT", "passage: ") + fake = _FakeModel() + embed_chunks([{"content": "hello world"}], model=fake) + assert fake.calls == [["passage: hello world"]] + + +def test_embed_chunks_empty_passage_prompt_leaves_text_unchanged(monkeypatch): + monkeypatch.setattr(embedder, "PASSAGE_PROMPT", "") + fake = _FakeModel() + embed_chunks([{"content": "hello world"}], model=fake) + assert fake.calls == [["hello world"]] -def test_embed_chunks_batches(monkeypatch): +def test_embed_chunks_batches(): fake = _FakeModel() chunks = [{"url": "u", "heading_path": "H", "chunk_index": i, "content": f"c{i}"} for i in range(70)] embed_chunks(chunks, model=fake, batch_size=32) diff --git a/ingestion/tests/test_llms_txt.py b/ingestion/tests/test_llms_txt.py index 7232c64..ad1fde8 100644 --- a/ingestion/tests/test_llms_txt.py +++ b/ingestion/tests/test_llms_txt.py @@ -8,7 +8,6 @@ from __future__ import annotations import httpx - from app.llms_txt import discover, split_llms_full diff --git a/ingestion/tests/test_main.py b/ingestion/tests/test_main.py index da9b0de..76a0fd5 100644 --- a/ingestion/tests/test_main.py +++ b/ingestion/tests/test_main.py @@ -16,7 +16,7 @@ import os import subprocess import sys -from datetime import datetime, timezone +from datetime import UTC, datetime import pytest from fastapi.testclient import TestClient @@ -54,7 +54,7 @@ def _make_record( enabled=True, status=status, proposed_by=proposed_by, - created_at=datetime.now(timezone.utc), + created_at=datetime.now(UTC), last_synced=None, last_status=None, llms_txt="auto", @@ -351,7 +351,6 @@ def crashing_sync_source(source, conn, **kwargs): def test_sync_second_call_returns_409_while_running(app_module, monkeypatch): - import asyncio import time # Make the sync worker slow so we can observe the "running" state. @@ -462,6 +461,7 @@ def close(self): def test_configure_logging_suppresses_third_party_loggers(): import logging + from app.logging_config import configure_logging configure_logging() diff --git a/ingestion/tests/test_scheduler.py b/ingestion/tests/test_scheduler.py index 674ef09..06adfcd 100644 --- a/ingestion/tests/test_scheduler.py +++ b/ingestion/tests/test_scheduler.py @@ -22,7 +22,6 @@ from datetime import datetime import pytest - from app import scheduler from app.sources_repo import SourceRecord diff --git a/ingestion/tests/test_sources_repo.py b/ingestion/tests/test_sources_repo.py index 436d81d..6898a82 100644 --- a/ingestion/tests/test_sources_repo.py +++ b/ingestion/tests/test_sources_repo.py @@ -22,9 +22,9 @@ import psycopg import pytest - from app import sources_repo from app.config import SourceConfig, load_sources +from app.embedder import EMBEDDING_DIM as _EMBEDDING_DIM from app.sources_repo import ( SOURCE_COLUMNS, ImportResult, @@ -34,7 +34,6 @@ _row_to_record, _select_due_records, cron_matches, - due_sources, parse_cron, validate_cron, ) @@ -619,7 +618,7 @@ def test_delete_source_cascades_to_pages_and_chunks(db_conn: psycopg.Connection) INSERT INTO doc_chunks (page_id, heading_path, chunk_index, content, embedding) VALUES (%s, %s, %s, %s, %s::vector) """, - (page_id, "Intro", 0, "hello world", "[" + ",".join(["0.0"] * 384) + "]"), + (page_id, "Intro", 0, "hello world", "[" + ",".join(["0.0"] * _EMBEDDING_DIM) + "]"), ) cur.execute("SELECT count(*) FROM doc_pages WHERE source_id = %s", (source_id,)) (pages_before,) = cur.fetchone() diff --git a/ingestion/tests/test_store.py b/ingestion/tests/test_store.py index 07db22b..06e9eec 100644 --- a/ingestion/tests/test_store.py +++ b/ingestion/tests/test_store.py @@ -16,7 +16,6 @@ import psycopg import pytest - from app import store from app.config import SourceConfig @@ -670,8 +669,10 @@ def test_replace_page_atomic_no_partial_chunks_on_failure(conn, monkeypatch): cur.execute("SELECT id FROM doc_sources WHERE name = %s", (source.name,)) (source_id,) = cur.fetchone() + from app.embedder import EMBEDDING_DIM + bad_chunks = [ - {"heading_path": ["H1"], "chunk_index": 0, "content": "Chunk 0 good", "embedding": [0.1] * 384}, + {"heading_path": ["H1"], "chunk_index": 0, "content": "Chunk 0 good", "embedding": [0.1] * EMBEDDING_DIM}, {"heading_path": ["H2"], "chunk_index": 1, "content": "Chunk 1 bad", "embedding": [0.1] * 10}, ] diff --git a/ingestion/tests/test_urlscope.py b/ingestion/tests/test_urlscope.py index f7151b3..bca12ab 100644 --- a/ingestion/tests/test_urlscope.py +++ b/ingestion/tests/test_urlscope.py @@ -1,7 +1,6 @@ import socket import pytest - from app.urlscope import ( _resolve_host_addrs, _resolve_is_private, diff --git a/ingestion/uv.lock b/ingestion/uv.lock index d0987b2..5aa069f 100644 --- a/ingestion/uv.lock +++ b/ingestion/uv.lock @@ -180,6 +180,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/7c/2e5dcf53909deddd0bf38cbe277ad9806be038276b1c6c436561b4d9b2e2/dateparser-1.4.1-py3-none-any.whl", hash = "sha256:f25d4e051a84be27a35bd297e3e1dc59ff78373701b89be352ba80372d22d0d0", size = 300503, upload-time = "2026-06-15T08:45:45.951Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "fastapi" version = "0.139.2" @@ -355,12 +364,15 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "defusedxml" }, { name = "fastapi" }, { name = "fastembed" }, { name = "httpx" }, + { name = "jinja2" }, { name = "prometheus-client" }, { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, + { name = "python-multipart" }, { name = "pyyaml" }, { name = "structlog" }, { name = "tokenizers" }, @@ -376,13 +388,16 @@ dev = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12" }, + { name = "defusedxml", specifier = ">=0.7" }, { name = "fastapi", specifier = ">=0.111" }, { name = "fastembed", specifier = ">=0.3" }, { name = "httpx", specifier = ">=0.27" }, + { name = "jinja2", specifier = ">=3.1" }, { name = "prometheus-client", specifier = ">=0.20" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.1" }, { name = "pydantic", specifier = ">=2.7" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "python-multipart", specifier = ">=0.0.9" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "structlog", specifier = ">=24.1" }, { name = "tokenizers", specifier = ">=0.19" }, @@ -400,6 +415,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "justext" version = "3.0.2" @@ -522,6 +549,69 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/bd/6e2b76a6c5dee10397db9c929f0c5066766ec1036046f0335b7ca7ca08b8/lxml_html_clean-0.4.5-py3-none-any.whl", hash = "sha256:c76fcadd1e5bfb9b8bafc2200d51e4e78eb0dad67f56881c21dfb6484c7e7746", size = 14573, upload-time = "2026-05-20T12:17:52.215Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mmh3" version = "5.2.1" @@ -1023,6 +1113,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + [[package]] name = "pytz" version = "2026.2" diff --git a/mcp-server/Dockerfile b/mcp-server/Dockerfile index 4ec4635..a503c9f 100644 --- a/mcp-server/Dockerfile +++ b/mcp-server/Dockerfile @@ -30,8 +30,13 @@ USER app # Pre-download the FastEmbed model at build time so the image works fully # offline at runtime and the first request isn't paying a multi-second # cold-load / download penalty (see IMPLEMENTATION_PLAN.md: "FastEmbed model -# cache — pre-baked into both images at build; runtime works offline"). -RUN python -c "from fastembed import TextEmbedding; TextEmbedding(model_name='BAAI/bge-small-en-v1.5')" +# cache — pre-baked into both images at build; runtime works offline"). The +# model is a build arg so `make configure` / compose can bake the selected +# model (default = config/models.yaml registry default); the runtime env var +# must match the ingestion service's model. +ARG EMBEDDING_MODEL_NAME=mixedbread-ai/mxbai-embed-large-v1 +ENV EMBEDDING_MODEL_NAME=${EMBEDDING_MODEL_NAME} +RUN python -c "import os; from fastembed import TextEmbedding; TextEmbedding(model_name=os.environ['EMBEDDING_MODEL_NAME'])" EXPOSE 8000 diff --git a/mcp-server/app/retrieval.py b/mcp-server/app/retrieval.py index 034fc98..2ebef14 100644 --- a/mcp-server/app/retrieval.py +++ b/mcp-server/app/retrieval.py @@ -34,7 +34,18 @@ logger = structlog.get_logger(__name__) -EMBEDDING_MODEL_NAME = "BAAI/bge-small-en-v1.5" +# Model, and the per-model QUERY prompt, are env-driven (set by `make configure` +# from config/models.yaml), defaulting to the registry default so a fresh +# checkout / CI works unconfigured. FastEmbed applies no prefix for the models +# we support, so the query instruction is prepended manually in `_embed_query` +# — the asymmetric counterpart to ingestion's EMBEDDING_PASSAGE_PROMPT. The two +# services MUST run the same model for the vectors to be comparable. +# Fallbacks used when EMBEDDING_* env is unset. These MUST equal the registry +# default row in config/models.yaml — the parity tests enforce it. +DEFAULT_MODEL_NAME = "mixedbread-ai/mxbai-embed-large-v1" +DEFAULT_QUERY_PROMPT = "Represent this sentence for searching relevant passages: " +EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", DEFAULT_MODEL_NAME) +QUERY_PROMPT = os.environ.get("EMBEDDING_QUERY_PROMPT", DEFAULT_QUERY_PROMPT) RRF_K = 60 ARM_CANDIDATE_LIMIT = 30 @@ -52,7 +63,7 @@ # arm using Reciprocal Rank Fusion (score = sum over arms of 1/(k + rank)). # `%(source)s` is NULL when no source filter is requested; the `IS NULL OR` # guard makes the filter optional without a second query string. -HYBRID_SEARCH_SQL = """ +HYBRID_SEARCH_SQL = f""" WITH vector_candidates AS ( SELECT dc.id, dc.embedding <=> %(query_vec)s::vector AS distance FROM doc_chunks dc @@ -60,7 +71,7 @@ JOIN doc_sources ds ON ds.id = dp.source_id WHERE %(source)s::text IS NULL OR ds.name = %(source)s ORDER BY dc.embedding <=> %(query_vec)s::vector - LIMIT {arm_limit} + LIMIT {ARM_CANDIDATE_LIMIT} ), vector_arm AS ( SELECT id, row_number() OVER (ORDER BY distance) AS rnk @@ -75,14 +86,14 @@ WHERE dc.fts @@ websearch_to_tsquery(dc.fts_config, %(query_text)s) AND (%(source)s::text IS NULL OR ds.name = %(source)s) ORDER BY rank_score DESC - LIMIT {arm_limit} + LIMIT {ARM_CANDIDATE_LIMIT} ), fts_arm AS ( SELECT id, row_number() OVER (ORDER BY rank_score DESC) AS rnk FROM fts_candidates ), fused AS ( - SELECT id, SUM(1.0 / ({rrf_k} + rnk)) AS rrf_score + SELECT id, SUM(1.0 / ({RRF_K} + rnk)) AS rrf_score FROM ( SELECT id, rnk FROM vector_arm UNION ALL @@ -96,7 +107,7 @@ JOIN doc_pages dp ON dp.id = dc.page_id ORDER BY fused.rrf_score DESC LIMIT %(limit)s; -""".format(arm_limit=ARM_CANDIDATE_LIMIT, rrf_k=RRF_K) +""" LIST_SOURCES_SQL = """ SELECT @@ -157,10 +168,12 @@ def _format_vector_literal(vec: Any) -> str: def _embed_query(query: str) -> str: - """Embed a search query with FastEmbed's `query_embed` (applies the BGE - `query:` prefix — asymmetric to `passage_embed` used at ingest time).""" + """Embed a search query, prepending EMBEDDING_QUERY_PROMPT (the per-model + query instruction — asymmetric to ingestion's EMBEDDING_PASSAGE_PROMPT). + FastEmbed applies no prefix for the supported models, so we prepend it and + use plain `embed()`.""" model = get_embedding_model() - (vector,) = list(model.query_embed([query])) + (vector,) = list(model.embed([QUERY_PROMPT + query])) return _format_vector_literal(vector) @@ -406,7 +419,7 @@ def _language_must_be_supported(cls, v: str) -> str: return normalized @model_validator(mode="after") - def _sitemap_shares_base_url_host(self) -> "ProposedSourceConfig": + def _sitemap_shares_base_url_host(self) -> ProposedSourceConfig: # SSRF guard (security review H1): `sitemap` is fetched BEFORE any of # its `` entries are host-filtered, and a `` fans # out to its children equally unvalidated. Constraining the sitemap to @@ -427,7 +440,7 @@ def _sitemap_shares_base_url_host(self) -> "ProposedSourceConfig": return self @model_validator(mode="after") - def _hosts_must_not_be_private(self) -> "ProposedSourceConfig": + def _hosts_must_not_be_private(self) -> ProposedSourceConfig: # SSRF guard (security review H2): source URLs are untrusted input (an # MCP tool callable by an AI agent), so reject a host that IS or # RESOLVES TO private/loopback/link-local/reserved space at proposal @@ -448,7 +461,7 @@ def _hosts_must_not_be_private(self) -> "ProposedSourceConfig": return self @model_validator(mode="after") - def _base_url_passes_own_prefix_filters(self) -> "ProposedSourceConfig": + def _base_url_passes_own_prefix_filters(self) -> ProposedSourceConfig: # Same rationale as SourceConfig's validator of the same name: a # sitemap-less source whose base_url path is excluded by its own # prefixes would crawl 0 pages if ever approved. Reject at proposal diff --git a/mcp-server/pyproject.toml b/mcp-server/pyproject.toml index db1c783..28fdc90 100644 --- a/mcp-server/pyproject.toml +++ b/mcp-server/pyproject.toml @@ -15,6 +15,10 @@ dependencies = [ [dependency-groups] dev = [ "pytest", + "pyyaml", + # Only for the cross-package parity test that loads ingestion/app/config.py + # (which imports ingestion/app/urlscope.py -> defusedxml) from disk. + "defusedxml", ] [build-system] diff --git a/mcp-server/tests/test_registry_defaults.py b/mcp-server/tests/test_registry_defaults.py new file mode 100644 index 0000000..6c36b31 --- /dev/null +++ b/mcp-server/tests/test_registry_defaults.py @@ -0,0 +1,35 @@ +"""Guards that the mcp-server's runtime embedding defaults match the shared +model registry (config/models.yaml). The ingestion side and the schema/SSRF +parity live in the top-level tests/test_model_registry.py; this covers the +retrieval service's own fallback constants. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +yaml = pytest.importorskip("yaml") + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +REGISTRY_PATH = REPO_ROOT / "config" / "models.yaml" + + +def _load_registry() -> dict: + with REGISTRY_PATH.open(encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def test_retrieval_defaults_match_registry_default(): + """The retrieval service's fallback constants (used when EMBEDDING_* env is + unset) must equal the registry default row. Checks the DEFAULT_* literals + directly so it holds regardless of the suite's EMBEDDING_* env, and avoids + reloading the module (which would re-register its Prometheus collectors).""" + from app import retrieval + + registry = _load_registry() + default_name = registry["default"] + default_row = registry["models"][default_name] + assert retrieval.DEFAULT_MODEL_NAME == default_name + assert retrieval.DEFAULT_QUERY_PROMPT == default_row["query_prompt"] diff --git a/mcp-server/tests/test_retrieval.py b/mcp-server/tests/test_retrieval.py index b6ff8de..eb8f297 100644 --- a/mcp-server/tests/test_retrieval.py +++ b/mcp-server/tests/test_retrieval.py @@ -1,6 +1,4 @@ import pytest -from pydantic import ValidationError - from app.retrieval import ( ARM_CANDIDATE_LIMIT, HYBRID_SEARCH_SQL, @@ -9,6 +7,7 @@ _format_vector_literal, format_hit, ) +from pydantic import ValidationError def test_format_hit_matches_contract(): diff --git a/mcp-server/tests/test_retrieval_integration.py b/mcp-server/tests/test_retrieval_integration.py index fc02e22..67171eb 100644 --- a/mcp-server/tests/test_retrieval_integration.py +++ b/mcp-server/tests/test_retrieval_integration.py @@ -1,8 +1,9 @@ """Integration tests for retrieval.py's hybrid RRF search against a live Postgres (the compose `db` service, T1 schema). -Seeds known chunks with *real* `passage_embed` embeddings (same asymmetric -FastEmbed model used at ingest time) and exercises the real `search()` +Seeds known chunks with *real* `embed()` embeddings (same FastEmbed model and +passage side used at ingest time — no passage prefix for the default model) and +exercises the real `search()` entrypoint end-to-end through the hybrid RRF SQL: - an exact-token query (a rare literal string that appears in only one @@ -158,7 +159,7 @@ def _insert_chunk( content: str, fts_config: str = "english", ) -> None: - (vec,) = list(model.passage_embed([content])) + (vec,) = list(model.embed([content])) literal = retrieval._format_vector_literal(vec) with conn.cursor() as cur: cur.execute( diff --git a/mcp-server/tests/test_server.py b/mcp-server/tests/test_server.py index da51007..cb425ad 100644 --- a/mcp-server/tests/test_server.py +++ b/mcp-server/tests/test_server.py @@ -26,16 +26,15 @@ os.environ.setdefault("MCP_TOKEN", "test-mcp-token-for-server-tests") +import app.server as server import httpx import psycopg import pytest +from app import retrieval from fastmcp import Client from fastmcp.client.transports import StreamableHttpTransport from fastmcp.server.auth import AccessToken -import app.server as server -from app import retrieval - TEST_TOKEN = os.environ["MCP_TOKEN"] diff --git a/mcp-server/uv.lock b/mcp-server/uv.lock index d13047d..efe7316 100644 --- a/mcp-server/uv.lock +++ b/mcp-server/uv.lock @@ -343,6 +343,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/df/a6e2a790f98e33dc0498ef79369bec5a8dfc2c4be05ac639cdf17c35ff15/cyclopts-4.21.1-py3-none-any.whl", hash = "sha256:9a02521b0961ae1adb374d1200a4bf1361428aeba4f63adebcd39cd2bb925949", size = 232354, upload-time = "2026-07-16T18:23:34.556Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -1693,7 +1702,9 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "defusedxml" }, { name = "pytest" }, + { name = "pyyaml" }, ] [package.metadata] @@ -1707,7 +1718,11 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest" }] +dev = [ + { name = "defusedxml" }, + { name = "pytest" }, + { name = "pyyaml" }, +] [[package]] name = "sse-starlette" diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..d65414e --- /dev/null +++ b/mypy.ini @@ -0,0 +1,31 @@ +[mypy] +python_version = 3.12 +# Runtime deps (fastembed, psycopg, fastapi, ...) ship partial/no stubs; we +# type-check our own code, not theirs. +ignore_missing_imports = True +namespace_packages = True +explicit_package_bases = True +warn_redundant_casts = True +warn_unused_ignores = False +no_implicit_optional = True + +# --- Typing backlog ------------------------------------------------------- +# These modules predate the type-check gate and carry pre-existing findings +# (mostly Optional-narrowing and getaddrinfo's str|int tuples). They are +# quarantined so the gate stays green and enforces types on NEW/changed code; +# tighten them incrementally by removing entries here. NOT a licence to add new +# untyped code — new modules are checked in full. +[mypy-app.admin] +ignore_errors = True +[mypy-app.crawler] +ignore_errors = True +[mypy-app.llms_txt] +ignore_errors = True +[mypy-app.main] +ignore_errors = True +[mypy-app.store] +ignore_errors = True +[mypy-app.urlscope] +ignore_errors = True +[mypy-app.retrieval] +ignore_errors = True diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..9b321b0 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,32 @@ +# Ruff config for the whole repo (both packages, scripts, tests). +# Run via `make lint` (ruff check + ruff format --check). +target-version = "py312" +line-length = 120 + +extend-exclude = [ + ".venv", + "*/.venv", + ".tooling-venv", + "backups", +] + +[lint] +# A pragmatic, low-noise set: pyflakes (F), a subset of pycodestyle (E/W), +# isort (I), pyupgrade (UP), and flake8-bugbear (B). This catches real bugs and +# unused/dead code without imposing opinionated style churn on a codebase with +# deliberately long, rationale-rich comments. +select = ["F", "E", "W", "I", "UP", "B"] +ignore = [ + "E501", # line length — long explanatory comments are intentional here + "E741", # ambiguous var names (e.g. `m` for model) used deliberately + "B008", # function calls in argument defaults (FastAPI/pydantic patterns) +] + +[lint.per-file-ignores] +# Tests deliberately assert on broad exceptions in a few negative cases. +"**/tests/**" = ["B011", "B017"] +# crawler.py must call warnings.filterwarnings() BEFORE importing the bs4-using +# submodules, so those imports are intentionally not at the top of the file. +"ingestion/app/crawler.py" = ["E402"] +# test_push_sources.py inserts the repo root on sys.path before importing scripts. +"tests/test_push_sources.py" = ["E402"] diff --git a/scripts/configure_model.py b/scripts/configure_model.py new file mode 100644 index 0000000..9e156df --- /dev/null +++ b/scripts/configure_model.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Resolve an embedding model selection into deploy config. + +`make configure MODEL=` (or `python scripts/configure_model.py `) +reads config/models.yaml — the single source of truth — and, for the chosen +model: + + 1. Upserts these keys into .env (creating it if absent, preserving every + other key already there): + EMBEDDING_MODEL_NAME, EMBEDDING_DIM, + EMBEDDING_QUERY_PROMPT, EMBEDDING_PASSAGE_PROMPT, + INGESTION_MEM_LIMIT, MCP_MEM_LIMIT + docker-compose interpolates the memory limits; the two services read the + rest at runtime. + 2. Renders db/init/01_schema.sql from 01_schema.sql.template with the model's + vector dimension. + +No model name is passed => the registry default is used. The services default +to the same registry-default values when a key is unset, so this step is only +required to (a) deviate from the default or (b) regenerate .env/schema. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +try: + import yaml +except ModuleNotFoundError as exc: # pragma: no cover - environment hint + raise SystemExit( + "configure_model.py needs PyYAML. Install it with `pip install pyyaml`, " + "or run this via the ingestion venv (which already has it): " + "ingestion/.venv/bin/python scripts/configure_model.py " + ) from exc + +REPO_ROOT = Path(__file__).resolve().parent.parent +REGISTRY_PATH = REPO_ROOT / "config" / "models.yaml" +ENV_PATH = REPO_ROOT / ".env" +SCHEMA_TEMPLATE = REPO_ROOT / "db" / "init" / "01_schema.sql.template" +SCHEMA_OUT = REPO_ROOT / "db" / "init" / "01_schema.sql" + +DIM_PLACEHOLDER = "__EMBEDDING_DIM__" + +# .env keys this script owns. Upserted on every run; everything else is left +# untouched. +MANAGED_KEYS = ( + "EMBEDDING_MODEL_NAME", + "EMBEDDING_DIM", + "EMBEDDING_QUERY_PROMPT", + "EMBEDDING_PASSAGE_PROMPT", + "INGESTION_MEM_LIMIT", + "MCP_MEM_LIMIT", +) + + +def load_registry() -> dict: + with REGISTRY_PATH.open(encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def resolve(registry: dict, model: str | None) -> tuple[str, dict]: + models = registry["models"] + name = model or registry["default"] + if name not in models: + available = "\n ".join(sorted(models)) + raise SystemExit( + f"unknown model {name!r}. Supported models (config/models.yaml):\n {available}" + ) + return name, models[name] + + +def derived_env(name: str, row: dict) -> dict[str, str]: + return { + "EMBEDDING_MODEL_NAME": name, + "EMBEDDING_DIM": str(row["dim"]), + "EMBEDDING_QUERY_PROMPT": row.get("query_prompt", ""), + "EMBEDDING_PASSAGE_PROMPT": row.get("passage_prompt", ""), + "INGESTION_MEM_LIMIT": row["mem_ingestion"], + "MCP_MEM_LIMIT": row["mem_mcp"], + } + + +def _quote(value: str) -> str: + """Quote a value for .env if it contains spaces or is empty, so + docker-compose / the Makefile's `-include .env` parse it as one token.""" + if value == "" or any(c in value for c in " \t#'\""): + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + return value + + +def upsert_env(updates: dict[str, str]) -> None: + """Update-or-append MANAGED_KEYS in .env, preserving all other lines.""" + lines = ENV_PATH.read_text(encoding="utf-8").splitlines() if ENV_PATH.exists() else [] + seen: set[str] = set() + out: list[str] = [] + for line in lines: + stripped = line.lstrip() + key = stripped.split("=", 1)[0].strip() if ("=" in stripped and not stripped.startswith("#")) else None + if key in updates: + out.append(f"{key}={_quote(updates[key])}") + seen.add(key) + else: + out.append(line) + trailing = [k for k in MANAGED_KEYS if k in updates and k not in seen] + if trailing: + if out and out[-1].strip(): + out.append("") + out.append("# --- Embedding model (managed by scripts/configure_model.py) ---") + out.extend(f"{k}={_quote(updates[k])}" for k in trailing) + ENV_PATH.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def render_schema_text(dim: int) -> str: + """Pure render of db/init/01_schema.sql for a given vector dimension. Kept + side-effect-free so tests can assert the committed file matches this output + (tests/test_model_registry.py).""" + template = SCHEMA_TEMPLATE.read_text(encoding="utf-8") + # Swap the TEMPLATE header for a GENERATED one FIRST — the template header + # text itself contains the __EMBEDDING_DIM__ token, so this must happen + # before the dimension substitution below or the match would break. + rendered = template.replace( + "-- TEMPLATE — do not edit db/init/01_schema.sql by hand.\n" + "--\n" + "-- scripts/configure_model.py renders this file into db/init/01_schema.sql,\n" + "-- substituting __EMBEDDING_DIM__ with the selected model's vector dimension\n" + "-- (see config/models.yaml). The committed 01_schema.sql is the rendering for\n" + "-- the registry's default model; `make configure MODEL=` re-renders it.\n" + "-- A parity test (tests/test_model_registry.py) fails CI if the two drift.", + "-- GENERATED from db/init/01_schema.sql.template by scripts/configure_model.py.\n" + "-- Do not edit by hand — run `make configure MODEL=` to change the vector\n" + f"-- dimension. Rendered for embedding dimension {dim}.", + ) + return rendered.replace(DIM_PLACEHOLDER, str(dim)) + + +def render_schema(dim: int) -> None: + SCHEMA_OUT.write_text(render_schema_text(dim), encoding="utf-8") + + +def main(argv: list[str]) -> int: + model = argv[1] if len(argv) > 1 and argv[1] else None + registry = load_registry() + name, row = resolve(registry, model) + env = derived_env(name, row) + upsert_env(env) + render_schema(int(row["dim"])) + print(f"Configured embedding model: {name}") + print(f" dim = {env['EMBEDDING_DIM']}") + print(f" query_prompt = {env['EMBEDDING_QUERY_PROMPT']!r}") + print(f" passage_prompt = {env['EMBEDDING_PASSAGE_PROMPT']!r}") + print(f" ingestion memory = {env['INGESTION_MEM_LIMIT']}") + print(f" mcp-server memory = {env['MCP_MEM_LIMIT']}") + print(f"Wrote {ENV_PATH.relative_to(REPO_ROOT)} and rendered {SCHEMA_OUT.relative_to(REPO_ROOT)}.") + print("Changing model invalidates existing vectors — run `make reindex` to re-embed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/push_sources.py b/scripts/push_sources.py index 9fc1784..0bfa47a 100755 --- a/scripts/push_sources.py +++ b/scripts/push_sources.py @@ -21,7 +21,7 @@ import sys from pathlib import Path from typing import Any -from urllib.parse import urljoin, urlparse +from urllib.parse import urlparse try: import httpx @@ -113,7 +113,7 @@ def main() -> int: return 1 try: - with open(args.file, "r", encoding="utf-8") as fp: + with open(args.file, encoding="utf-8") as fp: data = json.load(fp) except Exception as exc: print(f"FATAL: Failed to read/parse JSON file {args.file}: {exc}", file=sys.stderr) @@ -186,7 +186,7 @@ def main() -> int: # Trigger sync print(f" -> Triggering sync for {name!r}...") sync_resp = client.post( - f"/sync", + "/sync", json={"source": name}, headers={"Authorization": f"Bearer {sync_token}"}, ) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 2d9848f..305f874 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -30,7 +30,6 @@ import os import socket import subprocess -import sys import threading import time from pathlib import Path diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py new file mode 100644 index 0000000..9bc7576 --- /dev/null +++ b/tests/test_model_registry.py @@ -0,0 +1,100 @@ +"""Guards that keep the embedding-model registry, the two services' defaults, +the rendered schema, and the duplicated SSRF helper from drifting apart. + +Runs under the ingestion venv (see Makefile's `test` target), so `app` here is +the ingestion package and PyYAML is available. The mcp-server side has its own +`tests/test_registry_defaults.py` for its runtime defaults. +""" + +from __future__ import annotations + +import ast +import importlib.util +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +REGISTRY_PATH = REPO_ROOT / "config" / "models.yaml" +SCHEMA_FILE = REPO_ROOT / "db" / "init" / "01_schema.sql" +INGESTION_URLSCOPE = REPO_ROOT / "ingestion" / "app" / "urlscope.py" +MCP_RETRIEVAL = REPO_ROOT / "mcp-server" / "app" / "retrieval.py" + + +def _load_registry() -> dict: + with REGISTRY_PATH.open(encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def _load_configure_model(): + spec = importlib.util.spec_from_file_location( + "configure_model", REPO_ROOT / "scripts" / "configure_model.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +REGISTRY = _load_registry() + + +def test_registry_default_is_a_known_model(): + assert REGISTRY["default"] in REGISTRY["models"] + + +@pytest.mark.parametrize("name", list(REGISTRY["models"])) +def test_registry_rows_are_complete(name): + row = REGISTRY["models"][name] + assert isinstance(row["dim"], int) and row["dim"] > 0 + assert row["mem_ingestion"] and row["mem_mcp"] + # Prompts are required keys (may be empty strings) so the services and the + # configure script can rely on them existing. + assert "query_prompt" in row + assert "passage_prompt" in row + + +def test_committed_schema_matches_rendered_default(): + """db/init/01_schema.sql must equal render(template, default-model dim). If + this fails, run `make configure` (default model) to regenerate it.""" + configure_model = _load_configure_model() + default_dim = REGISTRY["models"][REGISTRY["default"]]["dim"] + expected = configure_model.render_schema_text(default_dim) + assert SCHEMA_FILE.read_text(encoding="utf-8") == expected + + +def test_committed_schema_vector_dim_matches_default(): + default_dim = REGISTRY["models"][REGISTRY["default"]]["dim"] + assert f"vector({default_dim})" in SCHEMA_FILE.read_text(encoding="utf-8") + + +def test_ingestion_embedder_defaults_match_registry_default(): + """The ingestion embedder's fallback constants (used when EMBEDDING_* env is + unset) must equal the registry default row, so an unconfigured deploy embeds + with the advertised default model/dim/prompt. Checks the DEFAULT_* literals + directly, so it holds regardless of what EMBEDDING_* env the suite runs with.""" + from app import embedder + + default_name = REGISTRY["default"] + default_row = REGISTRY["models"][default_name] + assert embedder.DEFAULT_MODEL_NAME == default_name + assert embedder.DEFAULT_EMBEDDING_DIM == default_row["dim"] + assert embedder.DEFAULT_PASSAGE_PROMPT == default_row["passage_prompt"] + + +def _extract_function_source(path: Path, func_name: str) -> str: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == func_name: + return ast.unparse(node) + raise AssertionError(f"{func_name} not found in {path}") + + +def test_addr_is_private_is_byte_identical_across_ssrf_copies(): + """`_addr_is_private` is hand-duplicated in ingestion/app/urlscope.py and + mcp-server/app/retrieval.py (the two services can't share an import). The + literal-address classifier must stay identical between them — a divergence + here is the exact SSRF drift the security review warned about.""" + ing = _extract_function_source(INGESTION_URLSCOPE, "_addr_is_private") + mcp = _extract_function_source(MCP_RETRIEVAL, "_addr_is_private") + assert ing == mcp diff --git a/tests/test_push_sources.py b/tests/test_push_sources.py index 1558d7c..317aa70 100644 --- a/tests/test_push_sources.py +++ b/tests/test_push_sources.py @@ -5,8 +5,6 @@ from pathlib import Path from unittest.mock import MagicMock, patch -import pytest - # Ensure repo root is on sys.path so we can import scripts.push_sources REPO_ROOT = Path(__file__).resolve().parent.parent if str(REPO_ROOT) not in sys.path: From 91963743db5087cdc7d283bc582f78916281490f Mon Sep 17 00:00:00 2001 From: ItayUliel Date: Tue, 21 Jul 2026 22:15:49 +0300 Subject: [PATCH 2/3] docs: document selectable embedding models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the docs in line with the registry-driven model selection: README and the runbook still described the model as fixed at BAAI/bge-small-en-v1.5 / 384-dim. - README: embeddings are selectable via config/models.yaml (default mxbai-embed-large-v1, 1024d); add `make configure` to the quickstart and `make lint`/`make typecheck` to development; link the LICENSE file. - Runbook: the offline-model FAQ now refers to the configured model and notes that switching it requires rebuilding both images (the model is baked in via the EMBEDDING_MODEL_NAME build arg). - ADR-004: new record for the registry decision — single source of truth, derived dimension/memory, the default upgrade, manual prompting, and the re-embed consequence. - ADR-001: inline supersede note pointing at ADR-004, matching the file's existing convention for the n8n note. The decision itself is left intact. - IMPLEMENTATION_PLAN: correct the claim that FastEmbed applies the BGE passage:/query: prefixes — it does not for any non-multitask model, so queries were previously embedded with no instruction prefix. --- IMPLEMENTATION_PLAN.md | 2 +- README.md | 24 ++++- .../001-custom-pipeline-over-off-the-shelf.md | 2 +- docs/adr/004-selectable-embedding-models.md | 99 +++++++++++++++++++ docs/runbook.md | 11 ++- 5 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 docs/adr/004-selectable-embedding-models.md diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 9c437bb..9bee08e 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -17,7 +17,7 @@ holds, and what should be adjusted: | Original choice | Verdict | Notes | |---|---|---| | PostgreSQL 16 + pgvector | ✅ Keep, pin ≥0.8.2 | Pin the image to `pgvector/pgvector:0.8.2-pg16` (not the floating `pg16` tag): **CVE-2026-3172** (CVSS 8.1) is a buffer overflow in the parallel HNSW build affecting 0.6.0–0.8.1, fixed in 0.8.2. RDS-compatible as intended. | -| FastEmbed + `BAAI/bge-small-en-v1.5` | ✅ Keep, with prefixes | 384-dim, CPU-friendly ONNX. **Must** use asymmetric prefixes: embed docs via `passage_embed()` and queries via `query_embed()` — FastEmbed applies the BGE `passage:`/`query:` prefixes for you. Skipping this measurably hurts recall. | +| FastEmbed + `BAAI/bge-small-en-v1.5` | ✅ Keep, with prefixes | 384-dim, CPU-friendly ONNX. **Must** use asymmetric prefixes: embed docs via `passage_embed()` and queries via `query_embed()` — FastEmbed applies the BGE `passage:`/`query:` prefixes for you. Skipping this measurably hurts recall. **CORRECTION (ADR-004):** the claim that FastEmbed applies these prefixes is FALSE for FastEmbed 0.8 — `passage_embed`/`query_embed` delegate straight to `embed()` for every non-multitask model (bge, mxbai, e5), so no prefix was ever applied. Prompts are now applied manually from `config/models.yaml`, and the model is registry-selectable (default `mixedbread-ai/mxbai-embed-large-v1`, 1024-dim). | | BeautifulSoup crawler | ⚠️ Upgrade | Keep BS4 for link discovery, but add **`trafilatura`** for main-content extraction (strips nav/sidebar/footer boilerplate far better than hand-rolled selectors) and prefer **sitemap.xml** discovery over recursive crawling when available. | | FastMCP (Python) | ✅ Keep — target **3.x**, pin `fastmcp>=3,<4` | FastMCP 3.0 is GA; **Streamable HTTP** (single `/mcp` endpoint) is the recommended remote transport; legacy SSE is deprecated. Breaking change vs 2.x: `stateless_http` (and host/port) moved off the constructor onto `run()`/`http_app()` — `FastMCP('self-docs')` then `mcp.run(transport="http", host="0.0.0.0", port=8000, stateless_http=True)`. Stateless because a search tool needs no session state and this survives restarts/load-balancing behind Traefik. | | Traefik routing | ✅ Keep | Route `Host(\`docs-mcp.\`)` → container port. MCP-over-HTTP is plain HTTP from Traefik's point of view; nothing special needed beyond normal labels (SSE-friendly: no response buffering). | diff --git a/README.md b/README.md index 748caa0..1735593 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,11 @@ API — and exposes hybrid semantic search as MCP tools over streamable HTTP. ## Why self-docs -- **Local-first embeddings.** FastEmbed (`BAAI/bge-small-en-v1.5`) runs - in-process; documentation never leaves your network. +- **Local-first embeddings.** FastEmbed runs in-process on CPU (ONNX, no + GPU/torch); documentation never leaves your network. The model is selectable + from a registry (`config/models.yaml`) — `make configure` derives the vector + dimension and container memory limits from your choice. Default: + `mixedbread-ai/mxbai-embed-large-v1` (1024-dim). - **Hybrid retrieval.** Vector similarity + per-source-language Postgres full-text search over `pgvector`, so exact terms and semantic matches both surface. @@ -72,7 +75,7 @@ API — and exposes hybrid semantic search as MCP tools over streamable HTTP. | Layer | Technology | |-------|------------| | Store | PostgreSQL 16 + pgvector 0.8.2 | -| Embeddings | FastEmbed · `BAAI/bge-small-en-v1.5` | +| Embeddings | FastEmbed · `mixedbread-ai/mxbai-embed-large-v1` (default, selectable — see `config/models.yaml`) | | MCP server | FastMCP 3.x (streamable HTTP) | | Ingestion | FastAPI crawler + chunker + scheduler | | Ingress | Traefik (production overlay) | @@ -89,10 +92,19 @@ cron scheduler (`app.scheduler`) for automated re-crawling; see the ```bash cp .env.example .env # fill in real values +make configure # optional — pick an embedding model (see below) make up # db + ingestion (:8080) + mcp-server (:8081) make sync # trigger the initial documentation sync ``` +`make configure` is optional: with no `.env` overrides both services already use +the registry default. Run it to choose a different model — +`make configure MODEL=BAAI/bge-base-en-v1.5` — and it resolves that model's +vector dimension, query/passage prompts, and per-service memory limits into +`.env`, then re-renders `db/init/01_schema.sql`. Switching models on an existing +deployment requires a re-embed; see +[Runbook → switch the embedding model](docs/runbook.md#switch-the-embedding-model). + Point local MCP clients at `http://127.0.0.1:8081/mcp` (streamable HTTP). The server requires an `Authorization: Bearer ` header — see [Client Setup](docs/client-setup.md) for per-client configuration. @@ -152,6 +164,10 @@ make test # Run the retrieval-quality eval (requires a synced db) make eval + +# Lint and static type checks (also enforced in CI) +make lint +make typecheck ``` Backup and restore are available via `make backup`, `make backup-prune`, and @@ -160,4 +176,4 @@ Backup and restore are available via `make backup`, `make backup-prune`, and ## License -Private — not published. +Private — not published. All rights reserved; see [LICENSE](LICENSE). diff --git a/docs/adr/001-custom-pipeline-over-off-the-shelf.md b/docs/adr/001-custom-pipeline-over-off-the-shelf.md index df5cb74..e3fabb3 100644 --- a/docs/adr/001-custom-pipeline-over-off-the-shelf.md +++ b/docs/adr/001-custom-pipeline-over-off-the-shelf.md @@ -22,7 +22,7 @@ Several off-the-shelf RAG platforms were evaluated during the research phase: Build a custom pipeline using: - **PostgreSQL 16 + pgvector 0.8.2** for vector + full-text storage -- **FastEmbed (BAAI/bge-small-en-v1.5)** for CPU-friendly ONNX embeddings +- **FastEmbed (BAAI/bge-small-en-v1.5)** for CPU-friendly ONNX embeddings *(Note: the model is no longer fixed — superseded by [ADR-004](004-selectable-embedding-models.md), which makes it registry-selectable and moves the default to `mixedbread-ai/mxbai-embed-large-v1`. FastEmbed/ONNX/CPU is unchanged.)* - **FastMCP 3.x** for MCP-over-HTTP serving (streamable HTTP, stateless) - **Traefik** for reverse proxy / TLS termination / rate limiting - **n8n** for weekly sync scheduling and failure alerting *(Note: superseded in Phase 6 by an in-process scheduler inside `ingestion` to eliminate external dependencies and double-scheduling hazards)* diff --git a/docs/adr/004-selectable-embedding-models.md b/docs/adr/004-selectable-embedding-models.md new file mode 100644 index 0000000..ebe7c01 --- /dev/null +++ b/docs/adr/004-selectable-embedding-models.md @@ -0,0 +1,99 @@ +# ADR-004: Registry-Selectable Embedding Models + +**Status:** Accepted +**Date:** 2026-07-21 +**Decision makers:** Project owner + architect + +## Context + +[ADR-001](001-custom-pipeline-over-off-the-shelf.md) fixed the embedding model at +`BAAI/bge-small-en-v1.5` (384-dim), hardcoded as a constant in two places that +cannot import each other: `ingestion/app/embedder.py` and +`mcp-server/app/retrieval.py`. Three problems accumulated: + +1. **Retrieval quality was capped** by the smallest model in the BGE family. + `bge-small` was chosen for CPU friendliness, not accuracy. +2. **Changing the model was a multi-file manual edit** — the two constants, the + `vector(N)` column in `db/init/01_schema.sql`, both Dockerfiles' pre-bake + step, and the compose memory limits all had to be changed together and kept + consistent by hand. Nothing enforced agreement. +3. **The documented asymmetric-prefix contract was false.** ADR-001 and the code + comments asserted that FastEmbed's `passage_embed()`/`query_embed()` apply the + BGE `passage:`/`query:` prefixes. Inspection of FastEmbed 0.8 shows both + methods delegate straight to `embed()` for every non-multitask model + (bge, mxbai, e5) — **no prefix was ever applied**. Queries were being embedded + with no instruction prefix at all. + +Larger models also need more memory than the previous limits allowed: the +mcp-server was capped at 1G, which a 1024-dim model's ONNX weights (~1.3G) would +OOM on first query. + +## Decision + +Introduce **`config/models.yaml` as the single source of truth** mapping each +supported model to its `dim`, `mem_ingestion`, `mem_mcp`, `query_prompt`, and +`passage_prompt`, with one row marked as the default. + +- **`make configure MODEL=`** (`scripts/configure_model.py`) resolves the + selection into `.env` (`EMBEDDING_MODEL_NAME`, `EMBEDDING_DIM`, + `EMBEDDING_QUERY_PROMPT`, `EMBEDDING_PASSAGE_PROMPT`, `INGESTION_MEM_LIMIT`, + `MCP_MEM_LIMIT`) and renders `db/init/01_schema.sql` from a new + `01_schema.sql.template` so `vector(N)` matches the model. +- **docker-compose derives** memory limits and the image build arg from those + vars; both services read the model/prompts from env, falling back to `DEFAULT_*` + constants that equal the registry default row. +- **The default moves to `mixedbread-ai/mxbai-embed-large-v1`** (1024-dim), with + `intfloat/multilingual-e5-large`, `BAAI/bge-base-en-v1.5`, and the previous + `BAAI/bge-small-en-v1.5` also supported. +- **Prompts are applied manually** around plain `embed()`, replacing the + `passage_embed`/`query_embed` calls that were silently no-ops. +- **Parity tests** (`tests/test_model_registry.py`, + `mcp-server/tests/test_registry_defaults.py`) assert the committed schema equals + the rendered template, both services' defaults equal the registry default, and + the duplicated SSRF helper stays byte-identical across services. + +## Rationale + +**Why a registry instead of just swapping the constant.** The model choice has +four downstream consequences (vector width, memory, prompts, baked image layer) +that must move together. Encoding them in one table and deriving the rest makes +an inconsistent deployment structurally hard rather than merely discouraged — +the same reasoning as ADR-002's single-source schema handling. + +**Why mxbai-embed-large as the default.** It is the strongest English model +FastEmbed supports that still runs CPU-only ONNX with no torch dependency, +preserving ADR-001's local-first, GPU-free constraint. The cost is memory +(1G → 2G per service) and ~3× query-embed latency (~100–150 ms), which is noise +next to the LLM round-trip that consumes the results. + +**Why keep the small models in the registry.** CI selects +`BAAI/bge-small-en-v1.5` so the test suite does not download 1.2GB on every run — +which also continuously exercises the `make configure` path. Operators on +constrained hardware get the same escape hatch. + +**Why manual prompts rather than fixing FastEmbed usage.** The asymmetric API is +a no-op for these models, so there is nothing to fix upstream-side; explicit +prefixes from the registry are unambiguous and make the per-model differences +visible (mxbai/bge instruct on the query only; e5 prefixes both sides). + +## Consequences + +- **Changing models requires a full re-embed.** Vectors from different models are + not comparable, and content-hash change detection would otherwise skip + unchanged pages. `make reindex` truncates `doc_pages`/`doc_chunks` and re-syncs; + a dimension change additionally requires recreating the schema + (ADR-002's nuke-and-rebuild path) and rebuilding both images. +- **Existing deployments must reindex on upgrade** — 384-dim vectors are invalid + against a `vector(1024)` column. +- **Retrieval quality changed** in both directions of risk: queries now carry the + instruction prefix they never had, and the model is larger. Validate with + `make eval` before and after rather than assuming improvement. +- **The registry is a new sync point.** A model added there without a matching + FastEmbed-supported name or wrong `dim` fails at runtime; the parity tests cover + default consistency but cannot validate a model FastEmbed does not ship. + +## Related + +- [ADR-001](001-custom-pipeline-over-off-the-shelf.md) — original fixed-model decision, superseded on this point +- [ADR-002](002-nuke-and-rebuild-schema-evolution.md) — the rebuild path a dimension change depends on +- [Runbook → switch the embedding model](../runbook.md#switch-the-embedding-model) diff --git a/docs/runbook.md b/docs/runbook.md index 3775878..691550c 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -662,10 +662,13 @@ docker compose logs ingestion | grep -E '"event": "(page_index_failed|sync_sourc ## Troubleshooting -- **Is the embedding model available offline?** Yes — `BAAI/bge-small-en-v1.5` - is pre-downloaded into both the `ingestion` and `mcp-server` images at - build time (see each Dockerfile). No network access is needed at runtime - for embedding; a fresh container start does not re-download the model. +- **Is the embedding model available offline?** Yes — the configured model is + pre-downloaded into both the `ingestion` and `mcp-server` images at build time + via the `EMBEDDING_MODEL_NAME` build arg (see each Dockerfile), which + docker-compose feeds from `.env`. No network access is needed at runtime for + embedding; a fresh container start does not re-download the model. Note this + means **changing the model requires rebuilding both images** so the new + weights are baked in — see [Switch the embedding model](#switch-the-embedding-model). - **Reading logs.** Both services emit structured JSON lines to stdout via `structlog` (fields: `ts`, `level`, `service`, `event`, plus context like From c0a9edbc33c8e6138766b03771cf431337822359 Mon Sep 17 00:00:00 2001 From: ItayUliel Date: Tue, 21 Jul 2026 22:26:31 +0300 Subject: [PATCH 3/3] fix(tests): don't treat `make configure` as schema drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The schema parity test asserted db/init/01_schema.sql always equals the registry-DEFAULT render, but CI legitimately runs `make configure MODEL=BAAI/bge-small-en-v1.5` (to avoid pulling the 1.2GB default model on every run), which rewrites that file to vector(384). The test therefore contradicted CI's own setup step and failed. Split it into the two invariants that actually matter: - The working-tree schema must be a faithful render of the template at the ACTIVE dimension (EMBEDDING_DIM, which `make configure` writes to .env and the Makefile exports; falls back to the registry default). This still catches hand-editing — verified by temporarily editing the SQL and watching it fail — while tolerating a reconfigured checkout. - The schema COMMITTED TO GIT must be the default-model render, so a fresh clone gets the advertised default. Read via `git show HEAD:...` so a rewritten working copy is never mistaken for drift; skips if git is unavailable. Verified at both dimensions: parity suite passes with EMBEDDING_DIM=1024 and, after configuring bge-small, with EMBEDDING_DIM=384. Full e2e suite: 22 passed, 2 skipped. --- tests/test_model_registry.py | 57 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py index 9bc7576..363537f 100644 --- a/tests/test_model_registry.py +++ b/tests/test_model_registry.py @@ -10,6 +10,8 @@ import ast import importlib.util +import os +import subprocess from pathlib import Path import pytest @@ -54,18 +56,59 @@ def test_registry_rows_are_complete(name): assert "passage_prompt" in row -def test_committed_schema_matches_rendered_default(): - """db/init/01_schema.sql must equal render(template, default-model dim). If - this fails, run `make configure` (default model) to regenerate it.""" +def _active_dim() -> int: + """The embedding dimension this checkout is currently configured for. + + `make configure` writes EMBEDDING_DIM into .env and the Makefile exports it, + so a checkout reconfigured for a non-default model (CI does exactly this to + avoid downloading the 1.2GB default) reports that model's dimension here. + Falls back to the registry default when unset. + """ + raw = os.environ.get("EMBEDDING_DIM") + if raw: + return int(raw) + return REGISTRY["models"][REGISTRY["default"]]["dim"] + + +def test_schema_is_faithful_render_of_template(): + """db/init/01_schema.sql must be exactly render(template, active dim) — i.e. + generated, never hand-edited. The active dim follows `make configure`, so + this holds both for a default checkout and for one reconfigured to another + model. If it fails, re-run `make configure` instead of editing the SQL.""" configure_model = _load_configure_model() - default_dim = REGISTRY["models"][REGISTRY["default"]]["dim"] - expected = configure_model.render_schema_text(default_dim) + expected = configure_model.render_schema_text(_active_dim()) assert SCHEMA_FILE.read_text(encoding="utf-8") == expected -def test_committed_schema_vector_dim_matches_default(): +def test_schema_vector_dim_matches_active_model(): + assert f"vector({_active_dim()})" in SCHEMA_FILE.read_text(encoding="utf-8") + + +def test_git_committed_schema_matches_registry_default(): + """The schema COMMITTED TO GIT must be the default-model render, so a fresh + clone gets the advertised default without running `make configure`. + + Deliberately reads git HEAD rather than the working tree: `make configure` + legitimately rewrites the working copy (CI reconfigures to a small model), + and that must not be mistaken for drift. What must never happen is + *committing* a schema rendered for a non-default model. + """ + try: + committed = subprocess.run( + ["git", "show", "HEAD:db/init/01_schema.sql"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: # pragma: no cover + pytest.skip(f"git unavailable: {exc}") + if committed.returncode != 0: # pragma: no cover - shallow/exportless checkout + pytest.skip(f"cannot read schema from git HEAD: {committed.stderr.strip()}") + + configure_model = _load_configure_model() default_dim = REGISTRY["models"][REGISTRY["default"]]["dim"] - assert f"vector({default_dim})" in SCHEMA_FILE.read_text(encoding="utf-8") + assert committed.stdout == configure_model.render_schema_text(default_dim) def test_ingestion_embedder_defaults_match_registry_default():