Feat/mxbai embeddings and hardening

.env.example

@@ -28,3 +28,16 @@ SYNC_TOKEN=change-me
 # Required — the mcp-server refuses tool calls without this and requests
 # must send `Authorization: Bearer $MCP_TOKEN`.
 MCP_TOKEN=change-me
+
+# --- Embedding model (managed by `make configure`) ---
+# These are written/overwritten by `make configure MODEL=<name>`, which resolves
+# them from config/models.yaml (the single source of truth). You normally do NOT
+# edit them by hand — run `make configure` to switch models, then `make reindex`.
+# The values below are the registry default (mixedbread-ai/mxbai-embed-large-v1);
+# both services fall back to these defaults if the vars are unset.
+#   EMBEDDING_MODEL_NAME=mixedbread-ai/mxbai-embed-large-v1
+#   EMBEDDING_DIM=1024                 # MUST match db/init/01_schema.sql's vector(N)
+#   EMBEDDING_QUERY_PROMPT=Represent this sentence for searching relevant passages:
+#   EMBEDDING_PASSAGE_PROMPT=          # empty for mxbai/bge; "passage: " for e5
+#   INGESTION_MEM_LIMIT=2g             # docker-compose memory limit, sized per model
+#   MCP_MEM_LIMIT=2g

.github/workflows/test.yml

@@ -7,6 +7,19 @@ on:
     branches: [main]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Lint (ruff)
+        run: make lint
+      - name: Type-check (mypy)
+        run: make typecheck
+
   test:
     runs-on: ubuntu-latest  # or: self-hosted
 
@@ -48,6 +61,16 @@ jobs:
             sleep 1
           done
 
+      - name: Configure embedding model (small/fast for CI)
+        run: |
+          # Use the small 384-dim model in CI so the suite doesn't download the
+          # 1.2GB production default (mxbai-large) on every run. This also
+          # exercises `make configure`: it writes EMBEDDING_* into .env (which
+          # the Makefile exports to the test venvs) and re-renders 01_schema.sql
+          # to vector(384) so the seeded test vectors match the column width.
+          python -m pip install --quiet pyyaml
+          make configure MODEL=BAAI/bge-small-en-v1.5
+
       - name: Init database schema
         run: |
           # Apply both init files, matching scripts/migrate.sh's production

.gitignore

@@ -10,7 +10,11 @@ __pycache__/
 *.pyc
 *.egg-info/
 .venv/
+.tooling-venv/
 .pytest_cache/
+.mypy_cache/
+.coverage
+.ruff_cache/
 
 # OS cruft
 .DS_Store

IMPLEMENTATION_PLAN.md

@@ -17,7 +17,7 @@ holds, and what should be adjusted:
 | Original choice | Verdict | Notes |
 |---|---|---|
 | PostgreSQL 16 + pgvector | ✅ Keep, pin ≥0.8.2 | Pin the image to `pgvector/pgvector:0.8.2-pg16` (not the floating `pg16` tag): **CVE-2026-3172** (CVSS 8.1) is a buffer overflow in the parallel HNSW build affecting 0.6.0–0.8.1, fixed in 0.8.2. RDS-compatible as intended. |
-| FastEmbed + `BAAI/bge-small-en-v1.5` | ✅ Keep, with prefixes | 384-dim, CPU-friendly ONNX. **Must** use asymmetric prefixes: embed docs via `passage_embed()` and queries via `query_embed()` — FastEmbed applies the BGE `passage:`/`query:` prefixes for you. Skipping this measurably hurts recall. |
+| FastEmbed + `BAAI/bge-small-en-v1.5` | ✅ Keep, with prefixes | 384-dim, CPU-friendly ONNX. **Must** use asymmetric prefixes: embed docs via `passage_embed()` and queries via `query_embed()` — FastEmbed applies the BGE `passage:`/`query:` prefixes for you. Skipping this measurably hurts recall. **CORRECTION (ADR-004):** the claim that FastEmbed applies these prefixes is FALSE for FastEmbed 0.8 — `passage_embed`/`query_embed` delegate straight to `embed()` for every non-multitask model (bge, mxbai, e5), so no prefix was ever applied. Prompts are now applied manually from `config/models.yaml`, and the model is registry-selectable (default `mixedbread-ai/mxbai-embed-large-v1`, 1024-dim). |
 | BeautifulSoup crawler | ⚠️ Upgrade | Keep BS4 for link discovery, but add **`trafilatura`** for main-content extraction (strips nav/sidebar/footer boilerplate far better than hand-rolled selectors) and prefer **sitemap.xml** discovery over recursive crawling when available. |
 | FastMCP (Python) | ✅ Keep — target **3.x**, pin `fastmcp>=3,<4` | FastMCP 3.0 is GA; **Streamable HTTP** (single `/mcp` endpoint) is the recommended remote transport; legacy SSE is deprecated. Breaking change vs 2.x: `stateless_http` (and host/port) moved off the constructor onto `run()`/`http_app()` — `FastMCP('self-docs')` then `mcp.run(transport="http", host="0.0.0.0", port=8000, stateless_http=True)`. Stateless because a search tool needs no session state and this survives restarts/load-balancing behind Traefik. |
 | Traefik routing | ✅ Keep | Route `Host(\`docs-mcp.<lan-domain>\`)` → container port. MCP-over-HTTP is plain HTTP from Traefik's point of view; nothing special needed beyond normal labels (SSE-friendly: no response buffering). |

LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2026 Itay Uliel
+
+All rights reserved.
+
+This software and its source code are proprietary and confidential. No license,
+express or implied, is granted to use, copy, modify, merge, publish, distribute,
+sublicense, or sell copies of this software, in whole or in part, without the
+prior written permission of the copyright holder.
+
+Unauthorized copying, distribution, or use of this software, via any medium, is
+strictly prohibited.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -1,7 +1,25 @@
 -include .env
 export
 
-.PHONY: up down up-prod down-prod sync test eval backup backup-prune backup-auto restore
+.PHONY: up down up-prod down-prod sync test eval lint typecheck configure reindex backup backup-prune backup-auto restore
+
+# Select the embedding model from config/models.yaml. Resolves the model's
+# vector dimension and per-service memory limits, writes them into .env, and
+# renders db/init/01_schema.sql. No MODEL => the registry default.
+# Usage: make configure                              (default model)
+#        make configure MODEL=BAAI/bge-base-en-v1.5  (a specific model)
+configure:
+	python3 scripts/configure_model.py "$(MODEL)"
+
+# Re-embed the entire corpus with the currently-configured model. Required
+# after `make configure` changes the model (content-hash change-detection would
+# otherwise skip unchanged pages and leave stale/mismatched vectors). Truncates
+# the crawled pages/chunks (NOT doc_sources) then triggers a fresh sync.
+reindex:
+	@echo "Truncating doc_pages/doc_chunks (sources preserved) then re-syncing..."
+	docker compose exec -T db psql -U $${POSTGRES_USER} -d $${POSTGRES_DB} \
+		-c "TRUNCATE doc_pages, doc_chunks RESTART IDENTITY CASCADE;"
+	$(MAKE) sync
 
 # Bring up the full stack locally (db + ingestion + mcp-server) using loopback ports.
 up:
@@ -54,11 +72,13 @@ test:
 	@test -d mcp-server/.venv || python3 -m venv mcp-server/.venv
 	@mcp-server/.venv/bin/pip install -q -U pip
 	@mcp-server/.venv/bin/pip install -q -e mcp-server
-	@mcp-server/.venv/bin/pip install -q pytest
+	@mcp-server/.venv/bin/pip install -q pytest pyyaml defusedxml
+	@ingestion/.venv/bin/pip install -q pytest-cov
+	@mcp-server/.venv/bin/pip install -q pytest-cov
 	@echo "=== ingestion test suite ==="
-	cd ingestion && ../ingestion/.venv/bin/pytest -q
+	cd ingestion && ../ingestion/.venv/bin/pytest -q --cov=app --cov-report=term-missing:skip-covered
 	@echo "=== mcp-server test suite ==="
-	cd mcp-server && ../mcp-server/.venv/bin/pytest -q
+	cd mcp-server && ../mcp-server/.venv/bin/pytest -q --cov=app --cov-report=term-missing:skip-covered
 	@echo "=== e2e (cross-package) test suite ==="
 	cd tests && ../ingestion/.venv/bin/python -m pytest -q
 	@echo "make test: all suites green (DB-dependent tests skip cleanly if 'docker compose up -d db' wasn't run first)."
@@ -75,6 +95,29 @@ eval:
 	@echo "=== retrieval quality eval ==="
 	cd tests/eval && ../../mcp-server/.venv/bin/python -m pytest -q -m eval
 
+# Tooling venv for lint/typecheck (ruff + mypy). Kept separate from the two
+# package venvs; ruff is a standalone binary, mypy runs with
+# --ignore-missing-imports so it needn't install every runtime dependency.
+TOOLS_VENV = .tooling-venv
+$(TOOLS_VENV):
+	python3 -m venv $(TOOLS_VENV)
+	@$(TOOLS_VENV)/bin/pip install -q -U pip ruff mypy
+
+# Lint across both packages, scripts, and tests. (Formatting is available via
+# `ruff format` but intentionally NOT gated — this codebase uses deliberate
+# hand-alignment in its long explanatory comments/tables.)
+lint: $(TOOLS_VENV)
+	$(TOOLS_VENV)/bin/ruff check .
+
+# Static type-check the application code and scripts. Each package is checked
+# separately (both use a top-level `app` package, so a single invocation would
+# see two modules named `app`). mypy.ini quarantines the pre-existing typing
+# backlog so the gate enforces types on new/changed code.
+typecheck: $(TOOLS_VENV)
+	cd ingestion && MYPYPATH=. ../$(TOOLS_VENV)/bin/mypy --config-file ../mypy.ini app
+	cd mcp-server && MYPYPATH=. ../$(TOOLS_VENV)/bin/mypy --config-file ../mypy.ini app
+	$(TOOLS_VENV)/bin/mypy --config-file mypy.ini scripts
+
 # Dump the docs database to a timestamped custom-format archive under ./backups.
 backup:
 	mkdir -p backups

README.md

@@ -39,8 +39,11 @@ API — and exposes hybrid semantic search as MCP tools over streamable HTTP.
 
 ## Why self-docs
 
-- **Local-first embeddings.** FastEmbed (`BAAI/bge-small-en-v1.5`) runs
-  in-process; documentation never leaves your network.
+- **Local-first embeddings.** FastEmbed runs in-process on CPU (ONNX, no
+  GPU/torch); documentation never leaves your network. The model is selectable
+  from a registry (`config/models.yaml`) — `make configure` derives the vector
+  dimension and container memory limits from your choice. Default:
+  `mixedbread-ai/mxbai-embed-large-v1` (1024-dim).
 - **Hybrid retrieval.** Vector similarity + per-source-language Postgres
   full-text search over `pgvector`, so exact terms and semantic matches both
   surface.
@@ -72,7 +75,7 @@ API — and exposes hybrid semantic search as MCP tools over streamable HTTP.
 | Layer | Technology |
 |-------|------------|
 | Store | PostgreSQL 16 + pgvector 0.8.2 |
-| Embeddings | FastEmbed · `BAAI/bge-small-en-v1.5` |
+| Embeddings | FastEmbed · `mixedbread-ai/mxbai-embed-large-v1` (default, selectable — see `config/models.yaml`) |
 | MCP server | FastMCP 3.x (streamable HTTP) |
 | Ingestion | FastAPI crawler + chunker + scheduler |
 | Ingress | Traefik (production overlay) |
@@ -89,10 +92,19 @@ cron scheduler (`app.scheduler`) for automated re-crawling; see the
 
 ```bash
 cp .env.example .env        # fill in real values
+make configure              # optional — pick an embedding model (see below)
 make up                     # db + ingestion (:8080) + mcp-server (:8081)
 make sync                   # trigger the initial documentation sync
 ```
 
+`make configure` is optional: with no `.env` overrides both services already use
+the registry default. Run it to choose a different model —
+`make configure MODEL=BAAI/bge-base-en-v1.5` — and it resolves that model's
+vector dimension, query/passage prompts, and per-service memory limits into
+`.env`, then re-renders `db/init/01_schema.sql`. Switching models on an existing
+deployment requires a re-embed; see
+[Runbook → switch the embedding model](docs/runbook.md#switch-the-embedding-model).
+
 Point local MCP clients at `http://127.0.0.1:8081/mcp` (streamable HTTP). The
 server requires an `Authorization: Bearer <MCP_TOKEN>` header — see
 [Client Setup](docs/client-setup.md) for per-client configuration.
@@ -152,6 +164,10 @@ make test
 
 # Run the retrieval-quality eval (requires a synced db)
 make eval
+
+# Lint and static type checks (also enforced in CI)
+make lint
+make typecheck
 ```
 
 Backup and restore are available via `make backup`, `make backup-prune`, and
@@ -160,4 +176,4 @@ Backup and restore are available via `make backup`, `make backup-prune`, and
 
 ## License
 
-Private — not published.
+Private — not published. All rights reserved; see [LICENSE](LICENSE).

config/models.yaml

@@ -0,0 +1,49 @@
+# Embedding model registry — SINGLE SOURCE OF TRUTH.
+#
+# Consumed by scripts/configure_model.py (`make configure MODEL=<name>`), which
+# resolves the selected row into .env vars and renders db/init/01_schema.sql's
+# vector(N) dimension. The two services (ingestion, mcp-server) never read this
+# file at runtime — they consume the resolved env vars, defaulting to the row
+# marked `default: true` below so a fresh clone / CI works without configuring.
+#
+# Every model here is verified FastEmbed-supported (ONNX, CPU, no torch/GPU).
+# `dim` MUST equal FastEmbed's reported dimension for the model (it drives the
+# pgvector column width). `mem_ingestion` / `mem_mcp` are docker-compose memory
+# limits sized for the model's ONNX weights plus runtime headroom.
+#
+# Prompting: FastEmbed's query_embed/passage_embed apply NO prefix for these
+# (non-multitask) models, so the services apply these prompts MANUALLY around
+# plain embed(). Leave a prompt empty ("") when the model wants no prefix on
+# that side. bge/mxbai want an instruction on the QUERY only; e5 wants a prefix
+# on BOTH sides.
+
+default: mixedbread-ai/mxbai-embed-large-v1
+
+models:
+  mixedbread-ai/mxbai-embed-large-v1:
+    dim: 1024
+    mem_ingestion: "2g"
+    mem_mcp: "2g"
+    query_prompt: "Represent this sentence for searching relevant passages: "
+    passage_prompt: ""
+
+  intfloat/multilingual-e5-large:
+    dim: 1024
+    mem_ingestion: "2g"
+    mem_mcp: "2g"
+    query_prompt: "query: "
+    passage_prompt: "passage: "
+
+  BAAI/bge-base-en-v1.5:
+    dim: 768
+    mem_ingestion: "1500m"
+    mem_mcp: "1g"
+    query_prompt: "Represent this sentence for searching relevant passages: "
+    passage_prompt: ""
+
+  BAAI/bge-small-en-v1.5:
+    dim: 384
+    mem_ingestion: "1500m"
+    mem_mcp: "1g"
+    query_prompt: "Represent this sentence for searching relevant passages: "
+    passage_prompt: ""

db/init/01_schema.sql

@@ -1,3 +1,6 @@
+-- GENERATED from db/init/01_schema.sql.template by scripts/configure_model.py.
+-- Do not edit by hand — run `make configure MODEL=<name>` to change the vector
+-- dimension. Rendered for embedding dimension 1024.
 CREATE EXTENSION IF NOT EXISTS vector;
 
 CREATE TABLE doc_sources (
@@ -28,7 +31,7 @@ CREATE TABLE doc_chunks (
     heading_path  TEXT,                      -- "Guide > Routing > Dynamic Routes"
     chunk_index   INT NOT NULL,
     content       TEXT NOT NULL,             -- markdown
-    embedding     vector(384) NOT NULL,
+    embedding     vector(1024) NOT NULL,
     fts_config    regconfig NOT NULL DEFAULT 'english',
     fts           tsvector GENERATED ALWAYS AS (to_tsvector(fts_config, content)) STORED
 );

db/init/01_schema.sql.template

@@ -0,0 +1,46 @@
+-- TEMPLATE — do not edit db/init/01_schema.sql by hand.
+--
+-- scripts/configure_model.py renders this file into db/init/01_schema.sql,
+-- substituting __EMBEDDING_DIM__ with the selected model's vector dimension
+-- (see config/models.yaml). The committed 01_schema.sql is the rendering for
+-- the registry's default model; `make configure MODEL=<name>` re-renders it.
+-- A parity test (tests/test_model_registry.py) fails CI if the two drift.
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE doc_sources (
+    id                 SERIAL PRIMARY KEY,
+    name               TEXT NOT NULL UNIQUE,      -- e.g. "nextjs", matches sources.yaml key
+    base_url           TEXT NOT NULL,
+    last_synced        TIMESTAMPTZ,
+    last_status        TEXT,                      -- ok | partial | failed
+    llms_txt           TEXT NOT NULL DEFAULT 'auto'
+                            CHECK (llms_txt IN ('auto', 'off', 'only')),
+    llms_etag          TEXT,
+    llms_last_modified TEXT
+);
+
+CREATE TABLE doc_pages (
+    id            SERIAL PRIMARY KEY,
+    source_id     INT NOT NULL REFERENCES doc_sources(id) ON DELETE CASCADE,
+    url           TEXT NOT NULL UNIQUE,
+    content_hash  CHAR(64) NOT NULL,         -- SHA-256 of extracted markdown
+    fetched_at    TIMESTAMPTZ NOT NULL DEFAULT now(),
+    etag          TEXT,
+    last_modified TEXT
+);
+
+CREATE TABLE doc_chunks (
+    id            BIGSERIAL PRIMARY KEY,
+    page_id       INT NOT NULL REFERENCES doc_pages(id) ON DELETE CASCADE,
+    heading_path  TEXT,                      -- "Guide > Routing > Dynamic Routes"
+    chunk_index   INT NOT NULL,
+    content       TEXT NOT NULL,             -- markdown
+    embedding     vector(__EMBEDDING_DIM__) NOT NULL,
+    fts_config    regconfig NOT NULL DEFAULT 'english',
+    fts           tsvector GENERATED ALWAYS AS (to_tsvector(fts_config, content)) STORED
+);
+
+CREATE INDEX doc_chunks_embedding_idx ON doc_chunks
+    USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
+CREATE INDEX doc_chunks_fts_idx ON doc_chunks USING gin (fts);
+CREATE INDEX doc_chunks_page_idx ON doc_chunks (page_id);