diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index ecffaa54..a4638803 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -40,6 +40,8 @@ services: - SPRING_DATASOURCE_PASSWORD=reports_password - OLLAMA_BASE_URL=http://ollama:11434 - FEEDBACK_SERVICE_URL=http://feedback-service:8080 + volumes: + - py_genai_helper_data:/code/vector-store expose: - 5000 labels: @@ -512,6 +514,7 @@ volumes: prometheus_data: grafana_data: loki_data: + py_genai_helper_data: networks: proxy: diff --git a/infra/helm/team-devoops/templates/deployment.yaml b/infra/helm/team-devoops/templates/deployment.yaml index 2ecb93eb..5d66f64c 100644 --- a/infra/helm/team-devoops/templates/deployment.yaml +++ b/infra/helm/team-devoops/templates/deployment.yaml @@ -19,6 +19,10 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" $name) | nindent 8 }} spec: + {{- if $svc.persistence }} + securityContext: + fsGroup: {{ $svc.persistence.fsGroup }} + {{- end }} {{- with $root.Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -29,6 +33,11 @@ spec: imagePullPolicy: {{ $root.Values.global.imagePullPolicy }} ports: - containerPort: {{ $svc.port }} + {{- if $svc.persistence }} + volumeMounts: + - name: data + mountPath: {{ $svc.persistence.mountPath }} + {{- end }} {{- $env := $svc.env | default dict }} {{- if $env }} env: @@ -82,5 +91,11 @@ spec: {{- end }} resources: {{- toYaml ($svc.resources | default $root.Values.resources) | nindent 12 }} + {{- if $svc.persistence }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ $name }}-data + {{- end }} --- {{- end }} diff --git a/infra/helm/team-devoops/templates/pvc.yaml b/infra/helm/team-devoops/templates/pvc.yaml new file mode 100644 index 00000000..703d30f6 --- /dev/null +++ b/infra/helm/team-devoops/templates/pvc.yaml @@ -0,0 +1,19 @@ +{{- range $name, $svc := .Values.services }} +{{- if $svc.persistence }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ $name }}-data + labels: + {{- include "team-devoops.labels" (dict "name" $name "root" $) | nindent 4 }} +spec: + accessModes: ["ReadWriteOnce"] + {{- if $.Values.global.storageClassName }} + storageClassName: {{ $.Values.global.storageClassName }} + {{- end }} + resources: + requests: + storage: {{ $svc.persistence.size }} +--- +{{- end }} +{{- end }} diff --git a/infra/helm/team-devoops/values.yaml b/infra/helm/team-devoops/values.yaml index f9152eb3..1262e910 100644 --- a/infra/helm/team-devoops/values.yaml +++ b/infra/helm/team-devoops/values.yaml @@ -100,11 +100,14 @@ monitoring: requests: cpu: 50m memory: 128Mi - # Actual usage is ~166Mi in practice (higher than first estimated) -- the earlier - # 200Mi trim left almost no headroom, so it's bumped back up here. + # Documented peak usage is ~166Mi (the earlier 200Mi trim against that left "almost no + # headroom" and had to be reverted -- see history). Live `kubectl top` right now shows only + # ~70Mi, but that's an idle snapshot, not the peak. Trimmed from 280Mi to 240Mi: still ~74Mi + # above the documented 166Mi peak, clearly more headroom than the rejected 200Mi/34Mi attempt, + # while freeing some of the namespace's shared 6Gi quota for py-genai-helper. limits: cpu: 100m - memory: 280Mi + memory: 240Mi # Same out-of-band ConfigMap treatment as prometheus above (infra/loki/loki-config.yaml # is the same file docker-compose uses). loki: @@ -115,10 +118,13 @@ monitoring: requests: cpu: 50m memory: 128Mi - # Actual usage is ~153Mi; trimmed to make room for ollama (see prometheus above). + # Documented peak usage is ~153Mi (previously trimmed to make room for ollama, see prometheus + # above). Live `kubectl top` right now shows only ~65Mi, but that's an idle snapshot, not the + # peak. Trimmed from 220Mi to 200Mi: keeps ~47Mi above the documented peak -- more than + # grafana's rejected 34Mi-headroom attempt above, so deliberately not pushed any lower. limits: cpu: 150m - memory: 220Mi + memory: 200Mi # Unlike loki/prometheus, alloy-config.alloy is Kubernetes-specific (loki.source.kubernetes, # not loki.source.docker) so it isn't shared with docker-compose -- it lives in this # chart's files/ and is embedded via .Files.Get like the Keycloak realm config, not @@ -260,6 +266,13 @@ resources: # image/tag -> optional: override the default ghcr image/tag for this service # env -> extra literal environment variables # envFromSecret -> inject all keys of an existing Secret as env +# persistence -> optional: mount a PVC (RWO) into the container. +# mountPath, size (PVC storage request), fsGroup (pod-level +# securityContext.fsGroup, needed when the image runs as +# non-root -- see loki/grafana/prometheus deployments for the +# same reasoning). Only safe with a single replica and a +# rolling strategy that fully terminates the old pod first +# (see .Values.strategy below), which is the case here. # --------------------------------------------------------------------------- services: organization-service: @@ -356,15 +369,32 @@ services: KEYCLOAK_ISSUER_URL: "https://ge83mom-devops26.stud.k8s.aet.cit.tum.de/auth/realms/devops" KEYCLOAK_JWKS_URL: "http://keycloak:8080/auth/realms/devops/protocol/openid-connect/certs" OLLAMA_BASE_URL: "http://ollama:11434" + # Persists the Chroma vector store (see services/py-genai-helper/rag.py) across pod + # restarts/reschedules. fsGroup 1000 matches the Dockerfile's `adduser` app user + # (confirmed uid/gid 1000 on python:3.12.9-slim-bookworm), which the CSI-provisioned + # PVC would otherwise mount root-owned and unwritable to that user. + persistence: + mountPath: /code/vector-store + size: 1Gi + fsGroup: 1000 resources: requests: cpu: 100m - memory: 256Mi - # Actual usage is ~135Mi; trimmed a bit (still ~3x actual) to make room for ollama - # (see monitoring.prometheus above). + memory: 350Mi + # The Chroma/onnxruntime RAG stack (see rag.py) alone -- before Flask/gunicorn/SQLAlchemy/auth + # are even loaded -- profiled at ~205Mi during a cold index build in a local container run, so + # the old 400Mi (sized for FAISS-era ~135Mi actual usage) is no longer enough headroom. Can't + # just raise it freely though: the namespace's ResourceQuota caps limits.memory at 6Gi total, + # and every other service's limit already commits ~5650Mi of that (verified live via + # `kubectl get resourcequota` + summing pod limits), leaving at most ~494Mi of quota room for + # this service -- a first attempt at 512Mi exceeded it, so the rollout couldn't schedule a + # single pod, hit progressDeadlineSeconds, and got auto-rolled-back by CD. 450Mi fits with a + # little quota slack left; re-check actual usage with `kubectl top` after this deploys, and if + # it needs to go higher, another service's limit has to come down first (same trimming pattern + # already used for prometheus/loki/grafana/alloy above). limits: cpu: 150m - memory: 400Mi + memory: 450Mi web-client: path: / port: 8080 diff --git a/services/py-genai-helper/.dockerignore b/services/py-genai-helper/.dockerignore index 1badf5e7..752da5e5 100644 --- a/services/py-genai-helper/.dockerignore +++ b/services/py-genai-helper/.dockerignore @@ -6,6 +6,7 @@ __pycache__/ .env .venv/ venv/ +vector-store/ *.egg-info/ .git/ .gitignore diff --git a/services/py-genai-helper/.gitignore b/services/py-genai-helper/.gitignore index 380124f4..a7b99855 100644 --- a/services/py-genai-helper/.gitignore +++ b/services/py-genai-helper/.gitignore @@ -12,3 +12,4 @@ __pycache__/ instance/ *.log include/ +vector-store/ diff --git a/services/py-genai-helper/Dockerfile b/services/py-genai-helper/Dockerfile index 6a688b97..6f9992d9 100644 --- a/services/py-genai-helper/Dockerfile +++ b/services/py-genai-helper/Dockerfile @@ -1,20 +1,22 @@ # Stage 1: builder -FROM python:3.12.9-alpine3.21 AS builder +FROM python:3.12.9-slim-bookworm AS builder WORKDIR /code -# Installs gcc and other dependencies (postgresql-dev provides libpq + pg_config for psycopg2) -RUN apk add --no-cache gcc musl-dev linux-headers postgresql-dev +# Installs gcc and other dependencies (libpq-dev provides libpq + pg_config for psycopg2) +RUN apt-get update && apt-get install --no-install-recommends -y gcc libpq-dev \ + && rm -rf /var/lib/apt/lists/* # Install dependencies COPY requirements.txt requirements.txt RUN pip install --no-cache-dir --prefix=/install -r requirements.txt # Stage 2: run -FROM python:3.12.9-alpine3.21 +FROM python:3.12.9-slim-bookworm WORKDIR /code -# Runtime shared library for psycopg2 (libpq) -RUN apk add --no-cache libpq +# Runtime shared library for psycopg2 (libpq) and wget for the healthcheck +RUN apt-get update && apt-get install --no-install-recommends -y libpq5 wget \ + && rm -rf /var/lib/apt/lists/* # Copy installed python packages from builder stage COPY --from=builder /install /usr/local @@ -23,7 +25,10 @@ COPY --from=builder /install /usr/local COPY . . # Switch user for increased security -RUN adduser -D user +RUN adduser --disabled-password --gecos "" user +# The persisted vector store (see rag.py) is written at runtime, so the app user needs write +# access to it; everything else stays read-only to that user. +RUN mkdir -p vector-store && chown -R user:user vector-store USER user # Set environment variables diff --git a/services/py-genai-helper/README.md b/services/py-genai-helper/README.md index 7c97f8eb..b130510b 100644 --- a/services/py-genai-helper/README.md +++ b/services/py-genai-helper/README.md @@ -73,3 +73,13 @@ feedback service's own visibility rules decide which entries feed the report. Th (OpenAI or Ollama, per `LLM_PROVIDER`/`uselocal` — see Local LLM above) writes the report text, which is then persisted to the `reports` schema. Configure the feedback service location via `FEEDBACK_SERVICE_URL` (default `http://feedback-service:8080`). + +## Knowledge base (RAG) + +The PDFs in `file-storage/` are chunked, embedded, and indexed with Chroma (see `rag.py`) so report +prompts can be augmented with relevant excerpts. The collection is persisted to `vector-store//` +(one subdirectory per embedding provider, since an OpenAI-embedded collection isn't valid to query with +Ollama embeddings and vice versa) so the embedding step — a real API/model cost — only happens once +per provider rather than on every process/worker start. A manifest of the source PDFs' filenames and +modification times is stored alongside the collection; if `file-storage/` changes, the manifest no longer +matches and the collection is rebuilt automatically. diff --git a/services/py-genai-helper/file-storage/Sportlerernaehrung.pdf b/services/py-genai-helper/file-storage/Sportlerernaehrung.pdf new file mode 100644 index 00000000..cc6829f2 Binary files /dev/null and b/services/py-genai-helper/file-storage/Sportlerernaehrung.pdf differ diff --git a/services/py-genai-helper/file-storage/Trainingsuebungen_fuer_ein_starkes_Handballteam.pdf b/services/py-genai-helper/file-storage/Trainingsuebungen_fuer_ein_starkes_Handballteam.pdf new file mode 100644 index 00000000..86d44d4c Binary files /dev/null and b/services/py-genai-helper/file-storage/Trainingsuebungen_fuer_ein_starkes_Handballteam.pdf differ diff --git a/services/py-genai-helper/file-storage/ernaehrung_fuer_sporttreibende.pdf b/services/py-genai-helper/file-storage/ernaehrung_fuer_sporttreibende.pdf new file mode 100644 index 00000000..53f5516c Binary files /dev/null and b/services/py-genai-helper/file-storage/ernaehrung_fuer_sporttreibende.pdf differ diff --git a/services/py-genai-helper/file-storage/faust.pdf b/services/py-genai-helper/file-storage/faust.pdf deleted file mode 100644 index 9f74428c..00000000 Binary files a/services/py-genai-helper/file-storage/faust.pdf and /dev/null differ diff --git a/services/py-genai-helper/rag.py b/services/py-genai-helper/rag.py index a85492a7..f0760b4a 100644 --- a/services/py-genai-helper/rag.py +++ b/services/py-genai-helper/rag.py @@ -1,9 +1,13 @@ +import json from functools import lru_cache from pathlib import Path +import chromadb from dotenv import load_dotenv +from langchain_chroma import Chroma from langchain_community.document_loaders import PyPDFLoader -from langchain_community.vectorstores import FAISS +from langchain_core.documents import Document +from langchain_core.embeddings import Embeddings from langchain_text_splitters import RecursiveCharacterTextSplitter from prometheus_client import Counter, Histogram @@ -15,35 +19,91 @@ RAG_QUERY_DURATION = Histogram("genai_rag_query_duration_seconds", "RAG query duration in seconds", ["provider"]) _FILE_STORAGE = Path(__file__).parent / "file-storage" +# Persisted Chroma collections, one subdirectory per embedding provider (see _load_pdfs). Embedding +# a PDF corpus costs real API calls/time, so the index is written to disk once and reloaded by every +# later process (e.g. every gunicorn worker) instead of re-embedding on each of them. +_VECTOR_STORE = Path(__file__).parent / "vector-store" +_MANIFEST_NAME = "manifest.json" -def _load_pdfs(use_local: bool) -> FAISS | None: +def _manifest(pdf_files: list[Path], model: str) -> dict[str, int | str]: + """Fingerprint of the PDFs and embedding model an index was built from, to detect a stale + on-disk index (also catches an EMBEDDING_MODEL change, not just a file-storage/ change -- + otherwise a differently-dimensioned model would load a collection it can't query).""" + manifest: dict[str, int | str] = {path.name: path.stat().st_mtime_ns for path in pdf_files} + manifest["_model"] = model + return manifest + + +def _load_persisted(provider: str, manifest: dict[str, int | str], embeddings: Embeddings) -> Chroma | None: + """Load the on-disk collection for `provider`, or None if missing/stale/unreadable.""" + store_dir = _VECTOR_STORE / provider + manifest_file = store_dir / _MANIFEST_NAME + if not manifest_file.exists(): + return None + try: + if json.loads(manifest_file.read_text()) != manifest: + return None + client = chromadb.PersistentClient(path=str(store_dir)) + return Chroma(collection_name=provider, embedding_function=embeddings, client=client) + except Exception: + # Corrupt or incompatible on-disk collection (e.g. written by a different Chroma/langchain + # version) — fall back to rebuilding rather than taking the service down. + return None + + +def _persist(provider: str, manifest: dict[str, int | str], docs: list[Document], embeddings: Embeddings) -> Chroma: + store_dir = _VECTOR_STORE / provider + store_dir.mkdir(parents=True, exist_ok=True) + # chromadb caches one PersistentClient per path in-process, so deleting/recreating store_dir + # out from under an already-cached client leaves it holding handles to the removed files + # (surfacing as "attempt to write a readonly database"). Go through the same cached client to + # drop the old collection instead, so a stale manifest (file-storage/ changed) rebuilds cleanly. + client = chromadb.PersistentClient(path=str(store_dir)) + try: + client.delete_collection(provider) + except Exception: + pass # first build for this provider — nothing to delete yet + vector_store = Chroma.from_documents(docs, embedding=embeddings, collection_name=provider, client=client) + (store_dir / _MANIFEST_NAME).write_text(json.dumps(manifest)) + return vector_store + + +def _load_pdfs(use_local: bool) -> Chroma | None: pdf_files = list(_FILE_STORAGE.glob("*.pdf")) if not pdf_files: return None + provider = "ollama" if use_local else "openai" + embeddings = get_embeddings(use_local) + manifest = _manifest(pdf_files, embeddings.model) + + vector_store = _load_persisted(provider, manifest, embeddings) + if vector_store is not None: + return vector_store + splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) docs = [] for path in pdf_files: loader = PyPDFLoader(str(path)) docs.extend(loader.load_and_split(splitter)) - return FAISS.from_documents(docs, embedding=get_embeddings(use_local)) + return _persist(provider, manifest, docs, embeddings) @lru_cache(maxsize=2) -def _get_vector_store(use_local: bool) -> FAISS | None: - """Build (once per provider) and cache the FAISS index over the PDFs in file-storage/. +def _get_vector_store(use_local: bool) -> Chroma | None: + """Build (once per provider) and cache the Chroma collection over the PDFs in file-storage/. - Cached per provider rather than globally: an index built from OpenAI embeddings isn't valid to - query with Ollama embeddings (and vice versa) — different model, different vector space. + Cached per provider rather than globally: a collection built from OpenAI embeddings isn't valid + to query with Ollama embeddings (and vice versa) — different model, different vector space. Deferred until first use rather than built at import time, so importing this module (directly, or transitively via ``reports``) doesn't require API credentials or make network calls. """ return _load_pdfs(use_local) -def retrieve_context(query: str, use_local: bool | None = None, k: int = 3) -> list[str]: +def retrieve_context(query: str, use_local: bool | None = None, k: int = 5) -> list[str]: """Return the text of the k knowledge-base chunks most relevant to the query. Returns an empty list if no PDFs are configured in file-storage/. ``use_local`` selects the diff --git a/services/py-genai-helper/requirements.txt b/services/py-genai-helper/requirements.txt index 3aa415e8..0dc28d3f 100644 --- a/services/py-genai-helper/requirements.txt +++ b/services/py-genai-helper/requirements.txt @@ -1,31 +1,48 @@ aiohappyeyeballs==2.6.2 aiohttp==3.13.5 aiosignal==1.4.0 +annotated-doc==0.0.4 annotated-types==0.7.0 anyio==4.13.0 attrs==26.1.0 +bcrypt==5.0.0 blinker==1.9.0 +build==1.5.0 certifi==2026.4.22 charset-normalizer==3.4.7 -click==8.3.3 +chromadb==1.5.9 +click==8.4.2 dataclasses-json==0.6.7 distro==1.9.0 -faiss-cpu==1.13.2 +durationpy==0.10 +filelock==3.29.6 Flask==3.1.3 +flatbuffers==25.12.19 frozenlist==1.8.0 +fsspec==2026.6.0 +googleapis-common-protos==1.75.0 greenlet==3.5.1 +grpcio==1.82.0 gunicorn==26.0.0 h11==0.16.0 +hf-xet==1.5.1 httpcore==1.0.9 +httptools==0.8.0 httpx==0.28.1 httpx-sse==0.4.3 +huggingface_hub==1.22.0 idna==3.14 +importlib_resources==7.1.0 itsdangerous==2.2.0 Jinja2==3.1.6 jiter==0.14.0 jsonpatch==1.33 jsonpointer==3.1.1 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +kubernetes==36.0.2 langchain==1.2.18 +langchain-chroma==1.1.0 langchain-classic==1.0.7 langchain-community==0.4.1 langchain-core==1.4.0 @@ -38,40 +55,71 @@ langgraph-checkpoint==4.1.0 langgraph-prebuilt==1.0.13 langgraph-sdk==0.3.14 langsmith==0.8.3 +markdown-it-py==4.2.0 MarkupSafe==3.0.3 marshmallow==3.26.2 +mdurl==0.1.2 +mmh3==5.2.1 multidict==6.7.1 mypy_extensions==1.1.0 numpy==2.4.6 +oauthlib==3.3.1 ollama==0.6.2 +onnxruntime==1.27.0 openai==2.36.0 +opentelemetry-api==1.43.0 +opentelemetry-exporter-otlp-proto-common==1.43.0 +opentelemetry-exporter-otlp-proto-grpc==1.43.0 +opentelemetry-proto==1.43.0 +opentelemetry-sdk==1.43.0 +opentelemetry-semantic-conventions==0.64b0 orjson==3.11.9 ormsgpack==1.12.2 +overrides==7.7.0 packaging==26.2 prometheus-client==0.21.1 prometheus-flask-exporter==0.23.1 propcache==0.5.2 +protobuf==7.35.1 psycopg2-binary==2.9.10 +pybase64==1.4.3 pydantic==2.13.4 pydantic-settings==2.14.1 pydantic_core==2.46.4 +Pygments==2.20.0 +PyJWT[crypto]==2.10.1 pypdf==5.6.0 +PyPika==0.51.1 +pyproject_hooks==1.2.0 +python-dateutil==2.9.0.post0 python-dotenv==1.2.2 -PyJWT[crypto]==2.10.1 PyYAML==6.0.3 +referencing==0.37.0 regex==2026.5.9 requests==2.34.0 +requests-oauthlib==2.0.0 requests-toolbelt==1.0.0 +rich==15.0.0 +rpds-py==2026.6.3 +shellingham==1.5.4 +six==1.17.0 sniffio==1.3.1 SQLAlchemy==2.0.49 tenacity==9.1.4 tiktoken==0.12.0 +tokenizers==0.23.1 tqdm==4.67.3 +typer==0.26.8 typing-inspect==0.9.0 typing-inspection==0.4.2 typing_extensions==4.15.0 urllib3==2.7.0 uuid_utils==0.15.0 +uvicorn==0.50.2 +uvloop==0.22.1 +watchfiles==1.2.0 +websocket-client==1.9.0 +websockets==16.0 Werkzeug==3.1.8 xxhash==3.7.0 yarl==1.24.2 diff --git a/services/py-genai-helper/tests/test_rag.py b/services/py-genai-helper/tests/test_rag.py index 50366d8b..27a43d32 100644 --- a/services/py-genai-helper/tests/test_rag.py +++ b/services/py-genai-helper/tests/test_rag.py @@ -1,10 +1,39 @@ """Tests for the RAG retrieval provider wiring in rag.py.""" import pytest +from langchain_core.documents import Document +from langchain_core.embeddings import Embeddings import rag +class _FakeEmbeddings(Embeddings): + """Dependency-free stand-in for a real embedding model: fixed vector for every input.""" + + def __init__(self, model="fake-model"): + self.model = model + + def embed_documents(self, texts): + return [[1.0, 0.0] for _ in texts] + + def embed_query(self, text): + return [1.0, 0.0] + + +def _stub_pdf_loader(monkeypatch, calls): + """Replace PyPDFLoader with one that records the paths it was asked to load.""" + + class _FakeLoader: + def __init__(self, path): + self.path = path + + def load_and_split(self, splitter): + calls.append(self.path) + return [Document(page_content=f"content of {self.path}")] + + monkeypatch.setattr(rag, "PyPDFLoader", _FakeLoader) + + @pytest.fixture(autouse=True) def _clear_vector_store_cache(): """The vector store is cached per-provider at module scope; isolate tests from each other.""" @@ -67,3 +96,95 @@ def test_retrieve_context_returns_empty_list_without_pdfs(monkeypatch): monkeypatch.setattr(rag, "_load_pdfs", lambda use_local: None) assert rag.retrieve_context("question") == [] + + +def _set_up_file_storage(monkeypatch, tmp_path): + file_storage = tmp_path / "file-storage" + file_storage.mkdir() + (file_storage / "a.pdf").write_bytes(b"content") + monkeypatch.setattr(rag, "_FILE_STORAGE", file_storage) + monkeypatch.setattr(rag, "_VECTOR_STORE", tmp_path / "vector-store") + monkeypatch.setattr(rag, "get_embeddings", lambda use_local: _FakeEmbeddings()) + return file_storage + + +def test_load_pdfs_persists_index_to_disk(monkeypatch, tmp_path): + _set_up_file_storage(monkeypatch, tmp_path) + calls = [] + _stub_pdf_loader(monkeypatch, calls) + + vector_store = rag._load_pdfs(use_local=False) + + assert vector_store is not None + assert len(calls) == 1 + store_dir = tmp_path / "vector-store" / "openai" + assert (store_dir / "manifest.json").exists() + assert (store_dir / "chroma.sqlite3").exists() + + +def test_load_pdfs_reuses_persisted_index_without_reloading_pdfs(monkeypatch, tmp_path): + _set_up_file_storage(monkeypatch, tmp_path) + calls = [] + _stub_pdf_loader(monkeypatch, calls) + rag._load_pdfs(use_local=False) + calls.clear() + + vector_store = rag._load_pdfs(use_local=False) + + assert vector_store is not None + assert calls == [] + + +def test_load_pdfs_rebuilds_when_file_storage_changes(monkeypatch, tmp_path): + file_storage = _set_up_file_storage(monkeypatch, tmp_path) + calls = [] + _stub_pdf_loader(monkeypatch, calls) + rag._load_pdfs(use_local=False) + calls.clear() + + (file_storage / "b.pdf").write_bytes(b"more content") + rag._load_pdfs(use_local=False) + + assert sorted(calls) == sorted(str(file_storage / name) for name in ("a.pdf", "b.pdf")) + + +def test_load_pdfs_rebuilds_when_embedding_model_changes(monkeypatch, tmp_path): + file_storage = _set_up_file_storage(monkeypatch, tmp_path) + calls = [] + _stub_pdf_loader(monkeypatch, calls) + rag._load_pdfs(use_local=False) + calls.clear() + + monkeypatch.setattr(rag, "get_embeddings", lambda use_local: _FakeEmbeddings(model="other-model")) + rag._load_pdfs(use_local=False) + + assert calls == [str(file_storage / "a.pdf")] + + +def test_load_pdfs_indexes_are_kept_separate_per_provider(monkeypatch, tmp_path): + _set_up_file_storage(monkeypatch, tmp_path) + calls = [] + _stub_pdf_loader(monkeypatch, calls) + + rag._load_pdfs(use_local=False) + rag._load_pdfs(use_local=True) + + assert (tmp_path / "vector-store" / "openai" / "manifest.json").exists() + assert (tmp_path / "vector-store" / "ollama" / "manifest.json").exists() + # Neither provider's build reused the other's on-disk index. + assert len(calls) == 2 + + +def test_load_persisted_returns_none_without_manifest(tmp_path, monkeypatch): + monkeypatch.setattr(rag, "_VECTOR_STORE", tmp_path / "vector-store") + + assert rag._load_persisted("openai", manifest={}, embeddings=_FakeEmbeddings()) is None + + +def test_load_persisted_returns_none_on_manifest_mismatch(tmp_path, monkeypatch): + monkeypatch.setattr(rag, "_VECTOR_STORE", tmp_path / "vector-store") + store_dir = tmp_path / "vector-store" / "openai" + store_dir.mkdir(parents=True) + (store_dir / rag._MANIFEST_NAME).write_text('{"a.pdf": 1}') + + assert rag._load_persisted("openai", manifest={"a.pdf": 2}, embeddings=_FakeEmbeddings()) is None