Implement weak topic detection module

.github/workflows/chatbot.yml

@@ -12,22 +12,39 @@ on:
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
+
     defaults:
       run:
         working-directory: chatbot
+
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
+
+
+
+
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
           cache: pip
           cache-dependency-path: chatbot/requirements.txt
+
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+
+      - name: Run unit and API tests
+        env:
+          JWT_SECRET_KEY: ${{ secrets.JWT_SECRET_KEY }}
+=======
       - name: Install dependencies
         run: |
          pip install -r requirements.txt
       - name: Run unit and API tests
         env:
           JWT_SECRET_KEY: "4d7d6b996c2e65e73b6c03056b45d675ab121910a01e49eb2a39b74aa6f8bae9"
+ main
         run: pytest rag-engine/tests -q --tb=short --ignore=rag-engine/tests/test_injection_real_pdf.py

ai-ml/embedding/chroma_store.py

@@ -1,14 +1,30 @@
 """
-Shared ChromaDB store for the ingestion pipeline.
-Uses the same persistent path, collection name, and embedding model
-as the chatbot (Team Mu) so ingested chunks are retrievable by it.
+Shared ChromaDB store — the single canonical content store for both
+the ingestion pipeline and quiz generation.
+
+[P0-1 fix] Previously, quiz generation queried Pinecone + MongoDB
+(see the old embedder.py) while ingestion wrote here. The two never
+overlapped, so newly ingested content was invisible to quiz
+generation. This file is now the ONLY storage path for both:
+ingestion calls store_chunks() (unchanged), and quiz generation's
+Embedder.search() calls query_chunks() (new) instead of touching
+Pinecone/Mongo at all.
+
+Also fixes a subtler bug: this module used to load its own separate
+SentenceTransformer instance with default (non-normalized) output,
+while embedding/model.py's shared model normalizes its vectors. Two
+different embedding configs writing into the same cosine-similarity
+space would have produced quietly wrong nearest-neighbor results.
+Both reads and writes now go through the one shared, normalized
+model in model.py.
 """
 import os
 from pathlib import Path
 
 import chromadb
 from dotenv import load_dotenv
-from sentence_transformers import SentenceTransformer
+
+from embedding.model import get_embedding_model
 
 load_dotenv(dotenv_path=Path(__file__).parent.parent.parent / ".env")
 
@@ -17,16 +33,6 @@
     r"C:\Dev\QuantumLearningWorkspace\shared_chroma_data",
 )
 DEFAULT_COLLECTION_NAME = "study_chunks"
-DEFAULT_MODEL_NAME = "all-MiniLM-L6-v2"
-
-_model = None
-
-
-def get_embedding_model() -> SentenceTransformer:
-    global _model
-    if _model is None:
-        _model = SentenceTransformer(DEFAULT_MODEL_NAME)
-    return _model
 
 
 def get_collection(name: str = DEFAULT_COLLECTION_NAME, path: str = None):
@@ -45,7 +51,7 @@ def store_chunks(chunks: list[dict], user_id: str, document_id: str, title: str)
         return 0
 
     collection = get_collection()
-    model = get_embedding_model()
+    model = get_embedding_model()  # shared, normalized model — same one queries use
 
     ids = [f"{document_id}_chunk{c['chunk_index']}" for c in chunks]
     documents = [c["text"] for c in chunks]
@@ -58,7 +64,7 @@ def store_chunks(chunks: list[dict], user_id: str, document_id: str, title: str)
         }
         for c in chunks
     ]
-    embeddings = model.encode(documents).tolist()
+    embeddings = model.encode(documents)  # list[list[float]], already normalized
 
     collection.upsert(
         ids=ids,
@@ -67,3 +73,65 @@ def store_chunks(chunks: list[dict], user_id: str, document_id: str, title: str)
         metadatas=metadatas,
     )
     return len(chunks)
+
+
+def query_chunks(
+    query_text: str,
+    top_k: int = 5,
+    user_id: str = None,
+    document_id: str = None,
+) -> list:
+    """
+    [P0-1 new] Embed a query and return the top_k nearest chunks from
+    the shared collection, in the shape quiz_service.py expects:
+        [{"score": float, "text": str, "title": str, "metadata": dict}, ...]
+
+    user_id / document_id, if given, are applied as an exact-match
+    metadata filter (ChromaDB's `where`). Both default to None (no
+    filter) today, which preserves current unscoped behavior — this
+    is deliberately plumbed through now so P0-3 (user-scoped quiz
+    retrieval) can pass user_id here without another storage change.
+    """
+    collection = get_collection()
+    model = get_embedding_model()
+
+    where = {}
+    if user_id is not None:
+        where["user_id"] = user_id
+    if document_id is not None:
+        where["document_id"] = document_id
+
+    query_embedding = model.encode(query_text)[0]
+
+    results = collection.query(
+        query_embeddings=[query_embedding],
+        n_results=top_k,
+        where=where or None,
+    )
+
+    ids = results.get("ids", [[]])[0]
+    documents = results.get("documents", [[]])[0]
+    metadatas = results.get("metadatas", [[]])[0]
+    distances = results.get("distances", [[]])[0]
+
+    output = []
+    for i in range(len(ids)):
+        meta = metadatas[i] or {}
+        output.append({
+            "score": distances[i],
+            "text": documents[i],
+            "title": meta.get("document", ""),
+            "metadata": meta,
+        })
+    return output
+
+
+def delete_chunks(document_id: str) -> None:
+    """
+    [P0-1 new] Remove all chunks belonging to a document from the
+    shared collection. Chroma's delete-by-filter is idempotent
+    (deleting a non-existent id/filter is a no-op), which is also
+    what P0-5's purge endpoint will need.
+    """
+    collection = get_collection()
+    collection.delete(where={"document_id": document_id})

ai-ml/embedding/embedder.py

@@ -1,129 +1,107 @@
 """
-Orchestrates the full embedding pipeline:
-
-  document (common ingestion schema)
-      -> chunk (chunker.py)
-      -> embed (model.py, free local sentence-transformers)
-      -> store full chunk text + metadata in MongoDB   (source of truth)
-      -> store vector + small pointer metadata in Pinecone (for search)
-
-MongoDB holds the full text because Pinecone metadata has size/type
-limits and isn't meant for large text blobs. Pinecone only stores the
-vector plus a small pointer (chunk id + a few filterable fields) so a
-similarity search can be resolved back to the full chunk via MongoDB.
+Orchestrates document embedding + the search interface quiz
+generation uses.
+
+[P0-1 fix] This used to run its own parallel storage path — full
+chunk text in MongoDB (source of truth) + vectors in Pinecone (for
+search) — completely separate from ingestion's shared ChromaDB. That
+meant anything ingested via ingestion/main.py was invisible to
+QuizService.generate_quiz_from_topic(), since it searched Pinecone.
+
+Everything now reads and writes the ONE shared ChromaDB collection
+via chroma_store.py — the same collection ingestion already writes
+to. No more Mongo/Pinecone clients, no more second indexing path.
 """
 
 import uuid
-from pymongo import MongoClient
-import certifi
 
-from embedding.config import settings
 from embedding.chunker import chunk_document
-from embedding.model import get_embedding_model
-from embedding.vector_store import PineconeVectorStore
+from embedding.chroma_store import store_chunks, query_chunks, delete_chunks
 
 import argparse
 import requests
 
 
 class Embedder:
-    def __init__(self, model=None, vector_store=None, mongo_client=None):
-        self.model = model or get_embedding_model()
-        self.vector_store = vector_store or PineconeVectorStore(dimension=self.model.dimension())
+    def __init__(self):
+        # No client setup needed here anymore — chroma_store.py owns
+        # the one shared collection + embedding model as module-level
+        # helpers, created fresh per call (see get_collection()).
+        pass
 
-        self._mongo_client = mongo_client or MongoClient(
-            settings.mongodb_uri, tlsCAFile=certifi.where()
-        )
-        self._db = self._mongo_client[settings.mongodb_db]
-        self._collection = self._db[settings.mongodb_collection]
-
-    def embed_document(self, document: dict) -> dict:
+    def embed_document(self, document: dict, user_id: str = "unknown") -> dict:
         """
         document: common ingestion schema
           { "source_type": ..., "title": ..., "text": ..., "metadata": {...} }
 
-        Chunks it, embeds every chunk, saves full text to MongoDB, and
-        upserts vectors to Pinecone. Returns a small summary dict.
+        Chunks it and stores it in the shared ChromaDB collection.
+        Returns a small summary dict.
+
+        Note: the normal ingestion request path (POST /ingest/pdf etc.)
+        goes through ingestion/main.py's own _chunk_and_store(), which
+        calls chroma_store.store_chunks() directly and doesn't use
+        this method. This method exists for standalone/manual use
+        (see the CLI at the bottom of this file) and now writes to the
+        exact same store, so both paths stay consistent.
         """
         chunks = chunk_document(document)
         if not chunks:
             return {"document_id": None, "chunks_stored": 0}
 
-        texts = [c["text"] for c in chunks]
-        vectors = self.model.encode(texts)
-
         document_id = str(uuid.uuid4())
-        mongo_docs = []
-        pinecone_vectors = []
-
-        for chunk, vector in zip(chunks, vectors):
-            chunk_id = f"{document_id}_{chunk['chunk_index']}"
-
-            mongo_docs.append({
-                "_id": chunk_id,
-                "document_id": document_id,
-                "chunk_index": chunk["chunk_index"],
-                "text": chunk["text"],
-                "title": chunk["title"],
-                "source_type": chunk["source_type"],
-                "metadata": chunk["metadata"],
-            })
-
-            pinecone_vectors.append({
-                "id": chunk_id,
-                "values": vector,
-                "metadata": {
-                    "document_id": document_id,
-                    "chunk_index": chunk["chunk_index"],
-                    "title": chunk["title"],
-                    "source_type": chunk["source_type"],
-                },
-            })
-
-        if mongo_docs:
-            self._collection.insert_many(mongo_docs)
-
-        self.vector_store.upsert(pinecone_vectors)
-
-        return {"document_id": document_id, "chunks_stored": len(mongo_docs)}
-
-    def search(self, query: str, top_k: int = 5) -> list:
+        stored_count = store_chunks(
+            chunks=chunks,
+            user_id=user_id,
+            document_id=document_id,
+            title=document.get("title", ""),
+        )
+        return {"document_id": document_id, "chunks_stored": stored_count}
+
+    def search(
+        self,
+        query: str,
+        top_k: int = 5,
+        user_id: str = None,
+        document_id: str = None,
+    ) -> list:
         """
-        Embeds the query, finds the nearest chunks in Pinecone, then
-        resolves each match back to its full text stored in MongoDB.
+        Searches the shared ChromaDB collection — the same store
+        ingestion writes to, so newly ingested content is immediately
+        queryable here (P0-1's Definition of Done).
+
+        user_id / document_id are optional scoping filters, passed
+        straight through to chroma_store.query_chunks(). Left as None
+        by default (unscoped), matching current QuizService behavior;
+        P0-3 is what wires the caller-side enforcement of these.
         """
-        query_vector = self.model.encode(query)[0]
-        matches = self.vector_store.query(query_vector, top_k=top_k)
-
-        results = []
-        for match in matches:
-            chunk_id = match["id"] if isinstance(match, dict) else match.id
-            score = match["score"] if isinstance(match, dict) else match.score
-
-            mongo_doc = self._collection.find_one({"_id": chunk_id})
-            if mongo_doc:
-                results.append({
-                    "score": score,
-                    "text": mongo_doc["text"],
-                    "title": mongo_doc["title"],
-                    "source_type": mongo_doc["source_type"],
-                    "metadata": mongo_doc["metadata"],
-                })
-        return results
-
-    def delete_document(self, document_id: str, chunk_count: int):
-        """Remove a document's chunks from both MongoDB and Pinecone."""
-        ids = [f"{document_id}_{i}" for i in range(chunk_count)]
-        self._collection.delete_many({"document_id": document_id})
-        self.vector_store.delete(ids)
-
-    def close(self):
-        self._mongo_client.close()
+        return query_chunks(query, top_k=top_k, user_id=user_id, document_id=document_id)
 
+    def delete_document(self, document_id: str, chunk_count: int = None):
+        """
+        Remove a document's chunks from the shared store.
+        chunk_count is no longer needed (Chroma deletes by
+        document_id metadata filter, not by reconstructing chunk ids)
+        but is accepted for backward compatibility with existing callers.
+        """
+        delete_chunks(document_id)
 
+    def close(self):
+        """
+        No-op now — chroma_store.py doesn't hold a persistent client
+        connection open between calls, so there's nothing to close.
+        Kept so existing callers (e.g. the CLI below) don't need to
+        change.
+        """
+        pass
 
 
-INGESTION_BASE_URL = "http://127.0.0.1:8000"
+INGESTION_BASE_URL = "http://127.0.0.1:8001"
+# NOTE: this was previously "http://127.0.0.1:8000", which is Mu's
+# confirmed port, not Lambda ingestion's. Per the confirmed port
+# scheme (Mu=8000, Pluto=5000, Lambda ingestion=8001, Lambda
+# quiz=8002), 8001 is correct here. Flagging in case this was
+# intentional for some other reason — worth a quick sanity check
+# against P1-6's verification pass.
 
 
 def _fetch_from_ingestion(pdf=None, youtube=None, article=None) -> dict:
@@ -171,12 +149,12 @@ def _fetch_from_ingestion(pdf=None, youtube=None, article=None) -> dict:
         }
 
     embedder = Embedder()
-    summary = embedder.embed_document(document)
+    summary = embedder.embed_document(document, user_id="cli-test-user")
     print("Embedded:", summary)
 
     query = document.get("title") or document.get("text", "")[:50]
     results = embedder.search(query, top_k=3)
     for r in results:
         print(f"[{r['score']:.3f}] {r['text'][:100]}...")
 
-    embedder.close()
+    embedder.close()