refactor: codebase improvements and bug fixes

README.md

@@ -76,24 +76,25 @@ The goal of this project is to demonstrate a production-style RAG architecture r
 RAG/
 │
 ├── app.py
+├── pyproject.toml
 ├── requirements.txt
 ├── README.md
 ├── .env.example
 │
 ├── data/
 │   └── pdf/
 │
-├── faiss_store/
+├── faiss_store/              ← created at runtime
 │   ├── faiss.index
 │   └── metadata.pkl
 │
 └── src/
+    ├── __init__.py
     ├── rag.py
     ├── vector_store.py
     ├── synchronizer.py
-    ├── embedding.py
-    ├── document_generator.py
-    └── search.py
+    ├── embedding_generator.py
+    └── document_generator.py
 ```
 
 ---

app.py

@@ -1,13 +1,20 @@
 from src.rag import RAG
 
-def main():
-    print("welcome!!!")
 
-if __name__ == "__main__":
+def main() -> None:
     rag = RAG()
-    query = ""
+    print("RAG system ready. Type 'exit' to quit.\n")
+
+    while True:
+        query = input("Enter your query: ").strip()
+        if query.lower() == "exit":
+            print("Goodbye!")
+            break
+        if not query:
+            continue
+        answer = rag.search_and_summarize(query)
+        print(f"\nAnswer:\n{answer}\n")
 
-    while query != "exit":
-        query = input("Enter your RAG query:")
-        res = rag.search_and_summarize(query)
-        print(res)
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -1,13 +1,11 @@
 [project]
 name = "rag"
 version = "0.1.0"
-description = "Add your description here"
+description = "Production-style RAG system with incremental FAISS indexing"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "chromadb>=1.5.9",
     "faiss-cpu>=1.14.3",
-    "ipykernel>=7.3.0",
     "langchain>=1.3.14",
     "langchain-community>=0.4.2",
     "langchain-core>=1.4.9",
@@ -17,3 +15,8 @@ dependencies = [
     "python-dotenv>=1.2.2",
     "sentence-transformers>=5.6.0",
 ]
+
+[project.optional-dependencies]
+dev = [
+    "ipykernel>=7.3.0",
+]

requirements.txt

@@ -1,10 +1,9 @@
+faiss-cpu
 langchain
 langchain-core
 langchain-community
-pypdf
+langchain-groq
 pymupdf
-faiss-cpu
+pypdf
+python-dotenv
 sentence-transformers
-chromadb
-langchain-groq
-python-dotenv

src/document_generator.py

@@ -1,25 +1,40 @@
-from langchain_community.document_loaders import PyPDFLoader,PyMuPDFLoader
+from langchain_community.document_loaders import PyMuPDFLoader
 from pathlib import Path
-from typing import List, Any
-import os
+from typing import List
 
-def generate_documents(path,pdf_files:List[str]):
-    ##for loading pdf
-    all_documents=[]
-    for  file in pdf_files:
+
+def generate_documents(path: Path, pdf_files: List[str]) -> List:
+    """Load and return LangChain documents from a list of PDF files.
+
+    Args:
+        path: Directory that contains the PDF files.
+        pdf_files: List of PDF filenames (not full paths) to load.
+
+    Returns:
+        A flat list of LangChain Document objects with ``source_name``
+        and ``format_type`` set in their metadata.
+    """
+    all_documents = []
+    for file in pdf_files:
         try:
-            documents = PyMuPDFLoader(str(path.joinpath(file))).load()
+            documents = PyMuPDFLoader(str(path / file)).load()
             for doc in documents:
-                doc.metadata["source_name"] = str(file)
-                doc.metadata["format_type"] = 'pdf'
+                doc.metadata["source_name"] = file
+                doc.metadata["format_type"] = "pdf"
             all_documents.extend(documents)
-            print(f"Total {len(all_documents)} loaded from {file}")
-
+            print(f"[INFO] Loaded {len(documents)} pages from '{file}' "
+                  f"(total so far: {len(all_documents)})")
         except Exception as e:
-            print(f"[ERROR] Failed to load PDF {file}: {e}")
+            print(f"[ERROR] Failed to load PDF '{file}': {e}")
             continue
-        
+
     return all_documents
 
+
 if __name__ == "__main__":
-    generate_documents("../data/pdf")
+    # Resolve the data directory relative to this file so the script works
+    # regardless of the working directory.
+    pdf_dir = Path(__file__).parent.parent / "data" / "pdf"
+    pdf_files = [f.name for f in pdf_dir.glob("*.pdf")]
+    docs = generate_documents(pdf_dir, pdf_files)
+    print(f"Total documents loaded: {len(docs)}")

src/embedding_generator.py

@@ -1,40 +1,56 @@
-from .document_generator import generate_documents
+from pathlib import Path
+from typing import List, Any
+
 import numpy as np
-from sentence_transformers import SentenceTransformer
-from typing import List,Tuple,Dict,Any
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from sentence_transformers import SentenceTransformer
+
+from .document_generator import generate_documents
+
 
 class Generate_Embedding:
-    def __init__(self,model_name: str = "all-MiniLM-L6-v2",chunk_size=1000, chunk_overlap=200,):
+    """Splits documents into chunks and produces vector embeddings.
+
+    A single :class:`SentenceTransformer` instance is created once and
+    reused for all calls, avoiding redundant model loads.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "all-MiniLM-L6-v2",
+        chunk_size: int = 1000,
+        chunk_overlap: int = 200,
+    ):
         self.chunk_size = chunk_size
         self.chunk_overlap = chunk_overlap
+        self.model_name = model_name
         self.model = SentenceTransformer(model_name)
 
-    def convert_chunks(self,docs:List):
+    def convert_chunks(self, docs: List[Any]) -> List[Any]:
+        """Split a list of LangChain documents into smaller chunks."""
         text_splitter = RecursiveCharacterTextSplitter(
-                        chunk_size=self.chunk_size,
-                        chunk_overlap=self.chunk_overlap,
-                        length_function=len,
-                        separators=["\n\n", "\n", " ", ""]
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=len,
+            separators=["\n\n", "\n", " ", ""],
         )
-
         chunks = text_splitter.split_documents(docs)
-        print(f"Split into {len(chunks)} chunks.")
+        print(f"[INFO] Split {len(docs)} documents into {len(chunks)} chunks.")
         return chunks
 
-    def embedd_chunks(self,texts: List[Any]):
-        text = [text.page_content for text in texts]
-        if self.model is None:
-            raise ValueError("Model is not loaded. Cannot generate embeddings.")
-        print(f"Generating embeddings for {len(text)} texts.")
-        embeddings = self.model.encode(text, show_progress_bar=True)
-        return embeddings
+    def embed_chunks(self, chunks: List[Any]) -> np.ndarray:
+        """Return a float32 embedding matrix for the given document chunks."""
+        texts = [chunk.page_content for chunk in chunks]
+        print(f"[INFO] Generating embeddings for {len(texts)} chunks.")
+        embeddings = self.model.encode(texts, show_progress_bar=True)
+        return np.array(embeddings, dtype="float32")
 
 
 if __name__ == "__main__":
-    docs = generate_documents("../data/pdf")
+    pdf_dir = Path(__file__).parent.parent / "data" / "pdf"
+    pdf_files = [f.name for f in pdf_dir.glob("*.pdf")]
+    docs = generate_documents(pdf_dir, pdf_files)
     pipeline = Generate_Embedding()
     chunks = pipeline.convert_chunks(docs)
-    embeded_chunks = pipeline.embedd_chunks(chunks)
-    print(embeded_chunks)
-
+    embeddings = pipeline.embed_chunks(chunks)
+    print(f"Embedding matrix shape: {embeddings.shape}")