Feature/ai/rag & prompt extraction

services/ai-service/.gitignore

@@ -0,0 +1,36 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+ENV/
+.venv
+
+# Data directories (user uploads and generated content)
+data/
+config/vector_db/
+config/frameworks/
+
+# Environment variables
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Model cache (if downloading models)
+.cache/
+models/

services/ai-service/README.md

@@ -0,0 +1,100 @@
+# Compliance Framework Evaluation System
+
+AI service for extracting compliance controls from framework documents and evaluating applicant documents against those frameworks.
+
+## Directory Structure
+
+### Input Directories (Place your PDFs here)
+
+```
+data/
+├── inputs/
+│   ├── frameworks/          # Place framework PDF files here
+│   └── applicants/         # Place applicant PDF files here
+```
+
+**Usage:**
+- **Framework PDFs**: Place compliance framework PDFs in `data/inputs/frameworks/`
+- **Applicant PDFs**: Place applicant documents to evaluate in `data/inputs/applicants/`
+
+### Output Directories (Generated automatically)
+
+```
+config/
+├── frameworks/             # Framework outputs (one JSON per section)
+│   └── {framework_name}/
+│       └── {section_name}.json
+└── vector_db/             # Qdrant vector database storage
+
+data/
+└── outputs/
+    └── evaluations/        # Evaluation reports (JSON files)
+```
+
+**What gets saved where:**
+- **Framework Data**: `config/frameworks/{framework_name}/`
+  - One JSON per section (e.g. `section_name.json`) — extracted compliance controls
+- **Vector Database**: `config/vector_db/` (Qdrant local storage)
+- **Evaluation Reports**: `data/outputs/evaluations/` (when using evaluator)
+
+## Quick Start
+
+### CLI: Setup a Framework
+
+```python
+from main import setup_framework
+
+# Single file, list of files, or directory path
+setup_framework('data/inputs/frameworks/my_framework.pdf', 'my_framework')
+# or
+setup_framework('data/inputs/frameworks/my_framework/', 'my_framework')
+```
+
+This extracts controls from each PDF and saves one JSON per PDF under `config/frameworks/{framework_name}/` (filename = PDF stem).
+
+### API: Setup a Framework
+
+Run the API server (from `services/ai-service/`):
+
+```bash
+python src/api/app.py
+# or
+python run.py
+```
+
+- **Docs**: [http://localhost:8000/api/docs](http://localhost:8000/api/docs)
+- **Health**: `GET /health`
+- **Setup framework**: `POST /api/v1/frameworks/setup`
+  - Form fields: `framework_name` (string), `section_names` (list of strings), `files` (list of PDFs). Same order for section_names and files. Each PDF is saved as `config/frameworks/{framework_name}/{section_name}.json`.
+
+### Other (from `src`)
+
+- **RAG indexing**: `from src.rag import index_framework` — index framework JSON + PDFs into Qdrant.
+- **Evaluation**: `from src.core import evaluate_applicant` — evaluate applicant docs (requires external evaluation prompt and controls JSON).
+
+## Dependencies
+
+Use **uv** from the project root (`services/ai-service/`):
+
+```bash
+# Install all dependencies (after cloning or when pyproject.toml changes)
+uv sync
+
+# Add a new runtime dependency
+uv add <package>
+
+# Add FastAPI and uvicorn
+uv add fastapi uvicorn[standard]
+
+# Dev dependencies are separate: use a dependency group
+uv add --group dev pytest
+```
+
+Run these in your terminal; dev dependencies stay in a separate group (e.g. `[project.optional-dependencies.dev]` or `[tool.uv]` dev-dependencies) so production installs stay lean.
+
+## Notes
+
+- All data directories are in `.gitignore` (user uploads and generated content)
+- The directory structure is designed to be API-friendly
+- Paths are relative to the project root (`services/ai-service/`)
+- Each folder under `src/` has a `README.md` for quick context (e.g. for code agents)

services/ai-service/main.py

@@ -0,0 +1,71 @@
+"""
+Main entry point for the Compliance Framework Extraction System.
+CLI: extract controls from each PDF and save one JSON per PDF (no compose, no prompt).
+Uses the same service layer as the API.
+"""
+
+import glob
+from pathlib import Path
+
+from src.services import FrameworkService
+from src.utils import get_input_paths
+
+
+def setup_framework(pdf_paths: str | list[str], framework_name: str) -> list[Path]:
+    """
+    Extract controls from each PDF and save one JSON per PDF under config/frameworks/{framework_name}/.
+
+    Each file is named after the source PDF stem (e.g. section-a.pdf -> section-a.json).
+    No composition or evaluation prompt generation.
+
+    Args:
+        pdf_paths: Single file path, list of files, or directory path
+        framework_name: Name of the framework
+
+    Returns:
+        List of paths to the saved JSON files
+    """
+    if isinstance(pdf_paths, list):
+        pdf_paths_list = pdf_paths
+    else:
+        p = Path(pdf_paths)
+        if p.is_dir():
+            pdf_paths_list = sorted(glob.glob(f"{pdf_paths}/*.pdf"))
+        else:
+            pdf_paths_list = [pdf_paths]
+
+    if not pdf_paths_list:
+        raise ValueError(f"No PDF files found: {pdf_paths}")
+
+    pdf_sections = [
+        (Path(p).stem, Path(p).read_bytes())
+        for p in pdf_paths_list
+    ]
+    service = FrameworkService()
+    result = service.setup_framework(framework_name=framework_name, pdf_sections=pdf_sections)
+
+    project_root = Path(__file__).resolve().parent
+    return [project_root / s["json_path"] for s in result["sections"]]
+
+
+def main():
+    """Main entry point."""
+    print("\n" + "=" * 60)
+    print("Compliance Framework Extraction System")
+    print("=" * 60)
+
+    paths = get_input_paths()
+    print("\nDirectory structure:")
+    print(f"  Input PDFs (Frameworks): {paths['frameworks']}")
+    print(f"  Input PDFs (Applicants): {paths['applicants']}")
+    print(f"  Framework outputs: config/frameworks/")
+
+    print("\nUsage:")
+    print("   setup_framework('data/inputs/frameworks/<dir>', 'framework_name')")
+    print("=" * 60 + "\n")
+
+
+if __name__ == "__main__":
+    framework_path = "data/inputs/frameworks/NDI"
+    framework_name = "NDI"
+    setup_framework(framework_path, framework_name)

services/ai-service/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "ai-service"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "openai>=1.0.0",
+    "pdfplumber>=0.10.0",
+    "python-dotenv>=1.0.0",
+    "pydantic>=2.0.0",
+    "pypdf>=3.0.0",
+    "ipykernel>=7.1.0",
+    "qdrant-client>=1.8.0",
+    "haystack-ai>=2.0.0",
+    "transformers>=4.35.0",
+    "torch>=2.0.0",
+    "sentence-transformers>=2.2.0",
+    "accelerate>=0.24.0",
+    "python-multipart>=0.0.9",
+    "aiofiles>=24.0.0",
+    "fastapi>=0.128.0",
+]
+
+[dependency-groups]
+dev = [
+    "fastapi[standard]>=0.128.0",
+]
+

services/ai-service/run.py

@@ -0,0 +1,3 @@
+"""Run the API: python run.py (from ai-service directory). Use --reload for auto-restart on code changes."""
+import uvicorn
+uvicorn.run("src.api.app:app", host="0.0.0.0", port=8000)