test_structured.py

"""
Test script for GhostTruth structured Q&A endpoint.
Tests the new /api/chat/structured endpoint with JSON validation and source attribution.
"""

import requests
import json
from pathlib import Path

# Configuration
BASE_URL = "http://localhost:3000"
STRUCTURED_ENDPOINT = f"{BASE_URL}/api/chat/structured"
LEGACY_ENDPOINT = f"{BASE_URL}/api/chat/"

def load_test_data(file_path: str):
    """Load test questions from JSON file"""
    with open(file_path, 'r', encoding='utf-8') as f:
        return json.load(f)

def test_structured_endpoint():
    """Test the new structured endpoint with source attribution"""
    print("=" * 80)
    print("TESTING STRUCTURED ENDPOINT: /api/chat/structured")
    print("=" * 80)
    
    # Load test data
    test_file = Path("test/algerie_telecom_questions.json")
    if not test_file.exists():
        print(f"❌ Test file not found: {test_file}")
        return
    
    payload = load_test_data(test_file)
    payload["include_sources"] = True
    
    print(f"\n📤 Sending request with {len(payload['question'])} categories...")
    print(f"Request payload preview:")
    print(json.dumps(payload, indent=2, ensure_ascii=False)[:500])
    print("...\n")
    
    try:
        response = requests.post(STRUCTURED_ENDPOINT, json=payload)
        response.raise_for_status()
        
        result = response.json()
        
        print("✅ Response received successfully!")
        print("\n" + "=" * 80)
        print("RESULTS")
        print("=" * 80)
        
        # Display results
        for category_id, questions in result.get("reponses", {}).items():
            print(f"\n📁 Category: {category_id}")
            print("-" * 80)
            
            for question_id, answer in questions.items():
                print(f"\n  🔹 {question_id}:")
                print(f"     Question: {payload['question'][category_id][question_id]}")
                print(f"     Answer: {answer[:200]}..." if len(answer) > 200 else f"     Answer: {answer}")
                
                # Display sources if available
                if result.get("sources") and category_id in result["sources"]:
                    sources = result["sources"][category_id].get(question_id, [])
                    if sources:
                        print(f"\n     📚 Sources ({len(sources)}):")
                        for idx, source in enumerate(sources, 1):
                            print(f"        {idx}. File: {source.get('file', 'N/A')}")
                            print(f"           Page: {source.get('page', 'N/A')}")
                            print(f"           Evidence: {source.get('evidence', 'N/A')[:150]}...")
        
        # Save full response
        output_file = Path("test/structured_response.json")
        with open(output_file, 'w', encoding='utf-8') as f:
            json.dump(result, f, indent=2, ensure_ascii=False)
        
        print(f"\n\n💾 Full response saved to: {output_file}")
        
        return result
        
    except requests.exceptions.RequestException as e:
        print(f"❌ Request failed: {e}")
        if hasattr(e, 'response') and e.response is not None:
            print(f"Response status: {e.response.status_code}")
            print(f"Response body: {e.response.text}")
        return None

def test_legacy_endpoint():
    """Test the legacy endpoint for comparison"""
    print("\n\n" + "=" * 80)
    print("TESTING LEGACY ENDPOINT: /api/chat/")
    print("=" * 80)
    
    test_file = Path("test/algerie_telecom_questions.json")
    payload = load_test_data(test_file)
    payload["include_sources"] = True
    
    print(f"\n📤 Sending request to legacy endpoint...")
    
    try:
        response = requests.post(LEGACY_ENDPOINT, json=payload)
        response.raise_for_status()
        
        result = response.json()
        print("✅ Legacy endpoint response received!")
        
        # Save response
        output_file = Path("test/legacy_response.json")
        with open(output_file, 'w', encoding='utf-8') as f:
            json.dump(result, f, indent=2, ensure_ascii=False)
        
        print(f"💾 Response saved to: {output_file}")
        
        return result
        
    except requests.exceptions.RequestException as e:
        print(f"❌ Request failed: {e}")
        return None

def compare_endpoints():
    """Compare structured vs legacy endpoints"""
    print("\n\n" + "=" * 80)
    print("COMPARISON: STRUCTURED vs LEGACY")
    print("=" * 80)
    
    structured = test_structured_endpoint()
    legacy = test_legacy_endpoint()
    
    if structured and legacy:
        print("\n📊 Comparison Summary:")
        print("-" * 80)
        print(f"Structured endpoint:")
        print(f"  - JSON validation: ✅ Enforced")
        print(f"  - Source format: SourceEvidence (file, page, evidence)")
        print(f"  - Temperature: 0.3 (more deterministic)")
        print(f"  - Top-K retrieval: 10")
        
        print(f"\nLegacy endpoint:")
        print(f"  - JSON validation: ❌ Not enforced")
        print(f"  - Source format: SourceInfo (score, file, chunk)")
        print(f"  - Temperature: 0.7 (default)")
        print(f"  - Top-K retrieval: 5")

if __name__ == "__main__":
    print("🚀 GhostTruth Structured Q&A System Test")
    print(f"Ensure the server is running at {BASE_URL}\n")
    
    choice = input("Choose test mode:\n  1. Structured endpoint only\n  2. Legacy endpoint only\n  3. Compare both\n\nEnter choice (1-3): ").strip()
    
    if choice == "1":
        test_structured_endpoint()
    elif choice == "2":
        test_legacy_endpoint()
    elif choice == "3":
        compare_endpoints()
    else:
        print("Invalid choice. Running structured endpoint test by default.")
        test_structured_endpoint()
    
    print("\n✨ Test completed!")