From c3844fe78eeb29c41487021e3467af64d4b18566 Mon Sep 17 00:00:00 2001 From: mahakagarwal7 Date: Sat, 15 Nov 2025 18:47:02 +0530 Subject: [PATCH 1/3] feat: complete JSON object analysis and storage simulation --- app/main.py | 76 ++++++++++- app/routers/database_routes.py | 122 ++++++++++++++++++ app/routers/json_routes.py | 212 +++++++++++++++++++++++++++++++ app/utils/json_analyzer.py | 223 +++++++++++++++++++++++++++++++++ requirements.txt | 7 +- 5 files changed, 635 insertions(+), 5 deletions(-) create mode 100644 app/routers/database_routes.py create mode 100644 app/routers/json_routes.py create mode 100644 app/utils/json_analyzer.py diff --git a/app/main.py b/app/main.py index e2976fe..3910635 100644 --- a/app/main.py +++ b/app/main.py @@ -2,9 +2,14 @@ from fastapi import FastAPI from app.routers import upload_router, retrieve_router +# ADD THESE IMPORTS +from app.routers import json_routes +from app.routers import database_routes import cloudinary from dotenv import load_dotenv import os +from pathlib import Path # ADD THIS + # Load environment variables from .env file load_dotenv() @@ -14,21 +19,84 @@ @app.on_event("startup") async def startup_event(): """ - Configure Cloudinary on app startup. + Configure Cloudinary on app startup and create storage directories. """ + # Cloudinary configuration (for images/videos) - KEEP EXISTING cloudinary.config( cloud_name = os.getenv("CLOUDINARY_CLOUD_NAME"), api_key = os.getenv("CLOUDINARY_API_KEY"), api_secret = os.getenv("CLOUDINARY_API_SECRET"), secure = True # Ensure all URLs are HTTPS ) - print("Cloudinary configuration loaded.") + print("✅ Cloudinary configuration loaded.") + + # CREATE STORAGE DIRECTORIES FOR JSON FILES - KEEP EXISTING + storage_dirs = [ + "app/storage/databases/sql", + "app/storage/databases/nosql", + "app/storage/temp" + ] + + for directory in storage_dirs: + Path(directory).mkdir(parents=True, exist_ok=True) + + print("✅ JSON storage directories created.") + + # ADD INTERNAL DATABASE DIRECTORIES - NEW + internal_db_dirs = [ + "app/storage/internal_databases/tables", + "app/storage/internal_databases/collections", + "app/storage/internal_databases/schemas" + ] + + for directory in internal_db_dirs: + Path(directory).mkdir(parents=True, exist_ok=True) + + print("✅ Internal database directories created.") +# INCLUDE ALL ROUTERS - KEEP EXISTING app.include_router(upload_router.router, prefix="/api", tags=["Upload"]) -# We will disable the local retrieve router for now, as Cloudinary handles delivery +# ADD JSON ROUTES - KEEP EXISTING +app.include_router(json_routes.router, prefix="/api", tags=["JSON Processing"]) +# ADD DATABASE ROUTES - NEW +app.include_router(database_routes.router, prefix="/api", tags=["Internal Databases"]) + +# We will disable the local retrieve router for now, as Cloudinary handles delivery - KEEP EXISTING # app.include_router(retrieve_router.router, prefix="/api", tags=["Retrieve"]) @app.get("/") async def root(): - return {"message": "Welcome to the Media Storage API. Use /api/upload to post files."} \ No newline at end of file + return { + "message": "Welcome to the Media Storage API", + "endpoints": { + "upload_media": "/api/upload - Upload images/videos to Cloudinary", + "upload_json": "/api/json/upload - Upload and analyze JSON files", + "list_json": "/api/json/files - List stored JSON files", + # ADD NEW ENDPOINTS + "internal_dbs": "/api/database - Internal database operations", + "list_tables": "/api/database/tables - List SQL tables", + "list_collections": "/api/database/collections - List NoSQL collections", + "db_stats": "/api/database/stats - Get database statistics" + } + } + +# ADD HEALTH CHECK ENDPOINT - NEW +@app.get("/health") +async def health_check(): + """ + System health check + """ + return { + "status": "healthy", + "cloudinary_configured": bool(os.getenv("CLOUDINARY_CLOUD_NAME")), + "storage_directories": { + "images": "Cloudinary (external)", + "json_files": "Internal storage", + "databases": "Internal simulation" + }, + "external_apis": { + "cloudinary": "For images/videos only", + "databases": "None - using internal storage" + } + } \ No newline at end of file diff --git a/app/routers/database_routes.py b/app/routers/database_routes.py new file mode 100644 index 0000000..cd809d2 --- /dev/null +++ b/app/routers/database_routes.py @@ -0,0 +1,122 @@ +# app/routers/database_routes.py + +from fastapi import APIRouter, HTTPException, Query +from typing import List, Dict, Any +import json +from pathlib import Path +import aiofiles + +from app.routers.json_routes import list_json_files + +router = APIRouter() + +@router.get("/database/tables") +async def list_sql_tables(): + """ + Lists files/entities stored in the simulated SQL database path. + """ + + return await list_json_files(category="sql", limit=1000, offset=0) + +@router.get("/database/collections") +async def list_nosql_collections(): + """ + Lists files/entities stored in the simulated NoSQL collection path. + """ + + return await list_json_files(category="nosql", limit=1000, offset=0) + +@router.get("/database/stats") +async def get_enhanced_stats(): + """ + Enhanced statistics with performance metrics + """ + try: + stats = { + "storage": await get_storage_stats(), + "performance": await get_performance_metrics(), + "recommendations": await get_optimization_recommendations() + } + + return stats + + except Exception as e: + raise HTTPException(500, f"Error getting stats: {str(e)}") + +async def get_storage_stats() -> Dict: + """Get detailed storage statistics""" + paths = { + "sql_json": Path("app/storage/databases/sql"), + "nosql_json": Path("app/storage/databases/nosql"), + "internal_tables": Path("app/storage/internal_databases/tables"), + "internal_collections": Path("app/storage/internal_databases/collections") + } + + stats = {} + total_size = 0 + + for name, path in paths.items(): + if path.exists(): + files = list(path.glob("*.json")) + size = sum(f.stat().st_size for f in files) + stats[name] = { + "file_count": len(files), + "total_size_bytes": size, + "total_size_mb": round(size / (1024 * 1024), 2) + } + total_size += size + + stats["total_storage_mb"] = round(total_size / (1024 * 1024), 2) + return stats + +async def get_performance_metrics() -> Dict: + """Get performance metrics""" + return { + "analysis_speed": "optimized", + "storage_efficiency": "high", + "memory_usage": "low", + "recommendations": [ + "Consider compressing large JSON files", + "Implement caching for frequent queries", + "Add background indexing for better search performance" + ] + } + +async def get_optimization_recommendations() -> List[str]: + """Get optimization recommendations based on current data""" + recommendations = [] + + sql_path = Path("app/storage/databases/sql") + if sql_path.exists(): + sql_files = list(sql_path.glob("*.json")) + if len(sql_files) > 50: + recommendations.append("Consider archiving old SQL JSON files") + + # Add more intelligent recommendations based on your data patterns + return recommendations + +@router.get("/database/cleanup") +async def cleanup_system(): + """ + Cleanup temporary files and optimize storage + """ + try: + temp_path = Path("app/storage/temp") + deleted_files = 0 + + if temp_path.exists(): + for temp_file in temp_path.glob("*"): + if temp_file.is_file(): + temp_file.unlink() + deleted_files += 1 + + return { + "message": "Cleanup completed", + "deleted_temp_files": deleted_files, + "freed_space": "System optimized" + } + + except Exception as e: + raise HTTPException(500, f"Cleanup error: {str(e)}") + + diff --git a/app/routers/json_routes.py b/app/routers/json_routes.py new file mode 100644 index 0000000..dff6321 --- /dev/null +++ b/app/routers/json_routes.py @@ -0,0 +1,212 @@ +# app/routers/json_routes.py + +from fastapi import APIRouter, UploadFile, File, HTTPException, BackgroundTasks +import aiofiles +from pathlib import Path +import asyncio + +from app.utils.json_analyzer import JSONAnalyzer + +from fastapi.responses import FileResponse + + +router = APIRouter(prefix="/json", tags=["JSON Files"]) +json_analyzer = JSONAnalyzer() + +@router.post("/upload") +async def upload_json( + background_tasks: BackgroundTasks, + file: UploadFile = File(...) +): + """ + Enhanced upload with background processing and better error handling + """ + if not file.filename.lower().endswith('.json'): + raise HTTPException(400, "Only JSON files are allowed") + + # Validate file size + content = await file.read() + if len(content) > 50 * 1024 * 1024: # 50MB limit + raise HTTPException(413, "File too large. Maximum size is 50MB") + + temp_dir = Path("app/storage/temp") + temp_dir.mkdir(exist_ok=True) + temp_file = temp_dir / f"temp_{file.filename}" + + try: + # Save uploaded file + async with aiofiles.open(temp_file, 'wb') as f: + await f.write(content) + + print(f"📥 Processing: {file.filename}") + + # Analyze JSON (quick operation) + analysis = json_analyzer.analyze_json_file(str(temp_file)) + + # Store based on analysis + result = json_analyzer.store_json_file(str(temp_file), file.filename, analysis) + + if result["success"]: + response = { + "message": "JSON processed successfully!", + "details": result, + "analysis": analysis + } + + # Add background task for additional processing + if not result.get("duplicate"): + background_tasks.add_task(process_additional_metadata, str(temp_file), analysis) + + return response + else: + raise HTTPException(500, result["error"]) + + except Exception as e: + # Cleanup on error + if temp_file.exists(): + temp_file.unlink() + raise HTTPException(500, f"Processing failed: {str(e)}") + +@router.post("/bulk-upload") +async def bulk_upload_json( + background_tasks: BackgroundTasks, + files: List[UploadFile] = File(...) +): + """ + Process multiple JSON files efficiently + """ + results = [] + tasks = [] + + for file in files: + if file.filename.lower().endswith('.json'): + # Process each file concurrently + task = process_single_file(file) + tasks.append(task) + + # Wait for all files to process + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + + successful = [r for r in results if not isinstance(r, Exception) and r.get("success")] + failed = [r for r in results if isinstance(r, Exception) or not r.get("success")] + + return { + "message": f"Bulk processing completed", + "summary": { + "total_files": len(files), + "successful": len(successful), + "failed": len(failed) + }, + "successful_files": successful, + "failed_files": failed + } + +async def process_single_file(file: UploadFile) -> Dict: + """Process a single file asynchronously""" + try: + content = await file.read() + temp_dir = Path("app/storage/temp") + temp_file = temp_dir / f"bulk_{file.filename}" + + async with aiofiles.open(temp_file, 'wb') as f: + await f.write(content) + + analysis = json_analyzer.analyze_json_file(str(temp_file)) + result = json_analyzer.store_json_file(str(temp_file), file.filename, analysis) + + return result + + except Exception as e: + return {"success": False, "error": str(e), "filename": file.filename} + +@router.get("/files") +async def list_json_files( + category: str = None, + limit: int = 100, + offset: int = 0 +): + """ + Enhanced file listing with filtering and pagination + """ + try: + sql_files = [] + nosql_files = [] + + # Get SQL files with metadata + sql_path = Path("app/storage/databases/sql") + if sql_path.exists(): + for file in sql_path.glob("*.json"): + if file.name.endswith('.meta.json'): + continue + + metadata = await get_file_metadata(file) + sql_files.append({ + "filename": file.name, + "size": file.stat().st_size, + "modified": file.stat().st_mtime, + "metadata": metadata + }) + + # Get NoSQL files with metadata + nosql_path = Path("app/storage/databases/nosql") + if nosql_path.exists(): + for file in nosql_path.glob("*.json"): + if file.name.endswith('.meta.json'): + continue + + metadata = await get_file_metadata(file) + nosql_files.append({ + "filename": file.name, + "size": file.stat().st_size, + "modified": file.stat().st_mtime, + "metadata": metadata + }) + + # Apply filtering + if category == "sql": + files = sql_files + elif category == "nosql": + files = nosql_files + else: + files = sql_files + nosql_files + + # Apply pagination + total_files = len(files) + paginated_files = files[offset:offset + limit] + + return { + "files": paginated_files, + "pagination": { + "total": total_files, + "limit": limit, + "offset": offset, + "has_more": (offset + limit) < total_files + }, + "summary": { + "sql_files": len(sql_files), + "nosql_files": len(nosql_files), + "total_files": total_files + } + } + + except Exception as e: + raise HTTPException(500, f"Error listing files: {str(e)}") + +async def get_file_metadata(file_path: Path) -> Dict: + """Get metadata for a file""" + metadata_path = file_path.with_suffix('.meta.json') + if metadata_path.exists(): + async with aiofiles.open(metadata_path, 'r') as f: + content = await f.read() + return json.loads(content) + return {} + +def process_additional_metadata(file_path: str, analysis: Dict): + """Background task for additional processing""" + # This runs in background - doesn't block the response + try: + # Could generate previews, create indexes, etc. + print(f"Background processing completed for {file_path}") + except Exception as e: + print(f"Background processing failed: {e}") \ No newline at end of file diff --git a/app/utils/json_analyzer.py b/app/utils/json_analyzer.py new file mode 100644 index 0000000..63ad6a0 --- /dev/null +++ b/app/utils/json_analyzer.py @@ -0,0 +1,223 @@ +# app/utils/json_analyzer.py + +import json +import os +import shutil +import uuid +from pathlib import Path +from typing import Dict, Any, List +from datetime import datetime +import hashlib + +class JSONAnalyzer: + def __init__(self): + self.base_dir = Path("app/storage") + self.sql_path = self.base_dir / "databases" / "sql" + self.nosql_path = self.base_dir / "databases" / "nosql" + self.temp_path = self.base_dir / "temp" + self.schema_path = self.base_dir / "internal_databases" / "schemas" + + # Create directories efficiently + for path in [self.sql_path, self.nosql_path, self.temp_path,self.schema_path]: + path.mkdir(parents=True, exist_ok=True) + + def analyze_json_file(self, file_path: str) -> Dict[str, Any]: + """ + Enhanced analysis with better performance and more insights + """ + try: + # Use file size for quick decisions + file_size = os.path.getsize(file_path) + if file_size > 10 * 1024 * 1024: # 10MB limit + return {"recommendation": "nosql", "reason": "File too large for SQL optimization"} + + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # Quick type check + if isinstance(data, list): + return self._analyze_array(data, file_path) + elif isinstance(data, dict): + return self._analyze_object(data, file_path) + else: + return {"recommendation": "nosql", "reason": "Simple data type"} + + except json.JSONDecodeError as e: + return {"recommendation": "nosql", "reason": f"Invalid JSON: {str(e)}"} + except Exception as e: + return {"recommendation": "nosql", "reason": f"Analysis error: {str(e)}"} + + def _analyze_array(self, data: List, file_path: str) -> Dict[str, Any]: + """Optimized array analysis""" + if not data: + return {"recommendation": "nosql", "reason": "Empty array"} + + # Quick sample analysis (first 10 items for performance) + sample_size = min(10, len(data)) + sample = data[:sample_size] + + if not all(isinstance(item, dict) for item in sample): + return {"recommendation": "nosql", "reason": "Array contains non-object items"} + + # Check structure consistency efficiently + first_keys = set(sample[0].keys()) + consistent_structure = all(set(item.keys()) == first_keys for item in sample) + + if consistent_structure: + # Additional checks for SQL optimization + if self._is_sql_optimized(sample, first_keys): + return { + "recommendation": "sql", + "reason": "Uniform structured data optimized for SQL", + "estimated_rows": len(data), + "columns": list(first_keys) + } + + return {"recommendation": "nosql", "reason": "Variable structure or complex data"} + + def _analyze_object(self, data: Dict, file_path: str) -> Dict[str, Any]: + """Optimized object analysis""" + # Check nesting depth efficiently + max_depth = self._calculate_depth(data) + + if max_depth > 2: + return { + "recommendation": "nosql", + "reason": f"Deeply nested structure (depth: {max_depth})", + "nesting_level": max_depth + } + else: + return { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": list(data.keys()) + } + + def _is_sql_optimized(self, sample: List[Dict], keys: set) -> bool: + """Check if data is optimized for SQL storage""" + # Rule 1: Reasonable number of columns + if len(keys) > 50: + return False + + # Rule 2: Check for common SQL patterns + has_id = any(key.lower() in ['id', '_id'] for key in keys) + has_foreign_keys = any(key.endswith('_id') for key in keys) + + return has_id or has_foreign_keys or len(keys) <= 20 + + def _calculate_depth(self, obj, current_depth=0) -> int: + """Calculate maximum nesting depth efficiently""" + if not isinstance(obj, dict): + return current_depth + + max_depth = current_depth + for value in obj.values(): + if isinstance(value, dict): + max_depth = max(max_depth, self._calculate_depth(value, current_depth + 1)) + elif isinstance(value, list) and value and isinstance(value[0], dict): + max_depth = max(max_depth, self._calculate_depth(value[0], current_depth + 1)) + + return max_depth + + def store_json_file(self, file_path: str, original_name: str, analysis: Dict) -> Dict[str, Any]: + """ + Enhanced storage with duplicate detection and metadata + """ + try: + # Generate content-based filename for deduplication + file_hash = self._get_file_hash(file_path) + name_stem = Path(original_name).stem + extension = Path(original_name).suffix + + # Check for duplicates + duplicate = self._check_duplicate(file_hash, analysis["recommendation"]) + if duplicate: + return { + "success": True, + "original_name": original_name, + "stored_name": duplicate, + "storage_type": analysis["recommendation"].upper(), + "final_path": duplicate, + "reason": analysis["reason"], + "duplicate": True, + "message": "File already exists (duplicate detected)" + } + + # Create unique filename with metadata + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + new_name = f"{name_stem}_{timestamp}_{file_hash[:8]}{extension}" + + # Choose storage location + if analysis["recommendation"] == "sql": + final_path = self.sql_path / new_name + storage_type = "SQL" + else: + final_path = self.nosql_path / new_name + storage_type = "NoSQL" + + # Move with metadata + shutil.move(file_path, str(final_path)) + + # Save analysis metadata + self._save_metadata(final_path, analysis, original_name) + + return { + "success": True, + "original_name": original_name, + "stored_name": new_name, + "storage_type": storage_type, + "final_path": str(final_path), + "reason": analysis["reason"], + "file_hash": file_hash, + "timestamp": timestamp + } + + except Exception as e: + return {"success": False, "error": str(e)} + + def _get_file_hash(self, file_path: str) -> str: + """Generate file hash for deduplication""" + hasher = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + + def _check_duplicate(self, file_hash: str, storage_type: str) -> str: + """Check if file already exists""" + storage_path = self.sql_path if storage_type == "sql" else self.nosql_path + + for existing_file in storage_path.glob("*.json"): + if file_hash in existing_file.name: + return str(existing_file) + return "" + + def _save_metadata(self, file_path: Path, analysis: Dict, original_name: str): + """Save analysis metadata alongside the file""" + metadata = { + "original_filename": original_name, + "analysis": analysis, + "analyzed_at": datetime.now().isoformat(), + "file_size": file_path.stat().st_size + } + + metadata_path = file_path.with_suffix('.meta.json') + with open(metadata_path, 'w') as f: + json.dump(metadata, f, indent=2) + + schema_info = { + "original_filename": original_name, + "storage_type": analysis["recommendation"], + # Use 'columns' for SQL arrays, 'keys' for objects + "columns": analysis.get("columns", analysis.get("keys", [])), + "reason": analysis["reason"], + "analyzed_at": datetime.now().isoformat() + } + + schema_file_name = file_path.stem + '.schema.json' + schema_path = self.schema_path / schema_file_name + + with open(schema_path, 'w') as f: + json.dump(schema_info, f, indent=2) + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index da19a6f..90c93ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,11 @@ +# requirements.txt + fastapi uvicorn[standard] python-multipart cloudinary werkzeug -cloudinary python-dotenv \ No newline at end of file +# Required for asynchronous file I/O operations (used in json_routes.py) +aiofiles +# Required for loading environment variables (used in main.py) +python-dotenv From 65c4ea2395169ea4ac78b47c51ca7b13d3634048 Mon Sep 17 00:00:00 2001 From: KrishLalakiya Date: Sat, 15 Nov 2025 21:11:01 +0530 Subject: [PATCH 2/3] Improve JSONAnalyzer logic and add schema/metadata output Enhanced JSONAnalyzer to recommend NoSQL for objects with immediate nesting and improved array analysis for structure consistency. Now saves both metadata and schema files for analyzed JSONs, and cleans up temp files on duplicate or error. Added missing imports and fixed directory creation for schema path. Updated related cached files and added new test data and schema/metadata outputs for both SQL and NoSQL recommendations. --- app/__pycache__/main.cpython-314.pyc | Bin 1553 -> 3743 bytes .../database_routes.cpython-314.pyc | Bin 0 -> 5922 bytes .../__pycache__/json_routes.cpython-314.pyc | Bin 0 -> 10805 bytes .../__pycache__/upload_router.cpython-314.pyc | Bin 2529 -> 2538 bytes app/routers/json_routes.py | 2 + .../nosql/arun_20251115_210256_3306320b.json | 18 +++++ .../arun_20251115_210256_3306320b.meta.json | 10 +++ .../nosql/r_20251115_210448_7d82d4ee.json | 29 +++++++ .../r_20251115_210448_7d82d4ee.meta.json | 10 +++ .../sql/arun_20251115_200713_07511889.json | 18 +++++ .../arun_20251115_200713_07511889.meta.json | 12 +++ .../sql/arun_20251115_200753_07511889.json | 18 +++++ .../arun_20251115_200753_07511889.meta.json | 12 +++ .../sql/arun_20251115_201824_3306320b.json | 18 +++++ .../arun_20251115_201824_3306320b.meta.json | 12 +++ .../sql/arun_20251115_201911_3306320b.json | 18 +++++ .../arun_20251115_201911_3306320b.meta.json | 12 +++ .../sql/flat_20251115_200447_71f45d7b.json | 4 + .../flat_20251115_200447_71f45d7b.meta.json | 13 ++++ .../sql/flat_20251115_210458_71f45d7b.json | 4 + .../flat_20251115_210458_71f45d7b.meta.json | 13 ++++ .../sql/r_20251115_200527_7d82d4ee.json | 29 +++++++ .../sql/r_20251115_200527_7d82d4ee.meta.json | 17 ++++ .../arun_20251115_200713_07511889.schema.json | 9 +++ .../arun_20251115_200753_07511889.schema.json | 9 +++ .../arun_20251115_201824_3306320b.schema.json | 9 +++ .../arun_20251115_201911_3306320b.schema.json | 9 +++ .../arun_20251115_210256_3306320b.schema.json | 7 ++ .../flat_20251115_200447_71f45d7b.schema.json | 10 +++ .../flat_20251115_210458_71f45d7b.schema.json | 10 +++ .../r_20251115_200527_7d82d4ee.schema.json | 14 ++++ .../r_20251115_210448_7d82d4ee.schema.json | 7 ++ .../__pycache__/file_utils.cpython-314.pyc | Bin 4419 -> 4428 bytes .../__pycache__/json_analyzer.cpython-314.pyc | Bin 0 -> 15088 bytes app/utils/json_analyzer.py | 73 ++++++++++++------ 35 files changed, 402 insertions(+), 24 deletions(-) create mode 100644 app/routers/__pycache__/database_routes.cpython-314.pyc create mode 100644 app/routers/__pycache__/json_routes.cpython-314.pyc create mode 100644 app/storage/databases/nosql/arun_20251115_210256_3306320b.json create mode 100644 app/storage/databases/nosql/arun_20251115_210256_3306320b.meta.json create mode 100644 app/storage/databases/nosql/r_20251115_210448_7d82d4ee.json create mode 100644 app/storage/databases/nosql/r_20251115_210448_7d82d4ee.meta.json create mode 100644 app/storage/databases/sql/arun_20251115_200713_07511889.json create mode 100644 app/storage/databases/sql/arun_20251115_200713_07511889.meta.json create mode 100644 app/storage/databases/sql/arun_20251115_200753_07511889.json create mode 100644 app/storage/databases/sql/arun_20251115_200753_07511889.meta.json create mode 100644 app/storage/databases/sql/arun_20251115_201824_3306320b.json create mode 100644 app/storage/databases/sql/arun_20251115_201824_3306320b.meta.json create mode 100644 app/storage/databases/sql/arun_20251115_201911_3306320b.json create mode 100644 app/storage/databases/sql/arun_20251115_201911_3306320b.meta.json create mode 100644 app/storage/databases/sql/flat_20251115_200447_71f45d7b.json create mode 100644 app/storage/databases/sql/flat_20251115_200447_71f45d7b.meta.json create mode 100644 app/storage/databases/sql/flat_20251115_210458_71f45d7b.json create mode 100644 app/storage/databases/sql/flat_20251115_210458_71f45d7b.meta.json create mode 100644 app/storage/databases/sql/r_20251115_200527_7d82d4ee.json create mode 100644 app/storage/databases/sql/r_20251115_200527_7d82d4ee.meta.json create mode 100644 app/storage/internal_databases/schemas/arun_20251115_200713_07511889.schema.json create mode 100644 app/storage/internal_databases/schemas/arun_20251115_200753_07511889.schema.json create mode 100644 app/storage/internal_databases/schemas/arun_20251115_201824_3306320b.schema.json create mode 100644 app/storage/internal_databases/schemas/arun_20251115_201911_3306320b.schema.json create mode 100644 app/storage/internal_databases/schemas/arun_20251115_210256_3306320b.schema.json create mode 100644 app/storage/internal_databases/schemas/flat_20251115_200447_71f45d7b.schema.json create mode 100644 app/storage/internal_databases/schemas/flat_20251115_210458_71f45d7b.schema.json create mode 100644 app/storage/internal_databases/schemas/r_20251115_200527_7d82d4ee.schema.json create mode 100644 app/storage/internal_databases/schemas/r_20251115_210448_7d82d4ee.schema.json create mode 100644 app/utils/__pycache__/json_analyzer.cpython-314.pyc diff --git a/app/__pycache__/main.cpython-314.pyc b/app/__pycache__/main.cpython-314.pyc index 6892a7b1e5c68769902f47f95939d340b956f7f7..fd340283d52833147547111e3947cccbf800bfb5 100644 GIT binary patch literal 3743 zcmcIm&2JmW6`$oUm*i@xFN*qbBx_~YvYAGrEI3w>xJ?aNw&+N-x{_Tepi8XCk+jv4 z%kC@qJJFJ|B*!KD>exqF(3{;;}Cj)evMQvhvqq(8)bAP7n$d6 zeqOKzZ_dL!YDYa<09v#~kB;VIb`12zTzo!ZCjy>iOTdfeI_70t9z}N(W9U-g4rbgK zQsZi32K=(ePTox}jrer?s7jXt4YgySWp8(Z_E@`~Pv7O2&OFamWi>g;shu;tF9Yu< zTStGuy6Z<-i$7r9{hal=6ZY8CI%jX&Ia9FHleAMU?Y=hcJ~et4Wzt`>@zV1RPRZ-`m-rv#>5WzE*sG!kf#JX7(OD4RB} z-h-*Iq}iKJ&mu1CnzC%0mbQTv=8mILTel5NW5m`hyIw1`pEm}R_Ye{eK87EA28{yj z3N86kq(!!fF;o!3$x;Z|kO5x+5*LQ!g_@Bct;vU)s-Q+^f*p*aLLzjnilN3RnnY^H zEv|;{pl)=16rmbd=m_=hw05@y>395r)de*+6P$Y#y#=sNpwsPs;z#_*;3sk?@_+D? zIP{}RXVI&?&SjF$oHT1z*Y%CMg_T*ute5qwW^F5GRnclSCHTXNRxK+f3u`t8mJfxp zZs8KpIwq46P0udex_vE|zpAblJ+hd;x;RIB+t`4e;#+g8M^qVad2UvnTUp5nR4TCz z6suYVz@^pn;uhYf5+eu(?=}_T3mdKf1j799KmERa!=>O9nytg$8I*B(5>g{p(=1%I z3610XIn{hIxhkYpRWp<@ZIrgtx770)-Wh_ZWrGl9 zLv4~JzcrkfP8L;7ChTq(+PG4Kwa&Hk^k56c!$l#34NRPuU!Ybp4FmoZgoQZcFVH2W zO^kr*d%eB1{QQghAS--G|#> z^Mr&IYeiA$&Dqxrw+XgLVF6doduHJpCR?^yE3lXwhQ0wmZ(^}-*!skxS=Nk+pcG9k zZxe`90S;8CXnJ+BwoQ`(T#HcpU~dYzNWn|~4#U?_h`yCkv};dDe=ek-qz67){czPC zy}X;AdD7eWQRc&pJ3O`9d+~^*(Ublz5siw>QgWrK$HK*LM4?Z7uy7!w{xc5~PvY{! z)jxFY#YcAHBOjL^t+?@#FXFH7^C10J^o%Y(7N)^yP;`YpW;Omlt;PeZ%U=jH-xF33 z-<_3bJ2^+lLLyM{wpZx^*kaAX>-v2v*xCkhx;!v1S!M|nqE|PZUKT$Ov)6)BPbil? zQmT7kaVXX3U>XL$RI5;I)riWoqNs{$jP-C;Qaf0WRXf!#wL2TjMuOteLt_>227SR% zeua&aS;2~JD)uH;p6ePkfve@32{0g34!Tu^wH9ZJoh$3JlnKQ@pQ2Y_S(3e{m$6CM zs)x0Sb_DjUr3L1Ud~OX^v^2=tZ3p|gzOF;Q(1gM2fMp*G;|Jh^HFzDR(&EkcZrt78~+_J2d-@ac+8?*@k@DRaP2uzdLxN*^}s( zno(I6n@Em|-xC2=dpAA6IkD^})(m^|2u@k61)K^81`q*GJ(L!LyCXt{*2Q9pt##~y zD})d>zwmn$-&sCDxTJPvBUvFpc(;?5{5qrf`%<7jmbN5lL=dVvp)sGmP19cM^2%i~ zY(=b5Z17U*ua2c zO|iiPhU|x+=tai50{q+XlDA>-03xGA&j*=CeDssPPuBjT|5CKkhj6=ee}q!zoIdcmNgf6cYxjKhw`?g{p$p|ANJjfYie9R9#zC!w8swPe)G z;ia02!EKv|j_4(ZclhJsiC?Lh<+_1?Ze0gA46x)Y7@9oCao>rE8~!J{{2z4oDLVfp z>UoMze~A)L(cpd@aq{ny{}PjK-}%Slg$MjIPI!O#o#BW2E;qOzIm^9vz@x}elTpn) zhd5ym_3ohF4}P|nI=_=T|496F^V7Fo)cY9CKSO*Vd}A+lZYOo_uj}rWMfZ)oC%$zk zeseFC*-2#{jk~{Cao@b{iSNL_h(zBXcxT{Y>b>FpNRkto=guahno1}e+T~LFk#1ku K_YI>wjQkf!xzK3< delta 743 zcmZWnOK1~O6uoc$`L<20NlMa8OidcJ2}rdI>nFHrU9{nWh>^4eGtoej2{V%}vN5(o zT^iofg==>%yLIa-1+g+kbSLi8M!|xh?{!KHc;Oxn=l&n>b*$(0U-J9hK&`qSuEm6& zKg|7t-u%U1hZ0Hv7O}L19K~{#T!m?ek0X{B z&oZzQ8r54KnjIO>b9jj_%*_8UV;6R35%w(cPK=Is6=J=?x#+05Z>;$Q=97@|lVfyc zaxH^wPhBrpHY>WNHMhtHcb)44dBwf=VmS#{`5MY7ofOFnbL6?uLLKs3NbqYJk=&Qk zt~W4bBkeqSCnW=Fqg}UZ3)Na(Q%&8lG=uz+qCs|upSpRM!guJx4>!rmwZ%07YFr7++TLO&%y(JyfA5T-uE>>;?0B=X5OyY5EWArD++z)?1s0VEti yU>^c6ZyW^E`@!^EY46Eiu@8X{urg496L&+;LN9_l(IXC_hlthsk58$=7WoUXZ<(C{ diff --git a/app/routers/__pycache__/database_routes.cpython-314.pyc b/app/routers/__pycache__/database_routes.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bda1cc11c735b896973e1de7b7c9dd419afcc5d9 GIT binary patch literal 5922 zcmcIoTW}NC89u9B-DOF(UH+I(qs}FHV zX5uvMq)BK)rkRN!=oH?_ByE}LWkMeNBu6GzHgrmxcBZ`9flNZ0hyMRr?Me_xrqk(} z(b==-{Cm#oKi~iT-`S1s8V7-5{%Mo=nVXQ$@J26expDh(hLCx3f<%}x5@r-8Y*I{N zR$<$*&ost`&5Ai}Q7mDrVx@iTm`$<47&m4QI}}H`?}&phS|*C;jZ3~OhVbzQ{ONFp_jB;{3n5jV6w zwC#_0p-?yx8;Dz{HeC@LGu%pK`tRBk-3%v@$vPYT9jC*^D=HYKJ6IqwZ7<4Sxo zE(<}pKu-b}(u)NeFA&1r-VTjU(heILExQz)%0x`CCG58HRv&;ZI|=Qx%shM8>>ypl zN)B^dS&`{8=R*#BE19RO28E0wDxx6svXYgABrj%oWk%p-F+H1#E6{Q5=wV)eF+KqN#Q_U$cEh?%dJ2fQ>$`J_>d70eCf1o}yvOW5gEJ$+n zu#m}~$wv1H@+l>oi;iU!A(axR;XMYPm`y3-z!TYIJT-6(UMD^+3>=%26(Jps=W
(@?dVR>`Zv8nC3dE0T!)+pWFh~EpXNy7pd#G)N@7oAhcY&f5o}K!0lgh z?!SB61qSEp0?c`)nftjpz~t*I`;f?{QbGdv^#8RPqyODz9LeH#R9;Kp5OXFAe_n#f zCz1y$&H0iK@BLWSL!~xQ$WhZ+We! zsW?bh;>x|Qv4I7}>EEHD;WPmFtYB5A3c#qT8lw?5T1GE12;y5dT+|HH((k?~9ltmZ(#|JI?6g?;?+yhD=)j48&ND*~5s8 zNOrKTL}X|@S@VOA(acOdlYm%3cR*~337LOZRA%^`AWdbZG!F74frs*4CER z!+0zFWCbdWw^fJ3Ji^>`)i1XW7F##1G;S)nnwL9&R*n1){@Bnf>j%aI_&&`L_>V1&+?E1Uw*POW$#}P*(nJ{||HZg205oZ*W11%F~y+7ibutaL2$e-GXCF1Ef zSb@M+>s%0IpZ4{F&C|X8#ykW(ap+k>?4*rU&0%`j91~2$jdM%@NGOwd^X&%&g-;4f z9P|K^38jfDQc-P~;ZQPs-3$paShg_eSPsIt>d*)t&GLZh_6Q@OY^lBRa@ZZ_Hn2*@E#}ofvQi}jvHXr zjVmc)9HZ;La+b`S+sPqb+l&Y#1L|Pr2v8b_l!hD}lf}FM9j3ueDl&uu#k@yM?+up6e6ngW_YiHg#yVAJ+wd^WAJDFMVNsH9A?&#IbHyvb4zKIy0|qjH&eT2MJ0 zRn1fsf|I~EG6o;770OQ%4;{3kwDd9DQKq*LyYKwILi143zqMrZpHCJ7!^MUllx)p! z4;A?ukQNEt{eWQ3;uVS-)deQE;jcU{R0c5H+&6C=-*<)wxVy_!oHi{mbVj&J-<9z z@^&unD|y>5)-NiTesq2Pj;lwD9eW_P1l$XwFCSWUkebHpuC^Z^T=S8}?mG>nqj&M- zl|$EiAOE1E$nTpUEjU`1xYm->JAe4Dj6r=a6c}k>u62fn;PSua0I3fh*AG=6BMrc% ziu5VX(<+PuI@dcH%Nl{|qCzj<=6VDG<@!!&=v*H$Mc4?3Y9?YGBoW(_Opq&s-=1$o zcBzsFx*}2Cao~VCS(IaPP7so+y(|@=S*Jr^n3@t3qL4|J)^qPLK;MWY*q#- z14zq0D?zNnxhUT=0_sIf3KHnlbWRdv8I?;aE=>#k!DCMz(GDK@-m!EpC4k;h_(VK0 zgEo*6`6)>_J&T=COK>j9Zw)1r{A4_FYMKHW4k$^$d)aytbP+5h3vnrNkKEFvFM6}0 zf29t*#frN65oqZ6ZN9LvWNW!FRkAf)FavNuCtQnT)oyoLR*BtgDTi;3=UyOS*25n? zlYO5Uh#ExIk5o}EC=n<;`b-i^RH+-P??MU692sReDgq}HKzIb+=iWc|8#`@&8!4D^ zDN_Cn4Gk%SBzMp0QO|%mW`g;yT&`#6$lJ9nJ|_qAKGWGqL&%?Nr-SI_L6_A)>2wEq|Y`1>DGxNwXO~F)K-kDoke5W|&vn z3RM+W5kTS@#8oz>9eOKR?Xyt6mGEBJ#&&!+is$?AFOx#yo-u{M2VZ9*b!dZ+cR z*2S~M=1oQa<^}r=r+Z=UmE9NiE<4*wuE2_`bIH|N=sLLUI<#sbu0VlXcUMM0oeOn@ zx|n>eArgX#ln~F%<`OVy6ywx(5K^s%k-&oy0-uTiCEuy|43r!!@8|!zRCOTLyBU-y!)d&Sqg?CUN0TUY$uOaAUadW#V-!Psk9In0HNS@tP;-d_zFH0X@|~T^vt=q=lv~L zPQUlql}8F4JBsd|1=~(~5}C me!;^z_=_PBNMEdj{I|N!n6ACZ@N1@eW*4s3@j13=O8gJ-Mo-58 literal 0 HcmV?d00001 diff --git a/app/routers/__pycache__/json_routes.cpython-314.pyc b/app/routers/__pycache__/json_routes.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..613faa68b0f07b764d50bd69d3c42ce6b943505f GIT binary patch literal 10805 zcmdT~U2q%Mb-s&T01GSt0t85c1W1BEA`Mw0B$E20CDHsR^+Pjj394P#3`9T_ED(Tq zm$Avj9!5?k^h_s~e`Z8g&WLW4nr-}`O#I-s>C_Z8k^0hrqHMq$b?P?CgWkZ%u@ff` zJ!cmSfHWODX*<(ia?hTHLR0#!?Z*XGZHhbm-O{`Pd8$a3~;7LjKi$N=Icz73F?>;^DrlI!xqVs zueVB8sMn9!hRdWfrOq&7A9hHNdh!I-KzJk1wo$xkpH9_5Du1G^fpq2-lAKSJkNGt< zi)Y^437TSgf`&dg?W%*~EmefKR*|?pP8E8o_?%vBTD#BmQU<-)H}z7uuWKTH&pqtSua`|dVI6Va3uOD5#9-tfe^Ng+9%h@1$E=fqJzB{O4T>8xKT zGsj}0Bs2Z72?=T~M;?7-v?mdc&&);zyvFYMsF+G7#OSO(@RXQL%$iZihj&Eiz>pH6 zXJXIFj1-;}XN`erMVYn1+X6lVG&Y<`3R7W8#78&EkM(thPKr@M z3>}Lml3z=P`lI4GDVYk5B&29O9-E9Nq@Bm7<5Fzr@nj?%-}$JN6vC6yosZ6lQgkX5 zPNhOHZ!pVZ2**CIT!?L{8QC&E9!?~ZQdo+Pj|&y>I42yU56B$(gmh3n)Vf0NRBp@B zczM_CTGu0UA7scxVUTda!tnwj%>4)qqmk4@7eQ(a#Ws~OoW@N>Tf}49CM~(oQ`&NC z&DTy^aneILXG-C9Jl&bsXYC0?uzjBb4W6au=wo^_N$EnICKc3aPvR^0 zzXmUN^KB*l10 z$gcG>R<3cya{~!7U>fqHe~22Vr*+fxxUMi>YSVc2LH!uM>P@9M_RTX?4>N{;o%#1R zq}$kIJqcB5o$;2>AD1>qAn(@X$G}K$E}>ejCdg=Qv~$W((zafcE1tJ%yR}9^gEpR& zF=*tiJ8Fu(XzMJ}i|dTH@nyUnWE??5fKIVN<5)vcR^#|`O-HTuBuII|P>eNcGR1SK zS>w0VxS$)X6}G8#r8_|nSE;Vrn?Fk#$pCdMW+cb%!*K_WfAx_EiSo8!N2=@HNX(?5 zAFLTKiRtP|E$mkOJNC1QC3ew0g@PB2?JDQ%r;y+#};4YChvJrvKH zEUW;dR?9p2isBrcFln=hJE1r>o=KUyO))A)R1NpgG0I;xJ7^wAoDC-?q7k2JvwhFR zq_e)FmGz~BI=u?$!DUG(b)G;q!}HDPf8|z@vtx%ZSx%uKO38xp7M#Y*{Clj`u6NP z-utN?EN?(mjd1`haq0xvb~gGfIIHpGIa!ZYP>ZSX`9ZFnT{RCd#G=OiOG6?EAfsKyg{G z41CsEe2(u7G>JyKd@(|Nw7*hjanOnIR8-dEkOj~<5fO1(1p}xlGeR^Rk!g?-(BjMb zsdJGSNQGlbWx8Z0nTjT4{WC&Lf<970EFsBO#qlU=fm4(#%3}tN`hR5vlT6^ zos3G*btIyAOWBhj`Z(0Y5Eg{v<5Mu`NLUKXrXpA{D3dJ8hUo;n-*bMmfHN$h^^}bh zN${JX6RXTW>TXgvMGfkV;6o{c5Ea3t$}F4};2={`5us&Ei!WaU7YcIfx>Ix(x}gRi zBBo)o?|}o%*@?ye(xDd*Ww^Rku5OL)ddbv3zIr zo1+W;bECJJhOFJaaB{Kp%|mY-S`4H;?eE#!=SFgjPY=G^>A865l|zfwOZ^$=Zur0L z+dN_)|XosTGz@OGnU4*rSTJ*xZ2_NM+RcDrR@!i&2R2}W9MRh+P3>P+YYs^ zhK#d))!F`b$JLo1A6|3z&a+TkUY~KatvcG?60aWk;q&iE|NPwFKX>!&n&Z^G;X}@G z!Jpy0w>j^J_Ug3fK-ONpus3V3Ss2b48MkeIIAOKQ{%VoU;WVOnSbPNdHo{27+f0q!T4JFp0s7puLdYm=394x;vWF9O9#ZUDONLwiA;-{@n|x z`rH6z5o4h*zds1%kAbow@FQv|^*}1X6%eH;`%Km=hQLp&pcb)M9@z>Cyo9|$8kOCn zKu(2>K%LkC1aYU2GOQy}P$!)4a%HcX-{U;%dbkiOL23i=M7Z<<0?%)NLsNlg8^Ag^ z=pbb$gMdL(Opy6sf}${yUUC#XThXNzVM|I6a1bY8Qwi!dH&;)N@`RX39e~1u3k6^U z@K%XrvEi6yQ**ck*heGm4Nwf(4A@e-pw3QC8m3_mrgejRVf=_~-Vf;bEsP5=~bu1}k%CJEPn4d4J!z2JPfK6ZW`D*-iv>i2u z=fhZWE>mpEz#A~?=1rjj*bNo}3Eo^%TLAAoS5nJrZzw=pA&mMi-_zk8c&zw2ySl-8U(=KHf2$4kdvJhtF^-%_`3 zB96*WY=pJmao1n5{iSV5pLP$v>soaWX37WWjsH>GyykfP(y0aNts_^T{)s=cqyM#r zn~tAV{7c2nnas#z7aJ}#Tsk#(B<*{CE<=chj3MVy2CBJK)w5q4@6QWHqOe6@L*V0DwCJh0u->Q)P^Jh6~Zwqg)bxt(9ZHS^o06vDxV4D=@7za zrHr9Qn~Gx38OAwC4&T zE#h^hKu8OA4nkTZ4+$OKSk&%B(HF2MwtTMvNUzkwOs0(8#-{vvQ&?)6ZPuJX7$QP6 zp+22Zm6{h1WSfD2Wkxrv#96A1-j0_TSPMIl zv>^!~*@a{`5;VbnRxz&vcw-_QK+=WeAtXK|2ay~GBH~iiLK+p59l?7@lnr+e7$uTn zZ2D0u>-JtTT{2zfGUZM4#yifcw0rxS^TByG>+&vEEJ|-a_r`OJ$#nhR_gs7DO@Bhb zu|;Tc$5HqBfkj!xymh^TIGfkmt+77@V1WH8)E#1uJ&Ubxbu2yo!@XC#)Bq)2cKki| z`0sKwO8j2@8yM|*sy{$|%h`93T%`Kk%t8+f?6QyQt6`TLXyltK`h5E315}@vUG8F# zKg>b>H51iW$6Pb_Sm9!&3cFcxQ^?mS)JuWrN}a;{41EVoD=k!?-?q|fM!sE#v=bk; za**raL0>o4^taO28z|%(_0Ze(Rz31NXy9iJ0mb8Qu+3;;aQEED_xr~Hd!8=ISJRNM zW)wRuvzR)VOajW#$7545Nj4shakp>g9#OEc`;cG*jxXH zQq3BIjr^5@H}zYlo(r}f;)uUM3=6mvXL(cE3gbgcI}SN1kC|D0JcX-W6y!Fe;3lw80;$rd^TV|@Y$9Sx=p39w)Wy8x>980T1LD*1 zS;KZEi~Om%4RewTKdQKGnZdlU%uXdEu`@A<-B>MYIqTWFyF?N?gq$c4y}}DUFk;zU z)C9sP4Euz~pIeI~h-&mKjV3tD&I-60%B1g}ke9R1t&#P=c6L8drx3%#;*^RuACgl&}!Z zrs6gHR^c(^3#to~czG6x%%avQ2rN|q6UI8Ts=iW-)1VANF3ZnFG?E{f0%y=(wiLog z$keJ4WEl>!m{%4B0py;8Lo~x$Rq##uVSOK!byd!rvX$<64%3|Ow7X-?*$L=|bH3F5 zVmD+kZ*ktN+jqr!$+~#`D-rY>say;}D`l6;me}{*yT7zr zo{YV5)!tZm_?K6!xu{lqu7+4#7p5V*n|0N`IFPkgEL7dL!eHxP9Qe>x`=0B;`2m>E zs+xsuuRb-;WozmezxIRY(=~hN%OLi(x-N`ExSO?CEqIVrU<$hhlf%L9J-PJtRc{x} zg2$G#5Nl=D?s@&u#p=sXX6?1Fi;EAf)dzrE__~rs&N-}hbB_Nu#d{;c=!Lr`Oa7VvGN@@-O#gU>rJ!03IOZ_0Qd;3iSyKO9rdlA?L!@8xs4iX zGb{)CD7d)3iyGQ(xW2phAY9zoiBdPSN z|J%{CY{IRo?vj(xAAm7L@n0A~{}hD88XyS9jFfF1$@irkp5A7Zt3MDt3zP=Cwho<3 zfERv&FNJRKNGuuG~LvQt@i3H#P=QRk&E;zV+z)FAFfEC-9eB)$htA?VIPjeVP z_-TkQtb~)olQ1I!4`kGDlsWWUN=5hq+`@LL;x(+fvLfcFyCS+#D} z#iYcC7&D5vOff1{BSH)W1tiJ}Iu8t5OxW2Omw(meU)r(e+BbJRYjkCdo>ikKW30Vp zti6-}+Eja$&Nw^a|F*O99V+Xsy)t-d@bd7*#QYG%j~BaM=~|dvbGCe9P~&_L*Sn8ewo<(=cG*TF?{M|j=$E%qy_M{8 zOEvP{R;a&bp?ckhYgQKdN($<)xhdpp485JEYd)&C$#$*5jQkEA(l!&^{U6`D%CsvX zcs_#q!cF@#7-q?)g(R1e>U^sMH*M!nnw6{1ei6xk;9>5go0}u>6B5|GYT>CMgi`Qn zJzoP^A;4UKwDMbw5I|aBBmimYN?Peo=r*>390;A#b;AaLEQrDt5tg5s^%sAx<-;7W zx^4?!ZA((xv&|)<`NT5%NcEf zfJB+wE@0q8t%1>Z-es+6d*{{1t{zR>9)6$g`OO{9g}H;yJ9wjQ>FK43w5>hEcHCk+ z6pzvQsfa<yZ$qI9PI3QWlw$xJElb{5w~}t@70-))gPGAMCZ=w@V92UrzoOh z6~QOhC@%=sH!XHtj(v>Z1Mh=eZB7T=ulp%X2or{v literal 0 HcmV?d00001 diff --git a/app/routers/__pycache__/upload_router.cpython-314.pyc b/app/routers/__pycache__/upload_router.cpython-314.pyc index 839f5d8274207b0f167e2d5d08b0904c5b8ea6bd..84c0c908639bf0615689bd0b4ae20ae000694d08 100644 GIT binary patch delta 58 zcmaDT{7RThn~#@^0SH)oH*zI0GDc6%WmJ~d^)1aQ$<+1DPf5(t4KB$qN=#4H4X!LM MNzL87h|!({0K^RuIsgCw delta 49 zcmaDQ{7{%nn~#@^0SJ@=HgY8~G6qe~WmFbY$Sut&$<$5D%t=)!&MzuSE!w=A(VhbU DGXM?Z diff --git a/app/routers/json_routes.py b/app/routers/json_routes.py index dff6321..3bd72ad 100644 --- a/app/routers/json_routes.py +++ b/app/routers/json_routes.py @@ -4,6 +4,8 @@ import aiofiles from pathlib import Path import asyncio +from typing import List, Dict +import json from app.utils.json_analyzer import JSONAnalyzer diff --git a/app/storage/databases/nosql/arun_20251115_210256_3306320b.json b/app/storage/databases/nosql/arun_20251115_210256_3306320b.json new file mode 100644 index 0000000..0a10eee --- /dev/null +++ b/app/storage/databases/nosql/arun_20251115_210256_3306320b.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "project_id":101, + "name": "Hackathon Storage System", + "lead": "Arun", + "deadline": "2025-11-20", + "budget": 50000 + }, + { + "project": 102, + "name": "Face Recognition Pipeline", + "lead": "Meera", + "deadline": "2025-12-05", + "budget": 75000 + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/nosql/arun_20251115_210256_3306320b.meta.json b/app/storage/databases/nosql/arun_20251115_210256_3306320b.meta.json new file mode 100644 index 0000000..0bbdf63 --- /dev/null +++ b/app/storage/databases/nosql/arun_20251115_210256_3306320b.meta.json @@ -0,0 +1,10 @@ +{ + "original_filename": "arun.json", + "analysis": { + "recommendation": "nosql", + "reason": "Contains nested dictionary or list (depth: 1), ideal for document storage.", + "nesting_level": 1 + }, + "analyzed_at": "2025-11-15T21:02:56.715489", + "file_size": 345 +} \ No newline at end of file diff --git a/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.json b/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.json new file mode 100644 index 0000000..6af1277 --- /dev/null +++ b/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.json @@ -0,0 +1,29 @@ +{ + "product_id": "SKU-A45B-11", + "name": "Wireless Noise-Canceling Headphones", + "in_stock": true, + "details": { + "brand": "AudioPhile", + "model": "X-1000", + "color": "Midnight Black", + "weight_grams": 250 + }, + "tags": [ + "audio", + "headphones", + "bluetooth", + "noise-canceling" + ], + "reviews": [ + { + "author": "user_123", + "rating": 5, + "comment": "Best headphones I've ever owned!" + }, + { + "author": "user_456", + "rating": 4, + "comment": "Great sound, but a bit tight on the ears." + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.meta.json b/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.meta.json new file mode 100644 index 0000000..56c76a1 --- /dev/null +++ b/app/storage/databases/nosql/r_20251115_210448_7d82d4ee.meta.json @@ -0,0 +1,10 @@ +{ + "original_filename": "r.json", + "analysis": { + "recommendation": "nosql", + "reason": "Contains nested dictionary or list (depth: 1), ideal for document storage.", + "nesting_level": 1 + }, + "analyzed_at": "2025-11-15T21:04:48.313979", + "file_size": 594 +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_200713_07511889.json b/app/storage/databases/sql/arun_20251115_200713_07511889.json new file mode 100644 index 0000000..f877973 --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_200713_07511889.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "project_id": 101, + "name": "Hackathon Storage System", + "lead": "Arun", + "deadline": "2025-11-20", + "budget": 50000 + }, + { + "project_id": 102, + "name": "Face Recognition Pipeline", + "lead": "Meera", + "deadline": "2025-12-05", + "budget": 75000 + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_200713_07511889.meta.json b/app/storage/databases/sql/arun_20251115_200713_07511889.meta.json new file mode 100644 index 0000000..25429ec --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_200713_07511889.meta.json @@ -0,0 +1,12 @@ +{ + "original_filename": "arun.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "data" + ] + }, + "analyzed_at": "2025-11-15T20:07:13.043189", + "file_size": 349 +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_200753_07511889.json b/app/storage/databases/sql/arun_20251115_200753_07511889.json new file mode 100644 index 0000000..f877973 --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_200753_07511889.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "project_id": 101, + "name": "Hackathon Storage System", + "lead": "Arun", + "deadline": "2025-11-20", + "budget": 50000 + }, + { + "project_id": 102, + "name": "Face Recognition Pipeline", + "lead": "Meera", + "deadline": "2025-12-05", + "budget": 75000 + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_200753_07511889.meta.json b/app/storage/databases/sql/arun_20251115_200753_07511889.meta.json new file mode 100644 index 0000000..f0cbb33 --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_200753_07511889.meta.json @@ -0,0 +1,12 @@ +{ + "original_filename": "arun.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "data" + ] + }, + "analyzed_at": "2025-11-15T20:07:53.568857", + "file_size": 349 +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_201824_3306320b.json b/app/storage/databases/sql/arun_20251115_201824_3306320b.json new file mode 100644 index 0000000..0a10eee --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_201824_3306320b.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "project_id":101, + "name": "Hackathon Storage System", + "lead": "Arun", + "deadline": "2025-11-20", + "budget": 50000 + }, + { + "project": 102, + "name": "Face Recognition Pipeline", + "lead": "Meera", + "deadline": "2025-12-05", + "budget": 75000 + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_201824_3306320b.meta.json b/app/storage/databases/sql/arun_20251115_201824_3306320b.meta.json new file mode 100644 index 0000000..e312056 --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_201824_3306320b.meta.json @@ -0,0 +1,12 @@ +{ + "original_filename": "arun.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "data" + ] + }, + "analyzed_at": "2025-11-15T20:18:24.763567", + "file_size": 345 +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_201911_3306320b.json b/app/storage/databases/sql/arun_20251115_201911_3306320b.json new file mode 100644 index 0000000..0a10eee --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_201911_3306320b.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "project_id":101, + "name": "Hackathon Storage System", + "lead": "Arun", + "deadline": "2025-11-20", + "budget": 50000 + }, + { + "project": 102, + "name": "Face Recognition Pipeline", + "lead": "Meera", + "deadline": "2025-12-05", + "budget": 75000 + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/sql/arun_20251115_201911_3306320b.meta.json b/app/storage/databases/sql/arun_20251115_201911_3306320b.meta.json new file mode 100644 index 0000000..f81c5d5 --- /dev/null +++ b/app/storage/databases/sql/arun_20251115_201911_3306320b.meta.json @@ -0,0 +1,12 @@ +{ + "original_filename": "arun.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "data" + ] + }, + "analyzed_at": "2025-11-15T20:19:11.867121", + "file_size": 345 +} \ No newline at end of file diff --git a/app/storage/databases/sql/flat_20251115_200447_71f45d7b.json b/app/storage/databases/sql/flat_20251115_200447_71f45d7b.json new file mode 100644 index 0000000..3dadca0 --- /dev/null +++ b/app/storage/databases/sql/flat_20251115_200447_71f45d7b.json @@ -0,0 +1,4 @@ +{ + "item_name": "Flat Test", + "item_value": 100 +} \ No newline at end of file diff --git a/app/storage/databases/sql/flat_20251115_200447_71f45d7b.meta.json b/app/storage/databases/sql/flat_20251115_200447_71f45d7b.meta.json new file mode 100644 index 0000000..6824778 --- /dev/null +++ b/app/storage/databases/sql/flat_20251115_200447_71f45d7b.meta.json @@ -0,0 +1,13 @@ +{ + "original_filename": "flat.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "item_name", + "item_value" + ] + }, + "analyzed_at": "2025-11-15T20:04:47.432046", + "file_size": 54 +} \ No newline at end of file diff --git a/app/storage/databases/sql/flat_20251115_210458_71f45d7b.json b/app/storage/databases/sql/flat_20251115_210458_71f45d7b.json new file mode 100644 index 0000000..3dadca0 --- /dev/null +++ b/app/storage/databases/sql/flat_20251115_210458_71f45d7b.json @@ -0,0 +1,4 @@ +{ + "item_name": "Flat Test", + "item_value": 100 +} \ No newline at end of file diff --git a/app/storage/databases/sql/flat_20251115_210458_71f45d7b.meta.json b/app/storage/databases/sql/flat_20251115_210458_71f45d7b.meta.json new file mode 100644 index 0000000..f94bd08 --- /dev/null +++ b/app/storage/databases/sql/flat_20251115_210458_71f45d7b.meta.json @@ -0,0 +1,13 @@ +{ + "original_filename": "flat.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat structure, ideal for relational querying.", + "keys": [ + "item_name", + "item_value" + ] + }, + "analyzed_at": "2025-11-15T21:04:58.674259", + "file_size": 54 +} \ No newline at end of file diff --git a/app/storage/databases/sql/r_20251115_200527_7d82d4ee.json b/app/storage/databases/sql/r_20251115_200527_7d82d4ee.json new file mode 100644 index 0000000..6af1277 --- /dev/null +++ b/app/storage/databases/sql/r_20251115_200527_7d82d4ee.json @@ -0,0 +1,29 @@ +{ + "product_id": "SKU-A45B-11", + "name": "Wireless Noise-Canceling Headphones", + "in_stock": true, + "details": { + "brand": "AudioPhile", + "model": "X-1000", + "color": "Midnight Black", + "weight_grams": 250 + }, + "tags": [ + "audio", + "headphones", + "bluetooth", + "noise-canceling" + ], + "reviews": [ + { + "author": "user_123", + "rating": 5, + "comment": "Best headphones I've ever owned!" + }, + { + "author": "user_456", + "rating": 4, + "comment": "Great sound, but a bit tight on the ears." + } + ] +} \ No newline at end of file diff --git a/app/storage/databases/sql/r_20251115_200527_7d82d4ee.meta.json b/app/storage/databases/sql/r_20251115_200527_7d82d4ee.meta.json new file mode 100644 index 0000000..7881110 --- /dev/null +++ b/app/storage/databases/sql/r_20251115_200527_7d82d4ee.meta.json @@ -0,0 +1,17 @@ +{ + "original_filename": "r.json", + "analysis": { + "recommendation": "sql", + "reason": "Flat or lightly nested structure", + "keys": [ + "product_id", + "name", + "in_stock", + "details", + "tags", + "reviews" + ] + }, + "analyzed_at": "2025-11-15T20:05:27.652281", + "file_size": 594 +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/arun_20251115_200713_07511889.schema.json b/app/storage/internal_databases/schemas/arun_20251115_200713_07511889.schema.json new file mode 100644 index 0000000..38e6903 --- /dev/null +++ b/app/storage/internal_databases/schemas/arun_20251115_200713_07511889.schema.json @@ -0,0 +1,9 @@ +{ + "original_filename": "arun.json", + "storage_type": "sql", + "columns": [ + "data" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:07:13.044088" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/arun_20251115_200753_07511889.schema.json b/app/storage/internal_databases/schemas/arun_20251115_200753_07511889.schema.json new file mode 100644 index 0000000..3411283 --- /dev/null +++ b/app/storage/internal_databases/schemas/arun_20251115_200753_07511889.schema.json @@ -0,0 +1,9 @@ +{ + "original_filename": "arun.json", + "storage_type": "sql", + "columns": [ + "data" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:07:53.569590" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/arun_20251115_201824_3306320b.schema.json b/app/storage/internal_databases/schemas/arun_20251115_201824_3306320b.schema.json new file mode 100644 index 0000000..0e5d7c6 --- /dev/null +++ b/app/storage/internal_databases/schemas/arun_20251115_201824_3306320b.schema.json @@ -0,0 +1,9 @@ +{ + "original_filename": "arun.json", + "storage_type": "sql", + "columns": [ + "data" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:18:24.764543" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/arun_20251115_201911_3306320b.schema.json b/app/storage/internal_databases/schemas/arun_20251115_201911_3306320b.schema.json new file mode 100644 index 0000000..5f8c6be --- /dev/null +++ b/app/storage/internal_databases/schemas/arun_20251115_201911_3306320b.schema.json @@ -0,0 +1,9 @@ +{ + "original_filename": "arun.json", + "storage_type": "sql", + "columns": [ + "data" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:19:11.868290" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/arun_20251115_210256_3306320b.schema.json b/app/storage/internal_databases/schemas/arun_20251115_210256_3306320b.schema.json new file mode 100644 index 0000000..008d78a --- /dev/null +++ b/app/storage/internal_databases/schemas/arun_20251115_210256_3306320b.schema.json @@ -0,0 +1,7 @@ +{ + "original_filename": "arun.json", + "storage_type": "nosql", + "columns": [], + "reason": "Contains nested dictionary or list (depth: 1), ideal for document storage.", + "analyzed_at": "2025-11-15T21:02:56.718434" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/flat_20251115_200447_71f45d7b.schema.json b/app/storage/internal_databases/schemas/flat_20251115_200447_71f45d7b.schema.json new file mode 100644 index 0000000..d8204f8 --- /dev/null +++ b/app/storage/internal_databases/schemas/flat_20251115_200447_71f45d7b.schema.json @@ -0,0 +1,10 @@ +{ + "original_filename": "flat.json", + "storage_type": "sql", + "columns": [ + "item_name", + "item_value" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:04:47.433815" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/flat_20251115_210458_71f45d7b.schema.json b/app/storage/internal_databases/schemas/flat_20251115_210458_71f45d7b.schema.json new file mode 100644 index 0000000..ce589fc --- /dev/null +++ b/app/storage/internal_databases/schemas/flat_20251115_210458_71f45d7b.schema.json @@ -0,0 +1,10 @@ +{ + "original_filename": "flat.json", + "storage_type": "sql", + "columns": [ + "item_name", + "item_value" + ], + "reason": "Flat structure, ideal for relational querying.", + "analyzed_at": "2025-11-15T21:04:58.675774" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/r_20251115_200527_7d82d4ee.schema.json b/app/storage/internal_databases/schemas/r_20251115_200527_7d82d4ee.schema.json new file mode 100644 index 0000000..53d47af --- /dev/null +++ b/app/storage/internal_databases/schemas/r_20251115_200527_7d82d4ee.schema.json @@ -0,0 +1,14 @@ +{ + "original_filename": "r.json", + "storage_type": "sql", + "columns": [ + "product_id", + "name", + "in_stock", + "details", + "tags", + "reviews" + ], + "reason": "Flat or lightly nested structure", + "analyzed_at": "2025-11-15T20:05:27.653407" +} \ No newline at end of file diff --git a/app/storage/internal_databases/schemas/r_20251115_210448_7d82d4ee.schema.json b/app/storage/internal_databases/schemas/r_20251115_210448_7d82d4ee.schema.json new file mode 100644 index 0000000..a216afd --- /dev/null +++ b/app/storage/internal_databases/schemas/r_20251115_210448_7d82d4ee.schema.json @@ -0,0 +1,7 @@ +{ + "original_filename": "r.json", + "storage_type": "nosql", + "columns": [], + "reason": "Contains nested dictionary or list (depth: 1), ideal for document storage.", + "analyzed_at": "2025-11-15T21:04:48.317670" +} \ No newline at end of file diff --git a/app/utils/__pycache__/file_utils.cpython-314.pyc b/app/utils/__pycache__/file_utils.cpython-314.pyc index 844d504ee549e40f2689bdde560a5e12f5833fe7..e6639868d68e9a9193bf49976d7224090c475a15 100644 GIT binary patch delta 58 zcmX@CbVi9wn~#@^0SH)oH*!s2WDJ`;n^9R>*S9pMBvaQnKP53oH@GCfC^0=%H@LF6 MBsF*Q1;%220Ly?8+yDRo delta 49 zcmX@3bXbW?n~#@^0SE*FH*!s2Wb~apn^9RvA-6QABvUsnGbdG{IKQYQwP^EA#$tW| DHVqDQ diff --git a/app/utils/__pycache__/json_analyzer.cpython-314.pyc b/app/utils/__pycache__/json_analyzer.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a866a61c357a08e7b6cb74da8d1493c54679f79b GIT binary patch literal 15088 zcmb_@eQ+Dsb?;qZ@kxLHzrkLu+If6Q-m`3{5fwW_daP;4Nnazsv=eLE&PYw%IumyWFh~R4s1wc9Gnvf1c|(<+ zGI9RtId?w*g0Pgdy%O*4-o5wi{W|CT&N&wy#da%!^#9*+aid!a`B!{U6RX^;Md9W; zIZwRI5E11oBQo?^C+g@`FY4i{8)AAq*6{LxH4OPTz+^LhlCfdAA3-OqKjgKchjKrP{h!Y;Y#13#lQPMvXy(+QKbAkx3 z%%Ol77P-lAP;M*ddgWp5W4H-G$IlUw@e=5$P1IMC7!0F|h^&wFFkW31iI}~5$ep5r zmaw!0x&$RgT4JOnCR$>mC1zS;p(SQ-B`vekvLY;tFrtN)*=U)a*0a)*Vp`&$B}KHv zNlQv-iH(+&(vmV-V)vGd#XcfBycIAhRg&$w;TKLk6AeVJ&V>0OlQV21+{4JtYYR|% zos4Pkykw_x?;}yI)JIM#6P`l@W@cMa_Svh;eS+3ni{GoYKCiX9F62vEvw8K^?e}H% z4Y~Eb#@r|9t3e$Hd`+XImJy_hGnSX2)SOoYk5EdPWaFD2)=35{35hd4=A?b&JACq* z;GelynX^W6%teTWG!U-^R?<3C6o|*W1u@14E`}uwY~;X&fDjfWz3_5GGDKr|Va4!d zTq@>R!-z~{zb++$NB7ZJuT39+a@ zpls%@_*Kd5_j6HB^!ufX+PXvNiY zC%od?M+<8j<~wh7E_STc?E1j8QgaXr9hvI-`TAS+id6GB3;K6T_2Gr@S7 zVdf|^}Qlx|Az zCp(oIas8Oa>4mtviTL!|H(9S4|Ls($HIgxn-_NjP#p+j86^65gDO8F*1nE&CKj-GB zpHXW=zcp$;b`09cNqxVGT7_C8d(EY$%!mbN#7#Pg+WsQUejVX-jijD5(tpo~_UY>h z$GAz(T!2K&Peu}|$F^~ULfbl4t=1VasIz1vJ9OQ;DP2ABn&XCph8E?@F=N_Z-bAVB z52M5pxyK&+%zo>U=tLkI42RrQgb1AAzQTzU?h9cN#HBkP<|z|E4X6-uPsaGLn~MtE z#R*ZcO2vFQ7@M37M?t1?u_%vAAOS}OKqk$!3<8t6#aPT83Gf%gZXo0yKJ~mi76-7w z&CnJS!_OF|#PQBP1R~*RFc#vX7f;Oe4s(-nXe|T-(AbS4+bv#=hqpn;1knv$4n?rf zg8M`aX@+9KDd>hcQ#=%XB@p34Ze+#1?inXl^Ej+9&&T+^ZjO)z;whGNF@eV!l}s1I zqQK3BB{mihM!6GC^`^rLhZ_!dQS` zAAq?Dg^%E8`8NEL<;Zj}41jnuF~-fVhLg^m$~=ykGcuCVx+&s;V0rdy_Y$1TUACFR;NJCt#_615%6j*g6L z%i>da`V#vuB&vc5XYjL zan{VW-e%K|woe>wnX1}!)z)Oy*2SHvs(lIPz6T|>a|5?q(R22| zRrM#F{SVa-zLKcAl5k#GuK<$pvEO~}BF^%(qb2ER`J6D85hhdKx^QuE`hx=>w|o@% zdHYAM#2#;=YDg60-Yj)uCR){n{?L|9fjY=GzUJ_ZA&CS$?83!~IXJ6$5VlPZ~=O@7Dk1 zX$J3i+aUj`9`Aq3mJHPDe_BEBYYkAcL2D@zo2(sz?><>;dG)BX6#b>o(_MPdT`b=O zg!uiK;l|-xAoH+1ZWOw~a+Md~j-@EIa`hFz1xr1U_28Rs$f!c9F}wx}3G;V%uF~uK zhvn6LA5~>EXd{}KmVv=BfZ+fYO~*?p zWHcYQgPoMuN)@~X`%J}ujzVP5lUzt~(xUb(dsTa*(v2B6^}kW>zt0G!6E%2#&ry^s zqi}*+E!bB5X58R2sN)Ik!mok)%rQyS-ul>K&0^H*QzZo|st5pq&1)~TO5WnEvIPpt zns&3GtYjeJK(*;{&bVHXWj%^&l$V;4;X9xYR9*1f;BKbq$YflE*I4NUkpfdm&rW zNxH75C6)tBO&O(Kk+zn8K;Yr&IDbIu+8|{$QBV8Y)gOt0iV_ZVY4!Votbm&3*U;5O z>Hhb}7en9PI~Te=kS_Bi$~-?B|Lq#?=&#i`GJ_*Ceb3N|3C5yg0F;e88jE(yTEtB| zwGFgq@Uq~ZigKul-2hUjf`C}Vg=Yy-VLzkI6G)7W^K?>UI;g0m{4G1I>pMz_GJh=_ z=Z`-EjQ|_#g|H~Gm%~>DkDh-P%6Mo(BNL6ml>mkTurW=^A;O^0 z2Z_nx;zB?SP9W+D^Hh18B>f~8mGlvS3;IAL!XJnJ@h2dYO8lJQMwx;1R&$rI>^5O1OyXg?Gz4tg~9=TH-4`6Fs;KdyqYMB<($TgszIZ9^*z<{F&qL7o;Y@j5qG1R= zbUV1%`W}}luYX5ecrMl0y=pPIN>_=Yq;$2Ilr$|owb+-~^=iU#En&VUt0xpl_=o8~ zqlJm0h4U>)(8By7pb1+Z_ol3LQ9yx6d4j&Bi?|i4!fLFOW<&8K%-$VX#moD{^m2FP ze#?E~-j=^RPhW0UwWw}@t88ocV070(*rJ*|sg4w0s!8Q&@U8qQNB}lRi07zNRZ?oA zAK*Kv3RZiDp@D6vi|ZT7@z4euh&gD=r*RSbS4WP)M$ghw%0NnFEhJFoY>p}i-{F@j zL8Y~<%5{n=XQ(RAYo#h8wf{=XuY(s!9)k`Q=$EL%tZfXiL!}U+GvZ}sJJ@Hifi<>6 z@3r_0-LMy++zR$_k=K?35ej;-)b&t%h^@i;`l>an9~GIeT47yQaQkrM97r`W=GM$C zMIxbW3Pi8EIk3G#90)ph6h&V2qI*jSEQ|^F0rz%~+X^jZOG&ejItB46NI$eOx~zjE z8*~V+Db=(n$utcDZjOx_B&%t&z@|s1?gK?@0EnPqMbWyl*3f75E|rS>AbcoS>ryQt zbr%+D@&px7xCrZoX2_^Q>ARE+afz`%`rDd&10U~PJ~jH$w)CFUi9M%NAx>5)Uu$V& z^6^KQBV6^?3pf=Z!o$S%pQEQUW4 zekB}{wO7|*Bp_zBPA&mH46pE85%L`-tYzIu zY)ucH?uDl^PFJGt*^eyuUQKx8iJF%)&c=o6RTEoYyh>Pmv8*!MuvX1;8E54??A*xw z*;{AdektYhWUFn~Bgh?yXrAbUMBR~VBv79uc-Np@Px)T4<4yxHx?&sbII33=Pe|;A zSS<2n#Uemmp&c@+SgeNqj|gN-?9#~pv999n-4=Se-%4L>Rx0>Ks#Nw;rIG`xb*i{J zzk|98sc6TzQdR`yiE#)7`E-RXvJGNfzd?-WwNk$d&P5H>=Y0plOaF=Q-ft+2*aJ=N(>3;$mA!XWYZw|7?i$> zZ!|-#;qcwi9bx|3t!s-%(v3S3jXVA(^jrQI)PXr1h712kcyEw_yvDqekxJK)`m<{RdQ(7_{-Xcw|kN}j#R_p`JVZn?`lt}+QSL6>!G6}?dVUy|7wqX z^B2Q+%71Y7gXSfF!rZ@E=HcF;GS3Y+kMm%4!wSGOH9<95nlX|d;%!iblIbySs6U3+ zLOEnqdf5!sj|gOU_4Kk?R%rnEFY{$p0SABz;$+F=1CPWAz(4@ZxG;H11!w@6|C|D5 z4OBxgL6br%$WbYWT2U@!!Qbhp3weZ2A3D#VlS54^74;g|AL|Ebocl#l^{y-vWlIH_HMqd3(h%LgK{>E2BYL zQ64r6?bYOj8yL22?W>SuP!xuXrm%)@3LR~T8*&WZY)xa9s(KfP11<&m z{xBQ_LH(j6jo(c^lgiJbb5w%h($SRlgctJHVvx3186Wt+S$H-;AlI1FQ1Z0z+R9WaX`%GZ!g1MVJ3NU$fX!3TO0SDsz({xe`xT+-t&LpXnPJ&9$ zPhPv))3})$*2O+EnAgQV*6Yaqo(+865EM1Wu_qhm)9dUz1L0wSNz}hxAmr+^~?S{ND2OPlJJ#*4Bc*OPU5)!`Am@XQ$a^5x{Ss^d}Dxe<3GGEiPR{hm*k z4T5J^(Px46De{^N(JZhIWiV6aJ~OU^n~Wik(geX_kdF{%wq?ZFw=&y@ z5t}+LB`jSPw;i;B^MhVFMhhmpX8N+hWW^SiW67bZc!Ue0mm(AvAv_AUt{hAbivjdR zSgAEW01q(sAE_;MgqWZLbBiiItUd{#s?(>4K<(eltQ=|g3Epy-^E(; zaUvEE@JL2s3bfrcTJdN#__eM`#;It8i(ZykI#$C}9D?vX>SjqFja`8=NFqOuU*pT6 zj7Qc2ViR~6NMa{ruY~1bbFp6tyb|_LDs#;5g?b))HtbBNmH#S|_$HFD;Mc(G3|+H? zr@;j+z%ucFitoRL@6FNh6*~L;pJ3V7F*^X604_3wxinG@t^+k(?Vt;m6Rh6oNXUCj zHN$-nZ5XQFZRvgJs!O{%lCF-$p_FU)?2vj;DeZbX>3TZ%c}EiC*l^aUx<2jdOu9N3 zxx^vw%AVnrYh-rlmk!rw&hi^)-*Tj#?MY|*oz{f2ec9Rju)Kaw%#^n*oXeCq-QKxq zdGBDlr8n8qn<;NzIGX6(ztVaj-8ztL9Z0nfX3E_Q9&ks`O)N~WG(4RtZ(Jw`cXXn0 zXtl(Gu4qe1DMYQSo91obwY^=Oc6BFR-FF-**H<9GUehvHyYS2cxA4*(=8orsLx0_Q zr}19(y`n_b^Rq{x+u(?BZCP=67DtyIJ2Lf6^XG1zTRgK;zw3rI)7bik?I9HTZ}}Ie zSL%DQsQrcwOapt_>&L!vY|a9KlbQDpF57y5lD#BtYfRc27p!+Cmu&-3Qc^Rw6a63c z8E5_6bjI1RV9Pk`z~{g9?HG7K=C&;Kt<-GGIBVz5(hP=!ZW4>V_&1+d0@V1N=qV5ZAMV5qZV94gW<^l2IGso8>%ib zbh|&<()bxv7Vyj##U4gO^CJS;l71C0?}Nbs7a3q^a33lo+d;Q%Yjx1k0DJujr>?Rp zFRmNY%nS%+GpbV9e1f>?en?|0{S~*K(m}z|$BBnfwc@%iyc@`#Q2WeQ9Z6Hx39e*L1+Z9~& zGWgRLez+in%i8{EN^k^_2j~kH#kq&!`+kH0&2NvNjfI$-4#czkoGV1bgJY-@ z;pq?@;Dp1~ino9QXmk`zgn7vjoS2GUmb)z5CvLYA)BE2@EH9?Fp~rL+qmKjn_gRVS zX6>7`Z`IElAJ|K7c;-gZ_U2`K^FrW3b^UU~jys1xt?s*F03YW~`y2MT@Tbn!M|x7T z2hQ#kx1ivuOgkErj>eRuDPe9}H$myA_U275z7TNl{J`_fHpBgGMTd$k8`u`%FlAdb zaG%V!)b1d;UQpCMvNCDGy9LOe%*#orMLqtMmo0|UM+CAZaCE{2?FhKsAEt#eOJf(+ zq(~au+9lX9a9Ty@Whjn>Wv4h&t(S8>4u{MO9kK+b)EDw5cr|+gm={!Rob|CMHS0P{ zSY<)g8c=L;kWx?|F-Kl1#KvCjfMp&PP|kk<73~0`N<}$A6CmVtd7-|b>rv@1c&)PM zL42%=7pLHmWQ!t@fN700b((*a3W$whOh#AC))_+=J%Ihrvd; zM)%^kq26`!aLdjQ49i>g-w5dncJ6=ef38&9oN}jotJ|iU02&;rlF~hob zLe4THy3gfHrmQaQ-jQ_g__S=tgW86<*HX2epOkk#GLeRzkKlA(sl5A9FybGkKa61+ z`A}Le!=fycVI$&lh=e0=Wy9f5X5I%-Kg+yzhNmA9$d=euyxf&9KLh54C%OB`uxdi1 z7CvZO`vkN!M!>-+#Iwx$F||fPM$PKwLhP!tp_c{c;A7ad5F2!}z}!qh#?)8}TnVUC zX!5~{YE`pu#sYC;_8jy{&A3+EqRDxB@VqX2A{(3ld1tEWj96esthq+Cw+Ju*YT3N> zrRBCRqV`g~dhI~fzJcn0%dB@B;wG3u=PCU6swlZA|Y^9Cc|k~7zq|KeP~ z@`SmjruXBp)AJvxXTu*?%L{ZA6y=!NISgi-c23T%+HcfgmL;G&5Q#-E3J`&IixV=~ zl1eo19EveCu`(P^hA3-KDR{%U2xo8NSrYB_e+kr1nVTR#lbp)0Y;dfyO^S2{bb@9rQ90^z z3mfM_sttxQ6-V_H3apxm-3`W)t#Zzsu~om*H{bu={RIYATtj|aaemP@DMAo@bmbSMo+uIgP9>5P>hQ2X$V`8ys#kLI* zSo^9~`B=@|(L~#>mAc&zZ521BZ(e)j+QQJ{^bfBsUHf40-l0_Yv#F}18C&^{iMi>O z$}NTZwQ4t2S!at@OG#bZ{DE5s7Ei3y?gck-rm`ko*`BOyPk8pMRPLEA`V4y2F(;($ zEz9rK!*==c}wZYY4d346;I0(#8ud#aD<3_oqGK2l-$ zP*-%M)be3P(UEow=vb63)D=NjSXR!+s}PIh(Vp@+@n=cY#o(8$e*a`FGzDwt#~;wV zJQav2ZybJ5!*GBQ{2%bED8HYtL%Ihs+l?8XNZ>~?16)G*?_q|ZnMXy<)3vUEU+BQj zS>1C8)W7*K{5IICyE0cld71ZbBxpQ&U_Bf1xI#;URmGjNtnM*pfZ-EuRBN z^~HZ5GaSbfp(>QJ0vJW|tRLW4;1vP_p$jrN9nUapM#9*BK`g%@_Fs^qe<7}4lIA37 W{ufgJ3sSt!M)b_nYXoyj?>_*qNgtU2 literal 0 HcmV?d00001 diff --git a/app/utils/json_analyzer.py b/app/utils/json_analyzer.py index 63ad6a0..2407c89 100644 --- a/app/utils/json_analyzer.py +++ b/app/utils/json_analyzer.py @@ -17,8 +17,8 @@ def __init__(self): self.temp_path = self.base_dir / "temp" self.schema_path = self.base_dir / "internal_databases" / "schemas" - # Create directories efficiently - for path in [self.sql_path, self.nosql_path, self.temp_path,self.schema_path]: + # Create directories efficiently (schema_path is correctly included now) + for path in [self.sql_path, self.nosql_path, self.temp_path, self.schema_path]: path.mkdir(parents=True, exist_ok=True) def analyze_json_file(self, file_path: str) -> Dict[str, Any]: @@ -38,9 +38,10 @@ def analyze_json_file(self, file_path: str) -> Dict[str, Any]: if isinstance(data, list): return self._analyze_array(data, file_path) elif isinstance(data, dict): - return self._analyze_object(data, file_path) + # Pass the file path (though not currently used in _analyze_object logic) + return self._analyze_object(data, file_path) else: - return {"recommendation": "nosql", "reason": "Simple data type"} + return {"recommendation": "nosql", "reason": "Simple scalar data type, best handled as NoSQL document"} except json.JSONDecodeError as e: return {"recommendation": "nosql", "reason": f"Invalid JSON: {str(e)}"} @@ -57,7 +58,7 @@ def _analyze_array(self, data: List, file_path: str) -> Dict[str, Any]: sample = data[:sample_size] if not all(isinstance(item, dict) for item in sample): - return {"recommendation": "nosql", "reason": "Array contains non-object items"} + return {"recommendation": "nosql", "reason": "Array contains non-object items, lacks uniform structure"} # Check structure consistency efficiently first_keys = set(sample[0].keys()) @@ -73,23 +74,41 @@ def _analyze_array(self, data: List, file_path: str) -> Dict[str, Any]: "columns": list(first_keys) } - return {"recommendation": "nosql", "reason": "Variable structure or complex data"} + return {"recommendation": "nosql", "reason": "Variable structure or complex data, better for NoSQL batch insertion"} def _analyze_object(self, data: Dict, file_path: str) -> Dict[str, Any]: - """Optimized object analysis""" - # Check nesting depth efficiently - max_depth = self._calculate_depth(data) + """ + Optimized object analysis. + Modified to recommend NoSQL for any immediate nesting (depth > 1) + to handle document-style data. + """ + + # NEW LOGIC: Check for any immediate nesting (nested dict/list at the top level) + has_immediate_nesting = False + for value in data.values(): + # Check for nested dictionaries + if isinstance(value, dict) and value: + has_immediate_nesting = True + break + # Check for lists that contain other complex structures + if isinstance(value, list) and value and any(isinstance(item, (dict, list)) for item in value): + has_immediate_nesting = True + break - if max_depth > 2: + # Calculate max depth for insight/metrics + max_depth = self._calculate_depth(data) + + # DECISION FIX: Prioritize nesting for NoSQL recommendation + if has_immediate_nesting: return { "recommendation": "nosql", - "reason": f"Deeply nested structure (depth: {max_depth})", + "reason": f"Contains nested dictionary or list (depth: {max_depth}), ideal for document storage.", "nesting_level": max_depth } else: return { "recommendation": "sql", - "reason": "Flat or lightly nested structure", + "reason": "Flat structure, ideal for relational querying.", "keys": list(data.keys()) } @@ -99,7 +118,7 @@ def _is_sql_optimized(self, sample: List[Dict], keys: set) -> bool: if len(keys) > 50: return False - # Rule 2: Check for common SQL patterns + # Rule 2: Check for common SQL patterns (optional aid) has_id = any(key.lower() in ['id', '_id'] for key in keys) has_foreign_keys = any(key.endswith('_id') for key in keys) @@ -132,10 +151,12 @@ def store_json_file(self, file_path: str, original_name: str, analysis: Dict) -> # Check for duplicates duplicate = self._check_duplicate(file_hash, analysis["recommendation"]) if duplicate: + # IMPORTANT: If duplicate, delete the temp file and return + Path(file_path).unlink(missing_ok=True) return { "success": True, "original_name": original_name, - "stored_name": duplicate, + "stored_name": Path(duplicate).name, "storage_type": analysis["recommendation"].upper(), "final_path": duplicate, "reason": analysis["reason"], @@ -155,10 +176,10 @@ def store_json_file(self, file_path: str, original_name: str, analysis: Dict) -> final_path = self.nosql_path / new_name storage_type = "NoSQL" - # Move with metadata + # Move with metadata (shutil.move handles the temp file deletion) shutil.move(file_path, str(final_path)) - # Save analysis metadata + # Save analysis metadata and schema self._save_metadata(final_path, analysis, original_name) return { @@ -173,6 +194,8 @@ def store_json_file(self, file_path: str, original_name: str, analysis: Dict) -> } except Exception as e: + # Ensure temp file is cleaned up if storage fails for other reasons + Path(file_path).unlink(missing_ok=True) return {"success": False, "error": str(e)} def _get_file_hash(self, file_path: str) -> str: @@ -187,13 +210,16 @@ def _check_duplicate(self, file_hash: str, storage_type: str) -> str: """Check if file already exists""" storage_path = self.sql_path if storage_type == "sql" else self.nosql_path - for existing_file in storage_path.glob("*.json"): - if file_hash in existing_file.name: - return str(existing_file) + # Only check files that are NOT metadata or schema files + for existing_file in storage_path.glob("*"): + if existing_file.is_file() and existing_file.name.endswith('.json') and not existing_file.name.endswith(('.meta.json', '.schema.json')): + if file_hash in existing_file.name: + return str(existing_file) return "" def _save_metadata(self, file_path: Path, analysis: Dict, original_name: str): - """Save analysis metadata alongside the file""" + """Save analysis metadata and schema alongside the file""" + # 1. Save Metadata metadata = { "original_filename": original_name, "analysis": analysis, @@ -205,10 +231,11 @@ def _save_metadata(self, file_path: Path, analysis: Dict, original_name: str): with open(metadata_path, 'w') as f: json.dump(metadata, f, indent=2) + # 2. Save Schema schema_info = { "original_filename": original_name, "storage_type": analysis["recommendation"], - # Use 'columns' for SQL arrays, 'keys' for objects + # Use 'columns' for array analysis, 'keys' for object analysis "columns": analysis.get("columns", analysis.get("keys", [])), "reason": analysis["reason"], "analyzed_at": datetime.now().isoformat() @@ -218,6 +245,4 @@ def _save_metadata(self, file_path: Path, analysis: Dict, original_name: str): schema_path = self.schema_path / schema_file_name with open(schema_path, 'w') as f: - json.dump(schema_info, f, indent=2) - - \ No newline at end of file + json.dump(schema_info, f, indent=2) \ No newline at end of file From 00af8431674587e59f61af7f7c997b3380f6b96a Mon Sep 17 00:00:00 2001 From: KrishLalakiya Date: Sat, 15 Nov 2025 22:32:52 +0530 Subject: [PATCH 3/3] 10 30 --- app/utils/file_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/utils/file_utils.py b/app/utils/file_utils.py index f102196..6dd479d 100644 --- a/app/utils/file_utils.py +++ b/app/utils/file_utils.py @@ -1,5 +1,4 @@ # app/utils/file_utils.py - import os import zipfile import io