scraping_config.py

#!/usr/bin/env python3
"""
Advanced Scraping Configuration
Centralized configuration for all scraping methods, CAPTCHA solving, and proxy management
"""

import os
from typing import Dict, List, Optional
from dotenv import load_dotenv

load_dotenv()

class ScrapingConfig:
    """
    Centralized configuration for advanced scraping system
    """
    
    # LinkedIn Authentication
    LINKEDIN_EMAIL = os.environ.get("LINKEDIN_EMAIL")
    LINKEDIN_PASSWORD = os.environ.get("LINKEDIN_PASSWORD")
    
    # CAPTCHA Solving Services
    CAPTCHA_SERVICES = {
        "2captcha": {
            "api_key": os.environ.get("TWOCAPTCHA_API_KEY"),
            "enabled": bool(os.environ.get("TWOCAPTCHA_API_KEY")),
            "priority": 1
        },
        "anti-captcha": {
            "api_key": os.environ.get("ANTICAPTCHA_API_KEY"),
            "enabled": bool(os.environ.get("ANTICAPTCHA_API_KEY")),
            "priority": 2
        },
        "capmonster": {
            "api_key": os.environ.get("CAPMONSTER_API_KEY"),
            "enabled": bool(os.environ.get("CAPMONSTER_API_KEY")),
            "priority": 3
        }
    }
    
    # OCR Configuration
    OCR_CONFIG = {
        "enabled": True,
        "tesseract_cmd": os.environ.get("TESSERACT_CMD", "tesseract"),
        "config": "--psm 8 -c tessedit_char_whitelist=0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
    }
    
    # Proxy Configuration
    PROXY_CONFIG = {
        "enabled": bool(os.environ.get("USE_PROXIES", "false").lower() == "true"),
        "rotation": True,
        "health_check": True,
        "sources": [
            "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
            "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt"
        ],
        "custom_list": os.environ.get("PROXY_LIST", "").split(",") if os.environ.get("PROXY_LIST") else []
    }
    
    # User Agent Configuration
    USER_AGENT_CONFIG = {
        "rotation": True,
        "custom_agents": [
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0'
        ]
    }
    
    # Scrapy Settings
    SCRAPY_SETTINGS = {
        'ROBOTSTXT_OBEY': False,
        'CONCURRENT_REQUESTS': 1,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 1,
        'DOWNLOAD_DELAY': 3,
        'RANDOMIZE_DOWNLOAD_DELAY': 0.5,
        'AUTOTHROTTLE_ENABLED': True,
        'AUTOTHROTTLE_START_DELAY': 1,
        'AUTOTHROTTLE_MAX_DELAY': 10,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 1.0,
        'COOKIES_ENABLED': True,
        'TELNETCONSOLE_ENABLED': False,
        'RETRY_TIMES': 3,
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 408, 429, 403, 999],
        
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy_linkedin_scraper.RotateUserAgentMiddleware': 400,
            'scrapy_linkedin_scraper.ProxyRotationMiddleware': 410,
            'scrapy_linkedin_scraper.CaptchaSolverMiddleware': 420,
            'scrapy_linkedin_scraper.AntiDetectionMiddleware': 430,
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
        }
    }
    
    # Playwright Configuration
    PLAYWRIGHT_CONFIG = {
        "headless": os.environ.get("PLAYWRIGHT_HEADLESS", "true").lower() == "true",
        "user_data_dir": os.environ.get("PLAYWRIGHT_USER_DATA_DIR", ".playwright/linkedin_session"),
        "timeout": 30000,
        "args": [
            "--disable-blink-features=AutomationControlled",
            "--disable-web-security",
            "--disable-features=VizDisplayCompositor",
            "--no-sandbox",
            "--disable-dev-shm-usage"
        ]
    }
    
    # Selenium Configuration
    SELENIUM_CONFIG = {
        "headless": os.environ.get("SELENIUM_HEADLESS", "true").lower() == "true",
        "user_data_dir": os.environ.get("SELENIUM_USER_DATA_DIR", ".selenium_profile"),
        "undetected": True,
        "options": [
            "--no-sandbox",
            "--disable-dev-shm-usage",
            "--disable-blink-features=AutomationControlled",
            "--disable-extensions",
            "--disable-plugins"
        ]
    }
    
    # Scraping Method Priority
    SCRAPING_METHODS = [
        {
            "name": "scrapy_advanced",
            "enabled": True,
            "priority": 1,
            "description": "Scrapy with advanced middlewares"
        },
        {
            "name": "authenticated_playwright",
            "enabled": True,
            "priority": 2,
            "description": "Playwright with LinkedIn authentication"
        },
        {
            "name": "selenium_undetected",
            "enabled": True,
            "priority": 3,
            "description": "Undetected Chrome with stealth mode"
        },
        {
            "name": "http_session",
            "enabled": True,
            "priority": 4,
            "description": "HTTP requests with session management"
        }
    ]
    
    # Cache Configuration
    CACHE_CONFIG = {
        "enabled": os.environ.get("CACHE_ENABLED", "false").lower() == "true",
        "ttl_minutes": int(os.environ.get("CACHE_TTL_MINUTES", "60")),
        "database": os.environ.get("CACHE_DB", "cache.duckdb")
    }
    
    # Rate Limiting
    RATE_LIMIT_CONFIG = {
        "requests_per_minute": int(os.environ.get("RATE_LIMIT_RPM", "10")),
        "burst_limit": int(os.environ.get("RATE_LIMIT_BURST", "5")),
        "backoff_factor": float(os.environ.get("RATE_LIMIT_BACKOFF", "2.0"))
    }
    
    @classmethod
    def get_enabled_captcha_services(cls) -> List[str]:
        """Get list of enabled CAPTCHA services sorted by priority"""
        enabled = []
        for service, config in cls.CAPTCHA_SERVICES.items():
            if config["enabled"]:
                enabled.append((service, config["priority"]))
        
        # Sort by priority
        enabled.sort(key=lambda x: x[1])
        return [service for service, _ in enabled]
    
    @classmethod
    def get_enabled_scraping_methods(cls) -> List[Dict]:
        """Get list of enabled scraping methods sorted by priority"""
        enabled = [method for method in cls.SCRAPING_METHODS if method["enabled"]]
        enabled.sort(key=lambda x: x["priority"])
        return enabled
    
    @classmethod
    def validate_config(cls) -> Dict[str, bool]:
        """Validate configuration and return status"""
        status = {
            "linkedin_auth": bool(cls.LINKEDIN_EMAIL and cls.LINKEDIN_PASSWORD),
            "captcha_services": len(cls.get_enabled_captcha_services()) > 0,
            "proxy_config": cls.PROXY_CONFIG["enabled"],
            "ocr_available": cls.OCR_CONFIG["enabled"],
            "scraping_methods": len(cls.get_enabled_scraping_methods()) > 0
        }
        return status
    
    @classmethod
    def print_config_status(cls):
        """Print current configuration status"""
        print("🔧 Advanced Scraping Configuration Status")
        print("=" * 50)
        
        status = cls.validate_config()
        
        print(f"🔐 LinkedIn Authentication: {'✅' if status['linkedin_auth'] else '❌'}")
        if status['linkedin_auth']:
            print(f"   Email: {cls.LINKEDIN_EMAIL}")
        
        print(f"🤖 CAPTCHA Services: {'✅' if status['captcha_services'] else '❌'}")
        for service in cls.get_enabled_captcha_services():
            print(f"   • {service}")
        
        print(f"📡 Proxy Rotation: {'✅' if status['proxy_config'] else '❌'}")
        if status['proxy_config']:
            proxy_count = len(cls.PROXY_CONFIG['custom_list'])
            print(f"   Custom Proxies: {proxy_count}")
        
        print(f"🔍 OCR Fallback: {'✅' if status['ocr_available'] else '❌'}")
        
        print(f"🕷️ Scraping Methods: {'✅' if status['scraping_methods'] else '❌'}")
        for method in cls.get_enabled_scraping_methods():
            print(f"   {method['priority']}. {method['name']} - {method['description']}")
        
        print(f"💾 Cache: {'✅ Enabled' if cls.CACHE_CONFIG['enabled'] else '❌ Disabled'}")
        
        return status

# Global configuration instance
config = ScrapingConfig()

if __name__ == "__main__":
    config.print_config_status()