scrapy_linkedin_scraper.py

#!/usr/bin/env python3
"""
Advanced Scrapy-based LinkedIn Scraper
Features:
- Scrapy core with custom middlewares
- Playwright integration for JavaScript-heavy sites
- Proxy rotation and user-agent rotation
- CAPTCHA solving integration
- Anti-detection techniques
- Robust error handling and retries
"""

import scrapy
import json
import random
import time
from typing import Dict, List, Optional
from urllib.parse import urljoin, urlparse
from scrapy.http import Request, Response
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.exceptions import NotConfigured
import requests
from fake_useragent import UserAgent

class LinkedInProfileSpider(scrapy.Spider):
    """
    Advanced LinkedIn profile spider with anti-detection
    """
    name = 'linkedin_profile'
    allowed_domains = [
        'linkedin.com', 'www.linkedin.com', 'in.linkedin.com',
        'uk.linkedin.com', 'ca.linkedin.com', 'au.linkedin.com'
    ]
    
    custom_settings = {
        'ROBOTSTXT_OBEY': False,
        'CONCURRENT_REQUESTS': 1,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 1,
        'DOWNLOAD_DELAY': 3,
        'RANDOMIZE_DOWNLOAD_DELAY': 0.5,
        'AUTOTHROTTLE_ENABLED': True,
        'AUTOTHROTTLE_START_DELAY': 1,
        'AUTOTHROTTLE_MAX_DELAY': 10,
        'AUTOTHROTTLE_TARGET_CONCURRENCY': 1.0,
        'COOKIES_ENABLED': True,
        'TELNETCONSOLE_ENABLED': False,
        
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy_linkedin_scraper.RotateUserAgentMiddleware': 400,
            'scrapy_linkedin_scraper.ProxyRotationMiddleware': 410,
            'scrapy_linkedin_scraper.CaptchaSolverMiddleware': 420,
            'scrapy_linkedin_scraper.AntiDetectionMiddleware': 430,
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
        },
        
        'RETRY_TIMES': 3,
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 408, 429, 403],
        
        'DEFAULT_REQUEST_HEADERS': {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
        }
    }
    
    def __init__(self, urls=None, *args, **kwargs):
        super(LinkedInProfileSpider, self).__init__(*args, **kwargs)
        self.start_urls = urls if urls else []
        self.results = []
    
    def start_requests(self):
        """Generate initial requests with anti-detection headers"""
        for url in self.start_urls:
            yield Request(
                url=url,
                callback=self.parse_profile,
                meta={
                    'playwright': True,  # Use Playwright for JavaScript
                    'playwright_include_page': True,
                    'playwright_page_methods': [
                        ('wait_for_load_state', 'networkidle'),
                        ('wait_for_timeout', 3000),
                    ],
                    'dont_cache': True,
                    'handle_httpstatus_list': [403, 429, 999],
                }
            )
    
    def parse_profile(self, response):
        """Parse LinkedIn profile page"""
        try:
            # Check if we hit a blocked page
            if self._is_blocked_page(response):
                self.logger.warning(f"Blocked page detected for {response.url}")
                yield self._handle_blocked_page(response)
                return
            
            # Extract profile data
            profile_data = self._extract_profile_data(response)
            
            # Validate extracted data
            if self._is_valid_profile_data(profile_data):
                profile_data['scraping_method'] = 'scrapy_advanced'
                profile_data['success'] = True
                self.results.append(profile_data)
                yield profile_data
            else:
                # Retry with different method if data is invalid
                yield Request(
                    url=response.url,
                    callback=self.parse_profile_fallback,
                    meta={'playwright': False, 'dont_cache': True},
                    dont_filter=True
                )
                
        except Exception as e:
            self.logger.error(f"Error parsing profile {response.url}: {e}")
            yield self._create_error_response(response.url, str(e))
    
    def parse_profile_fallback(self, response):
        """Fallback parser without JavaScript"""
        try:
            profile_data = self._extract_profile_data_simple(response)
            profile_data['scraping_method'] = 'scrapy_fallback'
            profile_data['success'] = True
            self.results.append(profile_data)
            yield profile_data
        except Exception as e:
            self.logger.error(f"Fallback parsing failed for {response.url}: {e}")
            yield self._create_error_response(response.url, str(e))
    
    def _is_blocked_page(self, response):
        """Check if we hit a security/login page"""
        blocked_indicators = [
            'security verification',
            'sign up',
            'join linkedin',
            'please sign in',
            'authwall',
            'challenge',
            'captcha'
        ]
        
        text = response.text.lower()
        return any(indicator in text for indicator in blocked_indicators)
    
    def _handle_blocked_page(self, response):
        """Handle blocked pages with retry logic"""
        return Request(
            url=response.url,
            callback=self.parse_profile,
            meta={
                'playwright': True,
                'captcha_solve': True,
                'proxy_rotate': True,
                'dont_cache': True
            },
            dont_filter=True
        )
    
    def _extract_profile_data(self, response):
        """Extract comprehensive profile data"""
        data = {}
        
        # Name extraction with multiple selectors
        name_selectors = [
            'h1.text-heading-xlarge::text',
            'h1.break-words::text',
            '.pv-text-details__left-panel h1::text',
            '.ph5 h1::text',
            'h1[data-test-id="profile-name"]::text',
            '.top-card-layout__title::text'
        ]
        
        for selector in name_selectors:
            name = response.css(selector).get()
            if name and name.strip():
                data['full_name'] = name.strip()
                break
        
        # Headline extraction
        headline_selectors = [
            '.text-body-medium.break-words::text',
            '.pv-text-details__left-panel .text-body-medium::text',
            '.top-card-layout__headline::text',
            '[data-test-id="profile-headline"]::text'
        ]
        
        for selector in headline_selectors:
            headline = response.css(selector).get()
            if headline and headline.strip():
                data['headline'] = headline.strip()
                break
        
        # Profile picture
        img_selectors = [
            '.pv-top-card-profile-picture__image::attr(src)',
            '.profile-photo-edit__preview::attr(src)',
            '.top-card-layout__entity-image img::attr(src)'
        ]
        
        for selector in img_selectors:
            img_url = response.css(selector).get()
            if img_url:
                data['profile_pic_url'] = img_url
                break
        
        # About section
        about_selectors = [
            '.pv-shared-text-with-see-more::text',
            '.inline-show-more-text::text',
            '.pv-about-section::text'
        ]
        
        for selector in about_selectors:
            about = response.css(selector).getall()
            if about:
                data['summary'] = ' '.join(about).strip()
                break
        
        # Location
        location_selectors = [
            '.pv-text-details__left-panel .text-body-small::text',
            '.top-card-layout__first-subline::text'
        ]
        
        for selector in location_selectors:
            location = response.css(selector).get()
            if location and location.strip():
                data['location'] = location.strip()
                break
        
        # Experience section
        experience_items = response.css('.pv-entity__summary-info')
        if experience_items:
            experiences = []
            for item in experience_items[:3]:  # Get top 3 experiences
                title = item.css('h3::text').get()
                company = item.css('.pv-entity__secondary-title::text').get()
                if title and company:
                    experiences.append({
                        'title': title.strip(),
                        'company': company.strip()
                    })
            data['experience'] = experiences
        
        # Set defaults
        data.setdefault('full_name', 'LinkedIn User')
        data.setdefault('headline', 'Professional')
        data.setdefault('summary', data.get('headline', 'LinkedIn Professional'))
        data.setdefault('profile_pic_url', '')
        
        return data
    
    def _extract_profile_data_simple(self, response):
        """Simple extraction for fallback"""
        data = {}
        
        # Try OpenGraph tags
        og_title = response.css('meta[property="og:title"]::attr(content)').get()
        if og_title:
            data['full_name'] = og_title.replace(' | LinkedIn', '').strip()
        
        og_description = response.css('meta[property="og:description"]::attr(content)').get()
        if og_description:
            data['headline'] = og_description.strip()
            data['summary'] = og_description.strip()
        
        og_image = response.css('meta[property="og:image"]::attr(content)').get()
        if og_image:
            data['profile_pic_url'] = og_image
        
        # Fallback to title
        if not data.get('full_name'):
            title = response.css('title::text').get()
            if title:
                data['full_name'] = title.replace(' | LinkedIn', '').strip()
        
        # Set defaults
        data.setdefault('full_name', 'LinkedIn User')
        data.setdefault('headline', 'Professional')
        data.setdefault('summary', data.get('headline', 'LinkedIn Professional'))
        data.setdefault('profile_pic_url', '')
        
        return data
    
    def _is_valid_profile_data(self, data):
        """Validate extracted profile data"""
        if not isinstance(data, dict):
            return False
        
        # Check for blocked indicators in name
        blocked_names = ['sign up', 'security verification', 'join linkedin']
        name = data.get('full_name', '').lower()
        if any(blocked in name for blocked in blocked_names):
            return False
        
        # Must have meaningful name or headline
        has_name = bool(data.get('full_name', '').strip())
        has_headline = bool(data.get('headline', '').strip())
        
        return has_name or has_headline
    
    def _create_error_response(self, url, error):
        """Create standardized error response"""
        return {
            'url': url,
            'success': False,
            'error': error,
            'full_name': 'Profile Not Accessible',
            'headline': 'Scraping Failed',
            'summary': f'Unable to access profile: {error}',
            'profile_pic_url': '',
            'scraping_method': 'scrapy_error'
        }

class RotateUserAgentMiddleware(UserAgentMiddleware):
    """Rotate user agents to avoid detection"""
    
    def __init__(self, user_agent=''):
        self.user_agent = user_agent
        try:
            self.ua = UserAgent()
            self.user_agent_list = [
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0'
            ]
        except:
            self.ua = None
            self.user_agent_list = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36']
    
    def process_request(self, request, spider):
        if self.ua:
            try:
                ua = self.ua.random
            except:
                ua = random.choice(self.user_agent_list)
        else:
            ua = random.choice(self.user_agent_list)
        
        request.headers['User-Agent'] = ua
        return None

class ProxyRotationMiddleware:
    """Advanced proxy rotation with health checking and automatic proxy sourcing"""
    
    def __init__(self):
        # Load proxies from environment or file
        self.proxy_list = self._load_proxies()
        self.proxy_index = 0
        self.failed_proxies = set()
        self.proxy_stats = {}
        
        # Proxy sources for automatic proxy fetching
        self.proxy_sources = [
            "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
            "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
            "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt"
        ]
    
    def _load_proxies(self):
        """Load proxies from various sources"""
        proxies = []
        
        # Load from environment variable
        env_proxies = os.environ.get("PROXY_LIST", "")
        if env_proxies:
            proxies.extend([p.strip() for p in env_proxies.split(",") if p.strip()])
        
        # Load from file if exists
        proxy_file = "proxies.txt"
        if os.path.exists(proxy_file):
            try:
                with open(proxy_file, 'r') as f:
                    file_proxies = [line.strip() for line in f if line.strip()]
                    proxies.extend(file_proxies)
            except Exception as e:
                print(f"⚠️ Could not load proxies from file: {e}")
        
        # If no proxies found, try to fetch from public sources
        if not proxies:
            proxies = self._fetch_public_proxies()
        
        # Format proxies properly
        formatted_proxies = []
        for proxy in proxies:
            if not proxy.startswith('http'):
                proxy = f"http://{proxy}"
            formatted_proxies.append(proxy)
        
        print(f"📡 Loaded {len(formatted_proxies)} proxies")
        return formatted_proxies
    
    def _fetch_public_proxies(self):
        """Fetch proxies from public sources"""
        proxies = []
        
        for source in self.proxy_sources:
            try:
                response = requests.get(source, timeout=10)
                if response.status_code == 200:
                    source_proxies = [line.strip() for line in response.text.split('\n') if line.strip()]
                    proxies.extend(source_proxies[:50])  # Limit to 50 per source
                    print(f"✅ Fetched {len(source_proxies[:50])} proxies from {source}")
            except Exception as e:
                print(f"❌ Failed to fetch proxies from {source}: {e}")
        
        return proxies
    
    def _test_proxy(self, proxy):
        """Test if a proxy is working"""
        try:
            test_response = requests.get(
                "http://httpbin.org/ip",
                proxies={"http": proxy, "https": proxy},
                timeout=10
            )
            return test_response.status_code == 200
        except:
            return False
    
    @classmethod
    def from_crawler(cls, crawler):
        return cls()
    
    def process_request(self, request, spider):
        if not self.proxy_list:
            return None
        
        # Always use proxy rotation for LinkedIn
        if 'linkedin.com' in request.url or request.meta.get('proxy_rotate', True):
            # Get next working proxy
            proxy = self._get_next_proxy()
            if proxy:
                request.meta['proxy'] = proxy
                spider.logger.debug(f"Using proxy: {proxy}")
        
        return None
    
    def _get_next_proxy(self):
        """Get next working proxy from the list"""
        attempts = 0
        max_attempts = len(self.proxy_list)
        
        while attempts < max_attempts:
            proxy = self.proxy_list[self.proxy_index]
            self.proxy_index = (self.proxy_index + 1) % len(self.proxy_list)
            
            # Skip failed proxies
            if proxy in self.failed_proxies:
                attempts += 1
                continue
            
            return proxy
        
        # If all proxies failed, reset failed list and try again
        if self.failed_proxies:
            print("🔄 Resetting failed proxy list")
            self.failed_proxies.clear()
            return self.proxy_list[0] if self.proxy_list else None
        
        return None
    
    def process_response(self, request, response, spider):
        # Track proxy performance
        proxy = request.meta.get('proxy')
        if proxy:
            if response.status in [403, 429, 503]:
                self.failed_proxies.add(proxy)
                spider.logger.warning(f"Proxy {proxy} failed with status {response.status}")
        
        return response

class CaptchaSolverMiddleware:
    """Advanced CAPTCHA solving with third-party services and OCR fallback"""
    
    def __init__(self):
        # Import our advanced CAPTCHA solver
        try:
            from captcha_solver import captcha_solver
            self.captcha_solver = captcha_solver
            self.solver_available = True
        except ImportError:
            self.captcha_solver = None
            self.solver_available = False
            
        # CAPTCHA solving statistics
        self.captcha_stats = {
            'detected': 0,
            'solved': 0,
            'failed': 0
        }
    
    @classmethod
    def from_crawler(cls, crawler):
        return cls()
    
    def process_response(self, request, response, spider):
        # Check if response contains CAPTCHA
        if self._has_captcha(response) and request.meta.get('captcha_solve'):
            self.captcha_stats['detected'] += 1
            spider.logger.info(f"🤖 CAPTCHA detected for {request.url}")
            
            if self.solver_available:
                # Determine CAPTCHA type and solve
                captcha_type = self._detect_captcha_type(response)
                solution = self._solve_captcha(response, captcha_type, spider)
                
                if solution:
                    self.captcha_stats['solved'] += 1
                    spider.logger.info(f"✅ CAPTCHA solved successfully!")
                    
                    # Create new request with CAPTCHA solution
                    return self._create_captcha_request(request, solution, captcha_type)
                else:
                    self.captcha_stats['failed'] += 1
                    spider.logger.warning("❌ CAPTCHA solving failed")
            
            # If no CAPTCHA service or solving failed, retry with delay
            spider.logger.warning("⏳ CAPTCHA solving not available, adding delay...")
            time.sleep(random.uniform(15, 30))
        
        return response
    
    def _has_captcha(self, response):
        """Enhanced CAPTCHA detection"""
        captcha_indicators = [
            'captcha', 'recaptcha', 'hcaptcha', 'challenge', 'verify',
            'g-recaptcha', 'h-captcha', 'cf-challenge', 'security check'
        ]
        
        text = response.text.lower()
        
        # Check for CAPTCHA indicators in text
        text_has_captcha = any(indicator in text for indicator in captcha_indicators)
        
        # Check for CAPTCHA-related elements in HTML
        html_indicators = [
            'data-sitekey', 'g-recaptcha-response', 'h-captcha-response',
            'cf-challenge-form', 'captcha-container'
        ]
        html_has_captcha = any(indicator in text for indicator in html_indicators)
        
        return text_has_captcha or html_has_captcha
    
    def _detect_captcha_type(self, response):
        """Detect the type of CAPTCHA present"""
        text = response.text.lower()
        
        if 'recaptcha' in text or 'g-recaptcha' in text:
            return 'recaptcha'
        elif 'hcaptcha' in text or 'h-captcha' in text:
            return 'hcaptcha'
        elif 'cf-challenge' in text or 'cloudflare' in text:
            return 'cloudflare'
        elif any(img_indicator in text for img_indicator in ['captcha.jpg', 'captcha.png', 'captcha.gif']):
            return 'image'
        else:
            return 'unknown'
    
    def _solve_captcha(self, response, captcha_type, spider):
        """Solve CAPTCHA using appropriate method"""
        
        if not self.captcha_solver:
            return None
        
        try:
            if captcha_type == 'recaptcha':
                # Extract site key and solve reCAPTCHA
                site_key = self._extract_site_key(response.text, 'data-sitekey')
                if site_key:
                    spider.logger.info(f"🔑 Solving reCAPTCHA with site key: {site_key}")
                    return self.captcha_solver.solve_recaptcha_v2(site_key, response.url)
            
            elif captcha_type == 'hcaptcha':
                # Extract site key and solve hCaptcha
                site_key = self._extract_site_key(response.text, 'data-sitekey')
                if site_key:
                    spider.logger.info(f"🔑 Solving hCaptcha with site key: {site_key}")
                    return self.captcha_solver.solve_hcaptcha(site_key, response.url)
            
            elif captcha_type == 'image':
                # Extract image and solve with OCR
                image_url = self._extract_captcha_image_url(response)
                if image_url:
                    spider.logger.info(f"🖼️ Solving image CAPTCHA: {image_url}")
                    # Download image and solve
                    image_data = self._download_captcha_image(image_url)
                    if image_data:
                        return self.captcha_solver.solve_image_captcha(image_data)
            
            return None
            
        except Exception as e:
            spider.logger.error(f"❌ CAPTCHA solving error: {e}")
            return None
    
    def _extract_site_key(self, html, attribute):
        """Extract CAPTCHA site key from HTML"""
        import re
        
        patterns = [
            rf'{attribute}="([^"]+)"',
            rf'{attribute}=\'([^\']+)\'',
            rf'{attribute}=([^\s>]+)'
        ]
        
        for pattern in patterns:
            match = re.search(pattern, html)
            if match:
                return match.group(1)
        
        return None
    
    def _extract_captcha_image_url(self, response):
        """Extract CAPTCHA image URL from response"""
        from scrapy import Selector
        
        selector = Selector(text=response.text)
        
        # Common CAPTCHA image selectors
        image_selectors = [
            'img[src*="captcha"]::attr(src)',
            'img[alt*="captcha"]::attr(src)',
            '.captcha img::attr(src)',
            '#captcha img::attr(src)'
        ]
        
        for selector_str in image_selectors:
            img_url = selector.css(selector_str).get()
            if img_url:
                # Make absolute URL
                from urllib.parse import urljoin
                return urljoin(response.url, img_url)
        
        return None
    
    def _download_captcha_image(self, image_url):
        """Download CAPTCHA image for OCR processing"""
        try:
            response = requests.get(image_url, timeout=10)
            if response.status_code == 200:
                return response.content
        except Exception as e:
            print(f"❌ Failed to download CAPTCHA image: {e}")
        
        return None
    
    def _create_captcha_request(self, original_request, solution, captcha_type):
        """Create new request with CAPTCHA solution"""
        
        # Clone the original request
        new_request = original_request.replace()
        
        if captcha_type == 'recaptcha':
            # Add reCAPTCHA solution to form data
            if hasattr(new_request, 'body') and new_request.body:
                # Parse existing form data and add solution
                form_data = new_request.body.decode('utf-8')
                form_data += f"&g-recaptcha-response={solution}"
                new_request = new_request.replace(body=form_data.encode('utf-8'))
            else:
                # Create new form data
                form_data = f"g-recaptcha-response={solution}"
                new_request = new_request.replace(
                    body=form_data.encode('utf-8'),
                    headers={'Content-Type': 'application/x-www-form-urlencoded'}
                )
        
        elif captcha_type == 'hcaptcha':
            # Add hCaptcha solution to form data
            if hasattr(new_request, 'body') and new_request.body:
                form_data = new_request.body.decode('utf-8')
                form_data += f"&h-captcha-response={solution}"
                new_request = new_request.replace(body=form_data.encode('utf-8'))
            else:
                form_data = f"h-captcha-response={solution}"
                new_request = new_request.replace(
                    body=form_data.encode('utf-8'),
                    headers={'Content-Type': 'application/x-www-form-urlencoded'}
                )
        
        elif captcha_type == 'image':
            # Add image CAPTCHA solution to form data
            if hasattr(new_request, 'body') and new_request.body:
                form_data = new_request.body.decode('utf-8')
                form_data += f"&captcha={solution}"
                new_request = new_request.replace(body=form_data.encode('utf-8'))
            else:
                form_data = f"captcha={solution}"
                new_request = new_request.replace(
                    body=form_data.encode('utf-8'),
                    headers={'Content-Type': 'application/x-www-form-urlencoded'}
                )
        
        # Mark request as having CAPTCHA solution
        new_request.meta['captcha_solved'] = True
        new_request.meta['captcha_solution'] = solution
        
        return new_request
    
    def get_stats(self):
        """Get CAPTCHA solving statistics"""
        return self.captcha_stats

class AntiDetectionMiddleware:
    """Advanced anti-detection techniques"""
    
    def process_request(self, request, spider):
        # Add realistic headers
        request.headers.update({
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Cache-Control': 'max-age=0'
        })
        
        # Add random delay
        if not request.meta.get('dont_delay'):
            delay = random.uniform(2, 5)
            time.sleep(delay)
        
        return None

def scrape_linkedin_profiles(urls: List[str]) -> List[Dict]:
    """
    Scrape LinkedIn profiles using advanced Scrapy setup
    
    Args:
        urls: List of LinkedIn profile URLs
        
    Returns:
        List of profile data dictionaries
    """
    
    # Configure Scrapy settings
    settings = get_project_settings()
    settings.update({
        'LOG_LEVEL': 'INFO',
        'FEEDS': {
            'results.json': {
                'format': 'json',
                'overwrite': True,
            },
        },
    })
    
    # Create and run spider
    process = CrawlerProcess(settings)
    spider = LinkedInProfileSpider(urls=urls)
    
    process.crawl(spider)
    process.start()
    
    return spider.results

def scrape_single_linkedin_profile(url: str) -> Dict:
    """
    Scrape a single LinkedIn profile
    
    Args:
        url: LinkedIn profile URL
        
    Returns:
        Profile data dictionary
    """
    results = scrape_linkedin_profiles([url])
    return results[0] if results else {
        'success': False,
        'error': 'No results returned',
        'full_name': 'Profile Not Found',
        'headline': 'Scraping Failed',
        'summary': 'Unable to scrape profile',
        'profile_pic_url': ''
    }

# Test function
def test_scrapy_scraper():
    """Test the Scrapy-based scraper"""
    test_urls = [
        "https://www.linkedin.com/in/liveankit",
        "https://in.linkedin.com/in/hiren-danecha-695a51110",
        "https://www.linkedin.com/in/williamhgates/"
    ]
    
    print("🕷️ Testing Advanced Scrapy LinkedIn Scraper")
    print("=" * 60)
    
    for url in test_urls:
        print(f"\n🔍 Testing: {url}")
        try:
            result = scrape_single_linkedin_profile(url)
            if result.get('success'):
                print(f"✅ Success! Method: {result.get('scraping_method', 'unknown')}")
                print(f"   Name: {result.get('full_name', 'N/A')}")
                print(f"   Headline: {result.get('headline', 'N/A')[:50]}...")
            else:
                print(f"❌ Failed: {result.get('error', 'Unknown error')}")
        except Exception as e:
            print(f"❌ Exception: {e}")

if __name__ == "__main__":
    test_scrapy_scraper()