verify_tutorials.py

#!/usr/bin/env python3
"""
Verification Script for Spark Tutorials
======================================

This script verifies that all tutorial files are present and have valid Python syntax.
It doesn't require Spark to be installed.

To run: python verify_tutorials.py
"""

import os
import ast
import sys

def verify_file(filename):
    """Verify a Python file exists and has valid syntax."""
    print(f"\nVerifying {filename}...")
    
    # Check if file exists
    if not os.path.exists(filename):
        print(f"  ✗ File not found: {filename}")
        return False
    
    # Check file size
    file_size = os.path.getsize(filename)
    print(f"  ✓ File exists ({file_size:,} bytes)")
    
    # Check Python syntax
    try:
        with open(filename, 'r') as f:
            content = f.read()
            ast.parse(content)
        print(f"  ✓ Valid Python syntax")
        
        # Count lines
        lines = content.split('\n')
        code_lines = [l for l in lines if l.strip() and not l.strip().startswith('#')]
        print(f"  ✓ {len(lines)} total lines, {len(code_lines)} code lines")
        
        # Check for key Spark imports
        spark_imports = ['pyspark', 'SparkSession', 'SparkContext']
        found_imports = []
        for imp in spark_imports:
            if imp in content:
                found_imports.append(imp)
        
        if found_imports:
            print(f"  ✓ Found Spark imports: {', '.join(found_imports)}")
        
        return True
        
    except SyntaxError as e:
        print(f"  ✗ Syntax error: {e}")
        return False
    except Exception as e:
        print(f"  ✗ Error reading file: {e}")
        return False

def main():
    """Main verification function."""
    print("=" * 60)
    print("SPARK TUTORIALS VERIFICATION")
    print("=" * 60)
    
    # List of files to verify
    files = [
        ("Tutorial 1: RDD Basics", "tutorial_1_spark_rdd_basics.py"),
        ("Tutorial 2: SQL and DataFrames", "tutorial_2_spark_sql_dataframes.py"),
        ("Tutorial 3: MLlib", "tutorial_3_spark_mllib.py"),
        ("Tutorial 4: ML Pipeline", "tutorial_4_spark_ml_pipeline.py"),
        ("Test Script", "test_all_tutorials.py"),
        ("README", "README.md")
    ]
    
    results = []
    
    # Verify each file
    for name, filename in files:
        print(f"\n{name}:")
        if filename.endswith('.py'):
            success = verify_file(filename)
        else:
            # Just check if non-Python files exist
            success = os.path.exists(filename)
            if success:
                size = os.path.getsize(filename)
                print(f"  ✓ File exists ({size:,} bytes)")
            else:
                print(f"  ✗ File not found")
        
        results.append((name, success))
    
    # Summary
    print("\n" + "=" * 60)
    print("SUMMARY")
    print("=" * 60)
    
    passed = sum(1 for _, success in results if success)
    total = len(results)
    
    for name, success in results:
        status = "✓" if success else "✗"
        print(f"{status} {name}")
    
    print(f"\nTotal: {passed}/{total} files verified successfully")
    
    if passed == total:
        print("\n✅ All tutorial files are present and valid!")
        print("\nTo run the tutorials, you'll need to:")
        print("1. Install PySpark: pip install pyspark numpy pandas")
        print("2. Run individual tutorials: python tutorial_1_spark_rdd_basics.py")
        print("3. Or use spark-submit: spark-submit tutorial_1_spark_rdd_basics.py")
    else:
        print(f"\n⚠️  {total - passed} file(s) have issues")

if __name__ == "__main__":
    main()