app.py

"""
Main Streamlit application for Afrobarometer Data Explorer.
"""

import streamlit as st
import pandas as pd
from pathlib import Path

# Import our custom modules
from app.utils.preprocessed_data_loader import (
    load_preprocessed_data,
    get_country_list,
    get_variable_info,
    filter_data_by_country,
    get_sample_data,
    get_data_summary
)
from app.utils.codebook_handler import display_codebook_info
from app.components.sidebar import render_sidebar
from app.pages.overview import render_overview_page
from app.pages.visualizations import render_visualizations_page
from app.utils.export import export_to_csv, export_to_excel, export_to_json, generate_summary_report, get_export_filename
from config.settings import STREAMLIT_CONFIG, DATA_FILE

# Configure page
st.set_page_config(**STREAMLIT_CONFIG)

# Custom CSS for better styling
st.markdown("""
<style>
    .main-header {
        font-size: 3rem;
        color: #1f77b4;
        text-align: center;
        margin-bottom: 2rem;
    }
    .metric-card {
        background-color: #f0f2f6;
        padding: 1rem;
        border-radius: 0.5rem;
        border-left: 4px solid #1f77b4;
    }
    .sidebar .sidebar-content {
        background-color: #f8f9fa;
    }
    .stSelectbox > div > div {
        background-color: white;
    }
</style>
""", unsafe_allow_html=True)


def main():
    """Main application function."""
    
    # Header
    st.markdown('<h1 class="main-header">🌍 Afrobarometer Data Explorer</h1>', unsafe_allow_html=True)
    st.markdown("### Interactive Dashboard for Afrobarometer Round 9 (39 Countries)")
    
    # Load preprocessed data
    with st.spinner("Loading preprocessed Afrobarometer data..."):
        df, metadata = load_preprocessed_data()
    
    if df is None:
        st.error("Failed to load preprocessed data. Please run the preprocessing script first.")
        st.info("💡 **To create the preprocessed dataset, run:**")
        st.code("python preprocess_data.py", language="bash")
        st.stop()
    
    # Render sidebar and get selections
    selected_country_groups, selected_countries, selected_variable, variable_type = render_sidebar(df, metadata)
    
    # Filter data by groupings and countries
    df_filtered = df.copy()
    if selected_country_groups or selected_countries:
        from app.utils.preprocessed_data_loader import filter_data_by_groupings_and_countries
        df_filtered = filter_data_by_groupings_and_countries(df, selected_country_groups, selected_countries)
    
    # Main content area with tabs
    tab1, tab2, tab3, tab4, tab5, tab6 = st.tabs(["📊 Overview", "📈 Visualizations", "🔍 Data Explorer", "📋 Summary Stats", "📚 Codebook", "💾 Export Data"])
    
    with tab1:
        render_overview_page(df_filtered, metadata)
    
    with tab2:
        render_visualizations_page(df_filtered, metadata, selected_variable)
    
    with tab3:
        render_data_explorer_page(df_filtered, metadata)
    
    with tab4:
        render_summary_stats_page(df_filtered, metadata)
    
    with tab5:
        display_codebook_info(metadata)
    
    with tab6:
        # Create a label for the export page
        if selected_countries:
            country_label = f"{len(selected_countries)} selected countries"
        elif selected_country_groups:
            country_label = f"Countries in {len(selected_country_groups)} grouping(s)"
        else:
            country_label = "All Countries"
        render_export_page(df_filtered, metadata, country_label)
    
    # Footer
    st.markdown("---")
    st.markdown("**Afrobarometer Data Explorer** | Built with Streamlit | Data: Round 9 (39 Countries)")


def render_data_explorer_page(df: pd.DataFrame, metadata: dict):
    """Render the data explorer page."""
    st.header("Interactive Data Explorer")
    
    # Variable comparison
    st.subheader("Variable Comparison")
    
    from app.utils.data_loader import get_variable_types
    
    var_types = get_variable_types(df)
    
    # Filter variables to show only Q3 to Q116 (same as sidebar)
    filtered_vars = []
    for var in var_types['all']:
        # Extract the Q number from the variable name (e.g., "Q3. Overall direction of the country (Q3)" -> "Q3")
        if 'Q' in var and '(' in var:
            # Look for pattern like "Q3)" or "Q116)"
            q_part = var.split('(')[-1].replace(')', '').strip()
            if q_part.startswith('Q'):
                try:
                    q_num = int(q_part[1:])  # Extract number after Q
                    if 3 <= q_num <= 116:
                        filtered_vars.append(var)
                except ValueError:
                    pass
    
    if not filtered_vars:
        st.warning("No Q3-Q116 variables found in the dataset.")
        return
    
    col1, col2 = st.columns(2)
    
    with col1:
        # Variable 1 selection - use same approach as sidebar
        selected_var1 = st.selectbox(
            "Variable 1:",
            options=filtered_vars,
            key="data_explorer_var1"
        )
    
    with col2:
        if selected_var1:
            # Variable 2 selection (exclude var1) - use same approach as sidebar
            var2_options = [var for var in filtered_vars if var != selected_var1]
            selected_var2 = st.selectbox(
                "Variable 2:",
                options=var2_options,
                key="data_explorer_var2"
            )
        else:
            selected_var2 = None
    
    # Create comparison plot
    if selected_var1 and selected_var2:
        from app.pages.visualizations import render_comparison_plot
        render_comparison_plot(df, metadata, selected_var1, selected_var2)


def render_summary_stats_page(df: pd.DataFrame, metadata: dict):
    """Render the summary statistics page."""
    st.header("Summary Statistics")
    
    # Overall statistics
    st.subheader("Dataset Summary")
    
    col1, col2 = st.columns(2)
    
    with col1:
        st.write("**Data Types:**")
        dtype_counts = df.dtypes.value_counts()
        st.dataframe(dtype_counts.to_frame('Count'), use_container_width=True)
    
    with col2:
        st.write("**Missing Values by Variable (Top 10):**")
        from app.utils.data_loader import get_missing_data_summary
        missing_summary = get_missing_data_summary(df)
        st.dataframe(missing_summary.head(10), use_container_width=True)
    
    # Numeric variables summary
    numeric_vars = df.select_dtypes(include=['number']).columns.tolist()
    if len(numeric_vars) > 0:
        st.subheader("Numeric Variables Summary")
        numeric_summary = df[numeric_vars].describe()
        st.dataframe(numeric_summary, use_container_width=True)
    
    # Categorical variables summary (now with labels)
    categorical_vars = df.select_dtypes(include=['object', 'category']).columns.tolist()
    if len(categorical_vars) > 0:
        st.subheader("Categorical Variables Summary (with Labels)")
        cat_summary = []
        for var in categorical_vars[:10]:  # Limit to first 10
            # var is now the labeled column name (e.g., "Country (COUNTRY)")
            unique_count = df[var].nunique()
            most_common = df[var].mode().iloc[0] if len(df[var].mode()) > 0 else "N/A"
            cat_summary.append({
                'Variable': var,  # Already labeled
                'Unique Values': unique_count,
                'Most Common': most_common
            })
        
        cat_df = pd.DataFrame(cat_summary)
        st.dataframe(cat_df, use_container_width=True)
        
        # Show detailed breakdown for a few categorical variables
        st.subheader("📋 Detailed Categorical Breakdown")
        
        # Select a few interesting categorical variables
        interesting_vars = []
        for var in categorical_vars[:5]:
            if df[var].nunique() <= 20:  # Only show variables with reasonable number of categories
                interesting_vars.append(var)
        
        for var in interesting_vars[:3]:  # Show max 3 variables
            value_counts = df[var].value_counts().head(10)
            
            with st.expander(f"{var} - Value Distribution"):
                st.write("**Top 10 values:**")
                for value, count in value_counts.items():
                    percentage = (count / len(df)) * 100
                    st.write(f"• **{value}**: {count:,} responses ({percentage:.1f}%)")


def render_export_page(df: pd.DataFrame, metadata: dict, country_label: str):
    """Render the enhanced export data page with country and variable selection."""
    st.header("Export Data")
    
    # Load the full dataset for selection (not just filtered)
    from app.utils.preprocessed_data_loader import load_preprocessed_data
    df_full, _ = load_preprocessed_data()
    
    if df_full is None:
        st.error("Could not load full dataset for export selection.")
        return
    
    st.subheader("📊 Custom Data Export")
    st.write("Select specific countries and variables to create a custom dataset for download.")
    
    # Create two columns for selection
    col1, col2 = st.columns(2)
    
    with col1:
        st.markdown("### 🌍 Country Selection")
        
        # Get all available countries
        from app.utils.preprocessed_data_loader import get_country_list
        all_countries = get_country_list(df_full)
        
        if all_countries:
            selected_countries = st.multiselect(
                "Select Countries:",
                options=all_countries,
                default=all_countries[:5] if len(all_countries) > 5 else all_countries,
                key="export_countries",
                help="Select one or more countries to include in the export"
            )
        else:
            st.warning("No countries found in the dataset.")
            selected_countries = []
    
    with col2:
        st.markdown("### 📈 Variable Selection")
        
        # Get all available variables
        from app.utils.data_loader import get_variable_types
        var_types = get_variable_types(df_full)
        all_variables = var_types['all']
        
        # Filter to Q3-Q116 variables (survey questions)
        survey_vars = []
        for var in all_variables:
            if 'Q' in var and '(' in var:
                q_part = var.split('(')[-1].replace(')', '').strip()
                if q_part.startswith('Q'):
                    try:
                        q_num = int(q_part[1:])
                        if 3 <= q_num <= 116:
                            survey_vars.append(var)
                    except ValueError:
                        pass
        
        # Add demographic and other important variables
        demo_vars = [var for var in all_variables if any(demo in var for demo in ['Country', 'GENDER', 'AGE', 'EDUC', 'URBRUR', 'REGION'])]
        
        # Combine and sort variables
        available_vars = sorted(survey_vars + demo_vars)
        
        if available_vars:
            selected_variables = st.multiselect(
                "Select Variables:",
                options=available_vars,
                default=available_vars[:10] if len(available_vars) > 10 else available_vars,
                key="export_variables",
                help="Select variables to include in the export. Start typing to search."
            )
        else:
            st.warning("No variables found in the dataset.")
            selected_variables = []
    
    # Show selection summary
    if selected_countries and selected_variables:
        st.markdown("---")
        st.subheader("📋 Export Summary")
        
        col1, col2, col3 = st.columns(3)
        with col1:
            st.metric("Selected Countries", len(selected_countries))
        with col2:
            st.metric("Selected Variables", len(selected_variables))
        
        # Calculate expected observations
        if selected_countries:
            country_data = df_full[df_full['Country (COUNTRY)'].isin(selected_countries)]
            expected_obs = len(country_data)
            with col3:
                st.metric("Expected Observations", f"{expected_obs:,}")
        
        # Show selected countries
        with st.expander("Selected Countries"):
            for country in selected_countries:
                st.write(f"• {country}")
        
        # Show selected variables
        with st.expander("Selected Variables"):
            for var in selected_variables:
                var_label = metadata.get('var_labels', {}).get(var, var)
                st.write(f"• {var_label} ({var})")
        
        # Export options
        st.markdown("---")
        st.subheader("💾 Download Options")
        
        export_format = st.radio("Export Format:", ["CSV", "Excel", "JSON"], horizontal=True)
        
        # Create custom dataset
        if st.button("🚀 Generate Custom Dataset", type="primary"):
            with st.spinner("Creating custom dataset..."):
                # Filter data by selected countries
                if selected_countries:
                    export_df = df_full[df_full['Country (COUNTRY)'].isin(selected_countries)].copy()
                else:
                    export_df = df_full.copy()
                
                # Select only chosen variables
                if selected_variables:
                    export_df = export_df[selected_variables]
                
                # Create country label for filename
                if len(selected_countries) == 1:
                    country_label = selected_countries[0]
                elif len(selected_countries) <= 3:
                    country_label = f"{len(selected_countries)}_countries"
                else:
                    country_label = f"{len(selected_countries)}_countries"
                
                # Generate filename
                filename = get_export_filename("afrobarometer_custom", export_format.lower(), country_label)
                
                # Export based on format
                if export_format == "CSV":
                    csv_data = export_to_csv(export_df)
                    st.download_button(
                        label="📥 Download CSV",
                        data=csv_data,
                        file_name=filename,
                        mime="text/csv",
                        type="primary"
                    )
                
                elif export_format == "Excel":
                    excel_data = export_to_excel(export_df, metadata)
                    st.download_button(
                        label="📥 Download Excel",
                        data=excel_data,
                        file_name=filename,
                        mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
                        type="primary"
                    )
                
                elif export_format == "JSON":
                    json_data = export_to_json(export_df)
                    st.download_button(
                        label="📥 Download JSON",
                        data=json_data,
                        file_name=filename,
                        mime="application/json",
                        type="primary"
                    )
                
                # Show dataset info
                st.success(f"✅ Custom dataset created with {len(export_df):,} observations and {len(export_df.columns)} variables!")
                
                # Show preview
                with st.expander("📊 Dataset Preview"):
                    st.dataframe(export_df.head(10), use_container_width=True)
    
    elif not selected_countries:
        st.warning("⚠️ Please select at least one country to export.")
    elif not selected_variables:
        st.warning("⚠️ Please select at least one variable to export.")
    
    # Legacy export section (for current filtered data)
    st.markdown("---")
    st.subheader("📥 Quick Export (Current Filtered Data)")
    st.write("Export the data currently filtered by the sidebar selections.")
    
    export_format_quick = st.radio("Export Format:", ["CSV", "Excel", "JSON"], key="quick_export", horizontal=True)
    
    if st.button("Generate Quick Download Link", key="quick_export_btn"):
        if export_format_quick == "CSV":
            csv_data = export_to_csv(df)
            filename = get_export_filename("afrobarometer_filtered", "csv", country_label)
            st.download_button(
                label="Download CSV",
                data=csv_data,
                file_name=filename,
                mime="text/csv"
            )
        
        elif export_format_quick == "Excel":
            excel_data = export_to_excel(df, metadata)
            filename = get_export_filename("afrobarometer_filtered", "xlsx", country_label)
            st.download_button(
                label="Download Excel",
                data=excel_data,
                file_name=filename,
                mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
            )
        
        elif export_format_quick == "JSON":
            json_data = export_to_json(df)
            filename = get_export_filename("afrobarometer_filtered", "json", country_label)
            st.download_button(
                label="Download JSON",
                data=json_data,
                file_name=filename,
                mime="application/json"
            )
    
    # Export summary report
    st.markdown("---")
    st.subheader("📄 Export Summary Report")
    if st.button("Generate Summary Report", key="summary_report_btn"):
        report = generate_summary_report(df, metadata, country_label)
        filename = get_export_filename("afrobarometer_summary", "md", country_label)
        st.download_button(
            label="Download Summary Report",
            data=report,
            file_name=filename,
            mime="text/markdown"
        )


if __name__ == "__main__":
    main()