analysis.py

import time

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from scipy.spatial.distance import pdist, squareform


def distance_correlation(x, y):
    """Calculate distance correlation between x and y"""
    n = len(x)
    x = np.asarray(x)
    y = np.asarray(y)

    a = squareform(pdist(x.reshape(-1, 1)))
    b = squareform(pdist(y.reshape(-1, 1)))

    A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean()
    B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean()

    dcov_AB = (A * B).sum() / n**2
    dvar_A = (A * A).sum() / n**2
    dvar_B = (B * B).sum() / n**2

    if dvar_A > 0 and dvar_B > 0:
        dcor = np.sqrt(dcov_AB) / np.sqrt(np.sqrt(dvar_A) * np.sqrt(dvar_B))
    else:
        dcor = 0

    return dcor


def generate_dataframe(length: int) -> pd.DataFrame:
    """Generate sample DataFrame with timestamps, vcpu, and mem"""
    start_timestamp = int(time.time())
    timestamps = [start_timestamp + i for i in range(length)]
    vcpu = np.random.randint(0, 101, size=length)
    mem = np.random.randint(1024, 16385, size=length)

    df = pd.DataFrame({"timestamps": timestamps, "vcpu": vcpu, "mem": mem})

    return df


def analyzer(dataframe: pd.DataFrame, interval: str, dataframe_name: str):
    # # Generate DataFrame
    # dataframe = generate_dataframe(100)
    interval = f"{interval}x10"

    # Group by timestamps (already at original intervals)
    grouped = dataframe.groupby("timestamps").agg({"vcpu": "median", "mem": "median"}).reset_index()

    # Calculate time offset for better visualization
    grouped["time_offset"] = grouped["timestamps"] - grouped["timestamps"].min()

    # Calculate mem/vcpu ratio (avoiding division by zero)
    grouped["mem_vcpu_ratio"] = grouped["mem"] / (grouped["vcpu"] + 0.1)

    # Create {interval} interval grouping
    grouped["time_bin"] = (grouped["time_offset"] // 10) * 10
    grouped_aggregated = (
        grouped.groupby("time_bin").agg({"vcpu": "median", "mem": "median", "mem_vcpu_ratio": "median"}).reset_index()
    )

    # Calculate correlations
    pearson_corr = grouped["vcpu"].corr(grouped["mem"])
    dist_corr = distance_correlation(grouped["vcpu"].values, grouped["mem"].values)

    print("DataFrame Statistics:")
    print(f"Total records (original intervals): {len(grouped)}")
    print(f"Total records ({interval} intervals): {len(grouped_aggregated)}")
    print(f"VCPU range: {grouped['vcpu'].min():.1f} - {grouped['vcpu'].max():.1f}")
    print(f"Memory range: {grouped['mem'].min():.1f} - {grouped['mem'].max():.1f}")
    print(f"Pearson correlation: {pearson_corr:.4f}")
    print(f"Distance correlation: {dist_corr:.4f}")
    print()

    # Create figure with 6 subplots
    fig = plt.figure(figsize=(18, 18))

    # Use GridSpec for better layout control
    gs = fig.add_gridspec(4, 2, hspace=0.35, wspace=0.3, height_ratios=[1, 1, 1, 1])

    # Plot 1: VCPU utilization over time (grouped by 1 second interval)
    ax1 = fig.add_subplot(gs[0, 0])
    ax1.plot(
        grouped["time_offset"],
        grouped["vcpu"],
        marker="o",
        linestyle="-",
        linewidth=1.5,
        markersize=4,
        alpha=0.5,
        color="#2E86AB",
        label="VCPU (original intervals)",
    )
    ax1.fill_between(grouped["time_offset"], grouped["vcpu"], alpha=0.2, color="#2E86AB")

    # Add {interval} median values
    ax1.plot(
        grouped_aggregated["time_bin"],
        grouped_aggregated["vcpu"],
        marker="o",
        linestyle="-",
        linewidth=3,
        markersize=8,
        alpha=0.9,
        color="#FF4444",
        label=f"median ({interval} intervals)",
    )

    ax1.set_xlabel("Time", fontsize=11)
    ax1.set_ylabel("VCPU Usage (%)", fontsize=11)
    ax1.set_title(
        f"1. VCPU Utilization Over Time\n(Original intervals + {interval} medians)", fontsize=12, fontweight="bold"
    )
    ax1.grid(True, alpha=0.3)
    # ax1.set_ylim([0, 105])
    ax1.legend(loc="upper right")

    # Plot 2: Memory utilization over time (grouped by 1 second interval)
    ax2 = fig.add_subplot(gs[0, 1])
    ax2.plot(
        grouped["time_offset"],
        grouped["mem"],
        marker="s",
        linestyle="-",
        linewidth=1.5,
        markersize=4,
        alpha=0.5,
        color="#A23B72",
        label="Memory (original intervals)",
    )
    ax2.fill_between(grouped["time_offset"], grouped["mem"], alpha=0.2, color="#A23B72")

    # Add {interval} median values
    ax2.plot(
        grouped_aggregated["time_bin"],
        grouped_aggregated["mem"],
        marker="s",
        linestyle="-",
        linewidth=3,
        markersize=8,
        alpha=0.9,
        color="#FF4444",
        label=f"median ({interval} intervals)",
    )

    ax2.set_xlabel("Time", fontsize=11)
    ax2.set_ylabel("Memory Usage (MB)", fontsize=11)
    ax2.set_title(
        f"2. Memory Utilization Over Time\n(original intervals + {interval} medians)", fontsize=12, fontweight="bold"
    )
    ax2.grid(True, alpha=0.3)
    # ax2.set_ylim([0, 17000])
    ax2.legend(loc="upper right")

    # Plot 3: Scatter plot with Pearson correlation
    ax3 = fig.add_subplot(gs[1, 0])
    scatter3 = ax3.scatter(
        grouped["vcpu"],
        grouped["mem"],
        c=grouped["time_offset"],
        cmap="viridis",
        s=80,
        alpha=0.7,
        edgecolors="black",
        linewidth=0.5,
    )

    # Add Pearson trend line
    z_pearson = np.polyfit(grouped["vcpu"], grouped["mem"], 1)
    p_pearson = np.poly1d(z_pearson)
    vcpu_sorted = np.sort(grouped["vcpu"])
    ax3.plot(
        vcpu_sorted,
        p_pearson(vcpu_sorted),
        "r--",
        linewidth=2.5,
        alpha=0.8,
        label=f"Linear Trend\n(r = {pearson_corr:.4f})",
    )

    cbar3 = plt.colorbar(scatter3, ax=ax3)
    cbar3.set_label("Time", fontsize=10)

    ax3.set_xlabel("VCPU Usage (%)", fontsize=11)
    ax3.set_ylabel("Memory Usage (MB)", fontsize=11)
    ax3.set_title(f"3. Pearson r = {pearson_corr:.4f}", fontsize=12, fontweight="bold")
    ax3.grid(True, alpha=0.3)
    ax3.legend(fontsize=9, loc="best")

    # Plot 4: Scatter plot with Distance correlation
    ax4 = fig.add_subplot(gs[1, 1])
    scatter4 = ax4.scatter(
        grouped["vcpu"],
        grouped["mem"],
        c=grouped["time_offset"],
        cmap="plasma",
        s=80,
        alpha=0.7,
        edgecolors="black",
        linewidth=0.5,
    )

    # Add same trend line for comparison
    ax4.plot(
        vcpu_sorted,
        p_pearson(vcpu_sorted),
        "r--",
        linewidth=2.5,
        alpha=0.8,
        label=f"Linear Trend\n(Distance = {dist_corr:.4f})",
    )

    cbar4 = plt.colorbar(scatter4, ax=ax4)
    cbar4.set_label("Time", fontsize=10)

    ax4.set_xlabel("VCPU Usage (%)", fontsize=11)
    ax4.set_ylabel("Memory Usage (MB)", fontsize=11)
    ax4.set_title(f"4. Distance Correlation = {dist_corr:.4f}", fontsize=12, fontweight="bold")
    ax4.grid(True, alpha=0.3)
    ax4.legend(fontsize=9, loc="best")

    # Plot 5: Memory/VCPU median ratio over time
    ax5 = fig.add_subplot(gs[2, :])

    # Calculate overall median ratio
    median_ratio = grouped["mem_vcpu_ratio"].median()
    std_ratio = grouped["mem_vcpu_ratio"].std()

    # Plot {interval} interval median ratios
    ax5.plot(
        grouped_aggregated["time_bin"],
        grouped_aggregated["mem_vcpu_ratio"],
        marker="D",
        linestyle="-",
        linewidth=2.5,
        markersize=8,
        alpha=0.8,
        color="#FF6B6B",
        label=f"median Ratio ({interval} intervals)",
    )
    ax5.fill_between(grouped_aggregated["time_bin"], grouped_aggregated["mem_vcpu_ratio"], alpha=0.3, color="#FF6B6B")

    # Plot overall median line (horizontal)
    ax5.axhline(
        y=median_ratio, color="green", linestyle="--", linewidth=2.5, alpha=0.8, label=f"Overall median = {median_ratio:.2f}"
    )

    # Add shaded region showing standard deviation
    ax5.axhspan(median_ratio - std_ratio, median_ratio + std_ratio, alpha=0.15, color="green", label=f"±1 Std Dev")

    ax5.set_xlabel("Time (seconds from start)", fontsize=11)
    ax5.set_ylabel("Memory/VCPU Ratio", fontsize=11)
    ax5.set_title(
        f"5. Memory to VCPU Ratio Over Time\n({interval} interval medians + overall median)", fontsize=12, fontweight="bold"
    )
    ax5.grid(True, alpha=0.3)
    ax5.legend(loc="upper right", fontsize=10)
    ax5.set_xlim([grouped["time_offset"].min(), grouped["time_offset"].max()])

    # Add statistics box
    stats_text = f"Overall median: {median_ratio:.2f}\n"
    stats_text += f'{interval} median Range: {grouped_aggregated["mem_vcpu_ratio"].min():.2f} - {grouped_aggregated["mem_vcpu_ratio"].max():.2f}\n'
    stats_text += f"Overall Std: {std_ratio:.2f}"
    ax5.text(
        0.02,
        0.98,
        stats_text,
        transform=ax5.transAxes,
        fontsize=9,
        verticalalignment="top",
        bbox=dict(boxstyle="round", facecolor="wheat", alpha=0.5),
    )

    # Plot 6: Memory/VCPU ratio ABSOLUTE VALUES over time ({interval} intervals)
    ax6 = fig.add_subplot(gs[3, :])

    # Plot absolute ratio values grouped by 10 seconds
    ax6.plot(
        grouped_aggregated["time_bin"],
        grouped_aggregated["mem_vcpu_ratio"],
        marker="o",
        linestyle="-",
        linewidth=2.5,
        markersize=8,
        alpha=0.8,
        color="#9C27B0",
        label=f"Absolute Ratio ({interval} intervals)",
    )
    ax6.fill_between(grouped_aggregated["time_bin"], grouped_aggregated["mem_vcpu_ratio"], alpha=0.3, color="#9C27B0")

    # Add overall median line for reference
    ax6.axhline(
        y=median_ratio, color="orange", linestyle="--", linewidth=2, alpha=0.7, label=f"Overall median = {median_ratio:.2f}"
    )

    ax6.set_xlabel("Time (seconds from start)", fontsize=11)
    ax6.set_ylabel("Memory/VCPU Ratio (Absolute)", fontsize=11)
    ax6.set_title(
        f"6. Memory to VCPU Ratio - Absolute Values\n({interval} interval grouping)", fontsize=12, fontweight="bold"
    )
    ax6.grid(True, alpha=0.3)
    ax6.legend(loc="upper right", fontsize=10)
    ax6.set_xlim([grouped["time_offset"].min(), grouped["time_offset"].max()])

    # Add statistics box
    stats_text_6 = f'median: {grouped_aggregated["mem_vcpu_ratio"].median():.2f}\n'
    stats_text_6 += f'Min: {grouped_aggregated["mem_vcpu_ratio"].min():.2f}\n'
    stats_text_6 += f'Max: {grouped_aggregated["mem_vcpu_ratio"].max():.2f}\n'
    stats_text_6 += f'Std: {grouped_aggregated["mem_vcpu_ratio"].std():.2f}'
    ax6.text(
        0.02,
        0.98,
        stats_text_6,
        transform=ax6.transAxes,
        fontsize=9,
        verticalalignment="top",
        bbox=dict(boxstyle="round", facecolor="lightblue", alpha=0.5),
    )

    # Add main title
    fig.suptitle("Complete Resource Utilization Analysis", fontsize=16, fontweight="bold", y=0.995)

    plt.savefig(f"plots/complete_plots_{interval}_{dataframe_name}.png", dpi=300, bbox_inches="tight")
    print("Complete 6-plot analysis with absolute values saved!")

    # Print detailed statistics
    print("\nDetailed Statistics:")
    print("=" * 60)
    print("\nVCPU Statistics (original intervals):")
    print(grouped["vcpu"].describe())
    print(f"\nVCPU Statistics ({interval} intervals):")
    print(grouped_aggregated["vcpu"].describe())
    print("\nMemory Statistics (original intervals):")
    print(grouped["mem"].describe())
    print(f"\nMemory Statistics ({interval} intervals):")
    print(grouped_aggregated["mem"].describe())
    print("\nMemory/VCPU Ratio Statistics (original intervals):")
    print(grouped["mem_vcpu_ratio"].describe())
    print(f"\nMemory/VCPU Ratio Statistics ({interval} intervals):")
    print(grouped_aggregated["mem_vcpu_ratio"].describe())
    print("\nCorrelation Summary:")
    print(f"  Pearson Correlation:   {pearson_corr:>8.4f}")
    print(f"  Distance Correlation:  {dist_corr:>8.4f}")
    print("=" * 60)

    # Save data to CSV for reference
    output_data_original = grouped[["time_offset", "vcpu", "mem", "mem_vcpu_ratio"]].copy()
    output_data_original.columns = ["Time_Offset_Seconds", "VCPU_Usage", "Memory_MB", "Mem_VCPU_Ratio"]
    output_data_original.to_csv(f"data/analysis_data_original_{interval}_{dataframe_name}.csv", index=False)

    output_data_aggregated = grouped_aggregated[["time_bin", "vcpu", "mem", "mem_vcpu_ratio"]].copy()
    output_data_aggregated.columns = ["Time_Bin_Seconds", "VCPU_Usage_median", "Memory_MB_median", "Mem_VCPU_Ratio_median"]
    output_data_aggregated.to_csv(f"data/analysis_data_{interval}_{dataframe_name}.csv", index=False)

    print("\nAnalysis data saved:")
    print("  - original intervals: analysis_data_original_6plots.csv")
    print(f"  - {interval} intervals: analysis_data_aggregated_6plots.csv")