dashboard.py

import glob
import os

import pandas as pd
import plotly.express as px
import streamlit as st

# Set page configuration
st.set_page_config(page_title="Lumen Engine Performance Dashboard", layout="wide")


# --- Data Loading Utilities ---
@st.cache_data
def load_micro_data(directory="results/micro"):
    """Loads all internal core engine metrics from the micro directory."""
    all_files = glob.glob(os.path.join(directory, "*.csv"))
    if not all_files:
        return pd.DataFrame()

    df_list = []
    required_columns = {
        "timestamp",
        "request_id",
        "queue_type",
        "alloc_type",
        "queue_ms",
        "preprocess_ms",
        "inference_ms",
        "postprocess_ms",
        "total_ms",
    }

    for filename in all_files:
        try:
            df = pd.read_csv(filename)
            if not required_columns.issubset(df.columns):
                st.warning(
                    f"Skipping micro file {os.path.basename(filename)}: Missing columns {required_columns - set(df.columns)}"
                )
                continue
            df_list.append(df)
        except Exception as e:
            st.error(f"Error reading micro file {os.path.basename(filename)}: {e}")

    if not df_list:
        return pd.DataFrame()

    combined_df = pd.concat(df_list, ignore_index=True)
    combined_df["Configuration"] = (
        combined_df["queue_type"] + " + " + combined_df["alloc_type"]
    )
    return combined_df


@st.cache_data
def load_macro_data(directory="results/macro"):
    """Loads all external network transit metrics from the macro directory."""
    all_files = glob.glob(os.path.join(directory, "*.csv"))
    if not all_files:
        return pd.DataFrame()

    df_list = []
    required_columns = {
        "timestamp",
        "request_id",
        "queue_type",
        "alloc_type",
        "bytes_sent",
        "connection_ms",
        "total_round_trip_ms",
    }

    for filename in all_files:
        try:
            df = pd.read_csv(filename)
            if not required_columns.issubset(df.columns):
                st.warning(
                    f"Skipping macro file {os.path.basename(filename)}: Missing columns {required_columns - set(df.columns)}"
                )
                continue
            df_list.append(df)
        except Exception as e:
            st.error(f"Error reading macro file {os.path.basename(filename)}: {e}")

    if not df_list:
        return pd.DataFrame()

    combined_df = pd.concat(df_list, ignore_index=True)
    combined_df["Configuration"] = (
        combined_df["queue_type"] + " + " + combined_df["alloc_type"]
    )
    return combined_df


# --- Throughput / RPS Calculator ---
def calculate_throughput(df):
    if df.empty or len(df) < 2:
        return 0
    duration = df["timestamp"].max() - df["timestamp"].min()
    if duration <= 0:
        duration = 1
    return len(df) / duration


# --- Main Application ---
st.title("⚡ Lumen Inference Engine Performance Lab")
st.markdown("""
This dashboard visualizes telemetry data generated by the **Lumen Inference Engine**.
Toggle between the tabs below to analyze isolated component execution timings (**Micro-Baselines**) 
and end-to-end socket round-trips over loopback interfaces (**Macro-Network Metrics**).
""")

# Load baseline frames
micro_df = load_micro_data("results/micro")
macro_df = load_macro_data("results/macro")

if micro_df.empty and macro_df.empty:
    st.info(
        "No baseline files found. Please ensure your metrics are generated inside 'results/micro' and 'results/macro' folders."
    )
else:
    # --- Sidebar Global Filter Sets ---
    st.sidebar.header("Filter Configuration")

    # Extract unique filters across both matrices if available
    unique_queues = set()
    unique_allocs = set()
    if not micro_df.empty:
        unique_queues.update(micro_df["queue_type"].unique())
        unique_allocs.update(micro_df["alloc_type"].unique())
    if not macro_df.empty:
        unique_queues.update(macro_df["queue_type"].unique())
        unique_allocs.update(macro_df["alloc_type"].unique())

    all_queues = list(unique_queues)
    all_allocs = list(unique_allocs)

    selected_queues = st.sidebar.multiselect(
        "Select Queue Types", all_queues, default=all_queues
    )
    selected_allocs = st.sidebar.multiselect(
        "Select Allocator Types", all_allocs, default=all_allocs
    )

    st.sidebar.markdown("---")
    st.sidebar.header("Chart Settings")
    chart_mode = st.sidebar.radio(
        "Jitter Chart Mode", ["Separate (Faceted)", "Overlaid (Comparison)"], index=0
    )

    # Apply global filters to data spaces
    if not micro_df.empty:
        micro_df = micro_df[
            (micro_df["queue_type"].isin(selected_queues))
            & (micro_df["alloc_type"].isin(selected_allocs))
        ]
    if not macro_df.empty:
        macro_df = macro_df[
            (macro_df["queue_type"].isin(selected_queues))
            & (macro_df["alloc_type"].isin(selected_allocs))
        ]

    # --- Setup Tab Views ---
    tab1, tab2 = st.tabs(["🔬 Micro-Core Benchmarks", "🌐 Macro-Network Benchmarks"])

    # ==========================================
    # TAB 1: MICRO PIPELINE ANALYTICS
    # ==========================================
    with tab1:
        st.header("Isolated Core Execution Performance")
        if micro_df.empty:
            st.warning("No micro data available for the selected filters.")
        else:
            # 1. Compute summary rows
            micro_metrics = (
                micro_df.groupby(["Configuration", "queue_type", "alloc_type"])
                .apply(
                    lambda x: pd.Series(
                        {
                            "RPS": calculate_throughput(x),
                            "P99 (ms)": x["total_ms"].quantile(0.99),
                            "Avg (ms)": x["total_ms"].mean(),
                            "Max (ms)": x["total_ms"].max(),
                            "Count": len(x),
                        }
                    )
                )
                .reset_index()
            )

            # Metrics Cards Row
            best_micro = micro_metrics.loc[micro_metrics["P99 (ms)"].idxmin()]
            mc1, mc2, mc3 = st.columns(3)
            mc1.metric(
                "Optimal Core Configuration (P99)",
                f"{best_micro['P99 (ms)']:.2f} ms",
                best_micro["Configuration"],
                delta_color="inverse",
            )
            mc2.metric("Peak Core Throughput", f"{micro_metrics['RPS'].max():.0f} RPS")
            mc3.metric("Micro Sample Volume", f"{len(micro_df):,}")

            # Charts Grid 1
            st.subheader("Core Latency Profile")
            c1, c2 = st.columns(2)
            with c1:
                fig_p99_micro = px.bar(
                    micro_metrics.sort_values("P99 (ms)"),
                    x="Configuration",
                    y="P99 (ms)",
                    color="alloc_type",
                    text_auto=".2f",
                    title="99th Percentile Internal Latency",
                )
                fig_p99_micro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_p99_micro, use_container_width=True)
            with c2:
                fig_box_micro = px.box(
                    micro_df,
                    x="Configuration",
                    y="total_ms",
                    color="alloc_type",
                    title="Hardware Latency Distributions",
                )
                fig_box_micro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_box_micro, use_container_width=True)

            # Charts Grid 2
            st.subheader("Throughput & Pipeline Stages")
            c3, c4 = st.columns(2)
            with c3:
                fig_rps_micro = px.bar(
                    micro_metrics.sort_values("RPS", ascending=False),
                    x="Configuration",
                    y="RPS",
                    color="queue_type",
                    text_auto=".0f",
                    title="Core Processing Throughput (RPS)",
                )
                fig_rps_micro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_rps_micro, use_container_width=True)
            with c4:
                melted_micro = micro_df.melt(
                    id_vars=["Configuration"],
                    value_vars=[
                        "queue_ms",
                        "preprocess_ms",
                        "inference_ms",
                        "postprocess_ms",
                    ],
                    var_name="Stage",
                    value_name="Time (ms)",
                )
                avg_stages = (
                    melted_micro.groupby(["Configuration", "Stage"])["Time (ms)"]
                    .mean()
                    .reset_index()
                )
                fig_stack_micro = px.bar(
                    avg_stages,
                    x="Configuration",
                    y="Time (ms)",
                    color="Stage",
                    title="Inference Lifecycle Cost per Configuration",
                    barmode="stack",
                )
                fig_stack_micro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_stack_micro, use_container_width=True)

            # Stability / Jitter Profile
            st.subheader("Core Pipeline Jitter Over Time")
            line_micro_data = micro_df.sort_values("timestamp")
            if len(line_micro_data) > 5000:
                line_micro_data = line_micro_data.iloc[::5, :]

            if chart_mode == "Separate (Faceted)":
                num_c = len(line_micro_data["Configuration"].unique())
                fig_line_micro = px.line(
                    line_micro_data,
                    x="request_id",
                    y="total_ms",
                    color="Configuration",
                    title="Core Latency Consistency Profile (Faceted)",
                    facet_row="Configuration",
                    height=max(400, 240 * num_c),
                )
                fig_line_micro.update_xaxes(
                    matches=None, showticklabels=False, title=None
                )
                fig_line_micro.for_each_annotation(
                    lambda a: a.update(text=a.text.split("=")[-1])
                )
            else:
                fig_line_micro = px.line(
                    line_micro_data,
                    x="request_id",
                    y="total_ms",
                    color="Configuration",
                    title="Core Latency Consistency Profile (Overlaid)",
                    height=550,
                )
                fig_line_micro.update_xaxes(title="Request Sequence ID")
            st.plotly_chart(fig_line_micro, use_container_width=True)

            with st.expander("Inspect Micro Baseline Data Matrix"):
                st.dataframe(
                    micro_metrics.style.highlight_min(
                        axis=0, subset=["P99 (ms)"], color="lightgreen"
                    )
                )

    # ==========================================
    # TAB 2: MACRO NETWORK ANALYTICS
    # ==========================================
    with tab2:
        st.header("End-to-End Socket Client Metrics")
        if macro_df.empty:
            st.warning("No macro network data available for the selected filters.")
        else:
            # 1. Compute network metrics summaries
            macro_metrics = (
                macro_df.groupby(["Configuration", "queue_type", "alloc_type"])
                .apply(
                    lambda x: pd.Series(
                        {
                            "Client RPS": calculate_throughput(x),
                            "P99 Round Trip (ms)": x["total_round_trip_ms"].quantile(
                                0.99
                            ),
                            "Avg Handshake (ms)": x["connection_ms"].mean(),
                            "Avg Round Trip (ms)": x["total_round_trip_ms"].mean(),
                            "Data Transferred (MB)": x["bytes_sent"].sum()
                            / (1024 * 1024),
                            "Count": len(x),
                        }
                    )
                )
                .reset_index()
            )

            # Metrics Cards Row
            best_macro = macro_metrics.loc[
                macro_metrics["P99 Round Trip (ms)"].idxmin()
            ]
            mac1, mac2, mac3 = st.columns(3)
            mac1.metric(
                "Optimal Client Configuration (RTT)",
                f"{best_macro['P99 Round Trip (ms)']:.2f} ms",
                best_macro["Configuration"],
                delta_color="inverse",
            )
            mac2.metric(
                "Peak Client Network Throughput",
                f"{macro_metrics['Client RPS'].max():.0f} RPS",
            )
            mac3.metric("Macro Network Sample Volume", f"{len(macro_df):,}")

            # Charts Grid 1
            st.subheader("Client-Observed Latency Profile")
            mac_c1, mac_c2 = st.columns(2)
            with mac_c1:
                fig_p99_macro = px.bar(
                    macro_metrics.sort_values("P99 Round Trip (ms)"),
                    x="Configuration",
                    y="P99 Round Trip (ms)",
                    color="alloc_type",
                    text_auto=".2f",
                    title="99th Percentile Network Round-Trip Time (RTT)",
                )
                fig_p99_macro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_p99_macro, use_container_width=True)
            with mac_c2:
                fig_box_macro = px.box(
                    macro_df,
                    x="Configuration",
                    y="total_round_trip_ms",
                    color="alloc_type",
                    title="End-to-End Client Latency Distributions",
                )
                fig_box_macro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_box_macro, use_container_width=True)

            # Charts Grid 2
            st.subheader("Network Throughput & Transport Cost Breakdown")
            mac_c3, mac_c4 = st.columns(2)
            with mac_c3:
                fig_rps_macro = px.bar(
                    macro_metrics.sort_values("Client RPS", ascending=False),
                    x="Configuration",
                    y="Client RPS",
                    color="queue_type",
                    text_auto=".0f",
                    title="Client System Throughput Rate (RPS)",
                )
                fig_rps_macro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_rps_macro, use_container_width=True)
            with mac_c4:
                # Construct stacked chart: Connection vs Application handling delay
                macro_copy = macro_df.copy()
                macro_copy["processing_transit_ms"] = (
                    macro_copy["total_round_trip_ms"] - macro_copy["connection_ms"]
                )

                melted_macro = macro_copy.melt(
                    id_vars=["Configuration"],
                    value_vars=["connection_ms", "processing_transit_ms"],
                    var_name="Transport Phase",
                    value_name="Time (ms)",
                )
                avg_transport = (
                    melted_macro.groupby(["Configuration", "Transport Phase"])[
                        "Time (ms)"
                    ]
                    .mean()
                    .reset_index()
                )

                fig_stack_macro = px.bar(
                    avg_transport,
                    x="Configuration",
                    y="Time (ms)",
                    color="Transport Phase",
                    title="TCP Handshake Cost vs Engine Runtime Latency",
                    barmode="stack",
                )
                fig_stack_macro.update_layout(xaxis_tickangle=-45)
                st.plotly_chart(fig_stack_macro, use_container_width=True)

            # Stability / Jitter Profile
            st.subheader("Network Stream Jitter Profile Over Time")
            line_macro_data = macro_df.sort_values("timestamp")
            if len(line_macro_data) > 5000:
                line_macro_data = line_macro_data.iloc[::5, :]

            if chart_mode == "Separate (Faceted)":
                num_c_mac = len(line_macro_data["Configuration"].unique())
                fig_line_macro = px.line(
                    line_macro_data,
                    x="request_id",
                    y="total_round_trip_ms",
                    color="Configuration",
                    title="Network Streaming Connection Jitter (Faceted)",
                    facet_row="Configuration",
                    height=max(400, 240 * num_c_mac),
                )
                fig_line_macro.update_xaxes(
                    matches=None, showticklabels=False, title=None
                )
                fig_line_macro.for_each_annotation(
                    lambda a: a.update(text=a.text.split("=")[-1])
                )
            else:
                fig_line_macro = px.line(
                    line_macro_data,
                    x="request_id",
                    y="total_round_trip_ms",
                    color="Configuration",
                    title="Network Streaming Connection Jitter (Overlaid)",
                    height=550,
                )
                fig_line_macro.update_xaxes(title="Request Sequence ID")
            st.plotly_chart(fig_line_macro, use_container_width=True)

            with st.expander("Inspect Macro Network Data Matrix"):
                st.dataframe(
                    macro_metrics.style.highlight_min(
                        axis=0, subset=["P99 Round Trip (ms)"], color="lightgreen"
                    )
                )