data_preprocessor_tool.py

import streamlit as st
import pandas as pd
from langchain_ollama import OllamaLLM
from preprocessing_tools import get_all_tools
import io
from fpdf import FPDF
import re

# --- LLM Setup (Ollama llama3.1:8b) ---
llm = OllamaLLM(model="llama3.1:8b", temperature=0.2)

def explain_step(step, details):
    prompt = f"""
    You are a data science assistant. Explain the following preprocessing step in simple terms for a user:
    Step: {step}
    Details: {details}
    """
    return llm(prompt)

tools = get_all_tools()

def add_heading(pdf, text, size=16):
    pdf.set_font("Arial", style="B", size=size)
    pdf.cell(0, 12, text, ln=True)
    pdf.ln(2)
    pdf.set_font("Arial", size=12)

def add_table(pdf, md_table):
    # Parse markdown table
    lines = [line for line in md_table.split('\n') if line.strip()]
    if len(lines) < 2:
        return
    headers = [h.strip() for h in lines[0].strip('|').split('|')]
    pdf.set_font("Arial", style="B", size=12)
    for h in headers:
        pdf.cell(40, 8, h, border=1)
    pdf.ln()
    pdf.set_font("Arial", size=12)
    for row in lines[2:]:
        cells = [c.strip() for c in row.strip('|').split('|')]
        for c in cells:
            pdf.cell(40, 8, c, border=1)
        pdf.ln()
    pdf.ln(2)

def add_section(pdf, text):
    # Detect markdown tables and render as tables, else as text
    table_pattern = r"\|.+\|\n\|[-| ]+\|[\s\S]+?\n(?=\n|$)"
    pos = 0
    for match in re.finditer(table_pattern, text):
        start, end = match.span()
        # Add text before table
        if start > pos:
            for line in text[pos:start].split('\n'):
                if line.strip():
                    pdf.multi_cell(0, 8, line)
            pdf.ln(1)
        # Add table
        add_table(pdf, text[start:end])
        pos = end
    # Add any remaining text
    if pos < len(text):
        for line in text[pos:].split('\n'):
            if line.strip():
                pdf.multi_cell(0, 8, line)
        pdf.ln(1)

st.set_page_config(page_title="AI Data Preprocessing Agent", layout="wide")
st.title("AI-Powered CSV Data Preprocessing Agent")

dataset_purpose = st.selectbox(
    "What is your goal with this dataset?",
    ["classification", "regression", "clustering", "text analysis", "other"]
)

uploaded_file = st.file_uploader("Upload your CSV file", type=["csv"])

if uploaded_file:
    df = pd.read_csv(uploaded_file)
    st.subheader("Raw Data Preview")
    st.dataframe(df.head())

    # --- Dataset Overview (pre-EDA) ---
    eda = tools['eda_report'](df)
    overview_md = tools['format_dataset_overview'](eda)
    st.markdown(overview_md)
    st.markdown("---")

    # --- EDA Report ---
    eda = tools['eda_by_purpose'](df, eda, dataset_purpose)
    eda_md = tools['format_eda_report'](eda)
    st.subheader("Exploratory Data Analysis (EDA) Report")
    st.markdown(eda_md)
    st.markdown("---")
    st.markdown("#### EDA Visualizations")
    visuals = tools['eda_visualizations'](df, eda, dataset_purpose)
    for i in range(0, len(visuals), 3):
        cols = st.columns(3)
        for j, (title, fig) in enumerate(visuals[i:i+3]):
            with cols[j]:
                st.markdown(f"**{title}**")
                st.pyplot(fig)
    st.markdown("---")

    # --- Dynamic Agent Pipeline ---
    steps_summary = []
    if dataset_purpose == "classification":
        df, steps = tools['impute_missing'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['encode_categorical'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_outliers'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['scale_numeric'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
    elif dataset_purpose == "regression":
        df, steps = tools['impute_missing'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_outliers'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['scale_numeric'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['power_transform'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
    elif dataset_purpose == "clustering":
        df, steps = tools['impute_missing'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_outliers'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['scale_numeric'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
    elif dataset_purpose == "text analysis":
        df, steps = tools['standardize_text'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['encode_categorical'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_duplicates'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
    else:
        df, steps = tools['impute_missing'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['standardize_text'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_outliers'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_duplicates'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['remove_low_variance'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['scale_numeric'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['encode_categorical'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})
        df, steps = tools['power_transform'](df, eda)
        for s in steps:
            steps_summary.append({'step': s[0], 'details': s[1], 'explanation': explain_step(s[0], s[1])})

    st.subheader("Preprocessing Steps & Explanations")
    steps_md = []
    for step in steps_summary:
        st.markdown(f"**{step['step']}**: {step['details']}")
        st.info(step['explanation'])
        steps_md.append(f"**{step['step']}**: {step['details']}\n> {step['explanation']}")

    st.subheader("Cleaned Data Preview")
    st.dataframe(df.head())

    # Download cleaned CSV
    csv = df.to_csv(index=False).encode('utf-8')
    st.download_button("Download Cleaned CSV", csv, "cleaned_data.csv", "text/csv")

    # --- Post-EDA Report ---
    post_eda = tools['eda_report'](df)
    post_eda_md = tools['format_post_eda_report'](post_eda)
    st.markdown(post_eda_md)
    st.markdown("---")
    st.subheader("EDA & Preprocessing Summary Report")
    summary_md = tools['eda_summary_markdown'](eda, dataset_purpose)
    st.markdown(summary_md)

    # --- Downloadable Report (PDF) ---
    report_sections = [
        ("AI-Powered CSV Data Preprocessing Agent Report", True),
        ("", False),
        ("Dataset Overview", True),
        (overview_md, False),
        ("Exploratory Data Analysis (EDA) Report", True),
        (eda_md, False),
        ("Preprocessing Steps & Explanations", True),
        ('\n'.join(steps_md), False),
        ("Post-EDA Report", True),
        (post_eda_md, False),
        ("EDA & Preprocessing Summary Report", True),
        (summary_md, False)
    ]
    pdf = FPDF()
    pdf.add_page()
    pdf.set_auto_page_break(auto=True, margin=15)
    pdf.set_font("Arial", size=12)
    for section, is_header in report_sections:
        if is_header:
            add_heading(pdf, section, size=16)
        else:
            add_section(pdf, section)
    pdf_bytes = pdf.output(dest='S').encode('latin1')
    st.download_button("Download Full EDA & Preprocessing Report (PDF)", data=pdf_bytes, file_name="eda_preprocessing_report.pdf", mime="application/pdf")