plotting

#!/usr/bin/env python3
"""
Unified tester script (fixed):
- Saves all plots to ./plots/
- Minor fixes: filename sanitization, avoid swarmplot warning, replace deprecated seaborn ci param,
  remove deprecated pandas infer_datetime_format usage.
- Logic and functionality otherwise unchanged.
"""

import os
from datetime import datetime
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")  # headless backend so saving works on servers/CI
import matplotlib.pyplot as plt
import seaborn as sns
import re

# ---------------- Helpers ----------------
def _match_column(df, col_name):
    cols = df.columns
    cols_lower = [c.lower() for c in cols]
    clue = col_name.lower()
    if clue not in cols_lower:
        raise KeyError(f"Column '{col_name}' not found. Available: {list(df.columns)}")
    return cols[cols_lower.index(clue)]

def _sanitize_basename(name: str) -> str:
    """
    Replace problematic filesystem characters with underscore so filenames are portable.
    Keeps letters, numbers, dash, underscore; converts everything else to underscore.
    """
    # strip path and extension if present
    base = os.path.basename(os.path.splitext(name)[0])
    # replace any non [a-zA-Z0-9-_] with underscore
    return re.sub(r'[^A-Za-z0-9_\-]', '_', base)

def _timestamped_filename(base, out_dir="plots"):
    """
    Create output directory (out_dir) if missing and return a timestamped filename
    inside that directory using the pattern: <out_dir>/<sanitized_base>_2_<YYYYMMDD_HHMMSS>.png
    """
    os.makedirs(out_dir, exist_ok=True)
    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    base_name = _sanitize_basename(base)
    filename = f"{base_name}_2_{ts}.png"
    return os.path.join(out_dir, filename)

def plot_one_column_Universal(csv_file, col, chart_type, output_file="chart.png", figsize=(8,6), max_categories=30):
    """
    Universal single column plotting function that treats ALL data types neutrally.
    Simply counts occurrences of unique values regardless of data type.

    Supported chart_types: bar, count, pie
    """
    try:
        print(f"[INFO] (Universal) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        actual_col = _match_column(df, col)
        print(f"[INFO] Using column: '{actual_col}' (treating all data types neutrally)")

        # Convert everything to string for neutral handling
        df[actual_col] = df[actual_col].astype("string").fillna("Missing")

        # Handle high cardinality by collapsing to top N + "Other"
        n_unique = df[actual_col].nunique()
        print(f"[INFO] '{actual_col}' unique values: {n_unique}")

        if n_unique > max_categories:
            print(f"[WARN] High cardinality ({n_unique}) — collapsing to top {max_categories} + 'Other'.")
            top_values = df[actual_col].value_counts().nlargest(max_categories).index
            df[actual_col] = df[actual_col].where(df[actual_col].isin(top_values), other="Other")
            print(f"[INFO] Values after collapsing: {df[actual_col].nunique()}")

        # Get value counts
        counts = df[actual_col].value_counts().sort_values(ascending=False)

        sns.set(style="whitegrid")
        plt.figure(figsize=figsize)
        ct = chart_type.lower()

        if ct == "bar":
            sns.barplot(x=counts.index, y=counts.values)
            plt.title(f"Bar Plot: {actual_col} (Universal Count)")
            plt.xlabel(actual_col)
            plt.ylabel("Count")
            plt.xticks(rotation=45, ha="right")

        elif ct == "count":
            sns.countplot(data=df, x=actual_col, order=counts.index)
            plt.title(f"Count Plot: {actual_col} (Universal Count)")
            plt.xticks(rotation=45, ha="right")

        elif ct == "pie":
            plt.pie(counts.values, labels=counts.index, autopct='%1.1f%%', startangle=140)
            plt.title(f"Pie Chart: {actual_col} (Universal Count)")

        else:
            raise ValueError(f"Unsupported chart type '{chart_type}'. Supported: bar, count, pie.")

        plt.tight_layout()
        filename = _timestamped_filename(output_file)
        plt.savefig(filename)
        plt.close()

        print(f"[SUCCESS] (Universal) Chart saved: {filename} (type: {chart_type})")
        return filename

    except Exception as e:
        print(f"[ERROR] (Universal) {e}")
        return None

# ---------------- Plotting functions (robust) ----------------
def plot_two_columns_NumericVsNumeric(csv_file, col_x, col_y, chart_type, output_file="chart.png", figsize=(8,6), bubble_size_col=None):
    try:
        print(f"[INFO] (NumericVsNumeric) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        cols = df.columns
        cols_lower = [c.lower() for c in cols]
        x_l = col_x.lower()
        y_l = col_y.lower()

        if x_l not in cols_lower or y_l not in cols_lower:
            raise KeyError(f"One or both columns not found. Available: {list(df.columns)}")

        actual_x = cols[cols_lower.index(x_l)]
        actual_y = cols[cols_lower.index(y_l)]
        print(f"[INFO] Using columns: '{actual_x}' (x) and '{actual_y}' (y)")

        # Optional bubble size column lookup (case-insensitive)
        size_col_actual = None
        if bubble_size_col:
            sz_l = bubble_size_col.lower()
            if sz_l in cols_lower:
                size_col_actual = cols[cols_lower.index(sz_l)]
                print(f"[INFO] Using bubble size column: '{size_col_actual}'")
            else:
                print(f"[WARN] bubble_size_col '{bubble_size_col}' not found; ignoring it.")

        # Coerce to numeric (safe) and drop invalid rows
        df[actual_x] = pd.to_numeric(df[actual_x], errors='coerce')
        df[actual_y] = pd.to_numeric(df[actual_y], errors='coerce')
        df = df.replace([np.inf, -np.inf], np.nan)
        before = len(df)
        df = df.dropna(subset=[actual_x, actual_y])
        after = len(df)
        print(f"[INFO] Rows before: {before}, after cleaning numeric cols: {after} (dropped {before-after})")
        if after == 0:
            raise ValueError("No valid numeric rows remain after cleaning NaNs/inf.")

        plt.figure(figsize=figsize)
        ct = chart_type.lower()

        if ct == "scatter":
            plt.scatter(df[actual_x], df[actual_y], alpha=0.7, edgecolors='none')
            plt.title(f"Scatter Plot: {actual_x} vs {actual_y}")

        elif ct == "line":
            df_sorted = df.sort_values(by=actual_x)
            plt.plot(df_sorted[actual_x], df_sorted[actual_y], marker='o')
            plt.title(f"Line Plot: {actual_x} vs {actual_y}")

        elif ct == "hexbin":
            plt.hexbin(df[actual_x], df[actual_y], gridsize=30)
            cb = plt.colorbar()
            cb.set_label("Counts")
            plt.title(f"Hexbin Plot: {actual_x} vs {actual_y}")

        elif ct == "bubble":
            # Use explicit bubble_size_col if provided; else derive from |y|
            if size_col_actual and size_col_actual in df.columns:
                df[size_col_actual] = pd.to_numeric(df[size_col_actual], errors='coerce').fillna(df[actual_y].abs())
                sizes_raw = df[size_col_actual].abs()
            else:
                sizes_raw = df[actual_y].abs().fillna(0.0)
            if sizes_raw.max() == 0:
                sizes = np.full(len(sizes_raw), 50.0)
            else:
                sizes = 50 + (sizes_raw / sizes_raw.max()) * 450
            plt.scatter(df[actual_x], df[actual_y], s=sizes, alpha=0.6, edgecolors='w', linewidth=0.3)
            plt.title(f"Bubble Chart (size ~ magnitude): {actual_x} vs {actual_y}")

        elif ct == "density":
            try:
                from scipy.stats import gaussian_kde
                xy = np.vstack([df[actual_x].values, df[actual_y].values])
                kde = gaussian_kde(xy)
                z = kde(xy)
                idx = z.argsort()
                x_sorted = df[actual_x].values[idx]
                y_sorted = df[actual_y].values[idx]
                z_sorted = z[idx]
                plt.scatter(x_sorted, y_sorted, c=z_sorted, cmap='viridis', s=20, edgecolors='none')
                cb = plt.colorbar()
                cb.set_label("Density")
                plt.title(f"2D Density (KDE): {actual_x} vs {actual_y}")
            except Exception as e:
                print(f"[WARN] gaussian_kde failed or scipy missing ({e}). Falling back to hexbin.")
                plt.hexbin(df[actual_x], df[actual_y], gridsize=30)
                cb = plt.colorbar()
                cb.set_label("Counts")
                plt.title(f"2D Density (hexbin fallback): {actual_x} vs {actual_y}")

        else:
            raise ValueError(f"Unsupported chart type '{chart_type}'. Supported: scatter, line, hexbin, bubble, density.")

        plt.xlabel(actual_x)
        plt.ylabel(actual_y)
        plt.tight_layout()

        filename = _timestamped_filename(output_file)
        plt.savefig(filename)
        plt.close()
        print(f"[SUCCESS] (NumericVsNumeric) Chart saved: {filename} (type: {chart_type})")
        return filename

    except Exception as e:
        print(f"[ERROR] (NumericVsNumeric) {e}")
        return None


def plot_two_columns_NumericVsCategorical(csv_file, col_cat, col_num, chart_type, output_file="chart.png", figsize=(8,6), max_categories=30):
    try:
        print(f"[INFO] (NumericVsCategorical) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        cols = df.columns
        cols_lower = [c.lower() for c in cols]
        cat_l = col_cat.lower()
        num_l = col_num.lower()

        if cat_l not in cols_lower or num_l not in cols_lower:
            raise KeyError(f"One or both columns not found. Available: {list(df.columns)}")

        actual_cat = cols[cols_lower.index(cat_l)]
        actual_num = cols[cols_lower.index(num_l)]
        print(f"[INFO] Using columns: '{actual_cat}' (category) and '{actual_num}' (numeric)")

        # Coerce numeric column
        df[actual_num] = pd.to_numeric(df[actual_num], errors='coerce')
        # Prepare categorical column as string and fill missing
        df[actual_cat] = df[actual_cat].astype("string").fillna("Missing")
        # Clean inf
        df = df.replace([np.inf, -np.inf], np.nan)

        before = len(df)
        df = df.dropna(subset=[actual_num])
        after = len(df)
        print(f"[INFO] Rows before: {before}, after dropping invalid numeric rows: {after} (dropped {before-after})")
        if after == 0:
            raise ValueError("No valid numeric rows remain after cleaning NaNs/inf.")

        # Collapse low-frequency categories if too many unique
        n_unique = df[actual_cat].nunique()
        print(f"[INFO] '{actual_cat}' unique categories: {n_unique}")
        if n_unique > max_categories:
            print(f"[WARN] High-cardinality categories ({n_unique}) — collapsing to top {max_categories} + 'Other'.")
            top = df[actual_cat].value_counts().nlargest(max_categories).index
            df[actual_cat] = df[actual_cat].where(df[actual_cat].isin(top), other="Other")
            print(f"[INFO] Categories after collapsing: {df[actual_cat].nunique()}")

        sns.set(style="whitegrid")
        plt.figure(figsize=figsize)
        ct = chart_type.lower()

        if ct.startswith("bar"):
            agg = "mean"
            if ":" in ct:
                _, suffix = ct.split(":",1)
                if suffix.strip() in ("sum","mean"):
                    agg = suffix.strip()
            if agg == "sum":
                agg_df = df.groupby(actual_cat, dropna=False)[actual_num].sum().sort_values(ascending=False)
            else:
                agg_df = df.groupby(actual_cat, dropna=False)[actual_num].mean().sort_values(ascending=False)
            sns.barplot(x=agg_df.index, y=agg_df.values)
            plt.xlabel(actual_cat)
            plt.ylabel(f"{agg}({actual_num})")
            plt.title(f"Bar Plot ({agg}): {actual_num} by {actual_cat}")
            plt.xticks(rotation=45, ha="right")

        elif ct == "box":
            sns.boxplot(x=actual_cat, y=actual_num, data=df)
            plt.title(f"Box Plot: {actual_num} by {actual_cat}")
            plt.xticks(rotation=45, ha="right")

        elif ct == "violin":
            sns.violinplot(x=actual_cat, y=actual_num, data=df, inner="quartile")
            plt.title(f"Violin Plot: {actual_num} by {actual_cat}")
            plt.xticks(rotation=45, ha="right")

        elif ct == "strip":
            sns.stripplot(x=actual_cat, y=actual_num, data=df, jitter=True, alpha=0.6)
            plt.title(f"Strip Plot: {actual_num} by {actual_cat}")
            plt.xticks(rotation=45, ha="right")

        elif ct == "swarm":
            # Avoid seaborn swarmplot warning when categories are very crowded.
            # If the largest category has too many points, fallback to stripplot (no warning).
            max_count_per_cat = df[actual_cat].value_counts().max()
            total_points = len(df)
            # thresholds chosen conservatively to avoid the warning Seaborn emits
            if max_count_per_cat > 250 or total_points > 5000:
                print(f"[INFO] Large category size (max {max_count_per_cat}) or total points ({total_points}) - using stripplot fallback to avoid swarm placement issues.")
                sns.stripplot(x=actual_cat, y=actual_num, data=df, jitter=True, alpha=0.6)
                plt.title(f"Swarm Plot (fallback - strip): {actual_num} by {actual_cat}")
                plt.xticks(rotation=45, ha="right")
            else:
                try:
                    sns.swarmplot(x=actual_cat, y=actual_num, data=df)
                    plt.title(f"Swarm Plot: {actual_num} by {actual_cat}")
                    plt.xticks(rotation=45, ha="right")
                except Exception as e:
                    print(f"[WARN] swarmplot failed ({e}); falling back to stripplot with jitter.")
                    sns.stripplot(x=actual_cat, y=actual_num, data=df, jitter=True, alpha=0.6)
                    plt.title(f"Swarm Plot (fallback - strip): {actual_num} by {actual_cat}")
                    plt.xticks(rotation=45, ha="right")

        elif ct == "point":
            # use new seaborn parameter errorbar instead of deprecated ci
            order = df[actual_cat].value_counts().index
            try:
                sns.pointplot(x=actual_cat, y=actual_num, data=df, order=order, errorbar='sd')
            except TypeError:
                # older seaborn might not support errorbar param; fallback to ci if available
                sns.pointplot(x=actual_cat, y=actual_num, data=df, order=order, ci="sd")
            plt.title(f"Point Plot (mean ± sd): {actual_num} by {actual_cat}")
            plt.xticks(rotation=45, ha="right")

        elif ct == "dot":
            categories = list(df[actual_cat].unique())
            cat_to_x = {c:i for i,c in enumerate(categories)}
            x = df[actual_cat].map(cat_to_x)
            jitter = np.random.normal(0, 0.08, size=len(x))
            plt.scatter(x + jitter, df[actual_num], s=10, alpha=0.6)
            plt.xticks(ticks=range(len(categories)), labels=categories, rotation=45, ha="right")
            plt.xlabel(actual_cat)
            plt.ylabel(actual_num)
            plt.title(f"Dot Plot: {actual_num} by {actual_cat}")

        else:
            raise ValueError(f"Unsupported chart type '{chart_type}'. Supported: bar, bar:sum, box, violin, strip, swarm, point, dot.")

        plt.tight_layout()
        filename = _timestamped_filename(output_file)
        plt.savefig(filename)
        plt.close()
        print(f"[SUCCESS] (NumericVsCategorical) Chart saved: {filename} (type: {chart_type})")
        return filename

    except Exception as e:
        print(f"[ERROR] (NumericVsCategorical) {e}")
        return None


def plot_two_columns_CategoricalVsCategorical(csv_file, col_a, col_b, chart_type, output_file="chart.png", figsize=(10,6), max_categories=30):
    try:
        print(f"[INFO] (CategoricalVsCategorical) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding="utf-8")
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        a = _match_column(df, col_a)
        b = _match_column(df, col_b)
        print(f"[INFO] Using columns: '{a}' and '{b}'")

        # treat categories as strings, fill missing
        df[a] = df[a].astype("string").fillna("Missing")
        df[b] = df[b].astype("string").fillna("Missing")
        df = df.replace([np.inf, -np.inf], np.nan)

        # collapse high-cardinality columns (applied to both)
        for c in (a, b):
            n_unique = df[c].nunique()
            if n_unique > max_categories:
                print(f"[WARN] Column '{c}' has {n_unique} categories; collapsing to top {max_categories} + 'Other'.")
                top = df[c].value_counts().nlargest(max_categories).index
                df[c] = df[c].where(df[c].isin(top), other="Other")

        counts = df.groupby([a, b]).size().unstack(fill_value=0)
        plt.figure(figsize=figsize)
        ct = chart_type.lower()

        if ct == "grouped_bar":
            counts.plot(kind="bar", figsize=figsize)
            plt.title(f"Grouped Bar: {a} vs {b}")
            plt.xlabel(a)
            plt.ylabel("Count")
            plt.xticks(rotation=45, ha="right")

        elif ct == "stacked_bar":
            counts.plot(kind="bar", stacked=True, figsize=figsize)
            plt.title(f"Stacked Bar: {a} vs {b}")
            plt.xlabel(a)
            plt.ylabel("Count")
            plt.xticks(rotation=45, ha="right")

        elif ct == "mosaic":
            try:
                from statsmodels.graphics.mosaicplot import mosaic
                props = {(str(k1), str(k2)): int(v) for k1, row in counts.iterrows() for k2, v in row.items()}
                plt.figure(figsize=figsize)
                mosaic(props, title=f"Mosaic Plot: {a} vs {b}")
            except Exception as e:
                print(f"[WARN] statsmodels mosaic unavailable or failed ({e}). Falling back to heatmap.")
                sns.heatmap(counts, annot=True, fmt="d")
                plt.title(f"Heatmap (mosaic fallback): {a} vs {b}")

        elif ct == "clustered_dot":
            flat = counts.reset_index().melt(id_vars=a, var_name=b, value_name="count")
            flat = flat[flat["count"] > 0].reset_index(drop=True)
            a_map = {v:i for i,v in enumerate(sorted(df[a].unique()))}
            b_map = {v:i for i,v in enumerate(sorted(df[b].unique()))}
            x = flat[a].map(a_map).astype(float)
            y = flat[b].map(b_map).astype(float)
            sizes = 50 + (flat["count"] / flat["count"].max()) * 450
            jitter_x = np.random.normal(0, 0.08, size=len(x))
            jitter_y = np.random.normal(0, 0.08, size=len(y))
            plt.figure(figsize=figsize)
            plt.scatter(x + jitter_x, y + jitter_y, s=sizes, alpha=0.7)
            plt.xticks(ticks=list(a_map.values()), labels=list(a_map.keys()), rotation=45, ha="right")
            plt.yticks(ticks=list(b_map.values()), labels=list(b_map.keys()))
            plt.xlabel(a)
            plt.ylabel(b)
            plt.title(f"Clustered dot (counts): {a} vs {b}")

        else:
            raise ValueError("Unsupported chart_type. Choose one of: grouped_bar, stacked_bar, mosaic, clustered_dot.")

        plt.tight_layout()
        out = _timestamped_filename(output_file)
        plt.savefig(out)
        plt.close()
        print(f"[SUCCESS] (CategoricalVsCategorical) Saved: {out}")
        return out

    except Exception as e:
        print(f"[ERROR] (CategoricalVsCategorical) {e}")
        return None


def plot_two_columns_NumericVsDatetime(csv_file, col_date, col_num, chart_type, output_file="chart.png", figsize=(10,5), date_parse_format=None, rolling_window=7):
    try:
        print(f"[INFO] (NumericVsDatetime) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        dcol = _match_column(df, col_date)
        ncol = _match_column(df, col_num)
        print(f"[INFO] Using columns: date='{dcol}', numeric='{ncol}'")

        # parse dates (no infer_datetime_format parameter)
        df[dcol] = pd.to_datetime(df[dcol], format=date_parse_format, errors="coerce")
        df[ncol] = pd.to_numeric(df[ncol], errors='coerce')
        df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=[dcol, ncol])
        if df.empty:
            raise ValueError("No valid rows after parsing datetime and numeric columns.")

        df = df.sort_values(by=dcol).set_index(dcol)

        plt.figure(figsize=figsize)
        ct = chart_type.lower()

        if ct == "line":
            df[ncol].plot.line()
            plt.title(f"Line: {ncol} over time")
            plt.xlabel("Date")
            plt.ylabel(ncol)

        elif ct == "area":
            df[ncol].plot.area()
            plt.title(f"Area: {ncol} over time")
            plt.xlabel("Date")
            plt.ylabel(ncol)

        elif ct == "scatter":
            plt.scatter(df.index, df[ncol], alpha=0.7)
            plt.title(f"Scatter: {ncol} vs Date")
            plt.xlabel("Date")
            plt.ylabel(ncol)

        elif ct == "rolling":
            daily = df[ncol].resample("D").mean()
            rolling = daily.rolling(window=rolling_window, min_periods=1).mean()
            plt.plot(daily.index, daily.values, marker='.', alpha=0.4, label="daily")
            plt.plot(rolling.index, rolling.values, marker=None, linewidth=2, label=f"rolling_{rolling_window}")
            plt.legend()
            plt.title(f"Rolling average (window={rolling_window}): {ncol}")
            plt.xlabel("Date")
            plt.ylabel(ncol)
        else:
            raise ValueError("Unsupported chart_type. Choose one of: line, area, scatter, rolling.")

        plt.tight_layout()
        out = _timestamped_filename(output_file)
        plt.savefig(out)
        plt.close()
        print(f"[SUCCESS] (NumericVsDatetime) Saved: {out}")
        return out

    except Exception as e:
        print(f"[ERROR] (NumericVsDatetime) {e}")
        return None


def plot_two_columns_CategoricalVsDatetime(csv_file, col_date, col_cat, chart_type, output_file="chart.png", figsize=(12,6), date_parse_format=None, resample_freq="D", max_categories=20):
    try:
        print(f"[INFO] (CategoricalVsDatetime) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        dcol = _match_column(df, col_date)
        ccol = _match_column(df, col_cat)
        print(f"[INFO] Using columns: date='{dcol}', category='{ccol}'")

        # parse dates (no infer_datetime_format)
        df[dcol] = pd.to_datetime(df[dcol], format=date_parse_format, errors="coerce")
        df[ccol] = df[ccol].astype("string").fillna("Missing")
        df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=[dcol])
        if df.empty:
            raise ValueError("No valid rows after parsing date column.")

        if df[ccol].nunique() > max_categories:
            print(f"[WARN] collapsing categories to top {max_categories}+Other")
            top = df[ccol].value_counts().nlargest(max_categories).index
            df[ccol] = df[ccol].where(df[ccol].isin(top), other="Other")

        ct = chart_type.lower()

        if ct == "timeline":
            categories = sorted(df[ccol].unique())
            cat_to_y = {c:i for i,c in enumerate(categories)}
            y = df[ccol].map(cat_to_y)
            x = df[dcol]
            plt.figure(figsize=figsize)
            jitter = np.random.normal(0, 0.08, size=len(y))
            plt.scatter(x, y + jitter, alpha=0.7)
            plt.yticks(ticks=list(cat_to_y.values()), labels=list(cat_to_y.keys()))
            plt.xlabel("Date")
            plt.title(f"Timeline of {ccol}")
            plt.tight_layout()

        elif ct == "stacked_area":
            df2 = df.set_index(dcol)
            counts = df2.groupby([pd.Grouper(freq=resample_freq), ccol]).size().unstack(fill_value=0)
            counts.plot.area(figsize=figsize)
            plt.title(f"Stacked area ({resample_freq}) of {ccol} over time")
            plt.xlabel("Time")
            plt.ylabel("Count")
            plt.tight_layout()

        else:
            raise ValueError("Unsupported chart_type. Choose 'timeline', or 'stacked_area'.")

        out = _timestamped_filename(output_file)
        plt.savefig(out)
        plt.close()
        print(f"[SUCCESS] (CategoricalVsDatetime) Saved: {out}")
        return out

    except Exception as e:
        print(f"[ERROR] (CategoricalVsDatetime) {e}")
        return None


def plot_two_columns_DatetimeVsDatetime(csv_file, col_start, col_end, chart_type, output_file="chart.png", figsize=(10,6), date_parse_format=None):
    try:
        print(f"[INFO] (DatetimeVsDatetime) Reading CSV: {csv_file}")
        try:
            df = pd.read_csv(csv_file, encoding='utf-8')
            print("[INFO] Read with encoding='utf-8'")
        except UnicodeDecodeError:
            df = pd.read_csv(csv_file, encoding='ISO-8859-1')
            print("[INFO] Read with encoding='ISO-8859-1'")

        s = _match_column(df, col_start)
        e = _match_column(df, col_end)
        print(f"[INFO] Using columns: start='{s}', end='{e}'")

        df[s] = pd.to_datetime(df[s], format=date_parse_format, errors='coerce')
        df[e] = pd.to_datetime(df[e], format=date_parse_format, errors='coerce')
        df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=[s,e])
        if df.empty:
            raise ValueError("No valid start/end rows after parsing dates.")

        ct = chart_type.lower()
        plt.figure(figsize=figsize)

        if ct == "gantt":
            df_sorted = df.sort_values(by=s).reset_index(drop=True)
            starts = pd.to_datetime(df_sorted[s])
            ends = pd.to_datetime(df_sorted[e])
            durations = (ends - starts).dt.total_seconds() / (60*60*24)  # days
            y = np.arange(len(df_sorted))
            # left must be numeric; convert to matplotlib datenum
            left = starts.map(datetime.toordinal)
            plt.barh(y, durations, left=left, height=0.6)
            xticks = plt.xticks()[0]
            xtick_labels = [datetime.fromordinal(int(x)).strftime("%Y-%m-%d") for x in xticks]
            plt.gca().set_yticks(y)
            plt.gca().set_yticklabels([f"row_{i}" for i in y])
            plt.xlabel("Date")
            plt.title("Gantt-like chart (start vs duration in days)")
            plt.tight_layout()

        elif ct == "duration_histogram":
            durations_days = (df[e] - df[s]).dt.total_seconds() / (60*60*24)
            plt.hist(durations_days.dropna(), bins=30)
            plt.xlabel("Duration (days)")
            plt.ylabel("Count")
            plt.title("Duration histogram (end - start)")
            plt.tight_layout()

        elif ct == "scatter":
            x = df[s].map(datetime.toordinal)
            y = df[e].map(datetime.toordinal)
            plt.scatter(x, y, alpha=0.7)
            plt.xlabel("Start (ordinal)")
            plt.ylabel("End (ordinal)")
            plt.title("Start vs End scatter (ordinal dates)")
            plt.tight_layout()
        else:
            raise ValueError("Unsupported chart_type. Choose 'gantt', 'duration_histogram', or 'scatter'.")

        out = _timestamped_filename(output_file)
        plt.savefig(out)
        plt.close()
        print(f"[SUCCESS] (DatetimeVsDatetime) Saved: {out}")
        return out

    except Exception as e:
        print(f"[ERROR] (DatetimeVsDatetime) {e}")
        return None

# ---------------- MAIN: run all scenarios using existing mydata.csv ----------------
if __name__ == "__main__":
    csv_path = "mydata.csv"
    if not os.path.exists(csv_path):
        print("[FATAL] mydata.csv not found in current directory. Place your dataset as 'mydata.csv' and re-run.")
        raise SystemExit(1)

    # Try to load once to inspect available columns and types
    try:
        df_master = pd.read_csv(csv_path, encoding='utf-8')
    except UnicodeDecodeError:
        df_master = pd.read_csv(csv_path, encoding='ISO-8859-1')

    cols = list(df_master.columns)
    cols_lower = [c.lower() for c in cols]
    print(f"[INFO] Found columns: {cols}")


    # Test all columns with universal function
    chart_types = ["bar", "count", "pie"]

    # Test a few sample columns (you can modify this to test specific columns)
    sample_columns = ['PRICEEACH','ORDERDATE','STATUS']

    for col in sample_columns:
        for ct in chart_types:
            out = plot_one_column_Universal(csv_path, col, ct, output_file=f"mydata_universal_{ct}_{col}.png")
            if out:
                print(f"[INFO] Produced universal chart: {out}")

    # Candidate column names from your provided schema (case-insensitive)
    candidates = {
        "numeric": ["QUANTITYORDERED", "PRICEEACH", "ORDERLINENUMBER", "SALES", "QTR_ID", "MONTH_ID", "YEAR_ID", "MSRP"],
        "categorical": ["STATUS", "PRODUCTLINE", "PRODUCTCODE", "CUSTOMERNAME", "COUNTRY", "TERRITORY", "DEALSIZE"],
        "datetime": ["ORDERDATE"],
        "text": ["ADDRESSLINE1", "CUSTOMERNAME", "CONTACTFIRSTNAME", "CONTACTLASTNAME"]
    }

    # helper: pick the first available from candidates
    def pick_first_available(lst):
        for name in lst:
            if name.lower() in cols_lower:
                return cols[cols_lower.index(name.lower())]
        return None

    # ---------- Numeric vs Numeric ----------
    nn_x = pick_first_available(candidates["numeric"])
    # pick a different numeric column for y if possible
    nn_y = None
    if nn_x:
        for n in candidates["numeric"]:
            if n.lower() in cols_lower and cols[cols_lower.index(n.lower())] != nn_x:
                nn_y = cols[cols_lower.index(n.lower())]
                break

    if nn_x and nn_y:
        nn_chart_types = ["scatter", "line", "hexbin", "bubble", "density"]
        for ct in nn_chart_types:
            out = plot_two_columns_NumericVsNumeric(csv_path, nn_x, nn_y, ct, output_file=f"mydata_nn_{ct}.png")
            if out:
                print(f"[INFO] Produced numeric-numeric: {out}")
    else:
        print("[WARN] Not enough numeric columns found for Numeric vs Numeric tests. Found:", nn_x, nn_y)

    # ---------- Numeric vs Categorical ----------
    cat_col = pick_first_available(candidates["categorical"])
    num_col = pick_first_available(candidates["numeric"])
    if cat_col and num_col:
        nc_chart_types = ["bar", "bar:sum", "box", "violin", "strip", "swarm", "point", "dot"]
        for ct in nc_chart_types:
            out = plot_two_columns_NumericVsCategorical(csv_path, cat_col, num_col, ct, output_file=f"mydata_nc_{ct}.png")
            if out:
                print(f"[INFO] Produced numeric-categorical: {out}")
    else:
        print("[WARN] Missing categorical or numeric column for Numeric vs Categorical tests. Found:", cat_col, num_col)

    # ---------- Categorical vs Categorical ----------
    cc_a = pick_first_available(candidates["categorical"])
    cc_b = None
    if cc_a:
        for c in candidates["categorical"]:
            if c.lower() in cols_lower and cols[cols_lower.index(c.lower())] != cc_a:
                cc_b = cols[cols_lower.index(c.lower())]
                break
    if cc_a and cc_b:
        cc_chart_types = ["grouped_bar", "stacked_bar", "mosaic", "clustered_dot"]
        for ct in cc_chart_types:
            out = plot_two_columns_CategoricalVsCategorical(csv_path, cc_a, cc_b, ct, output_file=f"mydata_cc_{ct}.png")
            if out:
                print(f"[INFO] Produced categorical-categorical: {out}")
    else:
        print("[WARN] Not enough categorical columns found for Categorical vs Categorical tests. Found:", cc_a, cc_b)

    # ---------- Numeric vs Datetime ----------
    dt_col = pick_first_available(candidates["datetime"])
    num_col = pick_first_available(candidates["numeric"])
    if dt_col and num_col:
        nd_chart_types = ["line", "area", "scatter", "rolling"]
        for ct in nd_chart_types:
            out = plot_two_columns_NumericVsDatetime(csv_path, dt_col, num_col, ct, output_file=f"mydata_nd_{ct}.png", rolling_window=7)
            if out:
                print(f"[INFO] Produced numeric-datetime: {out}")
    else:
        print("[WARN] Missing datetime or numeric column for Numeric vs Datetime tests. Found:", dt_col, num_col)

    # ---------- Categorical vs Datetime ----------
    dt_col = pick_first_available(candidates["datetime"])
    cat_col = pick_first_available(candidates["categorical"])
    if dt_col and cat_col:
        cd_chart_types = ["timeline", "stacked_area"]
        for ct in cd_chart_types:
            out = plot_two_columns_CategoricalVsDatetime(csv_path, dt_col, cat_col, ct, output_file=f"mydata_cd_{ct}.png", resample_freq="W")
            if out:
                print(f"[INFO] Produced categorical-datetime: {out}")
    else:
        print("[WARN] Missing datetime or categorical column for Categorical vs Datetime tests. Found:", dt_col, cat_col)

    # ---------- Datetime vs Datetime ----------
    date_cols = []
    for c in df_master.columns:
        if "date" in c.lower():
            date_cols.append(c)
    if len(date_cols) >= 2:
        d1, d2 = date_cols[0], date_cols[1]
        dd_chart_types = ["gantt", "duration_histogram", "scatter"]
        for ct in dd_chart_types:
            out = plot_two_columns_DatetimeVsDatetime(csv_path, d1, d2, ct, output_file=f"mydata_dd_{ct}.png")
            if out:
                print(f"[INFO] Produced datetime-datetime: {out}")
    else:
        print("[WARN] Not enough datetime columns for Datetime vs Datetime tests. Found date-like columns:", date_cols)


    print("[DONE] All requested chart attempts completed. Check saved PNG files in the 'plots' folder.")