Cross-Validation, Hyperparameter Tuning & Model Selection

import numpy as np

class KFoldCV:
    """K-Fold Cross-Validation implemented from scratch."""

    def __init__(self, n_splits=5, shuffle=False, random_state=None):
        self.n_splits = n_splits
        self.shuffle = shuffle
        self.random_state = random_state

    def split(self, X):
        """Generate train/test indices for K folds."""
        n_samples = len(X)
        indices = np.arange(n_samples)

        if self.shuffle:
            rng = np.random.RandomState(self.random_state)
            rng.shuffle(indices)

        # Compute fold sizes (handle uneven splits)
        fold_sizes = np.full(self.n_splits, n_samples // self.n_splits)
        fold_sizes[:n_samples % self.n_splits] += 1

        current = 0
        for fold_size in fold_sizes:
            test_idx = indices[current:current + fold_size]
            train_idx = np.concatenate([
                indices[:current],
                indices[current + fold_size:]
            ])
            yield train_idx, test_idx
            current += fold_size

    def cross_val_score(self, model_class, X, y, **model_params):
        """Compute cross-validated scores."""
        scores = []
        for fold, (train_idx, test_idx) in enumerate(self.split(X)):
            X_train, X_test = X[train_idx], X[test_idx]
            y_train, y_test = y[train_idx], y[test_idx]

            # Create fresh model for each fold
            model = model_class(**model_params)
            model.fit(X_train, y_train)

            score = np.mean(model.predict(X_test) == y_test)
            scores.append(score)
            print(f"  Fold {fold+1}: Accuracy = {score:.4f}")

        scores = np.array(scores)
        print(f"\n  Mean: {scores.mean():.4f} ± {scores.std():.4f}")
        return scores

# --- Demo with a simple model ---
class SimpleThresholdClassifier:
    """Classify based on feature mean threshold."""
    def fit(self, X, y):
        # Learn: for each class, compute mean feature value
        self.threshold = X[y == 1].mean() - X[y == 0].mean()
        self.global_mean = X.mean(axis=0)
        return self

    def predict(self, X):
        distances = np.linalg.norm(X - self.global_mean, axis=1)
        return (distances < np.median(distances)).astype(int)

# Generate sample data
np.random.seed(42)
X = np.random.randn(100, 3)
y = (X[:, 0] + X[:, 1] > 0).astype(int)

# Run 5-Fold CV
print("=" * 50)
print("5-Fold Cross-Validation from Scratch")
print("=" * 50)
kfold = KFoldCV(n_splits=5, shuffle=True, random_state=42)
scores = kfold.cross_val_score(SimpleThresholdClassifier, X, y)

class StratifiedKFold:
    """Stratified K-Fold: preserves class distribution in each fold."""

    def __init__(self, n_splits=5, shuffle=False, random_state=None):
        self.n_splits = n_splits
        self.shuffle = shuffle
        self.random_state = random_state

    def split(self, X, y):
        n_samples = len(y)
        classes = np.unique(y)
        rng = np.random.RandomState(self.random_state)

        # Group indices by class
        class_indices = {}
        for cls in classes:
            idx = np.where(y == cls)[0]
            if self.shuffle:
                rng.shuffle(idx)
            class_indices[cls] = idx

        # Split each class into K folds, then combine
        folds = [[] for _ in range(self.n_splits)]
        for cls in classes:
            idx = class_indices[cls]
            fold_sizes = np.full(self.n_splits, len(idx) // self.n_splits)
            fold_sizes[:len(idx) % self.n_splits] += 1

            current = 0
            for i, size in enumerate(fold_sizes):
                folds[i].extend(idx[current:current + size])
                current += size

        # Generate train/test pairs
        for i in range(self.n_splits):
            test_idx = np.array(folds[i])
            train_idx = np.concatenate([folds[j] for j in range(self.n_splits) if j != i])
            yield train_idx.astype(int), test_idx.astype(int)

# Demo: Imbalanced dataset
np.random.seed(42)
X_imb = np.random.randn(200, 4)
y_imb = np.array([0]*180 + [1]*20)  # 90% vs 10%

print("\n" + "=" * 50)
print("Stratified K-Fold (Imbalanced: 90/10 split)")
print("=" * 50)
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
for fold, (train_idx, test_idx) in enumerate(skf.split(X_imb, y_imb)):
    train_dist = np.bincount(y_imb[train_idx])
    test_dist = np.bincount(y_imb[test_idx])
    print(f"Fold {fold+1}: Train {train_dist} | Test {test_dist} | "
          f"Test class-1 ratio: {test_dist[1]/test_dist.sum():.2%}")

class TimeSeriesSplit:
    """Time-series CV with expanding training window."""

    def __init__(self, n_splits=5, min_train_size=None):
        self.n_splits = n_splits
        self.min_train_size = min_train_size

    def split(self, X):
        n_samples = len(X)
        test_size = n_samples // (self.n_splits + 1)
        min_train = self.min_train_size or test_size

        for i in range(self.n_splits):
            train_end = min_train + i * test_size
            test_start = train_end
            test_end = test_start + test_size

            if test_end > n_samples:
                break

            train_idx = np.arange(0, train_end)
            test_idx = np.arange(test_start, test_end)
            yield train_idx, test_idx

# Demo
print("\n" + "=" * 50)
print("Time-Series Split (60 months of data)")
print("=" * 50)
X_ts = np.arange(60)  # 60 months
tscv = TimeSeriesSplit(n_splits=5)
for fold, (train_idx, test_idx) in enumerate(tscv.split(X_ts)):
    print(f"Fold {fold+1}: Train months [0..{train_idx[-1]}] "
          f"({len(train_idx)} months) → Test months "
          f"[{test_idx[0]}..{test_idx[-1]}] ({len(test_idx)} months)")

from itertools import product
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import load_iris
from sklearn.metrics import accuracy_score

def grid_search_cv(model_class, param_grid, X, y, cv=5):
    """Grid Search with K-Fold Cross-Validation from scratch."""
    # Generate all combinations
    keys = list(param_grid.keys())
    values = list(param_grid.values())
    combinations = list(product(*values))

    print(f"Total combinations: {len(combinations)}")
    print(f"Total model fits: {len(combinations) * cv}\n")

    best_score = -np.inf
    best_params = None
    results = []

    for combo in combinations:
        params = dict(zip(keys, combo))

        # K-Fold CV for this combination
        fold_scores = []
        kf = KFoldCV(n_splits=cv, shuffle=True, random_state=42)
        for train_idx, test_idx in kf.split(X):
            model = model_class(**params)
            model.fit(X[train_idx], y[train_idx])
            pred = model.predict(X[test_idx])
            fold_scores.append(accuracy_score(y[test_idx], pred))

        mean_score = np.mean(fold_scores)
        std_score = np.std(fold_scores)
        results.append((params, mean_score, std_score))

        if mean_score > best_score:
            best_score = mean_score
            best_params = params

    # Sort and display top 5
    results.sort(key=lambda x: x[1], reverse=True)
    print("Top 5 Configurations:")
    print("-" * 60)
    for i, (params, mean, std) in enumerate(results[:5]):
        print(f"  {i+1}. Score: {mean:.4f} ± {std:.4f} | {params}")

    print(f"\n★ Best: {best_params} → {best_score:.4f}")
    return best_params, best_score

# Run Grid Search
iris = load_iris()
X_iris, y_iris = iris.data, iris.target

param_grid = {
    'max_depth': [2, 3, 5, 7, 10, None],
    'min_samples_split': [2, 5, 10, 20],
    'criterion': ['gini', 'entropy']
}

best_params, best_score = grid_search_cv(
    DecisionTreeClassifier, param_grid, X_iris, y_iris, cv=5
)

# pip install optuna
import optuna
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import load_breast_cancer

# Load data
data = load_breast_cancer()
X, y = data.data, data.target

def objective(trial):
    """Optuna objective: define search space and evaluate."""
    # Choose model type
    model_name = trial.suggest_categorical('model', ['rf', 'gb'])

    if model_name == 'rf':
        params = {
            'n_estimators': trial.suggest_int('rf_n_estimators', 50, 500),
            'max_depth': trial.suggest_int('rf_max_depth', 2, 20),
            'min_samples_split': trial.suggest_int('rf_min_samples_split', 2, 20),
            'max_features': trial.suggest_categorical('rf_max_features',
                                                       ['sqrt', 'log2']),
        }
        model = RandomForestClassifier(**params, random_state=42)
    else:
        params = {
            'n_estimators': trial.suggest_int('gb_n_estimators', 50, 500),
            'max_depth': trial.suggest_int('gb_max_depth', 2, 10),
            'learning_rate': trial.suggest_float('gb_lr', 0.01, 0.3, log=True),
            'subsample': trial.suggest_float('gb_subsample', 0.5, 1.0),
        }
        model = GradientBoostingClassifier(**params, random_state=42)

    # 5-Fold Stratified CV
    scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
    return scores.mean()

# Run optimization
study = optuna.create_study(direction='maximize',
                            sampler=optuna.samplers.TPESampler(seed=42))
study.optimize(objective, n_trials=50, show_progress_bar=True)

print(f"\nBest trial:")
print(f"  Value (accuracy): {study.best_trial.value:.4f}")
print(f"  Params: {study.best_trial.params}")

# Hyperparameter importance
importances = optuna.importance.get_param_importances(study)
print(f"\nHyperparameter Importance:")
for param, imp in importances.items():
    print(f"  {param}: {imp:.4f}")

import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve
from sklearn.svm import SVC

def plot_learning_curves(model, X, y, title="Learning Curve"):
    """Plot training and validation learning curves."""
    train_sizes, train_scores, val_scores = learning_curve(
        model, X, y,
        train_sizes=np.linspace(0.1, 1.0, 10),
        cv=5, scoring='accuracy',
        n_jobs=-1, random_state=42
    )

    train_mean = train_scores.mean(axis=1)
    train_std = train_scores.std(axis=1)
    val_mean = val_scores.mean(axis=1)
    val_std = val_scores.std(axis=1)

    plt.figure(figsize=(10, 6))
    plt.fill_between(train_sizes, train_mean - train_std,
                     train_mean + train_std, alpha=0.1, color='#059669')
    plt.fill_between(train_sizes, val_mean - val_std,
                     val_mean + val_std, alpha=0.1, color='#0891b2')
    plt.plot(train_sizes, train_mean, 'o-', color='#059669',
             label='Training Score', linewidth=2)
    plt.plot(train_sizes, val_mean, 'o-', color='#0891b2',
             label='Validation Score', linewidth=2)
    plt.xlabel('Training Set Size', fontsize=12)
    plt.ylabel('Accuracy', fontsize=12)
    plt.title(title, fontsize=14, fontweight='bold')
    plt.legend(loc='lower right', fontsize=11)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()

    # Diagnose
    gap = train_mean[-1] - val_mean[-1]
    if val_mean[-1] < 0.7:
        print("→ UNDERFITTING: Both scores are low. Try a more complex model.")
    elif gap > 0.1:
        print(f"→ OVERFITTING: Training-validation gap = {gap:.3f}. "
              f"Try regularization or more data.")
    else:
        print(f"→ GOOD FIT: Gap = {gap:.3f}. Model generalizes well.")

# Example usage
data = load_breast_cancer()
plot_learning_curves(SVC(kernel='rbf', C=1.0), data.data, data.target,
                     title="SVM (RBF) Learning Curve")

from scipy import stats

def compare_models(model_a, model_b, X, y, cv=10, alpha=0.05):
    """Compare two models using paired t-test and Wilcoxon test."""
    from sklearn.model_selection import StratifiedKFold

    skf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
    scores_a = []
    scores_b = []

    for train_idx, test_idx in skf.split(X, y):
        X_train, X_test = X[train_idx], X[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]

        # Model A
        ma = model_a.__class__(**model_a.get_params())
        ma.fit(X_train, y_train)
        scores_a.append(accuracy_score(y_test, ma.predict(X_test)))

        # Model B
        mb = model_b.__class__(**model_b.get_params())
        mb.fit(X_train, y_train)
        scores_b.append(accuracy_score(y_test, mb.predict(X_test)))

    scores_a = np.array(scores_a)
    scores_b = np.array(scores_b)
    differences = scores_a - scores_b

    print("=" * 60)
    print("MODEL COMPARISON REPORT")
    print("=" * 60)
    print(f"Model A: {model_a.__class__.__name__}")
    print(f"  Mean accuracy: {scores_a.mean():.4f} ± {scores_a.std():.4f}")
    print(f"Model B: {model_b.__class__.__name__}")
    print(f"  Mean accuracy: {scores_b.mean():.4f} ± {scores_b.std():.4f}")
    print(f"\nMean difference: {differences.mean():.4f}")

    # Paired t-test
    t_stat, p_value_t = stats.ttest_rel(scores_a, scores_b)
    print(f"\nPaired t-test: t={t_stat:.4f}, p={p_value_t:.4f}")

    # Wilcoxon signed-rank test
    try:
        w_stat, p_value_w = stats.wilcoxon(scores_a, scores_b)
        print(f"Wilcoxon test:  W={w_stat:.4f}, p={p_value_w:.4f}")
    except ValueError:
        print("Wilcoxon test: Cannot compute (all differences may be zero)")

    # Decision
    if p_value_t < alpha:
        winner = "A" if differences.mean() > 0 else "B"
        print(f"\n✓ SIGNIFICANT difference (p={p_value_t:.4f} < {alpha})")
        print(f"  → Model {winner} is statistically better.")
    else:
        print(f"\n✗ NO significant difference (p={p_value_t:.4f} ≥ {alpha})")
        print(f"  → Choose the simpler or faster model.")

    return scores_a, scores_b

# Usage
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression

data = load_breast_cancer()
scores_a, scores_b = compare_models(
    RandomForestClassifier(n_estimators=100, random_state=42),
    LogisticRegression(max_iter=1000, random_state=42),
    data.data, data.target, cv=10
)

# pip install keras-tuner
import tensorflow as tf
from tensorflow import keras
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_breast_cancer
import numpy as np

# Load data
data = load_breast_cancer()
X, y = data.data, data.target

# --- Manual K-Fold CV for Neural Networks ---
def nn_cross_validate(X, y, build_fn, epochs=50, n_splits=5):
    """Cross-validate a Keras model with proper data handling."""
    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)
    fold_scores = []

    for fold, (train_idx, test_idx) in enumerate(skf.split(X, y)):
        print(f"\n--- Fold {fold+1}/{n_splits} ---")

        X_train, X_test = X[train_idx], X[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]

        # Scale INSIDE the fold (no data leakage!)
        scaler = StandardScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        X_test_scaled = scaler.transform(X_test)  # transform only

        # Build fresh model for each fold
        model = build_fn(input_dim=X_train.shape[1])

        # Train with early stopping
        early_stop = keras.callbacks.EarlyStopping(
            monitor='val_loss', patience=10, restore_best_weights=True
        )
        model.fit(
            X_train_scaled, y_train,
            validation_split=0.15,
            epochs=epochs, batch_size=32,
            callbacks=[early_stop],
            verbose=0
        )

        # Evaluate
        _, accuracy = model.evaluate(X_test_scaled, y_test, verbose=0)
        fold_scores.append(accuracy)
        print(f"  Fold {fold+1} accuracy: {accuracy:.4f}")

    fold_scores = np.array(fold_scores)
    print(f"\n{'='*50}")
    print(f"CV Result: {fold_scores.mean():.4f} ± {fold_scores.std():.4f}")
    return fold_scores

# --- Model builder ---
def build_model(input_dim, units=64, dropout=0.3, lr=0.001):
    model = keras.Sequential([
        keras.layers.Dense(units, activation='relu',
                          input_shape=(input_dim,)),
        keras.layers.Dropout(dropout),
        keras.layers.Dense(units // 2, activation='relu'),
        keras.layers.Dropout(dropout / 2),
        keras.layers.Dense(1, activation='sigmoid')
    ])
    model.compile(
        optimizer=keras.optimizers.Adam(learning_rate=lr),
        loss='binary_crossentropy',
        metrics=['accuracy']
    )
    return model

# Run CV
scores = nn_cross_validate(X, y, build_model, epochs=100, n_splits=5)

import keras_tuner as kt

def build_tunable_model(hp):
    """Define hyperparameter search space for Keras Tuner."""
    model = keras.Sequential()

    # Tune number of layers
    for i in range(hp.Int('n_layers', 1, 4)):
        model.add(keras.layers.Dense(
            units=hp.Int(f'units_{i}', min_value=16, max_value=256, step=16),
            activation=hp.Choice(f'activation_{i}', ['relu', 'tanh', 'selu'])
        ))
        if hp.Boolean(f'dropout_{i}'):
            model.add(keras.layers.Dropout(
                rate=hp.Float(f'dropout_rate_{i}', 0.1, 0.5, step=0.1)
            ))

    model.add(keras.layers.Dense(1, activation='sigmoid'))

    # Tune learning rate
    lr = hp.Float('learning_rate', 1e-4, 1e-2, sampling='log')
    model.compile(
        optimizer=keras.optimizers.Adam(learning_rate=lr),
        loss='binary_crossentropy',
        metrics=['accuracy']
    )
    return model

# Bayesian optimization tuner
tuner = kt.BayesianOptimization(
    build_tunable_model,
    objective='val_accuracy',
    max_trials=30,
    directory='tuner_results',
    project_name='breast_cancer_nn'
)

# Scale data
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# Search
tuner.search(
    X_scaled, y,
    validation_split=0.2,
    epochs=50,
    batch_size=32,
    callbacks=[keras.callbacks.EarlyStopping(patience=5)]
)

# Results
print("\nTop 3 Models:")
tuner.results_summary(num_trials=3)

best_model = tuner.get_best_models(num_models=1)[0]
best_hp = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"\nBest hyperparameters: {best_hp.values}")

from sklearn.model_selection import (
    cross_val_score, StratifiedKFold, LeaveOneOut,
    TimeSeriesSplit, GridSearchCV, RandomizedSearchCV
)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.datasets import load_breast_cancer
import numpy as np

# Load data
X, y = load_breast_cancer(return_X_y=True)

# --- 1. Basic cross_val_score ---
print("=" * 60)
print("1. Basic cross_val_score (5-Fold)")
print("=" * 60)
models = {
    'Logistic Regression': LogisticRegression(max_iter=5000),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'SVM (RBF)': SVC(kernel='rbf', random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(random_state=42),
}

for name, model in models.items():
    scores = cross_val_score(model, X, y, cv=5, scoring='accuracy')
    print(f"  {name:25s}: {scores.mean():.4f} ± {scores.std():.4f}")

# --- 2. Stratified K-Fold with custom CV object ---
print(f"\n{'=' * 60}")
print("2. Stratified K-Fold (10 folds)")
print("=" * 60)
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
scores = cross_val_score(
    RandomForestClassifier(n_estimators=100, random_state=42),
    X, y, cv=skf, scoring='accuracy'
)
print(f"  RF (10-Fold Stratified): {scores.mean():.4f} ± {scores.std():.4f}")
print(f"  Individual fold scores: {np.round(scores, 4)}")

# --- 3. Multiple metrics ---
from sklearn.model_selection import cross_validate
print(f"\n{'=' * 60}")
print("3. Multiple Metrics with cross_validate")
print("=" * 60)
results = cross_validate(
    RandomForestClassifier(n_estimators=100, random_state=42),
    X, y, cv=5,
    scoring=['accuracy', 'precision', 'recall', 'f1', 'roc_auc'],
    return_train_score=True
)
for metric in ['accuracy', 'precision', 'recall', 'f1', 'roc_auc']:
    train = results[f'train_{metric}'].mean()
    test = results[f'test_{metric}'].mean()
    print(f"  {metric:12s}: Train={train:.4f} | Test={test:.4f} | "
          f"Gap={train-test:.4f}")

from sklearn.model_selection import GridSearchCV

# Define pipeline (prevents data leakage!)
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('clf', RandomForestClassifier(random_state=42))
])

# Define parameter grid (note: 'clf__' prefix for pipeline)
param_grid = {
    'clf__n_estimators': [50, 100, 200, 300],
    'clf__max_depth': [5, 10, 15, 20, None],
    'clf__min_samples_split': [2, 5, 10],
    'clf__max_features': ['sqrt', 'log2'],
}

# Run Grid Search
grid_search = GridSearchCV(
    pipeline,
    param_grid,
    cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=42),
    scoring='accuracy',
    n_jobs=-1,       # Use all CPU cores
    verbose=1,
    return_train_score=True,
    refit=True       # Refit best model on full training data
)

grid_search.fit(X, y)

# Results
print(f"\n{'='*60}")
print("GRID SEARCH RESULTS")
print(f"{'='*60}")
print(f"Best Score: {grid_search.best_score_:.4f}")
print(f"Best Params: {grid_search.best_params_}")
print(f"Total fits: {grid_search.n_splits_} × "
      f"{len(grid_search.cv_results_['mean_test_score'])} = "
      f"{grid_search.n_splits_ * len(grid_search.cv_results_['mean_test_score'])}")

# Top 5 combinations
import pandas as pd
results_df = pd.DataFrame(grid_search.cv_results_)
top5 = results_df.nsmallest(5, 'rank_test_score')[
    ['params', 'mean_test_score', 'std_test_score', 'mean_fit_time']
]
print("\nTop 5 Configurations:")
for _, row in top5.iterrows():
    print(f"  Score: {row['mean_test_score']:.4f} ± {row['std_test_score']:.4f} "
          f"| Time: {row['mean_fit_time']:.2f}s")

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform

# Distributions instead of fixed lists
param_distributions = {
    'clf__n_estimators': randint(50, 500),
    'clf__max_depth': randint(3, 30),
    'clf__min_samples_split': randint(2, 20),
    'clf__max_features': ['sqrt', 'log2'],
    'clf__min_samples_leaf': randint(1, 10),
}

random_search = RandomizedSearchCV(
    pipeline,
    param_distributions,
    n_iter=60,         # Only 60 random combinations (vs 1000s for grid)
    cv=StratifiedKFold(n_splits=5, shuffle=True, random_state=42),
    scoring='accuracy',
    n_jobs=-1,
    random_state=42,
    verbose=1,
    return_train_score=True
)

random_search.fit(X, y)

print(f"\n{'='*60}")
print("RANDOMIZED SEARCH RESULTS")
print(f"{'='*60}")
print(f"Best Score: {random_search.best_score_:.4f}")
print(f"Best Params: {random_search.best_params_}")
print(f"Evaluated {random_search.n_splits_ * 60} model fits "
      f"(Grid would need {4*5*3*2*4 * 5} = {4*5*3*2*4*5})")

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.decomposition import PCA

# The CORRECT way: everything inside the pipeline
proper_pipeline = Pipeline([
    ('scaler', StandardScaler()),              # Step 1: Scale
    ('feature_select', SelectKBest(f_classif)), # Step 2: Feature Selection
    ('pca', PCA()),                             # Step 3: Dimensionality Reduction
    ('clf', LogisticRegression(max_iter=5000))  # Step 4: Classifier
])

# Grid search over PIPELINE parameters
pipeline_param_grid = {
    'feature_select__k': [5, 10, 15, 20, 'all'],
    'pca__n_components': [3, 5, 10, 15],
    'clf__C': [0.01, 0.1, 1, 10, 100],
    'clf__penalty': ['l1', 'l2'],
    'clf__solver': ['saga'],
}

# This ensures NO data leakage:
# - StandardScaler is fit ONLY on training folds
# - SelectKBest chooses features ONLY from training folds
# - PCA components are learned ONLY from training folds
pipeline_search = GridSearchCV(
    proper_pipeline,
    pipeline_param_grid,
    cv=5, scoring='accuracy',
    n_jobs=-1, verbose=0
)
pipeline_search.fit(X, y)

print(f"Pipeline Best Score: {pipeline_search.best_score_:.4f}")
print(f"Pipeline Best Params: {pipeline_search.best_params_}")

# ⚠️ THE WRONG WAY (DATA LEAKAGE):
# scaler = StandardScaler()
# X_scaled = scaler.fit_transform(X)  ← Sees ALL data including test!
# selector = SelectKBest(k=10).fit_transform(X_scaled, y)  ← Leakage!
# scores = cross_val_score(LogisticRegression(), X_scaled, y, cv=5)
# ^^ This score is OPTIMISTICALLY BIASED!

from sklearn.linear_model import LogisticRegressionCV, RidgeCV, LassoCV
import matplotlib.pyplot as plt

# --- Logistic Regression with built-in CV for C ---
lr_cv = LogisticRegressionCV(
    Cs=np.logspace(-4, 4, 50),  # 50 values from 0.0001 to 10000
    cv=10,
    penalty='l2',
    scoring='accuracy',
    max_iter=5000,
    random_state=42
)
lr_cv.fit(X, y)
print(f"Best C (L2): {lr_cv.C_[0]:.4f}")
print(f"Best score: {lr_cv.scores_[1].mean(axis=0).max():.4f}")

# --- Visualize C vs CV Accuracy ---
mean_scores = lr_cv.scores_[1].mean(axis=0)
plt.figure(figsize=(10, 5))
plt.semilogx(np.logspace(-4, 4, 50), mean_scores, 'o-', color='#059669')
plt.axvline(x=lr_cv.C_[0], color='#f43f5e', linestyle='--',
            label=f'Best C = {lr_cv.C_[0]:.4f}')
plt.xlabel('Regularization Parameter C (log scale)')
plt.ylabel('CV Accuracy')
plt.title('L2 Regularization: C vs Cross-Validated Accuracy')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

# --- Ridge Regression with built-in CV ---
from sklearn.datasets import fetch_california_housing
X_reg, y_reg = fetch_california_housing(return_X_y=True)
ridge_cv = RidgeCV(
    alphas=np.logspace(-4, 4, 100),
    cv=10, scoring='neg_mean_squared_error'
)
ridge_cv.fit(X_reg, y_reg)
print(f"\nBest Ridge alpha: {ridge_cv.alpha_:.4f}")

# --- Lasso with CV (automatic feature selection) ---
lasso_cv = LassoCV(cv=10, random_state=42, max_iter=10000)
lasso_cv.fit(X_reg, y_reg)
print(f"Best Lasso alpha: {lasso_cv.alpha_:.6f}")
print(f"Features selected: {np.sum(lasso_cv.coef_ != 0)} / {X_reg.shape[1]}")

"""
Mini Project 1: AutoModelSelector
A complete framework for automated model selection with:
- Multiple CV strategies
- Statistical model comparison
- Leakage-free pipelines
- Results visualization
"""

import numpy as np
import pandas as pd
from sklearn.model_selection import (
    cross_val_score, StratifiedKFold, TimeSeriesSplit
)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier
)
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from scipy import stats
import time

class AutoModelSelector:
    """Automated model selection with statistical comparison."""

    def __init__(self, cv_strategy='stratified', n_folds=5,
                 random_state=42):
        self.cv_strategy = cv_strategy
        self.n_folds = n_folds
        self.random_state = random_state
        self.results_ = {}
        self.best_model_ = None

        # Define candidate models (all inside pipelines)
        self.candidates = {
            'LogisticRegression': Pipeline([
                ('scaler', StandardScaler()),
                ('clf', LogisticRegression(max_iter=5000,
                                           random_state=random_state))
            ]),
            'RandomForest': Pipeline([
                ('scaler', StandardScaler()),
                ('clf', RandomForestClassifier(n_estimators=100,
                                               random_state=random_state))
            ]),
            'GradientBoosting': Pipeline([
                ('scaler', StandardScaler()),
                ('clf', GradientBoostingClassifier(
                    random_state=random_state))
            ]),
            'SVM_RBF': Pipeline([
                ('scaler', StandardScaler()),
                ('clf', SVC(kernel='rbf', random_state=random_state))
            ]),
            'KNN': Pipeline([
                ('scaler', StandardScaler()),
                ('clf', KNeighborsClassifier(n_neighbors=5))
            ]),
        }

    def _get_cv(self, y=None):
        if self.cv_strategy == 'stratified':
            return StratifiedKFold(n_splits=self.n_folds,
                                   shuffle=True,
                                   random_state=self.random_state)
        elif self.cv_strategy == 'timeseries':
            return TimeSeriesSplit(n_splits=self.n_folds)
        else:
            return self.n_folds

    def fit(self, X, y, scoring='accuracy'):
        """Evaluate all candidate models."""
        cv = self._get_cv(y)
        print("=" * 70)
        print("AUTO MODEL SELECTION")
        print(f"Dataset: {X.shape[0]} samples, {X.shape[1]} features")
        print(f"CV Strategy: {self.cv_strategy} ({self.n_folds} folds)")
        print("=" * 70)

        for name, pipeline in self.candidates.items():
            start = time.time()
            scores = cross_val_score(pipeline, X, y, cv=cv,
                                     scoring=scoring, n_jobs=-1)
            elapsed = time.time() - start

            self.results_[name] = {
                'mean': scores.mean(),
                'std': scores.std(),
                'scores': scores,
                'time': elapsed
            }
            print(f"  {name:25s} | {scores.mean():.4f} ± {scores.std():.4f} "
                  f"| {elapsed:.2f}s")

        # Find best
        self.best_model_ = max(self.results_,
                               key=lambda k: self.results_[k]['mean'])
        print(f"\n★ Best Model: {self.best_model_} "
              f"({self.results_[self.best_model_]['mean']:.4f})")

        return self

    def compare_top2(self, alpha=0.05):
        """Statistical comparison of top 2 models."""
        sorted_models = sorted(self.results_,
                               key=lambda k: self.results_[k]['mean'],
                               reverse=True)
        m1, m2 = sorted_models[0], sorted_models[1]
        s1 = self.results_[m1]['scores']
        s2 = self.results_[m2]['scores']

        # Paired t-test
        t_stat, p_val = stats.ttest_rel(s1, s2)
        # Wilcoxon
        try:
            w_stat, p_wil = stats.wilcoxon(s1, s2)
        except:
            w_stat, p_wil = None, None

        print(f"\n{'='*70}")
        print(f"STATISTICAL COMPARISON: {m1} vs {m2}")
        print(f"{'='*70}")
        print(f"  {m1}: {s1.mean():.4f} ± {s1.std():.4f}")
        print(f"  {m2}: {s2.mean():.4f} ± {s2.std():.4f}")
        print(f"  Paired t-test: t={t_stat:.3f}, p={p_val:.4f}")
        if p_wil is not None:
            print(f"  Wilcoxon test: W={w_stat:.3f}, p={p_wil:.4f}")

        if p_val < alpha:
            print(f"  → Significant difference (p < {alpha})")
            print(f"  → Choose {m1}")
        else:
            print(f"  → No significant difference (p ≥ {alpha})")
            print(f"  → Choose simpler/faster model")

    def leaderboard(self):
        """Return results as a sorted DataFrame."""
        rows = []
        for name, res in self.results_.items():
            rows.append({
                'Model': name,
                'Mean Score': res['mean'],
                'Std': res['std'],
                'Time (s)': res['time'],
                'Score per Second': res['mean'] / max(res['time'], 0.01)
            })
        df = pd.DataFrame(rows).sort_values('Mean Score', ascending=False)
        return df.reset_index(drop=True)

# --- Run the framework ---
from sklearn.datasets import load_breast_cancer
X, y = load_breast_cancer(return_X_y=True)

selector = AutoModelSelector(cv_strategy='stratified', n_folds=10)
selector.fit(X, y)
selector.compare_top2()
print("\n" + selector.leaderboard().to_string(index=False))

"""
Mini Project 2: End-to-End AutoML Pipeline
Combines model selection + hyperparameter tuning using Optuna.
"""

import optuna
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier,
    AdaBoostClassifier
)
from sklearn.svm import SVC
from sklearn.datasets import load_breast_cancer
import numpy as np
import warnings
warnings.filterwarnings('ignore')

X, y = load_breast_cancer(return_X_y=True)

def automl_objective(trial):
    """Unified objective: select model AND tune hyperparameters."""

    # Step 1: Choose model family
    model_type = trial.suggest_categorical(
        'model_type',
        ['lr', 'rf', 'gb', 'svm', 'adaboost']
    )

    # Step 2: Define model-specific hyperparameters
    if model_type == 'lr':
        C = trial.suggest_float('lr_C', 1e-3, 100, log=True)
        penalty = trial.suggest_categorical('lr_penalty', ['l1', 'l2'])
        clf = LogisticRegression(
            C=C, penalty=penalty, solver='saga',
            max_iter=5000, random_state=42
        )

    elif model_type == 'rf':
        clf = RandomForestClassifier(
            n_estimators=trial.suggest_int('rf_n', 50, 300),
            max_depth=trial.suggest_int('rf_depth', 3, 20),
            min_samples_split=trial.suggest_int('rf_split', 2, 15),
            max_features=trial.suggest_categorical(
                'rf_feat', ['sqrt', 'log2']
            ),
            random_state=42
        )

    elif model_type == 'gb':
        clf = GradientBoostingClassifier(
            n_estimators=trial.suggest_int('gb_n', 50, 300),
            max_depth=trial.suggest_int('gb_depth', 2, 10),
            learning_rate=trial.suggest_float('gb_lr', 0.01, 0.3, log=True),
            subsample=trial.suggest_float('gb_sub', 0.5, 1.0),
            random_state=42
        )

    elif model_type == 'svm':
        clf = SVC(
            C=trial.suggest_float('svm_C', 0.1, 100, log=True),
            kernel=trial.suggest_categorical(
                'svm_kernel', ['rbf', 'poly']
            ),
            gamma=trial.suggest_categorical(
                'svm_gamma', ['scale', 'auto']
            ),
            random_state=42
        )

    else:  # adaboost
        clf = AdaBoostClassifier(
            n_estimators=trial.suggest_int('ada_n', 25, 200),
            learning_rate=trial.suggest_float('ada_lr', 0.01, 2.0, log=True),
            random_state=42
        )

    # Step 3: Wrap in pipeline (leak-proof!)
    pipeline = Pipeline([
        ('scaler', StandardScaler()),
        ('clf', clf)
    ])

    # Step 4: Cross-validate
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    scores = cross_val_score(pipeline, X, y, cv=cv,
                             scoring='accuracy', n_jobs=-1)
    return scores.mean()

# Run AutoML
optuna.logging.set_verbosity(optuna.logging.WARNING)
study = optuna.create_study(
    direction='maximize',
    sampler=optuna.samplers.TPESampler(seed=42)
)
study.optimize(automl_objective, n_trials=100)

# Results
print("=" * 70)
print("AutoML RESULTS")
print("=" * 70)
print(f"Best accuracy: {study.best_value:.4f}")
print(f"Best model type: {study.best_params['model_type']}")
print(f"Best params:")
for k, v in study.best_params.items():
    print(f"  {k}: {v}")

# Show model type distribution
model_counts = {}
for trial in study.trials:
    mt = trial.params.get('model_type', 'unknown')
    if mt not in model_counts:
        model_counts[mt] = {'count': 0, 'best_score': 0}
    model_counts[mt]['count'] += 1
    model_counts[mt]['best_score'] = max(
        model_counts[mt]['best_score'], trial.value or 0
    )

print(f"\nModel Type Distribution (out of {len(study.trials)} trials):")
for mt, info in sorted(model_counts.items(),
                       key=lambda x: x[1]['best_score'],
                       reverse=True):
    print(f"  {mt:12s}: {info['count']:3d} trials, "
          f"best = {info['best_score']:.4f}")

"""
Mini Project 3: Hyperparameter Importance Analysis
Determine which hyperparameters matter most for performance.
"""

import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_breast_cancer
from itertools import product

X, y = load_breast_cancer(return_X_y=True)

# Define parameter grid
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 7],
    'learning_rate': [0.01, 0.1, 0.2],
    'subsample': [0.7, 0.85, 1.0],
}

# Run all combinations
results = []
keys = list(param_grid.keys())
values = list(param_grid.values())

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

print("Running parameter sweep...")
for combo in product(*values):
    params = dict(zip(keys, combo))
    pipe = Pipeline([
        ('scaler', StandardScaler()),
        ('clf', GradientBoostingClassifier(**params, random_state=42))
    ])
    scores = cross_val_score(pipe, X, y, cv=cv, scoring='accuracy')
    results.append({**params, 'mean_score': scores.mean()})

# Analyze importance using variance-based method (fANOVA-like)
import pandas as pd
df = pd.DataFrame(results)

print(f"\n{'='*60}")
print("HYPERPARAMETER IMPORTANCE (Marginal Variance Method)")
print(f"{'='*60}")

overall_var = df['mean_score'].var()
importances = {}

for param in keys:
    # Group by this parameter, compute mean score per group
    group_means = df.groupby(param)['mean_score'].mean()
    # Importance = variance of group means / total variance
    param_var = group_means.var()
    importance = param_var / overall_var if overall_var > 0 else 0
    importances[param] = importance

# Sort and display
sorted_imp = sorted(importances.items(), key=lambda x: x[1], reverse=True)
total_imp = sum(v for _, v in sorted_imp)

print(f"\nOverall score range: {df['mean_score'].min():.4f} – "
      f"{df['mean_score'].max():.4f}")
print(f"\nRanked Importance:")
for param, imp in sorted_imp:
    bar = '█' * int(imp / max(importances.values()) * 30)
    normalized = imp / total_imp * 100 if total_imp > 0 else 0
    print(f"  {param:20s} {bar:30s} {normalized:5.1f}%")

# Show best value for each parameter
print(f"\nBest value for each hyperparameter:")
for param, _ in sorted_imp:
    group_means = df.groupby(param)['mean_score'].mean()
    best_val = group_means.idxmax()
    best_score = group_means.max()
    print(f"  {param:20s} = {str(best_val):10s} (avg score: {best_score:.4f})")