Neural Networks &
Forward Propagation

import numpy as np

class NeuralNetwork:
    """
    Configurable Multi-Layer Perceptron (MLP) from scratch.
    
    Supports:
    - Arbitrary number of layers and neurons per layer
    - Multiple activation functions (ReLU, sigmoid, tanh)
    - Xavier and He weight initialization
    - Vectorized forward propagation over batches
    
    Parameters
    ----------
    layer_dims : list of int
        Dimensions of each layer. E.g., [3, 4, 2] means
        3 inputs, 4 hidden neurons, 2 output neurons.
    activations : list of str
        Activation function for each layer (except input).
        E.g., ['relu', 'sigmoid'] for the above architecture.
    init_method : str
        Weight initialization method: 'xavier', 'he', or 'random'.
    seed : int or None
        Random seed for reproducibility.
    """
    
    def __init__(self, layer_dims, activations, init_method='he', seed=42):
        assert len(activations) == len(layer_dims) - 1, \
            f"Need {len(layer_dims)-1} activations, got {len(activations)}"
        
        self.layer_dims = layer_dims
        self.activations = activations
        self.L = len(layer_dims) - 1  # number of layers (excluding input)
        self.parameters = {}
        self.caches = []
        
        if seed is not None:
            np.random.seed(seed)
        
        # Initialize weights and biases
        self._initialize_parameters(init_method)
    
    def _initialize_parameters(self, method):
        """Initialize weights using specified method."""
        for l in range(1, self.L + 1):
            n_l = self.layer_dims[l]      # neurons in current layer
            n_prev = self.layer_dims[l-1]  # neurons in previous layer
            
            if method == 'xavier':
                # Var(W) = 2 / (n_in + n_out)
                std = np.sqrt(2.0 / (n_prev + n_l))
            elif method == 'he':
                # Var(W) = 2 / n_in
                std = np.sqrt(2.0 / n_prev)
            elif method == 'lecun':
                # Var(W) = 1 / n_in
                std = np.sqrt(1.0 / n_prev)
            else:
                std = 0.01  # small random
            
            self.parameters[f'W{l}'] = np.random.randn(n_l, n_prev) * std
            self.parameters[f'b{l}'] = np.zeros((n_l, 1))
            
            # Print shapes for verification
            print(f"Layer {l}: W{l} shape = {self.parameters[f'W{l}'].shape}, "
                  f"b{l} shape = {self.parameters[f'b{l}'].shape}")
    
    @staticmethod
    def _sigmoid(Z):
        """Sigmoid activation: σ(z) = 1/(1+e^(-z))"""
        A = 1.0 / (1.0 + np.exp(-np.clip(Z, -500, 500)))
        return A
    
    @staticmethod
    def _relu(Z):
        """ReLU activation: max(0, z)"""
        return np.maximum(0, Z)
    
    @staticmethod
    def _tanh(Z):
        """Tanh activation"""
        return np.tanh(Z)
    
    def _activate(self, Z, activation):
        """Apply activation function."""
        if activation == 'sigmoid':
            return self._sigmoid(Z)
        elif activation == 'relu':
            return self._relu(Z)
        elif activation == 'tanh':
            return self._tanh(Z)
        elif activation == 'linear':
            return Z
        else:
            raise ValueError(f"Unknown activation: {activation}")
    
    def forward(self, X):
        """
        Full forward propagation through the network.
        
        Parameters
        ----------
        X : ndarray of shape (n_features, m_examples)
            Input data matrix. Each column is one example.
        
        Returns
        -------
        A_L : ndarray of shape (n_output, m_examples)
            Network output (predictions).
        """
        self.caches = []
        A = X  # A[0] = X
        
        for l in range(1, self.L + 1):
            A_prev = A
            W = self.parameters[f'W{l}']
            b = self.parameters[f'b{l}']
            
            # Linear step: Z[l] = W[l] · A[l-1] + b[l]
            Z = np.dot(W, A_prev) + b
            
            # Activation step: A[l] = g(Z[l])
            A = self._activate(Z, self.activations[l-1])
            
            # Cache for backpropagation (Chapter 12)
            self.caches.append({
                'Z': Z,
                'A_prev': A_prev,
                'W': W,
                'b': b,
                'activation': self.activations[l-1]
            })
            
            # Verbose shape checking
            # print(f"Layer {l}: Z shape={Z.shape}, A shape={A.shape}")
        
        return A
    
    def predict(self, X, threshold=0.5):
        """Make predictions (for classification)."""
        A_L = self.forward(X)
        if self.activations[-1] == 'sigmoid':
            return (A_L > threshold).astype(int)
        return A_L
    
    def count_parameters(self):
        """Count total trainable parameters."""
        total = 0
        for l in range(1, self.L + 1):
            W = self.parameters[f'W{l}']
            b = self.parameters[f'b{l}']
            total += W.size + b.size
            print(f"Layer {l}: {W.shape[0]}×{W.shape[1]} weights + "
                  f"{b.shape[0]} biases = {W.size + b.size}")
        print(f"Total parameters: {total}")
        return total


# ===== DEMONSTRATION =====
if __name__ == "__main__":
    # Example 1: 2-layer network (3 → 4 → 2)
    print("=" * 60)
    print("Example 1: Forward Pass (3 → 4 → 2)")
    print("=" * 60)
    
    nn = NeuralNetwork(
        layer_dims=[3, 4, 2],
        activations=['relu', 'sigmoid'],
        init_method='he',
        seed=42
    )
    
    # Single example
    x = np.array([[1.0], [0.5], [-1.5]])  # shape (3, 1)
    output = nn.forward(x)
    print(f"\nInput x = {x.flatten()}")
    print(f"Output ŷ = {output.flatten()}")
    print(f"Predicted class: {nn.predict(x).flatten()}")
    
    # Batch of examples
    X_batch = np.array([
        [1.0, 0.0, -0.5, 2.0],  # feature 1 for 4 examples
        [0.5, 1.0, 0.0, -1.0],  # feature 2
        [-1.5, 0.5, 1.0, 0.0]   # feature 3
    ])  # shape (3, 4)
    
    print(f"\nBatch input shape: {X_batch.shape}")
    batch_output = nn.forward(X_batch)
    print(f"Batch output shape: {batch_output.shape}")
    print(f"Outputs:\n{batch_output}")
    
    # Parameter count
    print()
    nn.count_parameters()
    
    # Example 2: XOR Network
    print("\n" + "=" * 60)
    print("Example 2: XOR Network (2 → 2 → 1)")
    print("=" * 60)
    
    xor_nn = NeuralNetwork(
        layer_dims=[2, 2, 1],
        activations=['relu', 'sigmoid'],
        init_method='random',
        seed=None
    )
    
    # Manually set weights that solve XOR
    xor_nn.parameters['W1'] = np.array([[1.0, 1.0],
                                          [1.0, 1.0]])
    xor_nn.parameters['b1'] = np.array([[-0.5], [-1.5]])
    xor_nn.parameters['W2'] = np.array([[1.0, -2.0]])
    xor_nn.parameters['b2'] = np.array([[0.0]])
    
    # Test all XOR inputs
    XOR_X = np.array([[0, 0, 1, 1],
                       [0, 1, 0, 1]])  # shape (2, 4)
    
    xor_output = xor_nn.forward(XOR_X)
    print(f"XOR inputs:\n{XOR_X}")
    print(f"XOR outputs: {xor_output.flatten()}")
    print(f"XOR predictions: {xor_nn.predict(XOR_X).flatten()}")
    print(f"Expected:         [0 1 1 0]")
    

import numpy as np

def verify_forward_dimensions(layer_dims, m=1):
    """
    Verify all matrix dimensions for forward propagation.
    
    Parameters
    ----------
    layer_dims : list of int
        [n0, n1, n2, ..., nL] — neurons in each layer
    m : int
        Number of training examples (batch size)
    """
    print(f"Architecture: {' → '.join(map(str, layer_dims))}")
    print(f"Batch size: {m}")
    print(f"{'='*55}")
    
    total_params = 0
    total_flops = 0
    
    for l in range(1, len(layer_dims)):
        n_l = layer_dims[l]
        n_prev = layer_dims[l-1]
        
        # Weight matrix
        W_shape = (n_l, n_prev)
        b_shape = (n_l, 1)
        A_prev_shape = (n_prev, m)
        Z_shape = (n_l, m)
        A_shape = (n_l, m)
        
        params = n_l * n_prev + n_l
        flops = 2 * n_l * n_prev * m  # multiply-add for each
        total_params += params
        total_flops += flops
        
        print(f"\nLayer {l}:")
        print(f"  W[{l}] shape:     {W_shape}")
        print(f"  b[{l}] shape:     {b_shape}")
        print(f"  A[{l-1}] shape:   {A_prev_shape}")
        print(f"  Z[{l}] = W[{l}]·A[{l-1}] + b[{l}]")
        print(f"  ({n_l},{m}) = ({n_l},{n_prev})·({n_prev},{m}) + ({n_l},1) ✓")
        print(f"  A[{l}] shape:   {A_shape}")
        print(f"  Parameters: {params} ({n_l}×{n_prev} + {n_l})")
        print(f"  FLOPs:      {flops:,}")
    
    print(f"\n{'='*55}")
    print(f"Total parameters: {total_params:,}")
    print(f"Total FLOPs:      {total_flops:,}")
    print(f"{'='*55}")

# Test cases
verify_forward_dimensions([3, 4, 2], m=1)        # Worked Example 1
print()
verify_forward_dimensions([784, 256, 128, 10], m=64)  # MNIST classifier
    

import numpy as np
import matplotlib.pyplot as plt

def compare_initializations(n_layers=10, n_neurons=256, n_samples=1000):
    """
    Show how different initialization methods affect
    activation distributions through deep networks.
    """
    fig, axes = plt.subplots(2, 3, figsize=(15, 8))
    methods = {
        'Small Random (0.01)': 0.01,
        'Xavier': lambda n: np.sqrt(2.0 / (n + n)),
        'He': lambda n: np.sqrt(2.0 / n),
    }
    activations_fn = {
        'Tanh': np.tanh,
        'ReLU': lambda x: np.maximum(0, x),
    }
    
    for col, (init_name, init_val) in enumerate(methods.items()):
        for row, (act_name, act_fn) in enumerate(activations_fn.items()):
            ax = axes[row][col]
            A = np.random.randn(n_neurons, n_samples)  # input
            
            means, stds = [], []
            for l in range(n_layers):
                if callable(init_val):
                    std = init_val(n_neurons)
                else:
                    std = init_val
                
                W = np.random.randn(n_neurons, n_neurons) * std
                Z = W @ A
                A = act_fn(Z)
                means.append(np.mean(A))
                stds.append(np.std(A))
            
            ax.plot(range(1, n_layers+1), stds, 'o-', linewidth=2)
            ax.set_title(f"{init_name}\n{act_name}", fontsize=10)
            ax.set_xlabel("Layer")
            ax.set_ylabel("Std of activations")
            ax.set_ylim(0, max(stds) * 1.2 + 0.01)
            ax.grid(True, alpha=0.3)
    
    plt.suptitle("Effect of Initialization on Activation Variance", 
                 fontsize=14, fontweight='bold')
    plt.tight_layout()
    plt.savefig("initialization_comparison.png", dpi=150)
    plt.show()

compare_initializations()
    

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import numpy as np

# ===== Sequential API: Simple MLP =====
def build_sequential_mlp(input_dim, hidden_units, output_units, output_activation='sigmoid'):
    """
    Build an MLP using Keras Sequential API.
    
    Parameters
    ----------
    input_dim : int         → n[0]
    hidden_units : list     → [n[1], n[2], ...]
    output_units : int      → n[L]
    output_activation : str → 'sigmoid', 'softmax', 'linear'
    """
    model = keras.Sequential(name='MLP_Sequential')
    
    # First hidden layer (must specify input_shape)
    model.add(layers.Dense(
        units=hidden_units[0],
        activation='relu',
        kernel_initializer='he_normal',  # He initialization for ReLU
        input_shape=(input_dim,),
        name='hidden_1'
    ))
    
    # Additional hidden layers
    for i, units in enumerate(hidden_units[1:], start=2):
        model.add(layers.Dense(
            units=units,
            activation='relu',
            kernel_initializer='he_normal',
            name=f'hidden_{i}'
        ))
    
    # Output layer
    model.add(layers.Dense(
        units=output_units,
        activation=output_activation,
        kernel_initializer='glorot_uniform',  # Xavier for sigmoid/softmax
        name='output'
    ))
    
    return model

# Build a 784 → 256 → 128 → 10 network (MNIST-like)
model = build_sequential_mlp(
    input_dim=784,
    hidden_units=[256, 128],
    output_units=10,
    output_activation='softmax'
)

model.summary()

# Compile
model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

# Test forward pass with random data
X_test = np.random.randn(5, 784).astype(np.float32)
predictions = model.predict(X_test, verbose=0)
print(f"\nInput shape:  {X_test.shape}")
print(f"Output shape: {predictions.shape}")
print(f"Output (probabilities for 10 classes):\n{predictions}")
print(f"Predicted classes: {np.argmax(predictions, axis=1)}")

# Verify weight shapes
print("\n=== Weight Shapes ===")
for layer in model.layers:
    weights = layer.get_weights()
    if weights:
        print(f"{layer.name}: W={weights[0].shape}, b={weights[1].shape}")
    

import tensorflow as tf
from tensorflow.keras import layers, Model

def build_functional_mlp(input_dim, architecture):
    """
    Build MLP using Functional API for maximum flexibility.
    
    Parameters
    ----------
    input_dim : int
        Number of input features
    architecture : list of dict
        Each dict: {'units': int, 'activation': str, 'name': str}
    """
    # Input layer
    inputs = layers.Input(shape=(input_dim,), name='input')
    
    # Build hidden layers
    x = inputs
    for layer_config in architecture:
        x = layers.Dense(
            units=layer_config['units'],
            activation=layer_config['activation'],
            kernel_initializer='he_normal' if layer_config['activation'] == 'relu' 
                               else 'glorot_uniform',
            name=layer_config['name']
        )(x)
    
    # Create model
    model = Model(inputs=inputs, outputs=x, name='MLP_Functional')
    return model

# Define architecture
arch = [
    {'units': 256, 'activation': 'relu',    'name': 'hidden_1'},
    {'units': 128, 'activation': 'relu',    'name': 'hidden_2'},
    {'units': 64,  'activation': 'relu',    'name': 'hidden_3'},
    {'units': 10,  'activation': 'softmax', 'name': 'output'},
]

model = build_functional_mlp(784, arch)
model.summary()

# ===== Visualize forward pass layer by layer =====
# Create intermediate models to see outputs at each layer
import numpy as np

x_sample = np.random.randn(1, 784).astype(np.float32)

print("\n=== Forward Pass Layer by Layer ===")
for layer in model.layers:
    intermediate_model = Model(inputs=model.input, 
                                outputs=layer.output)
    intermediate_output = intermediate_model.predict(x_sample, verbose=0)
    print(f"{layer.name:12s} → shape: {intermediate_output.shape}, "
          f"mean: {intermediate_output.mean():.4f}, "
          f"std: {intermediate_output.std():.4f}")

# ===== Train on MNIST =====
print("\n=== Training on MNIST ===")
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.reshape(-1, 784).astype('float32') / 255.0
X_test = X_test.reshape(-1, 784).astype('float32') / 255.0

model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

history = model.fit(
    X_train, y_train,
    epochs=5,
    batch_size=128,
    validation_split=0.1,
    verbose=1
)

test_loss, test_acc = model.evaluate(X_test, y_test, verbose=0)
print(f"\nTest accuracy: {test_acc:.4f}")
    

from sklearn.neural_network import MLPClassifier, MLPRegressor
from sklearn.datasets import make_moons, make_circles, load_digits
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report
import numpy as np
import matplotlib.pyplot as plt

# ===== Example 1: XOR with MLPClassifier =====
print("=" * 50)
print("Example 1: XOR Problem")
print("=" * 50)

X_xor = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y_xor = np.array([0, 1, 1, 0])

mlp_xor = MLPClassifier(
    hidden_layer_sizes=(4,),      # One hidden layer with 4 neurons
    activation='relu',             # ReLU activation
    solver='adam',                 # Adam optimizer
    max_iter=1000,
    random_state=42,
    learning_rate_init=0.01
)

mlp_xor.fit(X_xor, y_xor)
predictions = mlp_xor.predict(X_xor)
print(f"Predictions: {predictions}")
print(f"Expected:    {y_xor}")
print(f"Accuracy:    {accuracy_score(y_xor, predictions):.2f}")

# ===== Example 2: Moons Dataset =====
print(f"\n{'='*50}")
print("Example 2: Moons Dataset (Non-linear)")
print("=" * 50)

X_moons, y_moons = make_moons(n_samples=1000, noise=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X_moons, y_moons, test_size=0.2, random_state=42
)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

mlp_moons = MLPClassifier(
    hidden_layer_sizes=(64, 32),   # Two hidden layers: 64 and 32 neurons
    activation='relu',
    solver='adam',
    max_iter=500,
    random_state=42,
    early_stopping=True,           # Stop when validation score plateaus
    validation_fraction=0.1,
    learning_rate='adaptive',      # Reduce LR when stuck
    verbose=False
)

mlp_moons.fit(X_train_scaled, y_train)
y_pred = mlp_moons.predict(X_test_scaled)
print(f"Test Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print(f"\nClassification Report:\n{classification_report(y_test, y_pred)}")

# ===== Example 3: Digits Recognition =====
print(f"\n{'='*50}")
print("Example 3: Handwritten Digits (8×8)")
print("=" * 50)

digits = load_digits()
X_digits, y_digits = digits.data, digits.target

X_train, X_test, y_train, y_test = train_test_split(
    X_digits, y_digits, test_size=0.2, random_state=42
)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

mlp_digits = MLPClassifier(
    hidden_layer_sizes=(128, 64),
    activation='relu',
    solver='adam',
    max_iter=300,
    random_state=42,
    batch_size=32,
    verbose=False
)

mlp_digits.fit(X_train_scaled, y_train)
print(f"Test Accuracy: {accuracy_score(y_test, mlp_digits.predict(X_test_scaled)):.4f}")

# Print architecture details
print(f"\nArchitecture: {X_digits.shape[1]} → "
      f"{' → '.join(map(str, mlp_digits.hidden_layer_sizes))} → "
      f"{len(np.unique(y_digits))}")

# Access learned weights
for i, (W, b) in enumerate(zip(mlp_digits.coefs_, mlp_digits.intercepts_)):
    print(f"Layer {i+1}: W shape = {W.shape}, b shape = {b.shape}")