Deep Learning

Perceptrons & Multilayer Networks

The Perceptron

A perceptron computes a weighted sum of inputs, adds a bias, and passes through an activation function. It's a single artificial neuron — the atom of deep learning.

Python
import numpy as np

class Perceptron:
    def __init__(self, n_inputs, lr=0.01):
        self.weights = np.random.randn(n_inputs) * 0.01
        self.bias = 0.0
        self.lr = lr

    def forward(self, x):
        return 1.0 if np.dot(self.weights, x) + self.bias > 0 else 0.0

    def train(self, X, y, epochs=100):
        for _ in range(epochs):
            for xi, yi in zip(X, y):
                pred = self.forward(xi)
                error = yi - pred
                self.weights += self.lr * error * xi
                self.bias += self.lr * error

# AND gate — linearly separable
X = np.array([[0,0],[0,1],[1,0],[1,1]])
y = np.array([0, 0, 0, 1])
p = Perceptron(2)
p.train(X, y)
print([p.forward(xi) for xi in X])  # [0, 0, 0, 1] ✓

Multilayer Perceptron (MLP)

Stacking layers of neurons creates an MLP — capable of learning any continuous function (Universal Approximation Theorem). The key insight: non-linear activations between layers allow the network to model complex, non-linear relationships.

Python
import torch
import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, output_dim)
        )

    def forward(self, x):
        return self.net(x)

model = MLP(784, 256, 10)  # MNIST: 784 pixels → 10 digits
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")

Python
import torch, torch.nn as nn
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

# Data
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,),(0.3081,))])
train_data = datasets.MNIST('./data', train=True, download=True, transform=transform)
test_data = datasets.MNIST('./data', train=False, transform=transform)
train_loader = DataLoader(train_data, batch_size=64, shuffle=True)
test_loader = DataLoader(test_data, batch_size=1000)

# Model
model = nn.Sequential(
    nn.Flatten(),
    nn.Linear(784, 512), nn.ReLU(), nn.Dropout(0.2),
    nn.Linear(512, 256), nn.ReLU(), nn.Dropout(0.2),
    nn.Linear(256, 10)
)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

# Train
for epoch in range(5):
    model.train()
    for X, y in train_loader:
        loss = criterion(model(X), y)
        optimizer.zero_grad(); loss.backward(); optimizer.step()

    # Evaluate
    model.eval()
    correct = sum(
        (model(X).argmax(1) == y).sum().item()
        for X, y in test_loader)
    print(f"Epoch {epoch+1}: Accuracy = {correct/len(test_data):.2%}")
# Achieves ~98% accuracy!

Activation Functions

Python
import torch
import torch.nn.functional as F
import numpy as np

x = torch.linspace(-5, 5, 100)

# Key activation functions
sigmoid = torch.sigmoid(x)            # σ(x) = 1/(1+e⁻ˣ) — range (0,1)
tanh = torch.tanh(x)                  # range (-1,1)
relu = F.relu(x)                       # max(0,x) — most popular
leaky_relu = F.leaky_relu(x, 0.01)    # allows small negative gradients
gelu = F.gelu(x)                       # Used in Transformers (GPT, BERT)
silu = F.silu(x)                       # x·σ(x) — "Swish", used in EfficientNet

Forward Pass & Backpropagation

The Forward Pass

Backpropagation: Chain Rule

Backpropagation computes ∂Loss/∂wᵢ for every weight by applying the chain rule backward through the network. This gradient tells each weight how much it contributed to the error — and which direction to adjust.

Python
import numpy as np

# 2-layer neural network from scratch
np.random.seed(42)
X = np.random.randn(100, 2)
y = (X[:, 0] * X[:, 1] > 0).astype(float).reshape(-1, 1)

# Initialize weights
W1 = np.random.randn(2, 16) * 0.5
b1 = np.zeros((1, 16))
W2 = np.random.randn(16, 1) * 0.5
b2 = np.zeros((1, 1))
lr = 0.1

def sigmoid(z): return 1 / (1 + np.exp(-z))

for epoch in range(1000):
    # FORWARD
    z1 = X @ W1 + b1
    a1 = np.maximum(0, z1)    # ReLU
    z2 = a1 @ W2 + b2
    a2 = sigmoid(z2)           # Output probability

    # Loss (binary cross-entropy)
    loss = -np.mean(y * np.log(a2 + 1e-8) + (1-y) * np.log(1-a2 + 1e-8))

    # BACKWARD (chain rule)
    m = len(X)
    dz2 = a2 - y                         # ∂L/∂z2
    dW2 = (1/m) * a1.T @ dz2             # ∂L/∂W2
    db2 = (1/m) * np.sum(dz2, axis=0)
    dz1 = (dz2 @ W2.T) * (z1 > 0)       # ReLU gradient
    dW1 = (1/m) * X.T @ dz1
    db1 = (1/m) * np.sum(dz1, axis=0)

    # UPDATE
    W2 -= lr * dW2; b2 -= lr * db2
    W1 -= lr * dW1; b1 -= lr * db1

    if epoch % 200 == 0:
        acc = ((a2 > 0.5) == y).mean()
        print(f"Epoch {epoch}: loss={loss:.4f}, acc={acc:.2%}")

Weight Initialization, Batch Normalization & Dropout

Weight Initialization

Python
import torch.nn as nn

# Proper initialization matters!
layer = nn.Linear(512, 256)

# He initialization for ReLU networks
nn.init.kaiming_normal_(layer.weight, mode='fan_in', nonlinearity='relu')
nn.init.zeros_(layer.bias)

# Xavier for sigmoid/tanh
nn.init.xavier_uniform_(layer.weight)

Batch Normalization

BatchNorm normalizes each layer's activations to zero mean and unit variance, then applies learnable scale (γ) and shift (β). This stabilizes training, allows higher learning rates, and acts as mild regularization.

Python
model = nn.Sequential(
    nn.Linear(784, 512),
    nn.BatchNorm1d(512),   # Normalize activations
    nn.ReLU(),
    nn.Dropout(0.3),       # Randomly zero 30% of neurons
    nn.Linear(512, 256),
    nn.BatchNorm1d(256),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(256, 10)
)
# BatchNorm + Dropout = stable training + regularization

Optimizers: SGD, Adam, AdamW & RMSProp

Python
import torch.optim as optim

model = MLP(784, 256, 10)

# SGD with momentum — classic, good generalization
opt_sgd = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

# Adam — adaptive learning rate per parameter
opt_adam = optim.Adam(model.parameters(), lr=1e-3, betas=(0.9, 0.999))

# AdamW — Adam with decoupled weight decay (standard for Transformers)
opt_adamw = optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.01)

# RMSProp — good for RNNs
opt_rms = optim.RMSprop(model.parameters(), lr=1e-3, alpha=0.99)

Learning Rate Schedules

Python
import torch.optim.lr_scheduler as lr_scheduler

optimizer = optim.AdamW(model.parameters(), lr=1e-3)

# Step Decay: reduce LR by 0.1 every 10 epochs
step_sched = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)

# Cosine Annealing: smoothly decays to near-zero
cosine_sched = lr_scheduler.CosineAnnealingLR(optimizer, T_max=50)

# Warmup + Cosine (used in all modern transformers)
def warmup_cosine(epoch, warmup_epochs=5, total_epochs=100):
    if epoch < warmup_epochs:
        return epoch / warmup_epochs  # Linear warmup
    progress = (epoch - warmup_epochs) / (total_epochs - warmup_epochs)
    return 0.5 * (1 + np.cos(np.pi * progress))  # Cosine decay

warmup_sched = lr_scheduler.LambdaLR(optimizer, warmup_cosine)

# One Cycle — aggressive: ramp up then decay
onecycle = lr_scheduler.OneCycleLR(optimizer, max_lr=1e-2, total_steps=1000)

Gradient Clipping, Mixed Precision & Early Stopping

Python
import torch
from torch.cuda.amp import autocast, GradScaler

# Gradient Clipping — prevents exploding gradients
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
optimizer.step()

# Mixed Precision Training (FP16/BF16)
scaler = GradScaler()
for X, y in train_loader:
    optimizer.zero_grad()
    with autocast():           # Forward pass in FP16
        output = model(X)
        loss = criterion(output, y)
    scaler.scale(loss).backward()  # Scaled backward pass
    scaler.step(optimizer)
    scaler.update()
# ~2x faster, ~50% less GPU memory!

# Early Stopping
class EarlyStopping:
    def __init__(self, patience=7, min_delta=0.001):
        self.patience = patience
        self.min_delta = min_delta
        self.counter = 0
        self.best_loss = None

    def __call__(self, val_loss):
        if self.best_loss is None or val_loss < self.best_loss - self.min_delta:
            self.best_loss = val_loss
            self.counter = 0
        else:
            self.counter += 1
        return self.counter >= self.patience

early_stop = EarlyStopping(patience=10)
# In training loop: if early_stop(val_loss): break

Convolutional Neural Networks (CNNs)

Python
import torch.nn as nn

class CNN(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.features = nn.Sequential(
            # Conv block 1: 1→32 channels
            nn.Conv2d(1, 32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2),      # 28×28 → 14×14

            # Conv block 2: 32→64
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2),      # 14×14 → 7×7

            # Conv block 3: 64→128
            nn.Conv2d(64, 128, kernel_size=3, padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.AdaptiveAvgPool2d(1)  # Global average pooling → 1×1
        )
        self.classifier = nn.Linear(128, num_classes)

    def forward(self, x):
        x = self.features(x)
        x = x.flatten(1)
        return self.classifier(x)

model = CNN()
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")

Python
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

transform = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomCrop(32, padding=4),
    transforms.ToTensor(),
    transforms.Normalize((0.4914,0.4822,0.4465), (0.2470,0.2435,0.2616))
])
train_ds = datasets.CIFAR10('./data', train=True, download=True, transform=transform)
train_loader = DataLoader(train_ds, batch_size=128, shuffle=True)

# Change first conv to accept 3 channels (RGB)
cnn = CNN(num_classes=10)
cnn.features[0] = nn.Conv2d(3, 32, 3, padding=1)
optimizer = torch.optim.AdamW(cnn.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

for epoch in range(10):
    cnn.train()
    total_loss = 0
    for X, y in train_loader:
        loss = criterion(cnn(X), y)
        optimizer.zero_grad(); loss.backward(); optimizer.step()
        total_loss += loss.item()
    print(f"Epoch {epoch+1}: loss={total_loss/len(train_loader):.4f}")

RNNs, LSTMs & GRUs

Python
import torch.nn as nn

# LSTM for sequence modeling
class SentimentLSTM(nn.Module):
    def __init__(self, vocab_size, embed_dim=128, hidden_dim=256):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers=2,
                           batch_first=True, dropout=0.3, bidirectional=True)
        self.fc = nn.Linear(hidden_dim * 2, 1)  # *2 for bidirectional

    def forward(self, x):
        embed = self.embedding(x)
        out, (hidden, cell) = self.lstm(embed)
        # Use last hidden state from both directions
        hidden = torch.cat([hidden[-2], hidden[-1]], dim=1)
        return torch.sigmoid(self.fc(hidden))

# GRU — simpler, faster, often comparable to LSTM
gru = nn.GRU(input_size=128, hidden_size=256, num_layers=2, batch_first=True)

Attention Mechanisms

Self-Attention

Python
import torch
import torch.nn as nn
import math

class SelfAttention(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_k = d_model // n_heads
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)

    def forward(self, x):
        B, T, C = x.shape

        # Project to queries, keys, values
        Q = self.W_q(x).view(B, T, self.n_heads, self.d_k).transpose(1,2)
        K = self.W_k(x).view(B, T, self.n_heads, self.d_k).transpose(1,2)
        V = self.W_v(x).view(B, T, self.n_heads, self.d_k).transpose(1,2)

        # Scaled dot-product attention
        scores = (Q @ K.transpose(-2,-1)) / math.sqrt(self.d_k)
        attn = torch.softmax(scores, dim=-1)
        out = attn @ V

        # Combine heads
        out = out.transpose(1,2).contiguous().view(B, T, C)
        return self.W_o(out)

# Test
attn = SelfAttention(d_model=512, n_heads=8)
x = torch.randn(2, 100, 512)  # batch=2, seq_len=100, dim=512
out = attn(x)
print(out.shape)  # [2, 100, 512]

Residual Networks (ResNet) & Skip Connections

Python
class ResidualBlock(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.block = nn.Sequential(
            nn.Conv2d(channels, channels, 3, padding=1),
            nn.BatchNorm2d(channels),
            nn.ReLU(),
            nn.Conv2d(channels, channels, 3, padding=1),
            nn.BatchNorm2d(channels)
        )
        self.relu = nn.ReLU()

    def forward(self, x):
        return self.relu(self.block(x) + x)  # Skip connection!

# Stack residual blocks to create ResNet
class SimpleResNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 64, 7, stride=2, padding=3)
        self.bn1 = nn.BatchNorm2d(64)
        self.pool = nn.MaxPool2d(3, stride=2, padding=1)
        self.layer1 = nn.Sequential(*[ResidualBlock(64) for _ in range(3)])
        self.avgpool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Linear(64, 10)

    def forward(self, x):
        x = self.pool(nn.functional.relu(self.bn1(self.conv1(x))))
        x = self.layer1(x)
        x = self.avgpool(x).flatten(1)
        return self.fc(x)

U-Net & Encoder-Decoder Designs

Python
class UNet(nn.Module):
    def __init__(self, in_ch=1, out_ch=1):
        super().__init__()
        # Encoder (downsampling)
        self.enc1 = self._block(in_ch, 64)
        self.enc2 = self._block(64, 128)
        self.pool = nn.MaxPool2d(2)

        # Bottleneck
        self.bottleneck = self._block(128, 256)

        # Decoder (upsampling)
        self.up2 = nn.ConvTranspose2d(256, 128, 2, stride=2)
        self.dec2 = self._block(256, 128)  # 256 because of skip concat
        self.up1 = nn.ConvTranspose2d(128, 64, 2, stride=2)
        self.dec1 = self._block(128, 64)

        self.final = nn.Conv2d(64, out_ch, 1)

    def _block(self, in_c, out_c):
        return nn.Sequential(
            nn.Conv2d(in_c, out_c, 3, padding=1), nn.ReLU(),
            nn.Conv2d(out_c, out_c, 3, padding=1), nn.ReLU())

    def forward(self, x):
        # Encode
        e1 = self.enc1(x)
        e2 = self.enc2(self.pool(e1))
        # Bottleneck
        b = self.bottleneck(self.pool(e2))
        # Decode with skip connections
        d2 = self.dec2(torch.cat([self.up2(b), e2], dim=1))
        d1 = self.dec1(torch.cat([self.up1(d2), e1], dim=1))
        return self.final(d1)

unet = UNet(3, 1)  # RGB input → binary mask output

PyTorch — Preferred for Research

Python
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

# Device setup
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Model
class Classifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(20, 128), nn.ReLU(), nn.Dropout(0.2),
            nn.Linear(128, 64), nn.ReLU(),
            nn.Linear(64, 2)
        )
    def forward(self, x): return self.net(x)

model = Classifier().to(device)
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

# Data
X = torch.randn(1000, 20)
y = (X[:, 0] > 0).long()
loader = DataLoader(TensorDataset(X, y), batch_size=32, shuffle=True)

# Training Loop
for epoch in range(20):
    model.train()
    total_loss = 0
    for xb, yb in loader:
        xb, yb = xb.to(device), yb.to(device)
        loss = criterion(model(xb), yb)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        total_loss += loss.item()

    if (epoch+1) % 5 == 0:
        print(f"Epoch {epoch+1}: loss={total_loss/len(loader):.4f}")

# Save & load
torch.save(model.state_dict(), 'model.pth')
model.load_state_dict(torch.load('model.pth'))

Python
import torch, torch.nn as nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms

# Config
EPOCHS, BATCH_SIZE, LR = 10, 128, 1e-3
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Data with augmentation
train_tf = transforms.Compose([
    transforms.RandomRotation(10),
    transforms.ToTensor(), transforms.Normalize((0.1307,),(0.3081,))])
test_tf = transforms.Compose([
    transforms.ToTensor(), transforms.Normalize((0.1307,),(0.3081,))])

train_loader = DataLoader(datasets.MNIST('data', train=True, download=True,
    transform=train_tf), batch_size=BATCH_SIZE, shuffle=True)
test_loader = DataLoader(datasets.MNIST('data', train=False,
    transform=test_tf), batch_size=1000)

# Model with best practices
model = nn.Sequential(
    nn.Flatten(),
    nn.Linear(784,512), nn.BatchNorm1d(512), nn.GELU(), nn.Dropout(0.2),
    nn.Linear(512,256), nn.BatchNorm1d(256), nn.GELU(), nn.Dropout(0.2),
    nn.Linear(256,10)
).to(device)

optimizer = torch.optim.AdamW(model.parameters(), lr=LR)
scheduler = torch.optim.lr_scheduler.OneCycleLR(
    optimizer, max_lr=LR*10, total_steps=EPOCHS*len(train_loader))
criterion = nn.CrossEntropyLoss()

best_acc = 0
for epoch in range(EPOCHS):
    model.train()
    for X, y in train_loader:
        X, y = X.to(device), y.to(device)
        loss = criterion(model(X), y)
        optimizer.zero_grad(); loss.backward(); optimizer.step(); scheduler.step()

    model.eval()
    correct = 0
    with torch.no_grad():
        for X, y in test_loader:
            X, y = X.to(device), y.to(device)
            correct += (model(X).argmax(1) == y).sum().item()
    acc = correct / len(test_loader.dataset)
    if acc > best_acc:
        best_acc = acc
        torch.save(model.state_dict(), 'best_model.pth')
    print(f"Epoch {epoch+1}: acc={acc:.2%} {'⭐ best!' if acc==best_acc else ''}")

device = torch.device('cuda')
model = model.to(device)      # Move model
X = X.to(device)              # Move data
# Both must be on same device!

TensorFlow/Keras, JAX & Custom Training

TensorFlow/Keras

Python
import tensorflow as tf

# Keras Sequential — high-level API
model = tf.keras.Sequential([
    tf.keras.layers.Flatten(input_shape=(28,28)),
    tf.keras.layers.Dense(512, activation='relu'),
    tf.keras.layers.BatchNormalization(),
    tf.keras.layers.Dropout(0.2),
    tf.keras.layers.Dense(256, activation='relu'),
    tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# Train in one line!
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()
x_train = x_train / 255.0

model.fit(x_train, y_train, epochs=5, batch_size=128,
          validation_split=0.1,
          callbacks=[tf.keras.callbacks.EarlyStopping(patience=3)])

JAX — Functional, XLA-Compiled

Python
import jax
import jax.numpy as jnp
from jax import grad, jit, vmap

# Automatic differentiation — the core of all DL
def loss_fn(w, x, y):
    pred = jnp.dot(x, w)
    return jnp.mean((pred - y) ** 2)

# grad() returns a function that computes gradients
grad_fn = jit(grad(loss_fn))  # JIT-compiled gradient function

# vmap — automatic vectorization (batch processing)
def predict_single(w, x):
    return jnp.dot(x, w)

predict_batch = vmap(predict_single, in_axes=(None, 0))
# Automatically handles batching without writing loops!

Custom Training Loop (PyTorch)

Python
# Full control training loop with all best practices
def train_one_epoch(model, loader, optimizer, scheduler, scaler, device):
    model.train()
    total_loss = 0
    for batch_idx, (X, y) in enumerate(loader):
        X, y = X.to(device), y.to(device)
        optimizer.zero_grad()

        # Mixed precision forward
        with torch.cuda.amp.autocast():
            output = model(X)
            loss = nn.functional.cross_entropy(output, y)

        # Scaled backward + gradient clipping
        scaler.scale(loss).backward()
        scaler.unscale_(optimizer)
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        scaler.step(optimizer)
        scaler.update()
        scheduler.step()

        total_loss += loss.item()
    return total_loss / len(loader)