Industry Problems & Solutions

Build Small Projects from Scratch

1. Implement Backpropagation from Scratch (No PyTorch)

Build a tiny autograd engine like Andrej Karpathy's micrograd. This proves you truly understand gradients — the foundation of all deep learning.

Python
class Value:
    """A scalar value with automatic gradient computation — like micrograd"""
    def __init__(self, data, _children=(), _op=''):
        self.data = data
        self.grad = 0.0
        self._backward = lambda: None
        self._prev = set(_children)
        self._op = _op

    def __add__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        out = Value(self.data + other.data, (self, other), '+')
        def _backward():
            self.grad += out.grad    # d(a+b)/da = 1
            other.grad += out.grad   # d(a+b)/db = 1
        out._backward = _backward
        return out

    def __mul__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        out = Value(self.data * other.data, (self, other), '*')
        def _backward():
            self.grad += other.data * out.grad  # d(a*b)/da = b
            other.grad += self.data * out.grad  # d(a*b)/db = a
        out._backward = _backward
        return out

    def relu(self):
        out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')
        def _backward():
            self.grad += (out.data > 0) * out.grad
        out._backward = _backward
        return out

    def backward(self):
        """Topological sort + reverse-mode autodiff"""
        topo, visited = [], set()
        def build_topo(v):
            if v not in visited:
                visited.add(v)
                for child in v._prev:
                    build_topo(child)
                topo.append(v)
        build_topo(self)
        self.grad = 1.0
        for v in reversed(topo):
            v._backward()

# Test it — this is literally how PyTorch works internally!
a = Value(2.0); b = Value(-3.0); c = Value(10.0)
d = a * b + c   # d = 2*(-3) + 10 = 4
d.backward()
print(f"a.grad = {a.grad}")  # -3.0 (dd/da = b = -3)
print(f"b.grad = {b.grad}")  #  2.0 (dd/db = a =  2)

Python
import random

class Neuron:
    def __init__(self, nin):
        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]
        self.b = Value(0)
    def __call__(self, x):
        act = sum((wi*xi for wi,xi in zip(self.w, x)), self.b)
        return act.relu()
    def parameters(self): return self.w + [self.b]

class MLP:
    def __init__(self, nin, nouts):
        sz = [nin] + nouts
        self.layers = [[Neuron(sz[i]) for _ in range(sz[i+1])] for i in range(len(nouts))]
    def __call__(self, x):
        for layer in self.layers:
            x = [n(x) for n in layer]
        return x[0] if len(x)==1 else x

# Train on XOR — the classic test!
model = MLP(2, [4, 4, 1])
X = [[0,0],[0,1],[1,0],[1,1]]
Y = [0,  1,  1,  0]
for epoch in range(100):
    preds = [model(x) for x in X]
    loss = sum((p - y)*(p - y) for p, y in zip(preds, Y))
    for p in model.parameters(): p.grad = 0.0
    loss.backward()
    for p in model.parameters(): p.data -= 0.05 * p.grad

2. Train a Character-Level Language Model (GPT-Style) from Scratch

Follow Andrej Karpathy's nanoGPT — build every layer yourself: embedding, attention, MLP, loss.

Python
import torch, torch.nn as nn, torch.nn.functional as F

class Head(nn.Module):
    """Single head of self-attention"""
    def __init__(self, head_size, n_embd, block_size):
        super().__init__()
        self.key   = nn.Linear(n_embd, head_size, bias=False)
        self.query = nn.Linear(n_embd, head_size, bias=False)
        self.value = nn.Linear(n_embd, head_size, bias=False)
        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))

    def forward(self, x):
        B, T, C = x.shape
        k, q = self.key(x), self.query(x)
        wei = q @ k.transpose(-2,-1) * C**-0.5
        wei = wei.masked_fill(self.tril[:T,:T]==0, float('-inf'))
        wei = F.softmax(wei, dim=-1)
        return wei @ self.value(x)

class GPT(nn.Module):
    def __init__(self, vocab_size, n_embd=64, n_head=4, n_layer=4, block_size=256):
        super().__init__()
        self.tok_emb = nn.Embedding(vocab_size, n_embd)
        self.pos_emb = nn.Embedding(block_size, n_embd)
        self.blocks = nn.Sequential(*[Block(n_embd, n_head, block_size) for _ in range(n_layer)])
        self.ln_f = nn.LayerNorm(n_embd)
        self.lm_head = nn.Linear(n_embd, vocab_size)

    def forward(self, idx, targets=None):
        B, T = idx.shape
        x = self.tok_emb(idx) + self.pos_emb(torch.arange(T, device=idx.device))
        x = self.ln_f(self.blocks(x))
        logits = self.lm_head(x)
        loss = F.cross_entropy(logits.view(-1,logits.size(-1)), targets.view(-1)) if targets is not None else None
        return logits, loss

3. Fine-Tune an Open-Source LLM on a Custom Dataset

Use LLaMA or Mistral with LoRA on a small domain dataset (e.g., physics Q&A for EduArtha).

4. Build and Deploy a Simple AI-Powered App

A chatbot, document summarizer, or quiz generator using your model via API.

🏭 Project 1: Question Difficulty Classifier from Scratch

No pre-trained models allowed. Raw text → trained neural network → deployed classifier.

What You Must Build from Scratch

1. Tokenizer — no NLTK, no spaCy. Write your own BPE (Byte Pair Encoding) tokenizer. Merge vocab pairs from a CBSE corpus. Handle Hinglish tokens.
2. Word embedding layer — from scratch. Implement Word2Vec skip-gram with negative sampling. Train on 10,000 CBSE questions. Understand what word vectors actually mean geometrically.
3. Text classification neural net — implement backprop manually. Build a 2-layer MLP in pure NumPy. Implement cross entropy loss, softmax output, and gradient descent by hand. No autograd.
4. Attention-based classifier — then compare. Now rebuild it in PyTorch with a simple self-attention layer. Compare accuracy. Understand why attention outperforms naive averaging.
5. Active learning loop. Your model should identify which unlabeled questions it is most uncertain about and ask a human to label only those. This is how real annotation pipelines work.

Python
# Step 1: BPE Tokenizer from scratch
class BPETokenizer:
    def __init__(self, vocab_size=5000):
        self.vocab_size = vocab_size
        self.merges = {}
        self.vocab = {}

    def _get_pair_counts(self, words):
        pairs = {}
        for word, freq in words.items():
            symbols = word.split()
            for i in range(len(symbols)-1):
                pair = (symbols[i], symbols[i+1])
                pairs[pair] = pairs.get(pair, 0) + freq
        return pairs

    def train(self, corpus):
        """Learn BPE merges from CBSE question corpus"""
        words = self._init_vocab(corpus)  # character-level split
        for i in range(self.vocab_size - len(self.vocab)):
            pairs = self._get_pair_counts(words)
            if not pairs: break
            best = max(pairs, key=pairs.get)
            words = self._merge_pair(words, best)
            self.merges[best] = i
        print(f"Learned {len(self.merges)} merges")

# Step 2: Word2Vec skip-gram from scratch
import numpy as np

class Word2Vec:
    def __init__(self, vocab_size, embed_dim=100):
        self.W_in = np.random.randn(vocab_size, embed_dim) * 0.01
        self.W_out = np.random.randn(embed_dim, vocab_size) * 0.01

    def forward(self, center_id, context_ids, negative_ids):
        # Center word embedding
        h = self.W_in[center_id]  # (embed_dim,)

        # Positive: maximize dot product with context words
        pos_score = self._sigmoid(h @ self.W_out[:, context_ids])

        # Negative: minimize dot product with random words
        neg_score = self._sigmoid(-h @ self.W_out[:, negative_ids])

        loss = -np.log(pos_score + 1e-7).sum() - np.log(neg_score + 1e-7).sum()
        return loss

# Step 3: MLP classifier with manual backprop (NumPy only)
class ManualMLP:
    def __init__(self, input_dim, hidden_dim, num_classes=3):
        self.W1 = np.random.randn(input_dim, hidden_dim) * np.sqrt(2/input_dim)
        self.b1 = np.zeros(hidden_dim)
        self.W2 = np.random.randn(hidden_dim, num_classes) * np.sqrt(2/hidden_dim)
        self.b2 = np.zeros(num_classes)

    def forward(self, X):
        self.z1 = X @ self.W1 + self.b1
        self.a1 = np.maximum(0, self.z1)  # ReLU
        self.z2 = self.a1 @ self.W2 + self.b2
        exp_z = np.exp(self.z2 - self.z2.max(axis=1, keepdims=True))
        self.probs = exp_z / exp_z.sum(axis=1, keepdims=True)  # softmax
        return self.probs

    def backward(self, X, y_onehot, lr=0.01):
        m = X.shape[0]
        dz2 = (self.probs - y_onehot) / m       # cross-entropy + softmax gradient
        dW2 = self.a1.T @ dz2
        db2 = dz2.sum(axis=0)
        da1 = dz2 @ self.W2.T
        dz1 = da1 * (self.z1 > 0)               # ReLU gradient
        dW1 = X.T @ dz1
        db1 = dz1.sum(axis=0)
        # Update weights
        self.W2 -= lr * dW2; self.b2 -= lr * db2
        self.W1 -= lr * dW1; self.b1 -= lr * db1

Tech Stack

Python   NumPy (no autograd)   PyTorch   BPE tokenizer   Word2Vec   Bloom's Taxonomy labels   Active Learning

How Industry Evaluates This

🏭 Project 2: Options Volatility Surface Prediction

Predict implied volatility for Nifty options across strikes and expiries.

What You Must Build from Scratch

1. Feature engineering for options data. Build features: moneyness (K/S), time to expiry (τ), VIX, open interest, Put-Call ratio, historical realized vol. Understand why each matters financially.
2. Implement a feedforward neural net for regression — backprop by hand. Predict IV as a continuous value. Use MSE loss. Implement gradient descent manually in NumPy first. Then port to PyTorch with Adam optimizer.
3. Add arbitrage-free constraints as a custom loss. A vol surface that allows arbitrage is useless. Add a penalty term to your loss function that enforces calendar spread no-arbitrage and butterfly no-arbitrage conditions.
4. Temporal model: LSTM over rolling vol windows. Markets have memory. Implement an LSTM from scratch (all 4 gates, manual backprop through time). Feed it 30-day rolling vol windows. Compare against the static MLP.
5. Backtest a butterfly spread using model predictions. When your model predicts IV significantly different from market IV, simulate entering a butterfly spread. Measure P&L over 3 months of NSE data.

Python
# Feature engineering for Nifty options
def build_options_features(option_chain, spot_price, vix):
    features = {
        "moneyness": option_chain["strike"] / spot_price,  # K/S
        "log_moneyness": np.log(option_chain["strike"] / spot_price),
        "time_to_expiry": option_chain["days_to_expiry"] / 365,
        "sqrt_tau": np.sqrt(option_chain["days_to_expiry"] / 365),
        "vix": vix,
        "open_interest": np.log1p(option_chain["oi"]),
        "put_call_ratio": option_chain["put_oi"] / option_chain["call_oi"],
        "realized_vol_30d": compute_realized_vol(spot_price, window=30),
    }
    return features

# Arbitrage-free loss constraint
def arbitrage_free_loss(predicted_iv, strikes, expiries, lambda_arb=10.0):
    """Enforce no-arbitrage conditions in the vol surface"""
    mse_loss = F.mse_loss(predicted_iv, target_iv)

    # Calendar spread: IV must increase with time (roughly)
    total_var = predicted_iv**2 * expiries  # total variance = σ²τ
    calendar_violation = F.relu(-torch.diff(total_var, dim=1)).sum()

    # Butterfly: convexity in strike → d²C/dK² ≥ 0
    d2_dK2 = torch.diff(predicted_iv, n=2, dim=0)
    butterfly_violation = F.relu(-d2_dK2).sum()

    return mse_loss + lambda_arb * (calendar_violation + butterfly_violation)

# LSTM for temporal vol prediction — from scratch
class LSTMCell:
    """Manual LSTM with all 4 gates"""
    def __init__(self, input_dim, hidden_dim):
        scale = np.sqrt(1/(input_dim + hidden_dim))
        # Forget, Input, Cell, Output gates
        self.Wf = np.random.randn(input_dim+hidden_dim, hidden_dim) * scale
        self.Wi = np.random.randn(input_dim+hidden_dim, hidden_dim) * scale
        self.Wc = np.random.randn(input_dim+hidden_dim, hidden_dim) * scale
        self.Wo = np.random.randn(input_dim+hidden_dim, hidden_dim) * scale
        self.bf = np.zeros(hidden_dim)
        self.bi = np.zeros(hidden_dim)
        self.bc = np.zeros(hidden_dim)
        self.bo = np.zeros(hidden_dim)

    def forward(self, x, h_prev, c_prev):
        concat = np.concatenate([h_prev, x])
        f = self._sigmoid(concat @ self.Wf + self.bf)  # Forget gate
        i = self._sigmoid(concat @ self.Wi + self.bi)  # Input gate
        c_tilde = np.tanh(concat @ self.Wc + self.bc)  # Candidate
        c = f * c_prev + i * c_tilde                    # Cell state
        o = self._sigmoid(concat @ self.Wo + self.bo)  # Output gate
        h = o * np.tanh(c)                              # Hidden state
        return h, c

Tech Stack

NumPy (backprop)   PyTorch   NSE options data   LSTM from scratch   Custom loss functions   Black-Scholes (baseline)   Backtesting engine

How Industry Evaluates This

🏭 Project 3: Physics-Informed Neural Network for Nuclear Binding Energy

Replace semi-empirical mass formula with a neural net that respects shell structure.

What You Must Build from Scratch

1. Implement the SEMF as your baseline model. Code Bethe-Weizsäcker formula. Evaluate on AME2020 (Atomic Mass Evaluation) database. Calculate residuals — these are what your neural net must learn.
2. Feature engineering with nuclear structure knowledge. Features: Z, N, A, pairing term, shell distance from magic numbers (2,8,20,28,50,82,126), deformation parameter β, isospin asymmetry. Physical domain knowledge goes into features.
3. Build a physics-informed neural network (PINN) in PyTorch. Your loss = data_loss + λ × physics_loss. The physics constraint: binding energy per nucleon must be concave with A (stability condition). Implement this as a differentiable penalty.
4. Implement uncertainty quantification. Use Monte Carlo Dropout or Deep Ensembles to get confidence intervals on each prediction. For nuclei far from stability, uncertainty should be large — your model must know what it doesn't know.
5. Predict 50 unknown nuclei and compare to experiment. Mask 50 nuclei from training. Predict their binding energies. Compare to experimental values. This is exactly how a real paper's validation section works.

Python
# Semi-Empirical Mass Formula — your baseline
def semf_binding_energy(Z, N):
    """Bethe-Weizsäcker formula (MeV)"""
    A = Z + N
    # Volume, Surface, Coulomb, Asymmetry terms
    a_v, a_s, a_c, a_a = 15.67, 17.23, 0.714, 23.29
    B = (a_v * A - a_s * A**(2/3) - a_c * Z*(Z-1) / A**(1/3)
         - a_a * (N-Z)**2 / A)
    # Pairing term
    if Z % 2 == 0 and N % 2 == 0: B += 12.0 / A**0.5
    elif Z % 2 == 1 and N % 2 == 1: B -= 12.0 / A**0.5
    return B

# Physics-informed features
def nuclear_features(Z, N):
    A = Z + N
    magic = [2, 8, 20, 28, 50, 82, 126]
    return {
        "Z": Z, "N": N, "A": A,
        "isospin_asymmetry": (N-Z)/A,
        "pairing": (1 if Z%2==0 and N%2==0 else -1 if Z%2==1 and N%2==1 else 0),
        "shell_dist_Z": min(abs(Z - m) for m in magic),
        "shell_dist_N": min(abs(N - m) for m in magic),
        "deformation_beta": estimate_deformation(Z, N),
        "semf_residual": experimental_BE(Z,N) - semf_binding_energy(Z,N),
    }

# PINN for binding energy
class NuclearPINN(nn.Module):
    def __init__(self, n_features=10):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(n_features, 128), nn.ReLU(),
            nn.Linear(128, 128), nn.ReLU(),
            nn.Linear(128, 64), nn.ReLU(),
            nn.Linear(64, 1))  # Predict B/A (binding energy per nucleon)

    def forward(self, x):
        return self.net(x)

def pinn_loss(model, features, targets, A_values, lambda_phys=0.5):
    predictions = model(features).squeeze()
    data_loss = F.mse_loss(predictions, targets)

    # Physics constraint: B/A should be concave with A
    # (stability: adding nucleons shouldn't increase B/A indefinitely)
    sorted_idx = A_values.argsort()
    ba_sorted = predictions[sorted_idx]
    d2_ba = torch.diff(ba_sorted, n=2)  # Second derivative
    physics_loss = F.relu(d2_ba).sum()    # Penalize convex regions

    return data_loss + lambda_phys * physics_loss

Tech Stack

PyTorch   AME2020 database   PINN: custom loss   MC Dropout   Deep Ensembles   Shell model features   Matplotlib 3D surfaces

How Industry Evaluates This

🏭 Project 4: Medical Image Segmentation — Built from Scratch

Detect and segment tumors in chest X-rays without using any pretrained weights.

What You Must Build from Scratch

1. Implement convolution operation in NumPy — no torch.nn.Conv2d. Implement forward pass AND backprop (gradient w.r.t. input and kernel). This is the hardest mathematical step.
2. Build a U-Net architecture in PyTorch. Encoder (downsampling with max-pool), bottleneck, decoder (upsampling with skip connections). Implement each block yourself — no torchvision models. Use CheXpert or NIH Chest X-ray dataset.
3. Custom loss: Dice + Focal loss combination. Standard BCE fails on class-imbalanced medical data (tumors are tiny). Implement Dice loss for overlap quality and Focal loss for hard example mining. Combine them with a learnable λ.
4. Grad-CAM explainability — implement from scratch. Implement Gradient-weighted Class Activation Maps. Given a prediction, compute which spatial regions drove it. Overlay heatmap on the original X-ray. This is what makes medical AI trustworthy.
5. Test-time augmentation + confidence calibration. Run inference 20 times with random augmentations. Mean = final prediction. Variance = uncertainty. Implement temperature scaling to calibrate confidence scores against a validation set.

Python
# Conv2D from scratch in NumPy
def conv2d_forward(X, W, stride=1, padding=0):
    """X: (B,C_in,H,W), W: (C_out,C_in,kH,kW)"""
    if padding > 0:
        X = np.pad(X, ((0,0),(0,0),(padding,padding),(padding,padding)))
    B, C_in, H, W_ = X.shape
    C_out, _, kH, kW = W.shape
    H_out = (H - kH) // stride + 1
    W_out = (W_ - kW) // stride + 1
    out = np.zeros((B, C_out, H_out, W_out))
    for i in range(H_out):
        for j in range(W_out):
            patch = X[:, :, i*stride:i*stride+kH, j*stride:j*stride+kW]
            out[:, :, i, j] = np.tensordot(patch, W, axes=([1,2,3],[1,2,3]))
    return out

# U-Net architecture
class UNet(nn.Module):
    def __init__(self, in_ch=1, out_ch=1):
        super().__init__()
        self.enc1 = self._block(1, 64)
        self.enc2 = self._block(64, 128)
        self.enc3 = self._block(128, 256)
        self.bottleneck = self._block(256, 512)
        self.dec3 = self._block(512+256, 256)  # skip connection!
        self.dec2 = self._block(256+128, 128)
        self.dec1 = self._block(128+64, 64)
        self.final = nn.Conv2d(64, out_ch, 1)
        self.pool = nn.MaxPool2d(2)
        self.up = nn.Upsample(scale_factor=2)

    def forward(self, x):
        e1 = self.enc1(x);  e2 = self.enc2(self.pool(e1))
        e3 = self.enc3(self.pool(e2))
        b = self.bottleneck(self.pool(e3))
        d3 = self.dec3(torch.cat([self.up(b), e3], 1))
        d2 = self.dec2(torch.cat([self.up(d3), e2], 1))
        d1 = self.dec1(torch.cat([self.up(d2), e1], 1))
        return torch.sigmoid(self.final(d1))

# Dice + Focal loss
def dice_focal_loss(pred, target, alpha=0.5, gamma=2.0):
    # Dice loss: penalizes low overlap
    smooth = 1e-5
    intersection = (pred * target).sum()
    dice = 1 - (2*intersection + smooth) / (pred.sum() + target.sum() + smooth)
    # Focal loss: focuses on hard examples
    bce = F.binary_cross_entropy(pred, target, reduction='none')
    focal = ((1-pred)**gamma * target + pred**gamma * (1-target)) * bce
    return alpha * dice + (1-alpha) * focal.mean()

# Grad-CAM from scratch
def grad_cam(model, image, target_layer):
    """Compute which regions drove the model's prediction"""
    activations, gradients = {}, {}
    def save_activation(module, inp, out): activations['value'] = out
    def save_gradient(module, inp, out): gradients['value'] = out[0]
    target_layer.register_forward_hook(save_activation)
    target_layer.register_full_backward_hook(save_gradient)

    output = model(image)
    output.backward()

    weights = gradients['value'].mean(dim=[2,3], keepdim=True)  # GAP of gradients
    cam = F.relu((weights * activations['value']).sum(dim=1))
    cam = F.interpolate(cam.unsqueeze(1), size=image.shape[-2:])
    return cam / cam.max()  # Normalize [0,1]

Tech Stack

NumPy (conv2d backprop)   PyTorch   U-Net from scratch   CheXpert dataset   Dice + Focal loss   Grad-CAM   Temperature scaling

How Industry Evaluates This

🏭 Project 5: Fine-Tune a Small LLM on CBSE Curriculum with RLHF

Train a 125M parameter model that generates pedagogically correct answers for Indian students.

What You Must Build from Scratch

1. Build a GPT-2 style transformer from scratch in PyTorch. Implement: token embedding, positional encoding, multi-head self-attention (manual QKV matrices), layer norm, feed-forward block, causal masking. 6 layers, 125M params.
2. Pre-train on CBSE/NCERT corpus. Scrape NCERT PDFs (Class 6-12), past year papers, CBSE sample papers. Build a domain-specific tokenizer. Pre-train with next-token prediction. Log perplexity on a held-out set.
3. Supervised fine-tuning (SFT) on question-answer pairs. Create 5,000 (question, ideal_answer) pairs with teacher annotations. Fine-tune your pre-trained model on these. Implement LoRA from scratch — modify only low-rank weight updates.
4. Train a reward model — implement from scratch. Collect human preference data: show teachers two model answers, ask which is better pedagogically. Train a reward model (same architecture + scalar head) on these preferences using Bradley-Terry model.
5. RLHF with PPO — implement the training loop. Use the reward model to fine-tune your SFT model using Proximal Policy Optimization. Implement the clipped surrogate objective. This is exactly how ChatGPT was trained — at small scale.

Python
# Step 1: GPT-2 from scratch — 125M params
class TransformerBlock(nn.Module):
    def __init__(self, d_model=768, n_head=12, d_ff=3072):
        super().__init__()
        self.ln1 = nn.LayerNorm(d_model)
        self.attn = nn.MultiheadAttention(d_model, n_head, batch_first=True)
        self.ln2 = nn.LayerNorm(d_model)
        self.ff = nn.Sequential(nn.Linear(d_model, d_ff), nn.GELU(),
                                nn.Linear(d_ff, d_model))
    def forward(self, x, mask=None):
        x = x + self.attn(self.ln1(x), self.ln1(x), self.ln1(x), attn_mask=mask)[0]
        x = x + self.ff(self.ln2(x))
        return x

# Step 4: Reward Model with Bradley-Terry
class RewardModel(nn.Module):
    def __init__(self, base_model):
        super().__init__()
        self.backbone = base_model
        self.reward_head = nn.Linear(768, 1)  # Scalar reward

    def forward(self, input_ids):
        hidden = self.backbone(input_ids)
        return self.reward_head(hidden[:, -1])  # Last token's reward

def reward_loss(reward_chosen, reward_rejected):
    """Bradley-Terry model: P(chosen > rejected) = σ(r_c - r_r)"""
    return -torch.log(torch.sigmoid(reward_chosen - reward_rejected)).mean()

# Step 5: PPO training loop
def ppo_step(model, ref_model, reward_model, prompts, beta=0.1, clip_eps=0.2):
    # Generate responses
    responses = model.generate(prompts, max_length=256)

    # Get rewards
    rewards = reward_model(responses)

    # KL divergence penalty (prevent reward hacking)
    log_probs = model.log_prob(responses)
    ref_log_probs = ref_model.log_prob(responses)
    kl_penalty = beta * (log_probs - ref_log_probs)
    adjusted_rewards = rewards - kl_penalty

    # PPO clipped surrogate objective
    ratio = torch.exp(log_probs - old_log_probs)
    clipped = torch.clamp(ratio, 1-clip_eps, 1+clip_eps)
    loss = -torch.min(ratio * adjusted_rewards, clipped * adjusted_rewards).mean()
    return loss

Tech Stack

PyTorch   GPT-2 from scratch   LoRA implementation   NCERT corpus   Bradley-Terry reward model   PPO from scratch   Weights & Biases

How Industry Evaluates This