Activation Functions Deep Dive

Step 1: Why Do We Need Activation Functions?

\text{Without activation: } f(x) = W_3(W_2(W_1x)) = W_{combined}x \text{ (still linear!)}
\text{With activation: } f(x) = \sigma(W_3\sigma(W_2\sigma(W_1x))) \text{ (non-linear!)}

Step 2: ReLU: The Modern Standard

f(x) = \max(0, x) = \begin{cases} x & \text{if } x > 0 \\ 0 & \text{if } x \leq 0 \end{cases}
f'(x) = \begin{cases} 1 & \text{if } x > 0 \\ 0 & \text{if } x \leq 0 \end{cases}

def relu(x):
    """ReLU activation function"""
    return max(0, x)

# Vectorized version for arrays
def relu_vectorized(x):
    return np.maximum(0, x)

# Example usage in a neural network layer
def forward_pass(inputs, weights, bias):
    z = np.dot(inputs, weights) + bias
    activation = relu_vectorized(z)
    return activation

Step 3: Sigmoid: The Classic Squashing Function

f(x) = \frac{1}{1 + e^{-x}} = \frac{e^x}{e^x + 1}
f'(x) = f(x)(1 - f(x)) = \frac{e^{-x}}{(1 + e^{-x})^2}

def sigmoid(x):
    """Sigmoid activation function"""
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
    """Derivative for backpropagation"""
    s = sigmoid(x)
    return s * (1 - s)

# Example: Binary classification output layer
def predict_probability(logits):
    """Convert network output to probability"""
    return sigmoid(logits)

# Decision boundary at 0.5
prediction = 1 if predict_probability(logits) > 0.5 else 0

Step 4: Tanh: Zero-Centered Sigmoid

f(x) = \tanh(x) = \frac{e^x - e^{-x}}{e^x + e^{-x}} = \frac{e^{2x} - 1}{e^{2x} + 1}
f'(x) = 1 - \tanh^2(x) = 1 - f(x)^2

def tanh(x):
    """Tanh activation function"""
    return np.tanh(x)

def tanh_derivative(x):
    """Derivative for backpropagation"""
    t = np.tanh(x)
    return 1 - t**2

# Example: RNN hidden state update
def rnn_step(x_t, h_prev, W_x, W_h, b):
    """One step of RNN with tanh activation"""
    z = np.dot(W_x, x_t) + np.dot(W_h, h_prev) + b
    h_t = tanh(z)  # New hidden state
    return h_t

# Comparison with sigmoid
print(f"Sigmoid(0) = {sigmoid(0):.3f}")  # 0.5 (not zero-centered)
print(f"Tanh(0) = {tanh(0):.3f}")        # 0.0 (zero-centered!)

Step 5: Leaky ReLU: Solving the Dead Neuron Problem

f(x) = \begin{cases} x & \text{if } x > 0 \\ \alpha x & \text{if } x \leq 0 \end{cases} \quad (\alpha = 0.01)
f'(x) = \begin{cases} 1 & \text{if } x > 0 \\ \alpha & \text{if } x \leq 0 \end{cases}

def leaky_relu(x, alpha=0.01):
    """Leaky ReLU activation function"""
    return np.where(x > 0, x, alpha * x)

def leaky_relu_derivative(x, alpha=0.01):
    """Derivative for backpropagation"""
    return np.where(x > 0, 1, alpha)

# Parametric ReLU (learnable alpha)
class PReLU:
    def __init__(self):
        self.alpha = 0.01  # Learnable parameter
    
    def forward(self, x):
        return np.where(x > 0, x, self.alpha * x)
    
    def backward(self, x, grad_output):
        grad_alpha = np.sum(grad_output * np.where(x > 0, 0, x))
        return grad_alpha

# Example: Preventing dead neurons
dead_neuron_input = -2.5
print(f"ReLU: {relu(dead_neuron_input)}")           # 0 (dead!)
print(f"Leaky ReLU: {leaky_relu(dead_neuron_input)}")  # -0.025 (alive!)

Step 6: Quick Comparison: Which One Should I Use?

Step 7: The Vanishing Gradient Problem (Why Sigmoid Fails in Deep Networks)

\text{Gradient after } n \text{ layers:}
\text{Sigmoid: } \prod_{i=1}^{n} f'(x_i) \approx (0.25)^n \rightarrow 0
\text{ReLU: } \prod_{i=1}^{n} f'(x_i) = 1^n = 1 \text{ (no decay!)}

Step 8: Dead Neurons in ReLU Networks

# Monitoring dead neurons during training
def count_dead_neurons(activations):
    """Count neurons that always output 0"""
    # activations shape: (batch_size, num_neurons)
    always_zero = np.all(activations == 0, axis=0)
    dead_count = np.sum(always_zero)
    dead_percentage = 100 * dead_count / activations.shape[1]
    return dead_count, dead_percentage

# Example: Check layer health
layer_output = relu(np.dot(X, weights) + bias)
dead, pct = count_dead_neurons(layer_output)
print(f"Dead neurons: {dead} ({pct:.1f}%)")

# Warning: More than 20% dead neurons is concerning!
if pct > 20:
    print("⚠️ Consider using Leaky ReLU or lowering learning rate")

Step 9: Practical Guidelines: Choosing Your Activation Function

# Complete example: Building a network with proper activations
import torch.nn as nn

class BinaryClassifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden1 = nn.Linear(10, 64)
        self.hidden2 = nn.Linear(64, 32)
        self.output = nn.Linear(32, 1)
        
        # Use ReLU for hidden layers
        self.relu = nn.ReLU()
        
        # Use Sigmoid for binary output
        self.sigmoid = nn.Sigmoid()
    
    def forward(self, x):
        x = self.relu(self.hidden1(x))  # Hidden layer 1
        x = self.relu(self.hidden2(x))  # Hidden layer 2
        x = self.sigmoid(self.output(x))  # Output probability
        return x

# Multi-class classifier
class MultiClassifier(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.hidden = nn.Linear(784, 128)
        self.output = nn.Linear(128, num_classes)
        self.relu = nn.ReLU()
        # Softmax is typically in loss function (CrossEntropyLoss)
    
    def forward(self, x):
        x = self.relu(self.hidden(x))
        x = self.output(x)  # Logits (no activation)
        return x

# Regression model
class Regressor(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden = nn.Linear(5, 32)
        self.output = nn.Linear(32, 1)
        self.relu = nn.ReLU()
    
    def forward(self, x):
        x = self.relu(self.hidden(x))
        x = self.output(x)  # No activation for regression
        return x

Step 10: What You've Learned: The Big Picture