第3课：激活函数

📖 本课概述

激活函数赋予神经网络非线性能力。没有激活函数，再深的网络也只等价于一个线性变换。本课深入分析各种激活函数的数学性质、优缺点和适用场景。

⚡ 一、为什么需要激活函数

1.1 线性瓶颈

激活函数引入非线性，使得神经网络可以逼近任意复杂的函数（万能逼近定理）。

1.2 万能逼近定理

具有非线性激活函数的单隐层前馈网络，只要隐层足够宽，可以逼近任意连续函数到任意精度。但实践中，深度比宽度更有效。

📊 二、常见激活函数详解

2.1 Sigmoid

优点：输出可解释为概率，适合二分类输出层

缺点：①梯度消失（两侧梯度趋0）②输出非零中心③指数运算慢

2.2 Tanh

优点：零中心化，收敛比Sigmoid快

缺点：仍有梯度消失问题

2.3 ReLU (Rectified Linear Unit)

优点：①计算极快②正区间梯度恒为1，缓解梯度消失③产生稀疏激活

缺点：①神经元死亡（Dead ReLU）②输出非零中心

2.4 Leaky ReLU / ELU / GELU

2.5 函数值与导数对比

import torch
import torch.nn as nn
import math

# 常见激活函数对比
activations = {
    "Sigmoid": lambda x: 1/(1+math.exp(-x)),
    "Tanh": lambda x: math.tanh(x),
    "ReLU": lambda x: max(0, x),
    "LeakyReLU(0.01)": lambda x: x if x>0 else 0.01*x,
    "ELU(1.0)": lambda x: x if x>0 else math.exp(x)-1,
    "GELU": lambda x: 0.5*x*(1+math.tanh(math.sqrt(2/math.pi)*(x+0.044715*x**3))),
}

# 函数值
print("=== 激活函数值对比 ===")
print(f"{'x':>6} | " + " | ".join(f"{name:>14}" for name in activations))
print("-" * 100)
for x_val in [-3, -1, -0.5, 0, 0.5, 1, 3]:
    vals = [f"{fn(x_val):>14.6f}" for fn in activations.values()]
    print(f"{x_val:>6} | " + " | ".join(vals))

# 梯度值（导数）
print("\n=== 激活函数导数对比 ===")
print(f"{'x':>6} | " + " | ".join(f"{name:>14}" for name in activations))
print("-" * 100)
for x_val in [-3, -1, -0.5, 0, 0.5, 1, 3]:
    x_t = torch.tensor(float(x_val), requires_grad=True)
    vals = []
    for name in activations:
        if name == "Sigmoid":
            y = torch.sigmoid(x_t)
        elif name == "Tanh":
            y = torch.tanh(x_t)
        elif name == "ReLU":
            y = torch.relu(x_t)
        elif name.startswith("LeakyReLU"):
            y = torch.nn.functional.leaky_relu(x_t, 0.01)
        elif name.startswith("ELU"):
            y = torch.nn.functional.elu(x_t, 1.0)
        elif name == "GELU":
            y = torch.nn.functional.gelu(x_t)
        y.backward()
        vals.append(f"{x_t.grad.item():>14.6f}")
        x_t = torch.tensor(float(x_val), requires_grad=True)
    print(f"{x_val:>6} | " + " | ".join(vals))

🔬 三、激活函数对训练的影响

import torch
import torch.nn as nn

# 激活函数对训练的影响实验
torch.manual_seed(42)
X = torch.randn(500, 20)
y = (X[:, 0] + X[:, 1] > 0).float().unsqueeze(1)

results = {}
for act_name, act_fn in [
    ("Sigmoid", nn.Sigmoid()),
    ("Tanh", nn.Tanh()),
    ("ReLU", nn.ReLU()),
    ("LeakyReLU", nn.LeakyReLU(0.01)),
    ("GELU", nn.GELU()),
]:
    model = nn.Sequential(
        nn.Linear(20, 64), act_fn,
        nn.Linear(64, 32), act_fn if act_name != "GELU" else nn.GELU(),
        nn.Linear(32, 1), nn.Sigmoid()
    )
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
    loss_fn = nn.BCELoss()
    
    losses = []
    for epoch in range(200):
        optimizer.zero_grad()
        output = model(X)
        loss = loss_fn(output, y)
        loss.backward()
        optimizer.step()
        losses.append(loss.item())
    
    results[act_name] = losses
    print(f"{act_name:>12}: 初始loss={losses[0]:.4f}, 最终loss={losses[-1]:.4f}, 改善={losses[0]-losses[-1]:.4f}")

# 梯度流分析
print("\n=== 梯度流分析（10层网络）===")
for act_name, act_cls in [
    ("Sigmoid", nn.Sigmoid),
    ("ReLU", nn.ReLU),
    ("Tanh", nn.Tanh),
]:
    layers = [nn.Linear(10, 10)]
    for _ in range(10):
        layers.extend([act_cls(), nn.Linear(10, 10)])
    model = nn.Sequential(*layers)
    
    x = torch.randn(2, 10)
    y = model(x).sum()
    y.backward()
    
    grad_norms = []
    for i, layer in enumerate(model):
        if isinstance(layer, nn.Linear) and layer.weight.grad is not None:
            grad_norms.append(layer.weight.grad.norm().item())
    
    print(f"{act_name:>8}: 梯度范数变化 = ", end="")
    for g in grad_norms[:5]:
        print(f"{g:.6f} → ", end="")
    print(f"... → {grad_norms[-1]:.6f}")

📋 四、选择指南

🔬 五、高级激活函数

5.1 Swish / SiLU

5.2 Mish

5.3 激活函数的选择策略

📊 六、激活函数发展时间线

📖 七、延伸阅读

📝 练习