第5课：优化器：SGD与Adam

📖 本课概述

优化器决定了模型参数如何更新。从最朴素的SGD到自适应的Adam，不同的优化器在收敛速度、稳定性和泛化能力上各有特点。本课深入理解优化器的数学原理和实践选择。

🏃 一、随机梯度下降 (SGD)

1.1 基本SGD

1.2 动量 (Momentum)

动量帮助SGD：①加速沿正确方向前进②抑制震荡③可能冲出浅的局部最优。

1.3 Nesterov动量

⚙️ 二、自适应学习率优化器

2.1 Adam

2.2 AdamW

AdamW将权重衰减与梯度更新解耦，避免了Adam+L2正则化的缺陷：

🔬 三、优化器对比实验

import torch
import torch.nn as nn
import numpy as np

# 对比不同优化器
torch.manual_seed(42)

# 创建一个有挑战性的损失景观
X = torch.randn(500, 10)
y = (X[:, 0]**2 + X[:, 1] * 2 - X[:, 2] > 0).float().unsqueeze(1)

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(10, 64), nn.ReLU(),
            nn.Linear(64, 32), nn.ReLU(),
            nn.Linear(32, 1), nn.Sigmoid()
        )
    def forward(self, x):
        return self.net(x)

optimizers_config = {
    "SGD(lr=0.1)": lambda p: torch.optim.SGD(p, lr=0.1),
    "SGD+Momentum(0.9)": lambda p: torch.optim.SGD(p, lr=0.1, momentum=0.9),
    "SGD+Nesterov": lambda p: torch.optim.SGD(p, lr=0.1, momentum=0.9, nesterov=True),
    "Adam(lr=0.001)": lambda p: torch.optim.Adam(p, lr=0.001),
    "AdamW(lr=0.001)": lambda p: torch.optim.AdamW(p, lr=0.001, weight_decay=0.01),
    "RMSprop(lr=0.01)": lambda p: torch.optim.RMSprop(p, lr=0.01),
}

loss_fn = nn.BCELoss()
results = {}

for name, opt_fn in optimizers_config.items():
    torch.manual_seed(42)
    model = Net()
    optimizer = opt_fn(model.parameters())
    
    losses = []
    for epoch in range(300):
        optimizer.zero_grad()
        output = model(X)
        loss = loss_fn(output, y)
        loss.backward()
        optimizer.step()
        if epoch % 50 == 0 or epoch == 299:
            losses.append((epoch, loss.item()))
    
    results[name] = losses
    print(f"{name:>25}: " + " → ".join([f"e{e}={l:.4f}" for e,l in losses]))

📊 四、Adam内部状态与学习率调度

import torch
import torch.nn as nn
import math

# Adam优化器内部状态可视化
torch.manual_seed(42)
model = nn.Linear(5, 1)
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

X = torch.randn(32, 5)
y = torch.randn(32, 1)
loss_fn = nn.MSELoss()

print("=== Adam优化器内部状态 ===")
print("参数: weight")
for step in range(10):
    optimizer.zero_grad()
    loss = loss_fn(model(X), y)
    loss.backward()
    optimizer.step()
    
    # 获取Adam内部状态
    state = optimizer.state_dict()['state']
    param = list(model.parameters())[0]
    
    if state:
        step_val = state[0].get('step', 0)
        exp_avg = state[0].get('exp_avg', torch.zeros_like(param))
        exp_avg_sq = state[0].get('max_exp_avg_sq' if 'max_exp_avg_sq' in state[0] else 'exp_avg_sq', torch.zeros_like(param))
        
        print(f"Step {step}: loss={loss.item():.6f}, grad_norm={param.grad.norm().item():.6f}, "
              f"m_norm={exp_avg.norm().item():.6f}, v_norm={exp_avg_sq.norm().item():.6f}")
    else:
        print(f"Step {step}: loss={loss.item():.6f}")

# 学习率调度器对比
print("\n=== 学习率调度 ===")
model2 = nn.Linear(5, 1)
base_lr = 0.1

schedulers = {
    "StepLR(step=5,γ=0.5)": lambda opt: torch.optim.lr_scheduler.StepLR(opt, step_size=5, gamma=0.5),
    "CosineAnnealing": lambda opt: torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=20),
    "ExponentialLR(γ=0.9)": lambda opt: torch.optim.lr_scheduler.ExponentialLR(opt, gamma=0.9),
}

for name, sched_fn in schedulers.items():
    opt = torch.optim.SGD(model2.parameters(), lr=base_lr)
    sched = sched_fn(opt)
    lrs = [base_lr]
    for _ in range(19):
        sched.step()
        lrs.append(opt.param_groups[0]['lr'])
    print(f"{name}: " + " → ".join([f"{lr:.5f}" for lr in lrs[::4]]) + f" → {lrs[-1]:.5f}")

📋 五、优化器选择建议

🔬 六、优化器进阶

6.1 LAMB优化器

Layer-wise Adaptive Moments optimizer for Batch training，专为超大batch训练设计。

6.2 Lion优化器

2023年Google Brain发现的更简单优化器：

6.3 优化器选择决策树

📖 七、延伸阅读

📝 练习