从零构建神经网络，理解梯度如何在层间流动
神经网络（Neural Network）是深度学习的基石，灵感来自大脑神经元连接。每一层对输入做线性变换+非线性激活，层层堆叠后可逼近任意复杂函数。
| 网络类型 | 结构 | 擅长 | 典型应用 |
|---|---|---|---|
| 全连接(MLP) | 稠密连接 | 表格数据 | 分类/回归 |
| CNN | 卷积+池化 | 图像 | 图像分类/检测 |
| RNN/LSTM | 循环连接 | 序列 | 时间序列/NLP |
| Transformer | 自注意力 | 通用 | LLM/多模态 |
| GAN | 生成+判别 | 生成 | 图像生成 |
神经网络的前向传播就是层层做线性变换+非线性激活：
| 激活函数 | 公式 | 值域 | 优点 | 缺点 |
|---|---|---|---|---|
| ReLU | max(0, z) | [0, +∞) | 计算快，缓解梯度消失 | Dead ReLU |
| Sigmoid | 1/(1+e⁻ᶻ) | (0, 1) | 输出概率 | 梯度消失 |
| Tanh | (eᶻ-e⁻ᶻ)/(eᶻ+e⁻ᶻ) | (-1, 1) | 零中心 | 梯度消失 |
| LeakyReLU | max(αz, z) | (-∞, +∞) | 无Dead ReLU | α需调参 |
| GELU | z·Φ(z) | (-0.17, +∞) | Transformer标配 | 计算稍慢 |
反向传播（Backpropagation）是训练神经网络的核心算法，利用链式法则高效计算损失对所有参数的梯度。
import numpy as np
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
np.random.seed(42)
# 生成月牙形非线性数据（线性分类器搞不定）
X, y = make_moons(n_samples=500, noise=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
class NeuralNetwork:
"""手写多层感知器（全连接神经网络）"""
def __init__(self, layer_sizes, lr=0.1):
self.layers = len(layer_sizes) - 1
self.lr = lr
self.weights = []
self.biases = []
# He初始化（ReLU推荐）
for i in range(self.layers):
w = np.random.randn(layer_sizes[i], layer_sizes[i+1]) * np.sqrt(2.0/layer_sizes[i])
b = np.zeros((1, layer_sizes[i+1]))
self.weights.append(w)
self.biases.append(b)
def relu(self, z):
return np.maximum(0, z)
def relu_deriv(self, z):
return (z > 0).astype(float)
def sigmoid(self, z):
z = np.clip(z, -500, 500)
return 1 / (1 + np.exp(-z))
def forward(self, X):
"""前向传播：X → 隐藏层(ReLU) → 输出层(Sigmoid)"""
self.z_list = []
self.a_list = [X]
a = X
for i in range(self.layers):
z = a @ self.weights[i] + self.biases[i]
self.z_list.append(z)
a = self.relu(z) if i < self.layers - 1 else self.sigmoid(z)
self.a_list.append(a)
return a
def backward(self, y):
"""反向传播：计算所有层的梯度"""
m = len(y)
y = y.reshape(-1, 1)
# 输出层: δ = ŷ - y (交叉熵+sigmoid)
dz = self.a_list[-1] - y
dw = self.a_list[-2].T @ dz / m
db = np.mean(dz, axis=0, keepdims=True)
self.d_weights = [None] * self.layers
self.d_biases = [None] * self.layers
self.d_weights[-1] = dw
self.d_biases[-1] = db
# 隐藏层: 反向传播δ
da = dz @ self.weights[-1].T
for i in range(self.layers - 2, -1, -1):
dz = da * self.relu_deriv(self.z_list[i])
dw = self.a_list[i].T @ dz / m
db = np.mean(dz, axis=0, keepdims=True)
self.d_weights[i] = dw
self.d_biases[i] = db
if i > 0:
da = dz @ self.weights[i].T
def update(self):
"""SGD参数更新"""
for i in range(self.layers):
self.weights[i] -= self.lr * self.d_weights[i]
self.biases[i] -= self.lr * self.d_biases[i]
def train(self, X, y, epochs=1000):
losses = []
for epoch in range(epochs):
pred = self.forward(X)
y_r = y.reshape(-1, 1)
loss = -np.mean(y_r*np.log(pred+1e-8) + (1-y_r)*np.log(1-pred+1e-8))
losses.append(loss)
self.backward(y)
self.update()
if epoch % 200 == 0:
print(f"Epoch {epoch}: loss={loss:.4f}")
return losses
def predict(self, X):
return (self.forward(X) >= 0.5).astype(int).flatten()
# 训练: 2 → 16 → 8 → 1
nn = NeuralNetwork([2, 16, 8, 1], lr=0.1)
losses = nn.train(X_train, y_train, epochs=1000)
print(f"\n训练集准确率: {accuracy_score(y_train, nn.predict(X_train)):.4f}")
print(f"测试集准确率: {accuracy_score(y_test, nn.predict(X_test)):.4f}")
print(f"损失下降: {losses[0]:.4f} → {losses[-1]:.4f}")反向传播容易写错！用数值梯度验证解析梯度的正确性：
# 梯度数值验证
nn.forward(X_train[:5])
nn.backward(y_train[:5])
eps = 1e-4
w0 = nn.weights[0][0, 0]
# 计算数值梯度
nn.weights[0][0, 0] = w0 + eps
loss_p = -np.mean(y_train[:5].reshape(-1,1)*np.log(nn.forward(X_train[:5])+1e-8)
+ (1-y_train[:5].reshape(-1,1))*np.log(1-nn.forward(X_train[:5])+1e-8))
nn.weights[0][0, 0] = w0 - eps
loss_m = -np.mean(y_train[:5].reshape(-1,1)*np.log(nn.forward(X_train[:5])+1e-8)
+ (1-y_train[:5].reshape(-1,1))*np.log(1-nn.forward(X_train[:5])+1e-8))
nn.weights[0][0, 0] = w0 # 恢复
num_grad = (loss_p - loss_m) / (2 * eps)
analytic_grad = nn.d_weights[0][0, 0]
ratio = abs(num_grad - analytic_grad) / (abs(num_grad) + abs(analytic_grad) + 1e-8)
print(f"数值梯度: {num_grad:.8f}")
print(f"解析梯度: {analytic_grad:.8f}")
print(f"相对误差: {ratio:.10f}")
print(f"验证结果: {'通过 ✅' if ratio < 1e-4 else '未通过 ❌'}")# 不同初始化对比
def init_comparison():
n_in, n_out = 100, 50
# 零初始化（❌ 最差）
w_zero = np.zeros((n_in, n_out))
# 随机初始化（⚠️ 可能梯度消失）
w_random = np.random.randn(n_in, n_out) * 0.01
# Xavier初始化（✅ sigmoid/tanh推荐）
w_xavier = np.random.randn(n_in, n_out) * np.sqrt(2.0/(n_in+n_out))
# He初始化（✅ ReLU推荐）
w_he = np.random.randn(n_in, n_out) * np.sqrt(2.0/n_in)
for name, w in [("零初始化", w_zero), ("随机(0.01)", w_random),
("Xavier", w_xavier), ("He", w_he)]:
print(f"{name}: 均值={w.mean():.4f}, 标准差={w.std():.4f}")
init_comparison()# L2正则化（权重衰减）
# 梯度更新时加入: W ← W - η*(dW + λ*W)
# Dropout（随机失活）
def dropout_forward(a, keep_prob=0.8):
mask = (np.random.rand(*a.shape) < keep_prob) / keep_prob
return a * mask, mask
def dropout_backward(da, mask):
return da * mask
# Batch Normalization
# 归一化每层输出 → 稳定训练 → 允许更大学习率
# BN(x) = γ * (x - μ) / √(σ² + ε) + β