集成学习的力量：三个臭皮匠，顶个诸葛亮
随机森林（Random Forest）是Breiman在2001年提出的集成学习方法，通过构建多棵决策树并汇总预测结果，显著提升泛化能力。
| 超参数 | 默认值 | 影响 |
|---|---|---|
| n_estimators | 100 | 树的数量，越多越好但边际递减 |
| max_depth | None(不限制) | 越深越容易过拟合 |
| max_features | √M(分类)/M/3(回归) | 控制多样性 |
| min_samples_split | 2 | 越大越保守 |
| min_samples_leaf | 1 | 越大越平滑 |
| bootstrap | True | 关闭=使用全量数据 |
import numpy as np
from collections import Counter
np.random.seed(42)
# 生成4特征2分类数据（含噪声特征）
n = 300
X = np.random.randn(n, 4)
y = np.where(X[:, 0] * 2 + X[:, 1] * 1.5 > 0, 1, 0)
# 添加20个噪声标签
noise_idx = np.random.choice(n, 20, replace=False)
y[noise_idx] = 1 - y[noise_idx]
# 信息熵
def entropy(y):
counts = Counter(y)
total = len(y)
return -sum((c/total)*np.log2(c/total) for c in counts.values() if c>0)
def info_gain(y, left_mask, right_mask):
n = len(y)
nl, nr = left_mask.sum(), right_mask.sum()
if nl == 0 or nr == 0: return 0
return entropy(y) - (nl/n*entropy(y[left_mask]) + nr/n*entropy(y[right_mask]))
class DecisionNode:
def __init__(self, feature=None, threshold=None, left=None, right=None, value=None):
self.feature, self.threshold = feature, threshold
self.left, self.right, self.value = left, right, value
def build_tree(X, y, depth=0, max_depth=5, min_samples=5, max_features=None):
if len(set(y)) == 1 or depth >= max_depth or len(y) < min_samples:
return DecisionNode(value=Counter(y).most_common(1)[0][0])
n_features = X.shape[1]
feat_candidates = np.random.choice(n_features,
max_features or n_features, replace=False)
best_gain, best_feat, best_thr = 0, 0, 0
for feat in feat_candidates:
for thr in np.percentile(X[:, feat], np.arange(10, 100, 10)):
left_mask = X[:, feat] <= thr
right_mask = ~left_mask
gain = info_gain(y, left_mask, right_mask)
if gain > best_gain:
best_gain, best_feat, best_thr = gain, feat, thr
if best_gain <= 0:
return DecisionNode(value=Counter(y).most_common(1)[0][0])
left_mask = X[:, best_feat] <= best_thr
return DecisionNode(
feature=best_feat, threshold=best_thr,
left=build_tree(X[left_mask], y[left_mask], depth+1, max_depth, min_samples, max_features),
right=build_tree(X[~left_mask], y[~left_mask], depth+1, max_depth, min_samples, max_features)
)
def predict_tree(node, x):
if node.value is not None: return node.value
return predict_tree(node.left if x[node.feature] <= node.threshold else node.right, x)class RandomForest:
def __init__(self, n_trees=20, max_depth=6, max_features='sqrt'):
self.n_trees = n_trees
self.max_depth = max_depth
self.max_features = max_features
self.trees = []
self.feature_importances_ = None
def fit(self, X, y):
n = len(X)
mf = int(np.sqrt(X.shape[1])) if self.max_features == 'sqrt' else X.shape[1]
self.trees = []
for i in range(self.n_trees):
# Bootstrap采样
idx = np.random.choice(n, n, replace=True)
tree = build_tree(X[idx], y[idx], max_depth=self.max_depth, max_features=mf)
self.trees.append(tree)
self._calc_importance(X, y)
def _calc_importance(self, X, y):
"""基于排列的特征重要性"""
baseline_acc = np.mean(self.predict(X) == y)
importances = np.zeros(X.shape[1])
for feat in range(X.shape[1]):
X_perm = X.copy()
X_perm[:, feat] = np.random.permutation(X_perm[:, feat])
perm_acc = np.mean(self.predict(X_perm) == y)
importances[feat] = baseline_acc - perm_acc
total = importances.sum()
self.feature_importances_ = importances / total if total > 0 else importances
def predict(self, X):
all_preds = np.array([[predict_tree(tree, x) for tree in self.trees] for x in X])
return np.array([Counter(row).most_common(1)[0][0] for row in all_preds])
# 训练
split = int(0.7 * n)
rf = RandomForest(n_trees=20, max_depth=6)
rf.fit(X[:split], y[:split])
y_pred = rf.predict(X[split:])
accuracy = np.mean(y_pred == y[split:])
print(f"随机森林准确率: {accuracy:.4f}") # 0.8556
print(f"\n特征重要性排序:")
for i, imp in enumerate(rf.feature_importances_):
bar = '█' * int(imp * 40)
print(f" 特征{i}: {imp:.4f} {bar}")
# 特征0: 0.5243 ████████████████████ (最重要)
# 特征1: 0.3479 █████████████ (次重要)
# 特征2: 0.0648 ██ (噪声)
# 特征3: 0.0630 ██ (噪声)Bootstrap采样中，每棵树约63.2%的样本被选中，剩余36.8%为OOB（Out-of-Bag）样本，可直接用于评估。
def oob_score(rf, X, y):
"""计算OOB分数"""
n = len(X)
oob_preds = [[] for _ in range(n)]
oob_mask = [[] for _ in range(n)]
for tree_idx, tree in enumerate(rf.trees):
# 重建每棵树的bootstrap索引
np.random.seed(42 + tree_idx)
idx = np.random.choice(n, n, replace=True)
in_bag = set(idx)
for i in range(n):
if i not in in_bag:
oob_preds[i].append(predict_tree(tree, X[i]))
correct = 0
total = 0
for i in range(n):
if len(oob_preds[i]) > 0:
pred = Counter(oob_preds[i]).most_common(1)[0][0]
if pred == y[i]:
correct += 1
total += 1
return correct / total if total > 0 else 0
print(f"OOB准确率: {oob_score(rf, X, y):.4f}")# 基尼重要性（基于分裂频率）
def gini_importance(rf, X, y):
importances = np.zeros(X.shape[1])
def count_splits(node, feat_counts):
if 'feature' in dir(node) and node.feature is not None:
feat_counts[node.feature] += 1
count_splits(node.left, feat_counts)
count_splits(node.right, feat_counts)
for tree in rf.trees:
counts = np.zeros(X.shape[1])
count_splits(tree, counts)
importances += counts
return importances / importances.sum()
# 三种重要性对比
print("特征重要性对比:")
print(f" 排列重要性: {rf.feature_importances_.round(4)}")
gini_imp = gini_importance(rf, X, y)
print(f" 基尼重要性: {gini_imp.round(4)}")| 方法 | 策略 | 优势 | 劣势 |
|---|---|---|---|
| Bagging | 并行多模型投票 | 降低方差 | 不降偏差 |
| 随机森林 | Bagging+特征随机 | 更多样性，更快 | 树多时内存大 |
| AdaBoost | 序列加权重采样 | 降低偏差 | 对噪声敏感 |
| GBDT | 序列残差拟合 | 精度最高 | 需调参，易过拟合 |
| XGBoost | GBDT+正则化 | 精度+速度 | 超参多 |