🎯 推荐系统 — 协同过滤+矩阵分解

📖 推荐系统概述

推荐系统是互联网最赚钱的AI应用——亚马逊35%营收、Netflix 80%观看内容、抖音全靠推荐。核心问题：给用户推荐他们可能喜欢但还没发现的物品。

🔢 协同过滤数学

协同过滤的核心是利用群体智慧：相似用户的评分模式相似，可以互相预测。

import numpy as np

def cosine_sim(v1, v2):
    """余弦相似度（处理NaN缺失值）"""
    valid = ~(np.isnan(v1) | np.isnan(v2))
    if valid.sum() < 2:
        return 0
    v1v, v2v = v1[valid], v2[valid]
    norm = np.linalg.norm(v1v) * np.linalg.norm(v2v)
    return np.dot(v1v, v2v) / norm if norm > 0 else 0

# 构建用户相似度矩阵
user_sim = np.zeros((n_users, n_users))
for i in range(n_users):
    for j in range(n_users):
        user_sim[i, j] = cosine_sim(ratings[i], ratings[j])

# 基于Top-K邻居预测
def predict_user_cf(ratings, user_sim, k=5):
    pred = np.full_like(ratings, np.nan, dtype=float)
    for u in range(n_users):
        neighbors = np.argsort(user_sim[u])[::-1][1:k+1]
        for i in range(n_items):
            if np.isnan(ratings[u, i]):
                sim_sum, weighted_sum = 0, 0
                for n in neighbors:
                    if not np.isnan(ratings[n, i]):
                        sim_sum += abs(user_sim[u, n])
                        weighted_sum += user_sim[u, n] * (ratings[n, i] - np.nanmean(ratings[n]))
                pred[u, i] = np.nanmean(ratings[u]) + (weighted_sum / sim_sum if sim_sum > 0 else 0)
    return np.clip(pred, 1, 5)

🔬 矩阵分解 (SGD)

矩阵分解是Netflix Prize大赛的冠军方法，将用户-物品评分矩阵分解为用户因子矩阵 × 物品因子矩阵，解决稀疏性问题。

class MatrixFactorization:
    """矩阵分解推荐引擎 (SGD优化)"""
    def __init__(self, n_users, n_items, n_factors=5, lr=0.01, reg=0.1):
        self.P = np.random.randn(n_users, n_factors) * 0.1  # 用户因子
        self.Q = np.random.randn(n_items, n_factors) * 0.1  # 物品因子
        self.bu = np.zeros(n_users)    # 用户偏置
        self.bi = np.zeros(n_items)    # 物品偏置
        self.mu = 0                     # 全局均值
        self.lr = lr
        self.reg = reg
    
    def fit(self, ratings, epochs=200):
        self.mu = np.nanmean(ratings)
        # 收集已知评分
        known = [(u, i, ratings[u, i]) for u in range(ratings.shape[0]) 
                 for i in range(ratings.shape[1]) if not np.isnan(ratings[u, i])]
        
        for epoch in range(epochs):
            np.random.shuffle(known)
            total_loss = 0
            for u, i, r in known:
                # 前向: 预测评分
                pred = self.mu + self.bu[u] + self.bi[i] + self.P[u] @ self.Q[i]
                err = r - pred
                total_loss += err**2
                
                # SGD更新
                self.bu[u] += self.lr * (err - self.reg * self.bu[u])
                self.bi[i] += self.lr * (err - self.reg * self.bi[i])
                Pu = self.P[u].copy()
                self.P[u] += self.lr * (err * self.Q[i] - self.reg * self.P[u])
                self.Q[i] += self.lr * (err * Pu - self.reg * self.Q[i])
            
            if epoch % 50 == 0:
                print(f"Epoch {epoch}: RMSE={np.sqrt(total_loss / len(known)):.4f}")
    
    def predict_all(self):
        return self.mu + self.bu[:, None] + self.bi[None, :] + self.P @ self.Q.T

📊 训练与评估

# 训练矩阵分解
mf = MatrixFactorization(n_users=20, n_items=15, n_factors=5, lr=0.01, reg=0.1)
mf.fit(ratings, epochs=200)

# 评估
from sklearn.metrics import mean_squared_error
known_mask = ~np.isnan(ratings)
true_vals = ratings[known_mask]
mf_pred_all = np.clip(mf.predict_all(), 1, 5)
mf_vals = mf_pred_all[known_mask]
mf_rmse = np.sqrt(mean_squared_error(true_vals, mf_vals))
mf_mae = np.mean(np.abs(true_vals - mf_vals))
print(f"矩阵分解RMSE={mf_rmse:.4f}, MAE={mf_mae:.4f}")

# 为用户0推荐Top-5
u = 0
pred_u = mf_pred_all[u]
seen = ~np.isnan(ratings[u])
unseen_items = np.where(~seen)[0]
recommend = sorted(unseen_items, key=lambda i: -pred_u[i])[:5]
print(f"\n用户0推荐: 物品{recommend}")
print(f"预测评分: {[f'{pred_u[i]:.2f}' for i in recommend]}")

📐 2024-2025 推荐系统前沿

• 双塔模型：用户塔+物品塔，DSSM/YouTube DNN，工业标配
• DeepFM/DCN：FM+深度学习融合，特征交叉自动化
• 序列推荐：SASRec/BERT4Rec，建模用户行为序列
• 图神经网络(GNN)：LightGCN，用户-物品二部图建模
• 多目标优化：同时优化点击率+转化率+停留时长
• 多模态推荐：文本+图像+视频特征融合
• 大模型+推荐：LLM做推荐解释、零样本推荐
• 因果推荐：去除位置偏差/流行度偏差，更公平推荐

🧮 推荐系统架构选择

📝 课后练习

1. 用MovieLens 100K数据集做推荐，对比User-CF和Item-CF
2. 实现ALS(交替最小二乘)矩阵分解，对比SGD的效果
3. 实现隐式反馈推荐（点击/浏览而非评分）
4. 用DeepFM做点击率预估（CTR预测）
5. 实现推荐多样性指标（ILS/coverage），评估推荐质量

🔧 推荐系统评估指标