🖼️ 第12课：多模态推荐

🎯 学习目标

🗂️ 课程定位

📖 核心理论

多模态推荐融合文本、图像、音频等多种媒体信息来增强推荐效果。在短视频、电商等场景中，物品往往包含丰富的多模态信息，单一模态难以全面刻画物品特征。

📐 数学公式

💻 代码实现

import numpy as np
np.random.seed(42)
print("="*60+"\n多模态推荐系统\n"+"="*60)
n_items=25;n_users=20
# 模拟多模态特征
text_dim=8;image_dim=8;audio_dim=4
text_features=np.random.randn(n_items,text_dim)
image_features=np.random.randn(n_items,image_dim)
audio_features=np.random.randn(n_items,audio_dim)
print(f"物品数:{n_items} 用户数:{n_users}")
print(f"文本特征:{text_dim}D 图像特征:{image_dim}D 音频特征:{audio_dim}D")
# 早期融合(Concatenation)
print("\n--- 早期融合 ---")
concat_features=np.concatenate([text_features,image_features,audio_features],axis=1)
print(f"融合后维度:{concat_features.shape[1]}")
# PCA降维
from numpy.linalg import svd
X_centered=concat_features-concat_features.mean(axis=0)
U,S,Vt=svd(X_centered,full_matrices=False)
pca_features=U[:,:10]@np.diag(S[:10])
print(f"PCA降维后:{pca_features.shape[1]}D")
# 晚期融合(分别建模后融合)
print("\n--- 晚期融合 ---")
# 为每种模态训练简单模型
interactions=np.random.random((n_users,n_items))>0.7
# 文本模型得分
w_text=np.random.randn(text_dim)
text_scores=text_features@w_text
# 图像模型得分
w_image=np.random.randn(image_dim)
image_scores=image_features@w_image
# 音频模型得分
w_audio=np.random.randn(audio_dim)
audio_scores=audio_features@w_audio
# 加权融合
alpha,beta,gamma=0.4,0.4,0.2
fusion_scores=alpha*text_scores+beta*image_scores+gamma*audio_scores
print(f"融合权重:文本={alpha} 图像={beta} 音频={gamma}")
# 注意力融合
print("\n--- 注意力融合 ---")
# 模态注意力权重
modality_embs=np.random.randn(3,4)
query=np.random.randn(4)
attn_weights=np.exp(modality_embs@query)
attn_weights=attn_weights/attn_weights.sum()
print(f"注意力权重:文本={attn_weights[0]:.3f} 图像={attn_weights[1]:.3f} 音频={attn_weights[2]:.3f}")
attn_scores=attn_weights[0]*text_scores+attn_weights[1]*image_scores+attn_weights[2]*audio_scores
# 推荐评估
print("\n--- 推荐评估 ---")
# 用交互数据评估
precisions=[];recalls=[]
for u in range(n_users):
    pos=np.where(interactions[u])[0]
    if len(pos)==0:continue
    hidden=np.random.choice(pos)
    train_pos=[i for i in pos if i!=hidden]
    # 基于训练物品的多模态特征构建用户画像
    user_text=text_features[train_pos].mean(axis=0)if train_pos else np.zeros(text_dim)
    user_image=image_features[train_pos].mean(axis=0)if train_pos else np.zeros(image_dim)
    user_audio=audio_features[train_pos].mean(axis=0)if train_pos else np.zeros(audio_dim)
    # 早期融合推荐
    user_concat=np.concatenate([user_text,user_image,user_audio])
    scores=pca_features@user_concat[:10]
    scores[train_pos]=-999
    top5=np.argsort(scores)[-5:]
    if hidden in top5:precisions.append(1/5)
    else:precisions.append(0)
    if hidden in top5:recalls.append(1)
    else:recalls.append(0)
print(f"早期融合: Precision@5={np.mean(precisions):.4f} Recall@5={np.mean(recalls):.4f}")
# 晚期融合推荐
precisions2=[];recalls2=[]
for u in range(n_users):
    pos=np.where(interactions[u])[0]
    if len(pos)==0:continue
    hidden=np.random.choice(pos)
    train_pos=[i for i in pos if i!=hidden]
    user_t=text_features[train_pos].mean(axis=0)if train_pos else np.zeros(text_dim)
    user_i=image_features[train_pos].mean(axis=0)if train_pos else np.zeros(image_dim)
    user_a=audio_features[train_pos].mean(axis=0)if train_pos else np.zeros(audio_dim)
    t_scores=text_features@user_t
    i_scores=image_features@user_i
    a_scores=audio_features@user_a
    fused=attn_weights[0]*t_scores+attn_weights[1]*i_scores+attn_weights[2]*a_scores
    fused[train_pos]=-999
    top5=np.argsort(fused)[-5:]
    if hidden in top5:precisions2.append(1/5)
    else:precisions2.append(0)
    if hidden in top5:recalls2.append(1)
    else:recalls2.append(0)
print(f"注意力融合: Precision@5={np.mean(precisions2):.4f} Recall@5={np.mean(recalls2):.4f}")
# 跨模态对比
print("\n跨模态相似度:")
modality_sims=[]
for i in range(min(5,n_items)):
    t_sim=np.dot(text_features[i],text_features[i])/np.linalg.norm(text_features[i])**2
    i_sim=np.dot(image_features[i],image_features[i])/np.linalg.norm(image_features[i])**2
    ti_cross=np.dot(text_features[i],image_features[i])/(np.linalg.norm(text_features[i])*np.linalg.norm(image_features[i]))
    print(f"  物品{i}: 文本自相似={t_sim:.3f} 图像自相似={i_sim:.3f} 跨模态={ti_cross:.3f}")

🔴 运行结果

✅ 验证通过

🔍 代码解读

💡 关键洞察

📝 本章小结

🧪 练习

📚 延伸阅读

💼 面试考点

🏭 工业案例