🎯 第17课：探索与利用（EE）

🎯 学习目标

🗂️ 课程定位

📖 核心理论

探索与利用（Explore-Exploit, EE）是推荐系统的核心权衡：利用已知最优策略获得高回报，还是探索新策略寻找更优解？多臂老虎机（Multi-Armed Bandit, MAB）是EE问题的经典框架。

📐 数学公式

💻 代码实现

import numpy as np
np.random.seed(42)
print("="*60+"\n探索与利用(EE)\n"+"="*60)
n_arms=10;n_rounds=1000
true_probs=np.random.random(n_arms)
print(f"臂数:{n_arms} 最优臂概率:{max(true_probs):.4f}")
print(f"各臂真实概率: {[f'{p:.3f}'for p in true_probs]}")
# ε-greedy
def epsilon_greedy(true_probs,n_rounds,epsilon=0.1):
    n_arms=len(true_probs)
    counts=np.zeros(n_arms);rewards=np.zeros(n_arms)
    total_reward=0;regrets=[]
    for t in range(n_rounds):
        if np.random.random()<epsilon:
            arm=np.random.randint(n_arms)
        else:
            arm=np.argmax(rewards/(counts+1e-10))
        reward=1 if np.random.random()<true_probs[arm] else 0
        counts[arm]+=1;rewards[arm]+=reward;total_reward+=reward
        regrets.append(max(true_probs)-true_probs[arm])
    return total_reward,np.cumsum(regrets),counts
# UCB
def ucb(true_probs,n_rounds,c=2):
    n_arms=len(true_probs)
    counts=np.zeros(n_arms);rewards=np.zeros(n_arms)
    total_reward=0;regrets=[]
    for t in range(n_rounds):
        if t<n_arms:arm=t
        else:
            avg=rewards/(counts+1e-10)
            ucb_vals=avg+c*np.sqrt(np.log(t+1)/(counts+1e-10))
            arm=np.argmax(ucb_vals)
        reward=1 if np.random.random()<true_probs[arm] else 0
        counts[arm]+=1;rewards[arm]+=reward;total_reward+=reward
        regrets.append(max(true_probs)-true_probs[arm])
    return total_reward,np.cumsum(regrets),counts
# Thompson Sampling
def thompson_sampling(true_probs,n_rounds):
    n_arms=len(true_probs)
    alpha=np.ones(n_arms);beta_param=np.ones(n_arms)
    total_reward=0;regrets=[]
    for t in range(n_rounds):
        samples=[np.random.beta(alpha[i],beta_param[i])for i in range(n_arms)]
        arm=np.argmax(samples)
        reward=1 if np.random.random()<true_probs[arm] else 0
        alpha[arm]+=reward;beta_param[arm]+=1-reward
        total_reward+=reward;regrets.append(max(true_probs)-true_probs[arm])
    return total_reward,np.cumsum(regrets),(alpha-1)/(alpha+beta_param-2+1e-10)
# 运行实验
r_eg,reg_eg,c_eg=epsilon_greedy(true_probs,n_rounds,0.1)
r_ucb,reg_ucb,c_ucb=ucb(true_probs,n_rounds)
r_ts,reg_ts,c_ts=thompson_sampling(true_probs,n_rounds)
print(f"\n{'方法':<20}{'总奖励':<10}{'最终累计遗憾':<15}{'最优臂次数'}")
print("-"*65)
best_arm=np.argmax(true_probs)
print(f"{'ε-greedy(ε=0.1)':<20}{r_eg:<10}{reg_eg[-1]:<15.2f}{int(c_eg[best_arm])}")
print(f"{'UCB(c=2)':<20}{r_ucb:<10}{reg_ucb[-1]:<15.2f}{int(c_ucb[best_arm])}")
print(f"{'Thompson Sampling':<20}{r_ts:<10}{reg_ts[-1]:<15.2f}{int(c_ts[best_arm])}")
# 不同ε影响
print("\nε对ε-greedy的影响:")
for eps in [0.01,0.05,0.1,0.2,0.5]:
    r,reg,_=epsilon_greedy(true_probs,n_rounds,eps)
    print(f"  ε={eps}:奖励={r} 遗憾={reg[-1]:.2f}")
# 推荐场景应用
print("\n--- 推荐场景EE应用 ---")
n_users=5;n_items_new=5;n_items_pop=15
item_scores=np.random.random(n_items_pop)*5+3  # 热门物品分数3-8
new_item_scores=np.random.random(n_items_new)*5+1  # 新物品分数1-6
print(f"热门物品平均分:{item_scores.mean():.2f} 新物品平均分:{new_item_scores.mean():.2f}")
# ε-greedy推荐
eps=0.2
rec_list=[]
for _ in range(10):
    if np.random.random()<eps:
        item=f"新物品{np.random.randint(n_items_new)}"
    else:
        item=f"热门{np.argmax(item_scores)}"
    rec_list.append(item)
print(f"ε-greedy推荐: {rec_list}")
# UCB推荐
all_scores=np.concatenate([item_scores,new_item_scores])
counts=np.ones(len(all_scores))
for i in range(len(item_scores)):counts[i]=10
ucb_vals=all_scores/counts+2*np.sqrt(np.log(100)/counts)
print(f"UCB推荐Top-5: {np.argsort(-ucb_vals)[:5]}")

🔴 运行结果

✅ 验证通过

🔍 代码解读

💡 关键洞察

📝 本章小结

🧪 练习

📚 延伸阅读

💼 面试考点

🏭 工业案例