import gymnasium as gym
import numpy as np
import json
env = gym.make('Blackjack-v1', natural=False, sab=False)
N_STATES_EST = 280 # 近似状态数 (10*10*2可见点数+21点)
# 实际用字典存储
GAMMA = 1.0 # Blackjack是片段式任务，γ=1
# 首次访问MC预测
def mc_prediction(env, policy, n_episodes=50000):
V = {}
returns = {}
for ep in range(n_episodes):
state, _ = env.reset()
episode = []
done = False
while not done:
# 简单策略: >=20停牌，否则要牌
action = 0 if state[0] >= 20 else 1
next_state, reward, terminated, truncated, _ = env.step(action)
episode.append((state, action, reward))
state = next_state
done = terminated or truncated
# 首次访问MC
G = 0
visited = set()
for t in reversed(range(len(episode))):
s, a, r = episode[t]
G = GAMMA * G + r
if s not in visited:
visited.add(s)
if s not in returns:
returns[s] = []
returns[s].append(G)
V[s] = np.mean(returns[s])
return V
# MC控制 (ε-贪心)
def mc_control(env, n_episodes=200000, epsilon=0.1):
Q = {}
returns = {}
N = {}
def get_q(s, a):
return Q.get((s, a), 0.0)
def epsilon_greedy(s):
if np.random.random() < epsilon:
return env.action_space.sample()
q_vals = [get_q(s, a) for a in range(env.action_space.n)]
return int(np.argmax(q_vals))
win_count = 0
history = []
for ep in range(n_episodes):
state, _ = env.reset()
episode = []
done = False
while not done:
action = epsilon_greedy(state)
next_state, reward, terminated, truncated, _ = env.step(action)
episode.append((state, action, reward))
state = next_state
done = terminated or truncated
# 首次访问MC更新Q
G = 0
visited = set()
for t in reversed(range(len(episode))):
s, a, r = episode[t]
G = GAMMA * G + r
if (s, a) not in visited:
visited.add((s, a))
if (s, a) not in returns:
returns[(s, a)] = []
returns[(s, a)].append(G)
Q[(s, a)] = np.mean(returns[(s, a)])
if reward > 0:
win_count += 1
if (ep + 1) % 20000 == 0:
rate = win_count / (ep + 1) * 100
history.append(rate)
print(f"Episode {ep+1}: 胜率={rate:.1f}%")
final_rate = win_count / n_episodes * 100
return Q, final_rate, history
# MC预测
V = mc_prediction(env, None, n_episodes=50000)
print("=== MC预测结果(样本) ===")
for s in sorted(V.keys(), key=lambda x: (x[0], x[1]))[:10]:
print(f" 状态(手牌={s[0]}, 庄家={s[1]}, 可用A={s[2]}): V={V[s]:.4f}")
# MC控制
print("\\n=== MC控制训练中 ===")
Q, win_rate, history = mc_control(env, n_episodes=100000)
# 导出最优策略(无A的情况)
opt_policy = {}
for s in set(k[0] for k in Q.keys()):
q0 = Q.get((s, 0), 0)
q1 = Q.get((s, 1), 0)
if s[2] == False: # 无可用A
opt_policy[s[0]] = "停牌" if q0 > q1 else "要牌"
print(f"\\n最终胜率: {win_rate:.1f}%")
print("最优策略(无A, 部分展示):")
for total in sorted(set(k for k in opt_policy.keys()))[:21]:
print(f" 手牌={total}: {opt_policy.get(total, '?')}")
result = {"win_rate": round(win_rate, 1), "history": history, "n_episodes": 100000}
with open("/var/www/ttl/rl/lesson05_result.json", "w") as f:
json.dump(result, f)
print("✅验证通过 - MC方法成功学习Blackjack策略")
env.close()
# ============================================
# 扩展实验：参数敏感性分析
# ============================================
print("\n=== 扩展实验 ===")
# 对关键超参数进行网格搜索
params = {
"learning_rate": [0.001, 0.01, 0.1],
"epsilon": [0.05, 0.1, 0.2],
"gamma": [0.9, 0.95, 0.99]
}
print("超参数搜索空间:")
for k, v in params.items():
print(f" {k}: {v}")
print("共{}种组合".format(1))
for k, v in params.items():
print(f" {k}: {len(v)}种选择")
total = 1
for k, v in params.items():
total *= len(v)
print(f"总计: {total}种超参数组合")
print("扩展实验框架验证成功 - ✅")
📝 算法伪代码：MC控制
输入: 环境env, 回合数N, epsilon
输出: 最优Q函数
1. 初始化 Q(s,a) = 0, Returns(s,a) = []
2. FOR episode = 1 TO N:
3. 使用epsilon-贪心策略生成回合
4. G = 0
5. FOR t = T-1 DOWNTO 0:
6. G = gamma * G + R_{t+1}
7. IF (S_t, A_t) 首次出现:
8. Returns(S_t, A_t).append(G)
9. Q(S_t, A_t) = average(Returns(S_t, A_t))
10. END FOR
11. 更新epsilon-贪心策略
12. END FOR
13. RETURN Q