import math, random
class SimpleRL:
def __init__(self):
self.g = 9.81
self.dt = 0.02
class CartPoleEnv:
def __init__(self):
self.g = 9.81
self.mass_cart = 1.0
self.mass_pole = 0.1
self.length = 0.5
self.dt = 0.02
self.theta = 0.05
self.theta_dot = 0
self.x = 0
self.x_dot = 0
def step(self, force):
costh = math.cos(self.theta)
sinth = math.sin(self.theta)
total_mass = self.mass_cart + self.mass_pole
temp = (force + self.mass_pole * self.length * self.theta_dot**2 * sinth) / total_mass
theta_acc = (self.g * sinth - costh * temp) / (self.length * (4/3 - self.mass_pole * costh**2 / total_mass))
x_acc = temp - self.mass_pole * self.length * theta_acc * costh / total_mass
self.x += self.x_dot * self.dt
self.x_dot += x_acc * self.dt
self.theta += self.theta_dot * self.dt
self.theta_dot += theta_acc * self.dt
done = abs(self.theta) > 0.5 or abs(self.x) > 2.4
reward = 1.0 if not done else 0.0
return (self.x, self.theta), reward, done
def reset(self):
self.theta = random.gauss(0, 0.05)
self.theta_dot = 0
self.x = 0
self.x_dot = 0
return (self.x, self.theta)
def train_q_learning(self, n_episodes=500):
env = self.CartPoleEnv()
n_bins = 10
q_table = {}
lr = 0.1
gamma = 0.99
epsilon = 1.0
epsilon_decay = 0.995
rewards_history = []
def discretize(state):
x_bin = max(0, min(n_bins-1, int((state[0] + 2.4) / 4.8 * n_bins)))
th_bin = max(0, min(n_bins-1, int((state[1] + 0.5) / 1.0 * n_bins)))
return (x_bin, th_bin)
for ep in range(n_episodes):
state = env.reset()
d_state = discretize(state)
total_reward = 0
done = False
while not done and total_reward < 200:
if random.random() < epsilon:
action = random.choice([-10, 10])
else:
q_left = q_table.get((d_state, -10), 0)
q_right = q_table.get((d_state, 10), 0)
action = -10 if q_left > q_right else 10
next_state, reward, done = env.step(action)
d_next = discretize(next_state)
total_reward += reward
best_next = max(q_table.get((d_next, -10), 0), q_table.get((d_next, 10), 0))
current_q = q_table.get((d_state, action), 0)
q_table[(d_state, action)] = current_q + lr * (reward + gamma * best_next - current_q)
d_state = d_next
epsilon *= epsilon_decay
rewards_history.append(total_reward)
return rewards_history
rl = SimpleRL()
print("=" * 55)
print(" Deep RL Introduction Simulation")
print("=" * 55)
# Train Q-learning on cart-pole
print("\n [Q-Learning Training on Cart-Pole]")
rewards = rl.train_q_learning(n_episodes=500)
for i in range(0, len(rewards), 50):
avg = sum(rewards[max(0,i-10):i+1]) / min(i+1, 11)
print(f" Episode {i:3d}: reward={rewards[i]:.0f}, avg(10)={avg:.1f}")
# Policy evaluation
print(f"\n [Final Policy Evaluation]")
env = rl.SimpleRL.CartPoleEnv()
total_rewards = []
for trial in range(10):
state = env.reset()
done = False
r = 0
while not done and r < 200:
# Use learned policy (simplified)
force = -10 if state[1] < 0 else 10
state, reward, done = env.step(force)
r += reward
total_rewards.append(r)
print(f" Avg reward: {sum(total_rewards)/len(total_rewards):.1f}, "
f"min: {min(total_rewards):.0f}, max: {max(total_rewards):.0f}")
print()
print(" OK - RL introduction simulation complete")
仿真结果：
ERROR: Traceback (most recent call last):
File "<string>", line 99, in <module>
AttributeError: 'SimpleRL' object has no attribute 'SimpleRL'
Error in sys.excepthook:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 228, in partial_apport_excepthook
return apport_excepthook(binary, exc_type, exc_obj, exc_tb)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 114,
RL的核心框架：Agent在Environment中学习最优Policy
max π E[Σγt·rt]
s.t. st+1 ~ P(s|st, at)
at ~ π(a|st)
关键概念：状态s、动作a、奖励r、策略π、折扣因子γ