【强化学习】用pandas 与 numpy 分别实现 q-learning, saras, saras(lambda)算法
本文作者：hhh5460
本文地址：https://www.cnblogs.com/hhh5460/p/10159331.html
特别感谢：本文的三幅图皆来自莫凡的教程 https://morvanzhou.github.io/
pandas是基于numpy的，但是两者之间的操作有区别，故在实现上述算法时的细节有出入。故记录之
几点说明：
1). 为了更好的说明问题，采用最简单的例一。
2). 分离了环境与个体，采用类编程的形式。
3). 调整了环境与个体的变量、函数的位置，使得Agent完全不需要改动！
4). 个体与环境的互动逻辑更符合实际
〇、效果图
一、pandas实现
1.q-learning
class RLQLearning(Agent): '''Agent的子类''' def __init__(self, env): super().__init__(env) def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4): '''学习''' print('q-learning算法') for _ in range(episode): s = self.env.reset() is_win = False while not is_win: a = self.observe(s, epsilon) r, s1, is_win = self.env.step(a) self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, self.env.get_valid_actions(s1)].max() - self.Q.ix[s, a]) s = s1
2.saras
class RLSaras(Agent): '''Agent的子类''' def __init__(self, env): super().__init__(env) def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4): '''学习''' print('saras算法') for _ in range(episode): s = self.env.reset() a = self.observe(s, epsilon) is_win = False while not is_win: r, s1, is_win = self.env.step(a) a1 = self.observe(s1, epsilon) self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a]) s, a = s1, a1
3.saras(lambda)
class RLSarasLambda(Agent): '''Agent的子类''' def __init__(self, env): super().__init__(env) self.E = self.Q.copy() # 复制Q table def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4): '''学习''' print('saras(lambda)算法，lambda_为衰减值') for _ in range(episode): self.E *= 0 s = self.env.reset() a = self.observe(s, epsilon) is_win = False while not is_win: r, s1, is_win = self.env.step(a) a1 = self.observe(s1, epsilon) delta = r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a] #self.E.ix[s, a] += 1 # 效果不如下两句 self.E.ix[s] *= 0 self.E.ix[s, a] = 1 for s_ in self.env.states: for a_ in self.env.actions: self.Q.ix[s_, a_] += alpha * delta * self.E.ix[s_, a_] self.E.ix[s_, a_] *= gamma * lambda_ s, a = s1, a1
4.完整代码
1 import pandas as pd 2 import random 3 import time 4 5 6 ''' 7 -o---T 8 # T 就是宝藏的位置, o 是探索者的位置 9 ''' 10 11 # 作者：hhh5460 12 # 时间：20181221 13 # 地点：Tai Zi Miao 14 15 class Env(object): 16 '''环境''' 17 def __init__(self): 18 '''初始化''' 19 self.board = list('-----T') 20 self.states = range(6) 21 self.actions = ['left', 'right'] 22 self.rewards = [0,0,0,0,0,1] 23 24 def get_valid_actions(self, state): 25 '''取当前状态下所有的合法动作''' 26 valid_actions = [] 27 if state != 5: # 除末状态（位置），皆可向右 28 valid_actions.append('right') 29 if state != 0: # 除首状态（位置），皆可向左 30 valid_actions.append('left') 31 return valid_actions 32 33 def _step(self, action): 34 '''执行动作，到达新状态''' 35 if action == 'right' and self.state != self.states[-1]: # 除末状态（位置），向右+1 36 self.state += 1 37 elif action == 'left' and self.state != self.states[0]: # 除首状态（位置），向左-1 38 self.state -= 1 39 40 def reset(self): 41 '''重置环境，返回状态0''' 42 self.board = list('-----T') 43 self.state = 0 44 self.board[self.state] = 'o' 45 print('\r ', end='') 46 print('\r{}'.format(''.join(self.board)), end='') 47 return self.state 48 49 def step(self, action, step_time=0.1): 50 '''执行动作 返回奖励、新状态、胜利标志''' 51 self.board[self.state] = '-' # 擦除旧位置'o' 52 self._step(action) # 到达新位置 53 self.board[self.state] = 'o' # 改变新位置 54 55 reward = self.rewards[self.state] # 奖励 56 is_win = [False, True][self.state == self.states[-1]] # 胜利标志 57 if is_win == True: 58 print('\r{} WIN!'.format(''.join(self.board)), end='') # 胜利，则加特写镜头 59 else: 60 print('\r{}'.format(''.join(self.board)), end='') 61 time.sleep(step_time) 62 63 return reward, self.state, is_win 64 65 66 class Agent(object): 67 '''智能体''' 68 def __init__(self, env): 69 '''初始化''' 70 # 环境 71 self.env = env 72 # 大脑 73 self.Q = pd.DataFrame(data=[[0 for _ in self.env.actions] for _ in self.env.states], 74 index=self.env.states, 75 columns=self.env.actions) 76 77 def observe(self, state, epsilon=0.4): 78 '''观察''' 79 # 根据自身所处状态，按某种策略选择相应的动作 80 if random.uniform(0,1) < epsilon: # 贪婪 81 s = self.Q.ix[state].filter(items=self.env.get_valid_actions(state)) 82 action = random.choice(s[s==s.max()].index) # 可能多个最大值！ 83 else: # 探索 84 action = random.choice(self.env.get_valid_actions(state)) 85 return action 86 87 def learn(self,*args, **kw): 88 '''学习''' 89 pass 90 91 def play(self, step_time=0.5): 92 '''玩耍''' 93 # 学有所成 94 s = self.env.reset() 95 is_win = False 96 while not is_win: 97 a = self.observe(s, epsilon=1.) # 1.，100%贪婪，即利用 98 _, s1, is_win = self.env.step(a, step_time) 99 s = s1 100 print() 101 102 class RLQLearning(Agent): 103 '''Agent的子类''' 104 def __init__(self, env): 105 super().__init__(env) 106 107 def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4): 108 '''学习''' 109 print('q-learning算法') 110 for _ in range(episode): 111 s = self.env.reset() 112 is_win = False 113 while not is_win: 114 a = self.observe(s, epsilon) 115 r, s1, is_win = self.env.step(a) 116 self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, self.env.get_valid_actions(s1)].max() - self.Q.ix[s, a]) 117 s = s1 118 119 class RLSaras(Agent): 120 '''Agent的子类''' 121 def __init__(self, env): 122 super().__init__(env) 123 124 def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4): 125 '''学习''' 126 print('saras算法') 127 for _ in range(episode): 128 s = self.env.reset() 129 a = self.observe(s, epsilon) 130 is_win = False 131 while not is_win: 132 r, s1, is_win = self.env.step(a) 133 a1 = self.observe(s1, epsilon) 134 self.Q.ix[s, a] += alpha * (r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a]) 135 s, a = s1, a1 136 137 class RLSarasLambda(Agent): 138 '''Agent的子类''' 139 def __init__(self, env): 140 super().__init__(env) 141 self.E = self.Q.copy() # 复制Q table 142 143 def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4): 144 '''学习''' 145 print('saras(lambda)算法，lambda_为衰减值') 146 for _ in range(episode): 147 self.E *= 0 148 s = self.env.reset() 149 a = self.observe(s, epsilon) 150 is_win = False 151 while not is_win: 152 r, s1, is_win = self.env.step(a) 153 a1 = self.observe(s1, epsilon) 154 delta = r + gamma * self.Q.ix[s1, a1] - self.Q.ix[s, a] 155 #self.E.ix[s, a] += 1 # 效果不如下两句 156 self.E.ix[s] *= 0 157 self.E.ix[s, a] = 1 158 for s_ in self.env.states: 159 for a_ in self.env.actions: 160 self.Q.ix[s_, a_] += alpha * delta * self.E.ix[s_, a_] 161 self.E.ix[s_, a_] *= gamma * lambda_ 162 s, a = s1, a1 163 164 165 if __name__ == '__main__': 166 env = Env() # 环境 167 168 agent = RLQLearning(env) # 个体 169 agent.learn(episode=13) # 先学 170 agent.play() # 再玩 171 172 agent2 = RLSaras(env) # 个体2 173 agent2.learn(episode=13) # 先学 174 agent2.play() # 再玩 175 176 agent3 = RLSarasLambda(env) # 个体3 177 agent3.learn(episode=13) # 先学 178 agent3.play() # 再玩
二、numpy实现
1.q-learning
2.saras
3.saras(lambda)
4.完整代码
1 import numpy as np 2 import time 3 4 5 ''' 6 -o---T 7 # T 就是宝藏的位置, o 是探索者的位置 8 ''' 9 10 # 作者：hhh5460 11 # 时间：20181221 12 # 地点：Tai Zi Miao 13 14 class Env(object): 15 '''环境''' 16 def __init__(self): 17 '''初始化''' 18 self.board = list('-----T') 19 self.states = range(6) 20 self.actions = ['left', 'right'] # 索引[0,1] 21 self.rewards = [0,0,0,0,0,1] 22 23 def get_valid_actions(self, state): 24 '''取当前状态下所有的合法动作（索引）''' 25 valid_actions = [] 26 if state != self.states[0]: # 除首状态（位置），皆可向左 27 valid_actions.append(self.actions.index('left')) 28 if state != self.states[-1]: # 除末状态（位置），皆可向右 29 valid_actions.append(self.actions.index('right')) 30 return valid_actions 31 32 def _step(self, action): 33 '''执行动作（索引），到达新状态''' 34 if self.actions[action] == 'left' and self.state > self.states[0]: # 除首状态（位置），向左-1 35 self.state = self.state - 1 36 elif self.actions[action] == 'right' and self.state < self.states[-1]: # 除末状态（位置），向右+1 37 self.state = self.state + 1 38 39 def reset(self): 40 '''重置环境，返回状态0''' 41 self.board = list('-----T') 42 self.state = 0 43 self.board[self.state] = 'o' 44 print('\r ', end='') 45 print('\r{}'.format(''.join(self.board)), end='') 46 return self.state 47 48 def step(self, action, step_time=0.1): 49 '''执行动作 返回奖励、新状态、胜利标志''' 50 self.board[self.state] = '-' # 擦除旧位置'o' 51 self._step(action) # 到达新位置 52 self.board[self.state] = 'o' # 改变新位置 53 54 reward = self.rewards[self.state] # 奖励 55 is_win = [False, True][self.state == self.states[-1]] # 胜利标志 56 if is_win == True: 57 print('\r{} WIN!'.format(''.join(self.board)), end='') # 胜利，则加特写镜头 58 else: 59 print('\r{}'.format(''.join(self.board)), end='') 60 time.sleep(step_time) 61 62 return reward, self.state, is_win 63 64 65 class Agent(object): 66 '''智能体''' 67 def __init__(self, env): 68 '''初始化''' 69 # 环境 70 self.env = env 71 # 大脑 72 self.Q = np.zeros((len(self.env.states), len(self.env.actions)), dtype=np.float32) 73 74 def observe(self, state, epsilon=0.8): 75 '''观察''' 76 # 根据自身所处状态，按某种策略选择相应的动作（索引） 77 valid_actions = self.env.get_valid_actions(state) 78 arr = self.Q[state, valid_actions] 79 if (np.random.uniform() > epsilon 80 or arr.max() == 0 81 or len(arr[arr==arr.max()]) > 1): 82 action = np.random.choice(valid_actions) # 探索 83 else: 84 action = self.Q[state].argmax() # 利用 85 return action 86 87 def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.8): 88 '''学习''' 89 pass 90 91 def play(self, step_time=0.5): 92 '''玩耍''' 93 # 学有所成 94 s = self.env.reset() 95 is_win = False 96 while not is_win: 97 a = self.observe(s, epsilon=1.) # 1.，100%贪婪，即利用 98 _, s1, is_win = self.env.step(a, step_time) 99 s = s1 100 print() 101 102 class RLQLearning(Agent): 103 '''智能体''' 104 def __init__(self, env): 105 '''初始化''' 106 super().__init__(env) 107 108 def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.8): 109 '''学习''' 110 print('q-learning算法') 111 for _ in range(episode): 112 s = self.env.reset() 113 is_win = False 114 while not is_win: 115 a = self.observe(s, epsilon) 116 r, s1, is_win = self.env.step(a) 117 self.Q[s, a] += alpha * (r + gamma * self.Q[s1, self.env.get_valid_actions(s1)].max() - self.Q[s, a]) 118 s = s1 119 120 class RLSaras(Agent): 121 '''Agent的子类''' 122 def __init__(self, env): 123 super().__init__(env) 124 125 def learn(self, alpha=0.01, gamma=0.9, episode=100, epsilon=0.4): 126 '''学习''' 127 print('saras算法') 128 for _ in range(episode): 129 s = self.env.reset() 130 a = self.observe(s, epsilon) 131 is_win = False 132 while not is_win: 133 r, s1, is_win = self.env.step(a) 134 a1 = self.observe(s1, epsilon) 135 self.Q[s, a] += alpha * (r + gamma * self.Q[s1, a1] - self.Q[s, a]) 136 s, a = s1, a1 137 138 class RLSarasLambda(Agent): 139 '''Agent的子类''' 140 def __init__(self, env): 141 super().__init__(env) 142 self.E = self.Q.copy() # 复制Q table 143 144 def learn(self, alpha=0.01, gamma=0.9, lambda_=0.9, episode=100, epsilon=0.4): 145 '''学习''' 146 print('saras(lambda)算法，lambda_为衰减值') 147 for _ in range(episode): 148 self.E *= 0 149 s = self.env.reset() 150 a = self.observe(s, epsilon) 151 is_win = False 152 while not is_win: 153 r, s1, is_win = self.env.step(a) 154 a1 = self.observe(s1, epsilon) 155 delta = r + gamma * self.Q[s1, a1] - self.Q[s, a] 156 #self.E.ix[s, a] += 1 # 效果不如下两句 157 self.E[s] *= 0 158 self.E[s, a] = 1 159 for s_ in self.env.states: 160 for a_ in range(len(self.env.actions)): # 遍历动作索引！！ 161 self.Q[s_, a_] += alpha * delta * self.E[s_, a_] 162 self.E[s_, a_] *= gamma * lambda_ 163 s, a = s1, a1 164 165 if __name__ == '__main__': 166 env = Env() # 环境 167 agent = RLQLearning(env) # 个体 168 agent.learn(episode=13) # 先学 169 agent.play() # 再玩 170 171 agent2 = RLSaras(env) # 个体2 172 agent2.learn(episode=13) # 先学 173 agent2.play() # 再玩 174 175 agent3 = RLSarasLambda(env) # 个体3 176 agent3.learn(episode=13) # 先学 177 agent3.play() # 再玩
