DQN算法
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
1 # coding=utf-8 2 3 import sys 4 import os 5 import gym 6 import pylab 7 import random 8 import numpy as np 9 from collections import deque 10 from keras.layers import Dense 11 from keras.optimizers import Adam 12 from keras.models import Sequential 13 from keras.callbacks import Callback 14 import matplotlib.pyplot as plt 15 16 class LossHistory(Callback): 17 def on_train_begin(self, logs={}): 18 self.losses = [] 19 20 def on_batch_end(self, batch, logs={}): 21 self.losses.append(logs.get('loss')) 22 23 class DQNAgent: 24 def __init__(self, state_size, action_size, 25 render=False, load_model=False, 26 gamma=0.99, learning_rate=0.001, 27 epsilon=1.0, epsilon_decay=0.999, 28 epsilon_min=0.01, batch_size=64, 29 train_start=100, memory_size=2000, 30 ): 31 # env 的状态空间的设置 32 self.state_size = state_size 33 self.action_size = action_size 34 35 # render表示是否打开gym下的动画展示,打开的话运行速度会大幅减慢 36 self.render = render 37 # load_model=True表示从文件中加载model 38 self.load_model = load_model 39 40 # 接下来的都是DQN的超参 41 self.gamma = gamma 42 self.learning_rate = learning_rate 43 self.epsilon = epsilon 44 self.epsilon_decay = epsilon_decay # e-贪心的e值随着步骤不断减小的比例 45 self.epsilon_min = epsilon_min # e-贪心的e值减小到一个阈值不再减小 46 47 self.train_start = train_start 48 self.batch_size = batch_size 49 50 # 记忆数据存储模块 51 self.memory = deque(maxlen=memory_size) 52 53 # 初始化模型 54 self.model = self.build_model() 55 56 # 记录损失值 57 self.history = LossHistory() 58 self.losses_list = [] 59 60 def build_model(self, units=128): 61 model = Sequential() 62 model.add(Dense(units, input_dim=self.state_size, 63 activation='sigmoid', kernel_initializer='he_uniform')) 64 model.add(Dense(units, activation='sigmoid', 65 kernel_initializer='he_uniform')) 66 model.add(Dense(self.action_size, activation='linear', 67 kernel_initializer='he_uniform')) 68 model.summary() 69 70 model.compile(loss='mean_squared_error', optimizer=Adam(lr=self.learning_rate)) 71 return model 72 73 def choose_action(self, state): 74 if np.random.rand() <= self.epsilon: 75 return random.randrange(self.action_size) 76 else: 77 q_value = self.model.predict(state) 78 return np.argmax(q_value[0]) 79 80 def add_memory(self, state, action, reward, done, next_state): 81 self.memory.append((state, action, reward, done, next_state)) 82 if self.epsilon > self.epsilon_min: 83 self.epsilon *= self.epsilon_decay 84 85 def train_model(self): 86 if len(self.memory) < self.train_start: 87 return 88 batch_size = min(self.batch_size, len(self.memory)) 89 min_batch = random.sample(self.memory, batch_size) 90 91 update_input = np.zeros((batch_size, self.state_size)) 92 update_target = np.zeros((batch_size, self.state_size)) 93 action, reward, done = [], [], [] 94 95 for i in range(batch_size): 96 update_input[i] = min_batch[i][0] 97 action.append(min_batch[i][1]) 98 reward.append(min_batch[i][2]) 99 done.append(min_batch[i][3]) 100 update_target[i] = min_batch[i][4] 101 102 target = self.model.predict(update_input, batch_size=batch_size) 103 target_val = self.model.predict(update_target, batch_size=batch_size) 104 105 for i in range(self.batch_size): 106 if done[i]: 107 target[i][action[i]] = reward[i] 108 else: 109 target[i][action[i]] = reward[i] + self.gamma * np.amax(target_val[i]) 110 111 self.model.fit(update_input, target, batch_size=batch_size, epochs=1, verbose=0, callbacks=[self.history]) 112 self.losses_list.append(self.history.losses[0]) 113 114 115 def draw_score_plot(scores, filename='graph.png'): 116 fig = plt.figure() 117 ax1 = fig.add_subplot(1, 1, 1) 118 ax1.set_title('mean score') 119 ax1.plot(range(len(scores)), scores, color='blue') 120 plt.savefig(filename) 121 122 123 def draw_plot(scores, losses, filename='graph.png'): 124 fig = plt.figure() 125 ax1 = fig.add_subplot(1, 2, 1) 126 ax1.set_title('mean score') 127 ax1.plot(range(len(scores)), scores, color='blue') 128 129 130 ax2 = fig.add_subplot(1, 2, 2) 131 ax2.set_title('mean loss-reward') 132 ax2.plot(range(len(losses)), losses, color='blue') 133 plt.savefig(filename)