用Q-learning算法实现自动走迷宫机器人的方法示例

import random 
import operator 

actions = ['u','r','d','l'] 
qline = {'u':1.2, 'r':-2.1, 'd':-24.5, 'l':27} 
epsilon = 0.3 # 以0.3的概率进行随机选择 

def choose_action(epsilon):     
  action = None 
   if random.uniform(0,1.0) <= epsilon: # 以某一概率 
    action = random.choice(actions)# 实现对动作的随机选择 
   else:  
     action = max(qline.items(), key=operator.itemgetter(1))[0] # 否则选择具有最大 Q 值的动作 
   return action 

range(100): 

  res += choose_action(epsilon) 

print(res) 

res = '' 

for i in range(100): 

   res += choose_action(epsilon) 

print(res) 
 ldllrrllllrlldlldllllllllllddulldlllllldllllludlldllllluudllllllulllllllllllullullllllllldlulllllrlr

from Maze import Maze 
%matplotlib inline 
%confer InlineBackend.figure_format = 'retina' 
  ## to-do: 创建迷宫并展示 
g=Maze(maze_size=(6,8), trap_number=1) 
g 
Maze of size (12, 12
)

rewards = []   
 ## 循环、随机移动机器人10次，记录下奖励 
for i in range(10): 
  res = g.move_robot(random. Choice(actions)) 
   rewards.append(res)   
 ## 输出机器人最后的位置 
print(g.sense_robot())   
## 打印迷宫，观察机器人位置 
g 

(0,9)

import random 
import operator    

 class Robot(object):  

  def __init__(self, maze, alpha=0.5, gamma=0.9, epsilon0=0.5):  

     self. Maze = maze 
     self.valid_actions = self.maze.valid_actions 

     self.state = None 
     self.action = None   

     # Set Parameters of the Learning Robot 
     self.alpha = alpha 
     self.gamma = gamma  

     self.epsilon0 = epsilon0 
     self. Epsilon = epsilon0 
     self.t = 0  

     self.Qtable = {} 
     self. Reset()  

  def. reset(self): 
     """ 
         Reset the robot 
     """ 
     self.state = self.sense_state() 
     self.create_Qtable_line(self.state)  

  def. set status(self, learning=False, testing=False): 
     """ 
     Determine whether the robot is learning its q table, or 
     executing the testing procedure. 
     """ 
     self. Learning = learning 
     self.testing = testing   

   def. update_parameter(self): 
     """ 
     Some of the paramters of the q learning robot can be altered, 
     update these parameters when necessary. 
     """ 
     if self.testing: 
       # TODO 1. No random choice when testing 
      self. Epsilon = 0 
     else: 
       # TODO 2. Update parameters when learning 
       self. Epsilon *= 0.95   

    return self. Epsilon   

   def. sense_state(self): 
     """ 
     Get the current state of the robot. In this 
     """ 

      # TODO 3. Return robot's current state 
          return self.maze.sense_robot()  

   def. create_Qtable_line(self, state): 
    """ 
     Create the qtable with the current state 
    """ 
     # TODO 4. Create qtable with current state 
     # Our qtable should be a two level dict, 
     # Qtable[state] ={'u':xx, 'd':xx, ...} 
     # If Qtable[state] already exits, then do 
     # not change it. 
     self.Qtable.setdefault(state, {a: 0.0 for a in self.valid_actions})       
   def. choose_action(self): 
     """ 
    Return an action according to given rules 
     """   
     def. is_random_exploration():  

       # TODO 5. Return whether do random choice 
       # hint: generate a random number, and compare 
       # it with epsilon 
      return random.uniform(0, 1.0) <= self. Epsilon 

     if self. Learning: 
       if is_random_exploration(): 
        # TODO 6. Return random choose aciton 
         return random. Choice(self.valid_actions) 
       else: 
         # TODO 7. Return action with highest q value 
         return max(self.Qtable[self.state].items(), key=operator.itemgetter(1))[0] 
     elif self.testing: 
       # TODO 7. choose action with highest q value 
       return max(self.Qtable[self.state].items(), key=operator.itemgetter(1))[0] 
     else: 
       # TODO 6. Return random choose aciton 
      return random. Choice(self.valid_actions)   

  def. update_Qtable(self, r, action, next_state): 
     """ 
     Update the qtable according to the given rule. 
     """ 
     if self. Learning: 
       # TODO 8. When learning, update the q table according 
       # to the given rules 
      self.Qtable[self.state][action] = (1 - self.alpha) * self.Qtable[self.state][action] + self.alpha * ( 
             r + self.gamma * max(self.Qtable[next_state].values())) 

  def. update(self): 
       """ 
     Describle the procedure what to do when update the robot. 
    Called every time in every epoch in training or testing. 
     Return current action and reward. 
     """ 
     self.state = self.sense_state() # Get the current state 
     self.create_Qtable_line(self.state) # For the state, create q table line 

    action = self.choose_action() # choose action for this state 
     reward = self.maze.move_robot(action) # move robot for given action 

    next_state = self.sense_state() # get next state 
     self.create_Qtable_line(next_state) # create q table line for next state 

     if self. Learning and not self.testing: 
       self.update_Qtable(reward, action, next_state) # update q table 
      self.update_parameter() # update parameters   

    return action, reward 
 # from Robot import Robot 
 # g=Maze(maze_size=(6,12), trap_number=2) 
 g=Maze("test_world\maze_01.txt") 
 robot = Robot(g) # 记得将 maze 变量修改为你创建迷宫的变量名 
 robot.set_status(learning=True,testing=False) 
 print(robot.update())  

 g 
（'d', -0.1）
Maze of size (12, 12)

from Runner import Runner 

g = Maze(maze_size=maze_size,trap_number=trap_number) 
r = Robot(g,alpha=alpha, epsilon0=epsilon0, gamma=gamma) 
r.set_status(learning=True) 

 runner = Runner(r, g) 
runner.run_training(epoch, display_direction=True) 
 #runner.generate_movie(filename = "final1.mp4") # 你可以注释该行代码，加快运行速度，不过你就无法观察到视频了。 
 g

 runner.plot_results()