From 912e3d42cb6b8021b015db3b3086e67ea7ac2a83 Mon Sep 17 00:00:00 2001 From: Brandon Rozek Date: Fri, 13 Sep 2019 19:48:24 -0400 Subject: [PATCH 1/3] Added new OpenAI Baseline Wrappers --- rltorch/env/wrappers.py | 132 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/rltorch/env/wrappers.py b/rltorch/env/wrappers.py index 065b931..2bd5b97 100644 --- a/rltorch/env/wrappers.py +++ b/rltorch/env/wrappers.py @@ -3,6 +3,111 @@ import torch from gym import spaces import cv2 from collections import deque +import numpy as np + +class EpisodicLifeEnv(gym.Wrapper): + def __init__(self, env=None): + """Make end-of-life == end-of-episode, but only reset on true game over. + Done by DeepMind for the DQN and co. since it helps value estimation. + """ + super(EpisodicLifeEnv, self).__init__(env) + self.lives = 0 + self.was_real_done = True + self.was_real_reset = False + + def step(self, action): + obs, reward, done, info = self.env.step(action) + self.was_real_done = done + # check current lives, make loss of life terminal, + # then update lives to handle bonus lives + lives = self.env.unwrapped.ale.lives() + if lives < self.lives and lives > 0: + # for Qbert somtimes we stay in lives == 0 condtion for a few frames + # so its important to keep lives > 0, so that we only reset once + # the environment advertises done. + done = True + self.lives = lives + return obs, reward, done, info + + def reset(self): + """Reset only when lives are exhausted. + This way all states are still reachable even though lives are episodic, + and the learner need not know about any of this behind-the-scenes. + """ + if self.was_real_done: + obs = self.env.reset() + self.was_real_reset = True + else: + # no-op step to advance from terminal/lost life state + obs, _, _, _ = self.env.step(0) + self.was_real_reset = False + self.lives = self.env.unwrapped.ale.lives() + return obs + + +class NoopResetEnv(gym.Wrapper): + def __init__(self, env=None, noop_max=30): + """Sample initial states by taking random number of no-ops on reset. + No-op is assumed to be action 0. + """ + super(NoopResetEnv, self).__init__(env) + self.noop_max = noop_max + self.override_num_noops = None + assert env.unwrapped.get_action_meanings()[0] == 'NOOP' + + def step(self, action): + return self.env.step(action) + + def reset(self): + """ Do no-op action for a number of steps in [1, noop_max].""" + self.env.reset() + if self.override_num_noops is not None: + noops = self.override_num_noops + else: + noops = np.random.randint(1, self.noop_max + 1) + assert noops > 0 + obs = None + for _ in range(noops): + obs, _, done, _ = self.env.step(0) + if done: + obs = self.env.reset() + return obs + + +class MaxAndSkipEnv(gym.Wrapper): + def __init__(self, env=None, skip=4): + """Return only every `skip`-th frame""" + super(MaxAndSkipEnv, self).__init__(env) + # most recent raw observations (for max pooling across time steps) + self._obs_buffer = deque(maxlen=2) + self._skip = skip + + def step(self, action): + total_reward = 0.0 + done = None + for _ in range(self._skip): + obs, reward, done, info = self.env.step(action) + self._obs_buffer.append(obs) + total_reward += reward + if done: + break + + max_frame = np.max(np.stack(self._obs_buffer), axis=0) + + return max_frame, total_reward, done, info + + def reset(self): + """Clear past frame buffer and init. to first obs. from inner env.""" + self._obs_buffer.clear() + obs = self.env.reset() + self._obs_buffer.append(obs) + return obs + +class ClippedRewardsWrapper(gym.RewardWrapper): + def reward(self, reward): + """Change all the positive rewards to 1, negative to -1 and keep zero.""" + return np.sign(reward) + # Mostly derived from OpenAI baselines class FireResetEnv(gym.Wrapper): @@ -126,4 +231,29 @@ class TorchWrap(gym.Wrapper): def _convert(self, frame): frame = torch.from_numpy(frame).unsqueeze(0).float() - return frame \ No newline at end of file + return frame + + + +class ProcessFrame84(gym.ObservationWrapper): + def __init__(self, env=None): + super(ProcessFrame84, self).__init__(env) + self.observation_space = spaces.Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8) + + def observation(self, obs): + return ProcessFrame84.process(obs) + + @staticmethod + def process(frame): + if frame.size == 210 * 160 * 3: + img = np.reshape(frame, [210, 160, 3]).astype(np.float32) + elif frame.size == 250 * 160 * 3: + img = np.reshape(frame, [250, 160, 3]).astype(np.float32) + else: + assert False, "Unknown resolution." + img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114 + resized_screen = cv2.resize(img, (84, 110), interpolation=cv2.INTER_AREA) + x_t = resized_screen[18:102, :] + x_t = np.reshape(x_t, [84, 84]) + return x_t.astype(np.uint8) + From dcf7cce30d8cb69ccea8dd9aa66dc2cb97a27edd Mon Sep 17 00:00:00 2001 From: Brandon Rozek Date: Fri, 13 Sep 2019 19:48:51 -0400 Subject: [PATCH 2/3] Flush out print text so I can keep track of the rewards by SSHing in --- rltorch/env/simulate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rltorch/env/simulate.py b/rltorch/env/simulate.py index ee3b16d..a792ccb 100644 --- a/rltorch/env/simulate.py +++ b/rltorch/env/simulate.py @@ -17,7 +17,7 @@ def simulateEnvEps(env, actor, config, total_episodes = 1, memory = None, logger if episode % config['print_stat_n_eps'] == 0: print("episode: {}/{}, score: {}" - .format(episode, total_episodes, episode_reward)) + .format(episode, total_episodes, episode_reward), flush=True) if logger is not None: logger.append(name + '/EpisodeReward', episode_reward) @@ -51,7 +51,7 @@ class EnvironmentRunSync(): if done: if self.episode_num % self.config['print_stat_n_eps'] == 0: print("episode: {}/{}, score: {}" - .format(self.episode_num, self.config['total_training_episodes'], self.episode_reward)) + .format(self.episode_num, self.config['total_training_episodes'], self.episode_reward), flush=True) if self.logwriter is not None: logger.append(self.name + '/EpisodeReward', self.episode_reward) @@ -92,10 +92,10 @@ class EnvironmentEpisodeSync(): if self.episode_num % self.config['print_stat_n_eps'] == 0: print("episode: {}/{}, score: {}" - .format(self.episode_num, self.config['total_training_episodes'], episodeReward)) + .format(self.episode_num, self.config['total_training_episodes'], episodeReward), flush=True) if self.logwriter is not None: logger.append(self.name + '/EpisodeReward', episodeReward) self.logwriter.write(logger) - self.episode_num += 1 \ No newline at end of file + self.episode_num += 1 From 7aa698c349e84740c846017d7b81665328d557db Mon Sep 17 00:00:00 2001 From: Brandon Rozek Date: Fri, 13 Sep 2019 19:49:04 -0400 Subject: [PATCH 3/3] Added save and load functionality --- rltorch/agents/DQNAgent.py | 11 ++++++++++- rltorch/agents/QEPAgent.py | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/rltorch/agents/DQNAgent.py b/rltorch/agents/DQNAgent.py index 3f20b52..6f47c05 100644 --- a/rltorch/agents/DQNAgent.py +++ b/rltorch/agents/DQNAgent.py @@ -4,6 +4,7 @@ import torch import torch.nn.functional as F from copy import deepcopy import numpy as np +from pathlib import Path class DQNAgent: def __init__(self, net , memory, config, target_net = None, logger = None): @@ -12,6 +13,12 @@ class DQNAgent: self.memory = memory self.config = deepcopy(config) self.logger = logger + def save(self, file_location): + torch.save(self.net.model.state_dict(), file_location) + def load(self, file_location): + self.net.model.state_dict(torch.load(file_location)) + self.net.model.to(self.net.device) + self.target_net.sync() def learn(self, logger = None): if len(self.memory) < self.config['batch_size']: @@ -57,8 +64,10 @@ class DQNAgent: # If we're sampling by TD error, multiply loss by a importance weight which helps decrease overfitting if (isinstance(self.memory, M.PrioritizedReplayMemory)): - loss = (torch.as_tensor(importance_weights, device = self.net.device) * ((obtained_values - expected_values)**2).squeeze(1)).mean() + # loss = (torch.as_tensor(importance_weights, device = self.net.device) * F.smooth_l1_loss(obtained_values, expected_values, reduction = 'none').squeeze(1)).mean() + loss = (torch.as_tensor(importance_weights, device = self.net.device) * ((obtained_values - expected_values)**2).squeeze(1)).mean() else: + # loss = F.smooth_l1_loss(obtained_values, expected_values) loss = F.mse_loss(obtained_values, expected_values) if self.logger is not None: diff --git a/rltorch/agents/QEPAgent.py b/rltorch/agents/QEPAgent.py index 6040e28..a21d886 100644 --- a/rltorch/agents/QEPAgent.py +++ b/rltorch/agents/QEPAgent.py @@ -3,6 +3,7 @@ import collections import numpy as np import torch from torch.distributions import Categorical +import torch.nn.functional as F import rltorch import rltorch.memory as M @@ -20,6 +21,19 @@ class QEPAgent: self.config = deepcopy(config) self.logger = logger self.policy_skip = 4 + + def save(self, file_location): + torch.save({ + 'policy': self.policy_net.model.state_dict(), + 'value': self.value_net.model.state_dict() + }, file_location) + def load(self, file_location): + checkpoint = torch.load(file_location) + self.value_net.model.state_dict(checkpoint['value']) + self.value_net.model.to(self.value_net.device) + self.policy_net.model.state_dict(checkpoint['policy']) + self.policy_net.model.to(self.policy_net.device) + self.target_net.sync() def fitness(self, policy_net, value_net, state_batch): batch_size = len(state_batch) @@ -37,7 +51,7 @@ class QEPAgent: value_importance = 1 - entropy_importance # entropy_loss = (action_probabilities * torch.log2(action_probabilities)).sum(1) # Standard entropy loss from information theory - entropy_loss = (action_probabilities - torch.tensor(1 / action_size).repeat(len(state_batch), action_size)).abs().sum(1) + entropy_loss = (action_probabilities - torch.tensor(1 / action_size, device = state_batch.device).repeat(len(state_batch), action_size)).abs().sum(1) return (entropy_importance * entropy_loss - value_importance * obtained_values).mean().item() @@ -116,6 +130,6 @@ class QEPAgent: self.policy_net.calc_gradients(self.target_value_net, state_batch) else: self.policy_net.calc_gradients(self.value_net, state_batch) - # self.policy_net.clamp_gradients() + ##### self.policy_net.clamp_gradients() self.policy_net.step()