brozek/rltorch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleaned up scripts, added more comments

Browse source
This commit is contained in:
Brandon Rozek 2019-03-04 17:09:46 -05:00
parent e42f5bba1b
commit a59f84b446
11 changed files with 103 additions and 436 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
examples/acrobot_a2c.py
View file

@ -1,5 +1,4 @@
import gym import gym
import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -9,10 +8,10 @@ import rltorch.memory as M
import rltorch.env as E import rltorch.env as E
from rltorch.action_selector import StochasticSelector from rltorch.action_selector import StochasticSelector
from tensorboardX import SummaryWriter from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
import signal
from copy import deepcopy
#
## Networks
#
class Value(nn.Module): class Value(nn.Module):
def __init__(self, state_size): def __init__(self, state_size):
super(Value, self).__init__() super(Value, self).__init__()
@ -28,11 +27,8 @@ class Value(nn.Module):
def forward(self, x): def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x))) x = F.relu(self.fc_norm(self.fc1(x)))
x = F.relu(self.fc2_norm(self.fc2(x))) x = F.relu(self.fc2_norm(self.fc2(x)))
x = self.fc3(x) x = self.fc3(x)
return x return x
class Policy(nn.Module): class Policy(nn.Module):
@ -48,50 +44,30 @@ class Policy(nn.Module):
self.fc2_norm = nn.LayerNorm(64) self.fc2_norm = nn.LayerNorm(64)
self.fc3 = rn.NoisyLinear(64, action_size) self.fc3 = rn.NoisyLinear(64, action_size)
# self.fc3_norm = nn.LayerNorm(action_size)
# self.value_fc = rn.NoisyLinear(64, 64)
# self.value_fc_norm = nn.LayerNorm(64)
# self.value = rn.NoisyLinear(64, 1)
# self.advantage_fc = rn.NoisyLinear(64, 64)
# self.advantage_fc_norm = nn.LayerNorm(64)
# self.advantage = rn.NoisyLinear(64, action_size)
def forward(self, x): def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x))) x = F.relu(self.fc_norm(self.fc1(x)))
x = F.relu(self.fc2_norm(self.fc2(x))) x = F.relu(self.fc2_norm(self.fc2(x)))
x = F.softmax(self.fc3(x), dim = 1) x = F.softmax(self.fc3(x), dim = 1)
# state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
# state_value = self.value(state_value)
# advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
# advantage = self.advantage(advantage)
# x = F.softmax(state_value + advantage - advantage.mean(), dim = 1)
return x return x
#
## Configuration
#
config = {} config = {}
config['seed'] = 901 config['seed'] = 901
config['environment_name'] = 'Acrobot-v1' config['environment_name'] = 'Acrobot-v1'
config['memory_size'] = 2000
config['total_training_episodes'] = 500 config['total_training_episodes'] = 500
config['total_evaluation_episodes'] = 10 config['total_evaluation_episodes'] = 10
config['batch_size'] = 32
config['learning_rate'] = 1e-3 config['learning_rate'] = 1e-3
config['target_sync_tau'] = 1e-1
config['discount_rate'] = 0.99 config['discount_rate'] = 0.99
config['replay_skip'] = 0
# How many episodes between printing out the episode stats # How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1 config['print_stat_n_eps'] = 1
config['disable_cuda'] = False config['disable_cuda'] = False
#
## Training Loop
#
def train(runner, agent, config, logger = None, logwriter = None): def train(runner, agent, config, logger = None, logwriter = None):
finished = False finished = False
while not finished: while not finished:
@ -103,9 +79,8 @@ def train(runner, agent, config, logger = None, logwriter = None):
logwriter.write(logger) logwriter.write(logger)
finished = runner.episode_num > config['total_training_episodes'] finished = runner.episode_num > config['total_training_episodes']
if __name__ == "__main__":
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
if __name__ == "__main__":
# Setting up the environment # Setting up the environment
rltorch.set_seed(config['seed']) rltorch.set_seed(config['seed'])
print("Setting up environment...", end = " ") print("Setting up environment...", end = " ")
@ -135,7 +110,6 @@ if __name__ == "__main__":
actor = StochasticSelector(policy_net, action_size, memory, device = device) actor = StochasticSelector(policy_net, action_size, memory, device = device)
# Agent is what performs the training # Agent is what performs the training
# agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger)
agent = rltorch.agents.A2CSingleAgent(policy_net, value_net, memory, config, logger = logger) agent = rltorch.agents.A2CSingleAgent(policy_net, value_net, memory, config, logger = logger)
# Runner performs one episode in the environment # Runner performs one episode in the environment

42
examples/acrobot_es.py
View file

@ -1,5 +1,4 @@
import gym import gym
import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -10,8 +9,10 @@ import rltorch.memory as M
import rltorch.env as E import rltorch.env as E
from rltorch.action_selector import StochasticSelector from rltorch.action_selector import StochasticSelector
from tensorboardX import SummaryWriter from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
#
## Networks
#
class Policy(nn.Module): class Policy(nn.Module):
def __init__(self, state_size, action_size): def __init__(self, state_size, action_size):
super(Policy, self).__init__() super(Policy, self).__init__()
@ -32,37 +33,37 @@ class Policy(nn.Module):
x = F.softmax(self.action_prob(x), dim = 1) x = F.softmax(self.action_prob(x), dim = 1)
return x return x
#
## Configuration
#
config = {} config = {}
config['seed'] = 901 config['seed'] = 901
config['environment_name'] = 'Acrobot-v1' config['environment_name'] = 'Acrobot-v1'
config['memory_size'] = 2000
config['total_training_episodes'] = 50 config['total_training_episodes'] = 50
config['total_evaluation_episodes'] = 5 config['total_evaluation_episodes'] = 5
config['batch_size'] = 32
config['learning_rate'] = 1e-1 config['learning_rate'] = 1e-1
config['target_sync_tau'] = 1e-1
config['discount_rate'] = 0.99 config['discount_rate'] = 0.99
config['replay_skip'] = 0
# How many episodes between printing out the episode stats # How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1 config['print_stat_n_eps'] = 1
config['disable_cuda'] = False config['disable_cuda'] = False
#
## Training Loop
def train(env, net, actor, config, logger = None, logwriter = None): #
def train(runner, net, config, logger = None, logwriter = None):
finished = False finished = False
episode_num = 1
while not finished: while not finished:
rltorch.env.simulateEnvEps(env, actor, config, logger = logger, name = "Training") runner.run()
episode_num += 1
net.calc_gradients() net.calc_gradients()
net.step() net.step()
# When the episode number changes, log network paramters
if logwriter is not None: if logwriter is not None:
net.log_named_parameters() net.log_named_parameters()
logwriter.write(logger) logwriter.write(logger)
finished = episode_num > config['total_training_episodes'] finished = runner.episode_num > config['total_training_episodes']
#
## Loss function
#
def fitness(model): def fitness(model):
env = gym.make("Acrobot-v1") env = gym.make("Acrobot-v1")
state = torch.from_numpy(env.reset()).float().unsqueeze(0) state = torch.from_numpy(env.reset()).float().unsqueeze(0)
@ -75,9 +76,12 @@ def fitness(model):
next_state, reward, done, _ = env.step(action) next_state, reward, done, _ = env.step(action)
total_reward += reward total_reward += reward
state = torch.from_numpy(next_state).float().unsqueeze(0) state = torch.from_numpy(next_state).float().unsqueeze(0)
return total_reward return -total_reward
if __name__ == "__main__": if __name__ == "__main__":
# Hide internal gym warnings
gym.logger.set_level(40)
# Setting up the environment # Setting up the environment
rltorch.set_seed(config['seed']) rltorch.set_seed(config['seed'])
print("Setting up environment...", end = " ") print("Setting up environment...", end = " ")
@ -90,21 +94,21 @@ if __name__ == "__main__":
# Logging # Logging
logger = rltorch.log.Logger() logger = rltorch.log.Logger()
# logwriter = rltorch.log.LogWriter(logger, SummaryWriter())
logwriter = rltorch.log.LogWriter(SummaryWriter()) logwriter = rltorch.log.LogWriter(SummaryWriter())
# Setting up the networks # Setting up the networks
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu") device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
net = rn.ESNetwork(Policy(state_size, action_size), net = rn.ESNetwork(Policy(state_size, action_size),
torch.optim.Adam, 100, fitness, config, device = device, name = "ES", logger = logger) torch.optim.Adam, 100, fitness, config, device = device, name = "ES", logger = logger)
net.model.share_memory()
# Actor takes a net and uses it to produce actions from given states # Actor takes a net and uses it to produce actions from given states
actor = StochasticSelector(net, action_size, device = device) actor = StochasticSelector(net, action_size, device = device)
print("Training...") # Runner performs an episode of the environment
runner = rltorch.env.EnvironmentEpisodeSync(env, actor, config, name = "Training", logwriter = logwriter)
train(env, net, actor, config, logger = logger, logwriter = logwriter) print("Training...")
train(runner, net, config, logger = logger, logwriter = logwriter)
# For profiling... # For profiling...
# import cProfile # import cProfile

43
examples/acrobot_ppo.py
View file

@ -1,5 +1,4 @@
import gym import gym
import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -9,10 +8,10 @@ import rltorch.memory as M
import rltorch.env as E import rltorch.env as E
from rltorch.action_selector import StochasticSelector from rltorch.action_selector import StochasticSelector
from tensorboardX import SummaryWriter from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
import signal
from copy import deepcopy
#
## Networks
#
class Value(nn.Module): class Value(nn.Module):
def __init__(self, state_size): def __init__(self, state_size):
super(Value, self).__init__() super(Value, self).__init__()
@ -28,11 +27,8 @@ class Value(nn.Module):
def forward(self, x): def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x))) x = F.relu(self.fc_norm(self.fc1(x)))
x = F.relu(self.fc2_norm(self.fc2(x))) x = F.relu(self.fc2_norm(self.fc2(x)))
x = self.fc3(x) x = self.fc3(x)
return x return x
class Policy(nn.Module): class Policy(nn.Module):
@ -48,50 +44,30 @@ class Policy(nn.Module):
self.fc2_norm = nn.LayerNorm(64) self.fc2_norm = nn.LayerNorm(64)
self.fc3 = rn.NoisyLinear(64, action_size) self.fc3 = rn.NoisyLinear(64, action_size)
# self.fc3_norm = nn.LayerNorm(action_size)
# self.value_fc = rn.NoisyLinear(64, 64)
# self.value_fc_norm = nn.LayerNorm(64)
# self.value = rn.NoisyLinear(64, 1)
# self.advantage_fc = rn.NoisyLinear(64, 64)
# self.advantage_fc_norm = nn.LayerNorm(64)
# self.advantage = rn.NoisyLinear(64, action_size)
def forward(self, x): def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x))) x = F.relu(self.fc_norm(self.fc1(x)))
x = F.relu(self.fc2_norm(self.fc2(x))) x = F.relu(self.fc2_norm(self.fc2(x)))
x = F.softmax(self.fc3(x), dim = 1) x = F.softmax(self.fc3(x), dim = 1)
# state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
# state_value = self.value(state_value)
# advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
# advantage = self.advantage(advantage)
# x = F.softmax(state_value + advantage - advantage.mean(), dim = 1)
return x return x
#
## Configuration
#
config = {} config = {}
config['seed'] = 901 config['seed'] = 901
config['environment_name'] = 'Acrobot-v1' config['environment_name'] = 'Acrobot-v1'
config['memory_size'] = 2000
config['total_training_episodes'] = 500 config['total_training_episodes'] = 500
config['total_evaluation_episodes'] = 10 config['total_evaluation_episodes'] = 10
config['batch_size'] = 32
config['learning_rate'] = 1e-3 config['learning_rate'] = 1e-3
config['target_sync_tau'] = 1e-1
config['discount_rate'] = 0.99 config['discount_rate'] = 0.99
config['replay_skip'] = 0
# How many episodes between printing out the episode stats # How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1 config['print_stat_n_eps'] = 1
config['disable_cuda'] = False config['disable_cuda'] = False
#
## Training Loop
#
def train(runner, agent, config, logger = None, logwriter = None): def train(runner, agent, config, logger = None, logwriter = None):
finished = False finished = False
while not finished: while not finished:
@ -133,7 +109,6 @@ if __name__ == "__main__":
actor = StochasticSelector(policy_net, action_size, memory, device = device) actor = StochasticSelector(policy_net, action_size, memory, device = device)
# Agent is what performs the training # Agent is what performs the training
# agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger)
agent = rltorch.agents.PPOAgent(policy_net, value_net, memory, config, logger = logger) agent = rltorch.agents.PPOAgent(policy_net, value_net, memory, config, logger = logger)
# Runner performs a certain number of steps in the environment # Runner performs a certain number of steps in the environment

21
examples/acrobot_qep.py
View file

@ -1,5 +1,4 @@
import gym import gym
import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -9,9 +8,11 @@ import rltorch.memory as M
import rltorch.env as E import rltorch.env as E
from rltorch.action_selector import StochasticSelector from rltorch.action_selector import StochasticSelector
from tensorboardX import SummaryWriter from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
from copy import deepcopy from copy import deepcopy
#
## Networks
#
class Value(nn.Module): class Value(nn.Module):
def __init__(self, state_size, action_size): def __init__(self, state_size, action_size):
super(Value, self).__init__() super(Value, self).__init__()
@ -39,7 +40,6 @@ class Value(nn.Module):
advantage = self.advantage(advantage) advantage = self.advantage(advantage)
x = state_value + advantage - advantage.mean() x = state_value + advantage - advantage.mean()
return x return x
@ -63,6 +63,9 @@ class Policy(nn.Module):
x = F.softmax(self.action_prob(x), dim = 1) x = F.softmax(self.action_prob(x), dim = 1)
return x return x
#
## Configuration
#
config = {} config = {}
config['seed'] = 901 config['seed'] = 901
config['environment_name'] = 'Acrobot-v1' config['environment_name'] = 'Acrobot-v1'
@ -88,7 +91,9 @@ config['prioritized_replay_sampling_priority'] = 0.6
config['prioritized_replay_weight_importance'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.4, end_value = 1, iterations = 5000) config['prioritized_replay_weight_importance'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.4, end_value = 1, iterations = 5000)
#
## Training Loop
#
def train(runner, agent, config, logger = None, logwriter = None): def train(runner, agent, config, logger = None, logwriter = None):
finished = False finished = False
last_episode_num = 1 last_episode_num = 1
@ -103,6 +108,7 @@ def train(runner, agent, config, logger = None, logwriter = None):
logwriter.write(logger) logwriter.write(logger)
finished = runner.episode_num > config['total_training_episodes'] finished = runner.episode_num > config['total_training_episodes']
if __name__ == "__main__": if __name__ == "__main__":
# Setting up the environment # Setting up the environment
rltorch.set_seed(config['seed']) rltorch.set_seed(config['seed'])
@ -116,7 +122,6 @@ if __name__ == "__main__":
# Logging # Logging
logger = rltorch.log.Logger() logger = rltorch.log.Logger()
# logwriter = rltorch.log.LogWriter(logger, SummaryWriter())
logwriter = rltorch.log.LogWriter(SummaryWriter()) logwriter = rltorch.log.LogWriter(SummaryWriter())
# Setting up the networks # Setting up the networks
@ -127,13 +132,11 @@ if __name__ == "__main__":
torch.optim.Adam, 500, None, config2, sigma = 0.1, device = device, name = "ES", logger = logger) torch.optim.Adam, 500, None, config2, sigma = 0.1, device = device, name = "ES", logger = logger)
value_net = rn.Network(Value(state_size, action_size), value_net = rn.Network(Value(state_size, action_size),
torch.optim.Adam, config, device = device, name = "DQN", logger = logger) torch.optim.Adam, config, device = device, name = "DQN", logger = logger)
target_net = rn.TargetNetwork(value_net, device = device) target_net = rn.TargetNetwork(value_net, device = device)
value_net.model.share_memory()
target_net.model.share_memory()
# Actor takes a net and uses it to produce actions from given states # Actor takes a net and uses it to produce actions from given states
actor = StochasticSelector(policy_net, action_size, device = device) actor = StochasticSelector(policy_net, action_size, device = device)
# Memory stores experiences for later training # Memory stores experiences for later training
memory = M.PrioritizedReplayMemory(capacity = config['memory_size'], alpha = config['prioritized_replay_sampling_priority']) memory = M.PrioritizedReplayMemory(capacity = config['memory_size'], alpha = config['prioritized_replay_sampling_priority'])
@ -141,11 +144,9 @@ if __name__ == "__main__":
runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter) runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
# Agent is what performs the training # Agent is what performs the training
# agent = TestAgent(policy_net, value_net, memory, config, target_value_net = target_net, logger = logger)
agent = rltorch.agents.QEPAgent(policy_net, value_net, memory, config, target_value_net = target_net, logger = logger) agent = rltorch.agents.QEPAgent(policy_net, value_net, memory, config, target_value_net = target_net, logger = logger)
print("Training...") print("Training...")
train(runner, agent, config, logger = logger, logwriter = logwriter) train(runner, agent, config, logger = logger, logwriter = logwriter)
# For profiling... # For profiling...

61
examples/acrobot_reinforce.py
View file

@ -1,5 +1,4 @@
import gym import gym
import numpy as np
import torch import torch
import torch.nn as nn import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
@ -9,69 +8,57 @@ import rltorch.memory as M
import rltorch.env as E import rltorch.env as E
from rltorch.action_selector import StochasticSelector from rltorch.action_selector import StochasticSelector
from tensorboardX import SummaryWriter from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
import signal
from copy import deepcopy
class Value(nn.Module): #
## Networks
#
class Policy(nn.Module):
def __init__(self, state_size, action_size): def __init__(self, state_size, action_size):
super(Value, self).__init__() super(Policy, self).__init__()
self.state_size = state_size self.state_size = state_size
self.action_size = action_size self.action_size = action_size
self.fc1 = rn.NoisyLinear(state_size, 64) self.fc1 = rn.NoisyLinear(state_size, 64)
self.fc_norm = nn.LayerNorm(64) self.fc_norm = nn.LayerNorm(64)
self.value_fc = rn.NoisyLinear(64, 64) self.fc2 = rn.NoisyLinear(64, 64)
self.value_fc_norm = nn.LayerNorm(64) self.fc2_norm = nn.LayerNorm(64)
self.value = rn.NoisyLinear(64, 1)
self.advantage_fc = rn.NoisyLinear(64, 64) self.fc3 = rn.NoisyLinear(64, action_size)
self.advantage_fc_norm = nn.LayerNorm(64)
self.advantage = rn.NoisyLinear(64, action_size)
def forward(self, x): def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x))) x = F.relu(self.fc_norm(self.fc1(x)))
x = F.relu(self.fc2_norm(self.fc2(x)))
state_value = F.relu(self.value_fc_norm(self.value_fc(x))) x = F.softmax(self.fc3(x), dim = 1)
state_value = self.value(state_value)
advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
advantage = self.advantage(advantage)
x = F.softmax(state_value + advantage - advantage.mean(), dim = 1)
return x return x
#
## Configuration
#
config = {} config = {}
config['seed'] = 901 config['seed'] = 901
config['environment_name'] = 'Acrobot-v1' config['environment_name'] = 'Acrobot-v1'
config['memory_size'] = 2000
config['total_training_episodes'] = 500 config['total_training_episodes'] = 500
config['total_evaluation_episodes'] = 10 config['total_evaluation_episodes'] = 10
config['batch_size'] = 32
config['learning_rate'] = 1e-3 config['learning_rate'] = 1e-3
config['target_sync_tau'] = 1e-1
config['discount_rate'] = 0.99 config['discount_rate'] = 0.99
config['replay_skip'] = 0
# How many episodes between printing out the episode stats # How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1 config['print_stat_n_eps'] = 1
config['disable_cuda'] = False config['disable_cuda'] = False
def train(env, agent, actor, memory, config, logger = None, logwriter = None): #
## Training Loop
#
def train(runner, agent, config, logger = None, logwriter = None):
finished = False finished = False
episode_num = 1
while not finished: while not finished:
rltorch.env.simulateEnvEps(env, actor, config, memory = memory, logger = logger, name = "Training") runner.run()
episode_num += 1
agent.learn() agent.learn()
# When the episode number changes, log network paramters # When the episode number changes, log network paramters
if logwriter is not None: if logwriter is not None:
agent.net.log_named_parameters() agent.net.log_named_parameters()
logwriter.write(logger) logwriter.write(logger)
finished = episode_num > config['total_training_episodes'] finished = runner.episode_num > config['total_training_episodes']
if __name__ == "__main__": if __name__ == "__main__":
@ -93,11 +80,9 @@ if __name__ == "__main__":
# Setting up the networks # Setting up the networks
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu") device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
net = rn.Network(Value(state_size, action_size), net = rn.Network(Policy(state_size, action_size),
torch.optim.Adam, config, device = device, name = "DQN") torch.optim.Adam, config, device = device, name = "DQN")
target_net = rn.TargetNetwork(net, device = device) target_net = rn.TargetNetwork(net, device = device)
net.model.share_memory()
target_net.model.share_memory()
# Memory stores experiences for later training # Memory stores experiences for later training
memory = M.EpisodeMemory() memory = M.EpisodeMemory()
@ -108,9 +93,11 @@ if __name__ == "__main__":
# Agent is what performs the training # Agent is what performs the training
agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger) agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger)
print("Training...") # Runner performs one episode in the environment
runner = rltorch.env.EnvironmentEpisodeSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
train(env, agent, actor, memory, config, logger = logger, logwriter = logwriter) print("Training...")
train(runner, agent, config, logger = logger, logwriter = logwriter)
# For profiling... # For profiling...
# import cProfile # import cProfile

135
examples/acrobot_single_process.py
View file

@ -1,135 +0,0 @@
import gym
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import rltorch
import rltorch.network as rn
import rltorch.memory as M
import rltorch.env as E
from rltorch.action_selector import ArgMaxSelector
from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
class Value(nn.Module):
def __init__(self, state_size, action_size):
super(Value, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.fc1 = rn.NoisyLinear(state_size, 255)
self.fc_norm = nn.LayerNorm(255)
self.value_fc = rn.NoisyLinear(255, 255)
self.value_fc_norm = nn.LayerNorm(255)
self.value = rn.NoisyLinear(255, 1)
self.advantage_fc = rn.NoisyLinear(255, 255)
self.advantage_fc_norm = nn.LayerNorm(255)
self.advantage = rn.NoisyLinear(255, action_size)
def forward(self, x):
x = F.relu(self.fc_norm(self.fc1(x)))
state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
state_value = self.value(state_value)
advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
advantage = self.advantage(advantage)
x = state_value + advantage - advantage.mean()
return x
config = {}
config['seed'] = 901
config['environment_name'] = 'Acrobot-v1'
config['memory_size'] = 2000
config['total_training_episodes'] = 50
config['total_evaluation_episodes'] = 5
config['batch_size'] = 32
config['learning_rate'] = 1e-3
config['target_sync_tau'] = 1e-1
config['discount_rate'] = 0.99
config['replay_skip'] = 0
# How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1
config['disable_cuda'] = False
# Prioritized vs Random Sampling
# 0 - Random sampling
# 1 - Only the highest prioirities
config['prioritized_replay_sampling_priority'] = 0.6
# How important are the weights for the loss?
# 0 - Treat all losses equally
# 1 - Lower the importance of high losses
# Should ideally start from 0 and move your way to 1 to prevent overfitting
config['prioritized_replay_weight_importance'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.4, end_value = 1, iterations = 5000)
def train(runner, agent, config, logger = None, logwriter = None):
finished = False
last_episode_num = 1
while not finished:
runner.run(config['replay_skip'] + 1)
agent.learn()
if logwriter is not None:
if last_episode_num < runner.episode_num:
last_episode_num = runner.episode_num
agent.net.log_named_parameters()
logwriter.write(logger)
finished = runner.episode_num > config['total_training_episodes']
if __name__ == "__main__":
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
# Setting up the environment
rltorch.set_seed(config['seed'])
print("Setting up environment...", end = " ")
env = E.TorchWrap(gym.make(config['environment_name']))
env.seed(config['seed'])
print("Done.")
state_size = env.observation_space.shape[0]
action_size = env.action_space.n
# Logging
logger = rltorch.log.Logger()
# logwriter = rltorch.log.LogWriter(logger, SummaryWriter())
logwriter = rltorch.log.LogWriter(SummaryWriter())
# Setting up the networks
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
net = rn.Network(Value(state_size, action_size),
torch.optim.Adam, config, device = device, name = "DQN", logger = logger)
target_net = rn.TargetNetwork(net, device = device)
net.model.share_memory()
target_net.model.share_memory()
# Actor takes a net and uses it to produce actions from given states
actor = ArgMaxSelector(net, action_size, device = device)
# Memory stores experiences for later training
memory = M.PrioritizedReplayMemory(capacity = config['memory_size'], alpha = config['prioritized_replay_sampling_priority'])
# memory = M.ReplayMemory(capacity = config['memory_size'])
# Runner performs a certain number of steps in the environment
runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
# Agent is what performs the training
agent = rltorch.agents.DQNAgent(net, memory, config, target_net = target_net, logger = logger)
print("Training...")
train(runner, agent, config, logger = logger, logwriter = logwriter)
# For profiling...
# import cProfile
# cProfile.run('train(runner, agent, config, logger = logger, logwriter = logwriter )')
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
print("Training Finished.")
print("Evaluating...")
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
print("Evaulations Done.")
logwriter.close() # We don't need to write anything out to disk anymore

147
examples/pong.py
View file

@ -1,147 +0,0 @@
import gym
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import rltorch
import rltorch.network as rn
import rltorch.memory as M
import rltorch.env as E
from rltorch.action_selector import ArgMaxSelector
from tensorboardX import SummaryWriter
import torch.multiprocessing as mp
class Value(nn.Module):
def __init__(self, state_size, action_size):
super(Value, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.conv1 = nn.Conv2d(4, 32, kernel_size = (8, 8), stride = (4, 4))
self.conv_norm1 = nn.LayerNorm([32, 19, 19])
self.conv2 = nn.Conv2d(32, 64, kernel_size = (4, 4), stride = (2, 2))
self.conv_norm2 = nn.LayerNorm([64, 8, 8])
self.conv3 = nn.Conv2d(64, 64, kernel_size = (3, 3), stride = (1, 1))
self.conv_norm3 = nn.LayerNorm([64, 6, 6])
self.fc1 = rn.NoisyLinear(64 * 6 * 6, 384)
self.fc_norm = nn.LayerNorm(384)
self.value_fc = rn.NoisyLinear(384, 384)
self.value_fc_norm = nn.LayerNorm(384)
self.value = rn.NoisyLinear(384, 1)
self.advantage_fc = rn.NoisyLinear(384, 384)
self.advantage_fc_norm = nn.LayerNorm(384)
self.advantage = rn.NoisyLinear(384, action_size)
def forward(self, x):
x = F.relu(self.conv_norm1(self.conv1(x)))
x = F.relu(self.conv_norm2(self.conv2(x)))
x = F.relu(self.conv_norm3(self.conv3(x)))
# Makes batch_size dimension again
x = x.view(-1, 64 * 6 * 6)
x = F.relu(self.fc_norm(self.fc1(x)))
state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
state_value = self.value(state_value)
advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
advantage = self.advantage(advantage)
x = state_value + advantage - advantage.mean()
# For debugging purposes...
if torch.isnan(x).any().item():
print("WARNING NAN IN MODEL DETECTED")
return x
config = {}
config['seed'] = 901
config['environment_name'] = 'PongNoFrameskip-v4'
config['memory_size'] = 5000
config['total_training_episodes'] = 500
config['total_evaluation_episodes'] = 10
config['learning_rate'] = 1e-4
config['target_sync_tau'] = 1e-3
config['discount_rate'] = 0.99
config['exploration_rate'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.1, end_value = 0.01, iterations = 5000)
config['replay_skip'] = 4
config['batch_size'] = 32 * (config['replay_skip'] + 1)
# How many episodes between printing out the episode stats
config['print_stat_n_eps'] = 1
config['disable_cuda'] = False
# Prioritized vs Random Sampling
# 0 - Random sampling
# 1 - Only the highest prioirities
config['prioritized_replay_sampling_priority'] = 0.6
# How important are the weights for the loss?
# 0 - Treat all losses equally
# 1 - Lower the importance of high losses
# Should ideally start from 0 and move your way to 1 to prevent overfitting
config['prioritized_replay_weight_importance'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.4, end_value = 1, iterations = 5000)
if __name__ == "__main__":
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
# Setting up the environment
rltorch.set_seed(config['seed'])
print("Setting up environment...", end = " ")
env = E.FrameStack(E.TorchWrap(
E.ProcessFrame(E.FireResetEnv(gym.make(config['environment_name'])),
resize_shape = (80, 80), crop_bounds = [34, 194, 15, 145], grayscale = True))
, 4)
env.seed(config['seed'])
print("Done.")
state_size = env.observation_space.shape[0]
action_size = env.action_space.n
# Logging
logger = rltorch.log.Logger()
logwriter = rltorch.log.LogWriter(SummaryWriter())
# Setting up the networks
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
net = rn.Network(Value(state_size, action_size),
torch.optim.Adam, config, device = device, name = "DQN")
target_net = rn.TargetNetwork(net, device = device)
net.model.share_memory()
target_net.model.share_memory()
# Actor takes a net and uses it to produce actions from given states
actor = ArgMaxSelector(net, action_size, device = device)
# Memory stores experiences for later training
memory = M.PrioritizedReplayMemory(capacity = config['memory_size'], alpha = config['prioritized_replay_sampling_priority'])
# memory = M.ReplayMemory(capacity = config['memory_size'])
# Runner performs a certain number of steps in the environment
runner = rltorch.mp.EnvironmentRun(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
# Agent is what performs the training
agent = rltorch.agents.DQNAgent(net, memory, config, target_net = target_net, logger = logger)
print("Training...")
train(runner, agent, config, logger = logger, logwriter = logwriter)
# For profiling...
# import cProfile
# cProfile.run('train(runner, agent, config, logger = logger, logwriter = logwriter )')
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
print("Training Finished.")
runner.terminate() # We don't need the extra process anymore
print("Evaluating...")
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
print("Evaulations Done.")
logwriter.close() # We don't need to write anything out to disk anymore

13
rltorch/agents/A2CSingleAgent.py
View file

@ -1,12 +1,8 @@
from copy import deepcopy from copy import deepcopy
import numpy as np
import torch import torch
import torch.nn.functional as F import torch.nn.functional as F
from torch.distributions import Categorical
import rltorch import rltorch
import rltorch.memory as M import rltorch.memory as M
import collections
import random
class A2CSingleAgent: class A2CSingleAgent:
def __init__(self, policy_net, value_net, memory, config, logger = None): def __init__(self, policy_net, value_net, memory, config, logger = None):
@ -25,11 +21,11 @@ class A2CSingleAgent:
return discounted_rewards return discounted_rewards
def learn(self): def learn(self):
episode_batch = self.memory.recall() episode_batch = self.memory.recall()
state_batch, action_batch, reward_batch, next_state_batch, done_batch, log_prob_batch = zip(*episode_batch) state_batch, action_batch, reward_batch, next_state_batch, done_batch, log_prob_batch = zip(*episode_batch)
# Send batches to the appropriate device
state_batch = torch.cat(state_batch).to(self.value_net.device) state_batch = torch.cat(state_batch).to(self.value_net.device)
reward_batch = torch.tensor(reward_batch).to(self.value_net.device) reward_batch = torch.tensor(reward_batch).to(self.value_net.device)
not_done_batch = ~torch.tensor(done_batch).to(self.value_net.device) not_done_batch = ~torch.tensor(done_batch).to(self.value_net.device)
@ -37,12 +33,16 @@ class A2CSingleAgent:
log_prob_batch = torch.cat(log_prob_batch).to(self.value_net.device) log_prob_batch = torch.cat(log_prob_batch).to(self.value_net.device)
## Value Loss ## Value Loss
# In A2C, the value loss is the difference between the discounted reward and the value from the first state
# The value of the first state is supposed to tell us the expected reward from the current policy of the whole episode
value_loss = F.mse_loss(self._discount_rewards(reward_batch).sum(), self.value_net(state_batch[0])) value_loss = F.mse_loss(self._discount_rewards(reward_batch).sum(), self.value_net(state_batch[0]))
self.value_net.zero_grad() self.value_net.zero_grad()
value_loss.backward() value_loss.backward()
self.value_net.step() self.value_net.step()
## Policy Loss ## Policy Loss
# Increase probabilities of advantageous states
# and decrease the probabilities of non-advantageous ones
with torch.no_grad(): with torch.no_grad():
state_values = self.value_net(state_batch) state_values = self.value_net(state_batch)
next_state_values = torch.zeros_like(state_values) next_state_values = torch.zeros_like(state_values)
@ -61,8 +61,7 @@ class A2CSingleAgent:
policy_loss.backward() policy_loss.backward()
self.policy_net.step() self.policy_net.step()
# Memory under the old policy is not needed for future training
# Memory is irrelevant for future training
self.memory.clear() self.memory.clear()

2
rltorch/agents/DQNAgent.py
View file

@ -55,6 +55,7 @@ class DQNAgent:
expected_values = (reward_batch + (self.config['discount_rate'] * best_next_state_value)).unsqueeze(1) expected_values = (reward_batch + (self.config['discount_rate'] * best_next_state_value)).unsqueeze(1)
# If we're sampling by TD error, multiply loss by a importance weight which helps decrease overfitting
if (isinstance(self.memory, M.PrioritizedReplayMemory)): if (isinstance(self.memory, M.PrioritizedReplayMemory)):
loss = (torch.as_tensor(importance_weights, device = self.net.device) * ((obtained_values - expected_values)**2).squeeze(1)).mean() loss = (torch.as_tensor(importance_weights, device = self.net.device) * ((obtained_values - expected_values)**2).squeeze(1)).mean()
else: else:
@ -74,6 +75,7 @@ class DQNAgent:
else: else:
self.target_net.sync() self.target_net.sync()
# If we're sampling by TD error, readjust the weights of the experiences
if (isinstance(self.memory, M.PrioritizedReplayMemory)): if (isinstance(self.memory, M.PrioritizedReplayMemory)):
td_error = (obtained_values - expected_values).detach().abs() td_error = (obtained_values - expected_values).detach().abs()
self.memory.update_priorities(batch_indexes, td_error) self.memory.update_priorities(batch_indexes, td_error)

8
rltorch/agents/PPOAgent.py
View file

@ -31,6 +31,7 @@ class PPOAgent:
episode_batch = self.memory.recall() episode_batch = self.memory.recall()
state_batch, action_batch, reward_batch, next_state_batch, done_batch, log_prob_batch = zip(*episode_batch) state_batch, action_batch, reward_batch, next_state_batch, done_batch, log_prob_batch = zip(*episode_batch)
# Send batches to the appropriate device
state_batch = torch.cat(state_batch).to(self.value_net.device) state_batch = torch.cat(state_batch).to(self.value_net.device)
action_batch = torch.tensor(action_batch).to(self.value_net.device) action_batch = torch.tensor(action_batch).to(self.value_net.device)
reward_batch = torch.tensor(reward_batch).to(self.value_net.device) reward_batch = torch.tensor(reward_batch).to(self.value_net.device)
@ -39,12 +40,16 @@ class PPOAgent:
log_prob_batch = torch.cat(log_prob_batch).to(self.value_net.device) log_prob_batch = torch.cat(log_prob_batch).to(self.value_net.device)
## Value Loss ## Value Loss
# In PPO, the value loss is the difference between the discounted reward and the value from the first state
# The value of the first state is supposed to tell us the expected reward from the current policy of the whole episode
value_loss = F.mse_loss(self._discount_rewards(reward_batch).sum(), self.value_net(state_batch[0])) value_loss = F.mse_loss(self._discount_rewards(reward_batch).sum(), self.value_net(state_batch[0]))
self.value_net.zero_grad() self.value_net.zero_grad()
value_loss.backward() value_loss.backward()
self.value_net.step() self.value_net.step()
## Policy Loss ## Policy Loss
# Increase probabilities of advantageous states
# and decrease the probabilities of non-advantageous ones
with torch.no_grad(): with torch.no_grad():
state_values = self.value_net(state_batch) state_values = self.value_net(state_batch)
next_state_values = torch.zeros_like(state_values) next_state_values = torch.zeros_like(state_values)
@ -56,6 +61,7 @@ class PPOAgent:
distributions = list(map(Categorical, action_probabilities)) distributions = list(map(Categorical, action_probabilities))
old_log_probs = torch.stack(list(map(lambda distribution, action: distribution.log_prob(action), distributions, action_batch))) old_log_probs = torch.stack(list(map(lambda distribution, action: distribution.log_prob(action), distributions, action_batch)))
# For PPO we want to stay within a certain ratio of the old policy
policy_ratio = torch.exp(log_prob_batch - old_log_probs) # Equivalent to (log_prob / old_log_prob) policy_ratio = torch.exp(log_prob_batch - old_log_probs) # Equivalent to (log_prob / old_log_prob)
policy_loss1 = policy_ratio * advantages policy_loss1 = policy_ratio * advantages
policy_loss2 = policy_ratio.clamp(min = 0.8, max = 1.2) * advantages # From original paper policy_loss2 = policy_ratio.clamp(min = 0.8, max = 1.2) * advantages # From original paper
@ -72,7 +78,7 @@ class PPOAgent:
self.policy_net.step() self.policy_net.step()
# Memory is irrelevant for future training # Memory under the old policy is not needed for future training
self.memory.clear() self.memory.clear()

3
rltorch/network/ESNetwork.py
View file

@ -3,7 +3,8 @@ import torch
from .Network import Network from .Network import Network
from copy import deepcopy from copy import deepcopy
# [TODO] See if you need to move network to device # [TODO] Should we torch.no_grad the __call__?
# What if we want to sometimes do gradient descent as well?
class ESNetwork(Network): class ESNetwork(Network):
""" """
Network that functions from the paper Evolutionary Strategies (https://arxiv.org/abs/1703.03864) Network that functions from the paper Evolutionary Strategies (https://arxiv.org/abs/1703.03864)