Added Evolutionary Strategies Network and added more example scripts
This commit is contained in:
parent
26084d4c7c
commit
76a044ace9
14 changed files with 695 additions and 41 deletions
161
examples/acrobot_a2c.py
Normal file
161
examples/acrobot_a2c.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import gym
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import rltorch
|
||||
import rltorch.network as rn
|
||||
import rltorch.memory as M
|
||||
import rltorch.env as E
|
||||
from rltorch.action_selector import StochasticSelector
|
||||
from tensorboardX import SummaryWriter
|
||||
import torch.multiprocessing as mp
|
||||
import signal
|
||||
from copy import deepcopy
|
||||
|
||||
class Value(nn.Module):
|
||||
def __init__(self, state_size):
|
||||
super(Value, self).__init__()
|
||||
self.state_size = state_size
|
||||
|
||||
self.fc1 = rn.NoisyLinear(state_size, 64)
|
||||
self.fc_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc2 = rn.NoisyLinear(64, 64)
|
||||
self.fc2_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc3 = rn.NoisyLinear(64, 1)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
|
||||
x = self.fc3(x)
|
||||
|
||||
return x
|
||||
|
||||
class Policy(nn.Module):
|
||||
def __init__(self, state_size, action_size):
|
||||
super(Policy, self).__init__()
|
||||
self.state_size = state_size
|
||||
self.action_size = action_size
|
||||
|
||||
self.fc1 = rn.NoisyLinear(state_size, 64)
|
||||
self.fc_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc2 = rn.NoisyLinear(64, 64)
|
||||
self.fc2_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc3 = rn.NoisyLinear(64, action_size)
|
||||
# self.fc3_norm = nn.LayerNorm(action_size)
|
||||
|
||||
# self.value_fc = rn.NoisyLinear(64, 64)
|
||||
# self.value_fc_norm = nn.LayerNorm(64)
|
||||
# self.value = rn.NoisyLinear(64, 1)
|
||||
|
||||
# self.advantage_fc = rn.NoisyLinear(64, 64)
|
||||
# self.advantage_fc_norm = nn.LayerNorm(64)
|
||||
# self.advantage = rn.NoisyLinear(64, action_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
|
||||
x = F.softmax(self.fc3(x), dim = 1)
|
||||
|
||||
# state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
|
||||
# state_value = self.value(state_value)
|
||||
|
||||
# advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
|
||||
# advantage = self.advantage(advantage)
|
||||
|
||||
# x = F.softmax(state_value + advantage - advantage.mean(), dim = 1)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
config = {}
|
||||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 500
|
||||
config['total_evaluation_episodes'] = 10
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-3
|
||||
config['target_sync_tau'] = 1e-1
|
||||
config['discount_rate'] = 0.99
|
||||
config['replay_skip'] = 0
|
||||
# How many episodes between printing out the episode stats
|
||||
config['print_stat_n_eps'] = 1
|
||||
config['disable_cuda'] = False
|
||||
|
||||
|
||||
def train(runner, agent, config, logger = None, logwriter = None):
|
||||
finished = False
|
||||
last_episode_num = 1
|
||||
while not finished:
|
||||
runner.run(config['replay_skip'] + 1)
|
||||
agent.learn()
|
||||
if logwriter is not None:
|
||||
if last_episode_num < runner.episode_num:
|
||||
last_episode_num = runner.episode_num
|
||||
agent.value_net.log_named_parameters()
|
||||
agent.policy_net.log_named_parameters()
|
||||
logwriter.write(logger)
|
||||
finished = runner.episode_num > config['total_training_episodes']
|
||||
|
||||
if __name__ == "__main__":
|
||||
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
|
||||
|
||||
# Setting up the environment
|
||||
rltorch.set_seed(config['seed'])
|
||||
print("Setting up environment...", end = " ")
|
||||
env = E.TorchWrap(gym.make(config['environment_name']))
|
||||
env.seed(config['seed'])
|
||||
print("Done.")
|
||||
|
||||
state_size = env.observation_space.shape[0]
|
||||
action_size = env.action_space.n
|
||||
|
||||
# Logging
|
||||
logger = rltorch.log.Logger()
|
||||
logwriter = rltorch.log.LogWriter(SummaryWriter())
|
||||
|
||||
# Setting up the networks
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
|
||||
policy_net = rn.Network(Policy(state_size, action_size),
|
||||
torch.optim.Adam, config, device = device, name = "Policy")
|
||||
value_net = rn.Network(Value(state_size),
|
||||
torch.optim.Adam, config, device = device, name = "DQN")
|
||||
|
||||
|
||||
# Memory stores experiences for later training
|
||||
memory = M.EpisodeMemory()
|
||||
|
||||
# Actor takes a net and uses it to produce actions from given states
|
||||
actor = StochasticSelector(policy_net, action_size, memory, device = device)
|
||||
|
||||
# Agent is what performs the training
|
||||
# agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger)
|
||||
agent = rltorch.agents.A2CSingleAgent(policy_net, value_net, memory, config, logger = logger)
|
||||
|
||||
# Runner performs a certain number of steps in the environment
|
||||
runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
|
||||
|
||||
print("Training...")
|
||||
train(runner, agent, config, logger = logger, logwriter = logwriter)
|
||||
|
||||
# For profiling...
|
||||
# import cProfile
|
||||
# cProfile.run('train(runner, agent, config, logger = logger, logwriter = logwriter )')
|
||||
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
|
||||
|
||||
print("Training Finished.")
|
||||
|
||||
print("Evaluating...")
|
||||
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
|
||||
print("Evaulations Done.")
|
||||
|
||||
logwriter.close() # We don't need to write anything out to disk anymore
|
||||
120
examples/acrobot_es.py
Normal file
120
examples/acrobot_es.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import gym
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.distributions import Categorical
|
||||
import rltorch
|
||||
import rltorch.network as rn
|
||||
import rltorch.memory as M
|
||||
import rltorch.env as E
|
||||
from rltorch.action_selector import StochasticSelector
|
||||
from tensorboardX import SummaryWriter
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
class Policy(nn.Module):
|
||||
def __init__(self, state_size, action_size):
|
||||
super(Policy, self).__init__()
|
||||
self.state_size = state_size
|
||||
self.action_size = action_size
|
||||
|
||||
self.fc1 = nn.Linear(state_size, 125)
|
||||
self.fc_norm = nn.LayerNorm(125)
|
||||
|
||||
self.fc2 = nn.Linear(125, 125)
|
||||
self.fc2_norm = nn.LayerNorm(125)
|
||||
|
||||
self.action_prob = nn.Linear(125, action_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
x = F.softmax(self.action_prob(x), dim = 1)
|
||||
return x
|
||||
|
||||
config = {}
|
||||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 50
|
||||
config['total_evaluation_episodes'] = 5
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-1
|
||||
config['target_sync_tau'] = 1e-1
|
||||
config['discount_rate'] = 0.99
|
||||
config['replay_skip'] = 0
|
||||
# How many episodes between printing out the episode stats
|
||||
config['print_stat_n_eps'] = 1
|
||||
config['disable_cuda'] = False
|
||||
|
||||
|
||||
|
||||
def train(env, net, actor, config, logger = None, logwriter = None):
|
||||
finished = False
|
||||
episode_num = 1
|
||||
while not finished:
|
||||
rltorch.env.simulateEnvEps(env, actor, config, logger = logger, name = "Training")
|
||||
episode_num += 1
|
||||
net.calc_gradients()
|
||||
net.step()
|
||||
# When the episode number changes, log network paramters
|
||||
if logwriter is not None:
|
||||
net.log_named_parameters()
|
||||
logwriter.write(logger)
|
||||
finished = episode_num > config['total_training_episodes']
|
||||
|
||||
def fitness(model):
|
||||
env = gym.make("Acrobot-v1")
|
||||
state = torch.from_numpy(env.reset()).float().unsqueeze(0)
|
||||
total_reward = 0
|
||||
done = False
|
||||
while not done:
|
||||
action_probabilities = model(state)
|
||||
distribution = Categorical(action_probabilities)
|
||||
action = distribution.sample().item()
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
total_reward += reward
|
||||
state = torch.from_numpy(next_state).float().unsqueeze(0)
|
||||
return total_reward
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Setting up the environment
|
||||
rltorch.set_seed(config['seed'])
|
||||
print("Setting up environment...", end = " ")
|
||||
env = E.TorchWrap(gym.make(config['environment_name']))
|
||||
env.seed(config['seed'])
|
||||
print("Done.")
|
||||
|
||||
state_size = env.observation_space.shape[0]
|
||||
action_size = env.action_space.n
|
||||
|
||||
# Logging
|
||||
logger = rltorch.log.Logger()
|
||||
# logwriter = rltorch.log.LogWriter(logger, SummaryWriter())
|
||||
logwriter = rltorch.log.LogWriter(SummaryWriter())
|
||||
|
||||
# Setting up the networks
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
|
||||
net = rn.ESNetwork(Policy(state_size, action_size),
|
||||
torch.optim.Adam, 100, fitness, config, device = device, name = "ES", logger = logger)
|
||||
net.model.share_memory()
|
||||
|
||||
# Actor takes a net and uses it to produce actions from given states
|
||||
actor = StochasticSelector(net, action_size, device = device)
|
||||
|
||||
print("Training...")
|
||||
|
||||
train(env, net, actor, config, logger = logger, logwriter = logwriter)
|
||||
|
||||
# For profiling...
|
||||
# import cProfile
|
||||
# cProfile.run('train(runner, agent, config, logger = logger, logwriter = logwriter )')
|
||||
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
|
||||
|
||||
print("Training Finished.")
|
||||
|
||||
print("Evaluating...")
|
||||
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
|
||||
print("Evaulations Done.")
|
||||
|
||||
logwriter.close() # We don't need to write anything out to disk anymore
|
||||
161
examples/acrobot_ppo.py
Normal file
161
examples/acrobot_ppo.py
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import gym
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import rltorch
|
||||
import rltorch.network as rn
|
||||
import rltorch.memory as M
|
||||
import rltorch.env as E
|
||||
from rltorch.action_selector import StochasticSelector
|
||||
from tensorboardX import SummaryWriter
|
||||
import torch.multiprocessing as mp
|
||||
import signal
|
||||
from copy import deepcopy
|
||||
|
||||
class Value(nn.Module):
|
||||
def __init__(self, state_size):
|
||||
super(Value, self).__init__()
|
||||
self.state_size = state_size
|
||||
|
||||
self.fc1 = rn.NoisyLinear(state_size, 64)
|
||||
self.fc_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc2 = rn.NoisyLinear(64, 64)
|
||||
self.fc2_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc3 = rn.NoisyLinear(64, 1)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
|
||||
x = self.fc3(x)
|
||||
|
||||
return x
|
||||
|
||||
class Policy(nn.Module):
|
||||
def __init__(self, state_size, action_size):
|
||||
super(Policy, self).__init__()
|
||||
self.state_size = state_size
|
||||
self.action_size = action_size
|
||||
|
||||
self.fc1 = rn.NoisyLinear(state_size, 64)
|
||||
self.fc_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc2 = rn.NoisyLinear(64, 64)
|
||||
self.fc2_norm = nn.LayerNorm(64)
|
||||
|
||||
self.fc3 = rn.NoisyLinear(64, action_size)
|
||||
# self.fc3_norm = nn.LayerNorm(action_size)
|
||||
|
||||
# self.value_fc = rn.NoisyLinear(64, 64)
|
||||
# self.value_fc_norm = nn.LayerNorm(64)
|
||||
# self.value = rn.NoisyLinear(64, 1)
|
||||
|
||||
# self.advantage_fc = rn.NoisyLinear(64, 64)
|
||||
# self.advantage_fc_norm = nn.LayerNorm(64)
|
||||
# self.advantage = rn.NoisyLinear(64, action_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
|
||||
x = F.softmax(self.fc3(x), dim = 1)
|
||||
|
||||
# state_value = F.relu(self.value_fc_norm(self.value_fc(x)))
|
||||
# state_value = self.value(state_value)
|
||||
|
||||
# advantage = F.relu(self.advantage_fc_norm(self.advantage_fc(x)))
|
||||
# advantage = self.advantage(advantage)
|
||||
|
||||
# x = F.softmax(state_value + advantage - advantage.mean(), dim = 1)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
config = {}
|
||||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 500
|
||||
config['total_evaluation_episodes'] = 10
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-3
|
||||
config['target_sync_tau'] = 1e-1
|
||||
config['discount_rate'] = 0.99
|
||||
config['replay_skip'] = 0
|
||||
# How many episodes between printing out the episode stats
|
||||
config['print_stat_n_eps'] = 1
|
||||
config['disable_cuda'] = False
|
||||
|
||||
|
||||
def train(runner, agent, config, logger = None, logwriter = None):
|
||||
finished = False
|
||||
last_episode_num = 1
|
||||
while not finished:
|
||||
runner.run(config['replay_skip'] + 1)
|
||||
agent.learn()
|
||||
if logwriter is not None:
|
||||
if last_episode_num < runner.episode_num:
|
||||
last_episode_num = runner.episode_num
|
||||
agent.value_net.log_named_parameters()
|
||||
agent.policy_net.log_named_parameters()
|
||||
logwriter.write(logger)
|
||||
finished = runner.episode_num > config['total_training_episodes']
|
||||
|
||||
if __name__ == "__main__":
|
||||
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
|
||||
|
||||
# Setting up the environment
|
||||
rltorch.set_seed(config['seed'])
|
||||
print("Setting up environment...", end = " ")
|
||||
env = E.TorchWrap(gym.make(config['environment_name']))
|
||||
env.seed(config['seed'])
|
||||
print("Done.")
|
||||
|
||||
state_size = env.observation_space.shape[0]
|
||||
action_size = env.action_space.n
|
||||
|
||||
# Logging
|
||||
logger = rltorch.log.Logger()
|
||||
logwriter = rltorch.log.LogWriter(SummaryWriter())
|
||||
|
||||
# Setting up the networks
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
|
||||
policy_net = rn.Network(Policy(state_size, action_size),
|
||||
torch.optim.Adam, config, device = device, name = "Policy")
|
||||
value_net = rn.Network(Value(state_size),
|
||||
torch.optim.Adam, config, device = device, name = "DQN")
|
||||
|
||||
|
||||
# Memory stores experiences for later training
|
||||
memory = M.EpisodeMemory()
|
||||
|
||||
# Actor takes a net and uses it to produce actions from given states
|
||||
actor = StochasticSelector(policy_net, action_size, memory, device = device)
|
||||
|
||||
# Agent is what performs the training
|
||||
# agent = rltorch.agents.REINFORCEAgent(net, memory, config, target_net = target_net, logger = logger)
|
||||
agent = rltorch.agents.PPOAgent(policy_net, value_net, memory, config, logger = logger)
|
||||
|
||||
# Runner performs a certain number of steps in the environment
|
||||
runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
|
||||
|
||||
print("Training...")
|
||||
train(runner, agent, config, logger = logger, logwriter = logwriter)
|
||||
|
||||
# For profiling...
|
||||
# import cProfile
|
||||
# cProfile.run('train(runner, agent, config, logger = logger, logwriter = logwriter )')
|
||||
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
|
||||
|
||||
print("Training Finished.")
|
||||
|
||||
print("Evaluating...")
|
||||
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
|
||||
print("Evaulations Done.")
|
||||
|
||||
logwriter.close() # We don't need to write anything out to disk anymore
|
||||
|
|
@ -7,9 +7,10 @@ import rltorch
|
|||
import rltorch.network as rn
|
||||
import rltorch.memory as M
|
||||
import rltorch.env as E
|
||||
from rltorch.action_selector import ArgMaxSelector
|
||||
from rltorch.action_selector import StochasticSelector
|
||||
from tensorboardX import SummaryWriter
|
||||
import torch.multiprocessing as mp
|
||||
from copy import deepcopy
|
||||
|
||||
class Value(nn.Module):
|
||||
def __init__(self, state_size, action_size):
|
||||
|
|
@ -17,16 +18,16 @@ class Value(nn.Module):
|
|||
self.state_size = state_size
|
||||
self.action_size = action_size
|
||||
|
||||
self.fc1 = rn.NoisyLinear(state_size, 64)
|
||||
self.fc_norm = nn.LayerNorm(64)
|
||||
self.fc1 = rn.NoisyLinear(state_size, 255)
|
||||
self.fc_norm = nn.LayerNorm(255)
|
||||
|
||||
self.value_fc = rn.NoisyLinear(64, 64)
|
||||
self.value_fc_norm = nn.LayerNorm(64)
|
||||
self.value = rn.NoisyLinear(64, 1)
|
||||
self.value_fc = rn.NoisyLinear(255, 255)
|
||||
self.value_fc_norm = nn.LayerNorm(255)
|
||||
self.value = rn.NoisyLinear(255, 1)
|
||||
|
||||
self.advantage_fc = rn.NoisyLinear(64, 64)
|
||||
self.advantage_fc_norm = nn.LayerNorm(64)
|
||||
self.advantage = rn.NoisyLinear(64, action_size)
|
||||
self.advantage_fc = rn.NoisyLinear(255, 255)
|
||||
self.advantage_fc_norm = nn.LayerNorm(255)
|
||||
self.advantage = rn.NoisyLinear(255, action_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
|
|
@ -42,12 +43,32 @@ class Value(nn.Module):
|
|||
return x
|
||||
|
||||
|
||||
class Policy(nn.Module):
|
||||
def __init__(self, state_size, action_size):
|
||||
super(Policy, self).__init__()
|
||||
self.state_size = state_size
|
||||
self.action_size = action_size
|
||||
|
||||
self.fc1 = nn.Linear(state_size, 125)
|
||||
self.fc_norm = nn.LayerNorm(125)
|
||||
|
||||
self.fc2 = nn.Linear(125, 125)
|
||||
self.fc2_norm = nn.LayerNorm(125)
|
||||
|
||||
self.action_prob = nn.Linear(125, action_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.fc_norm(self.fc1(x)))
|
||||
x = F.relu(self.fc2_norm(self.fc2(x)))
|
||||
x = F.softmax(self.action_prob(x), dim = 1)
|
||||
return x
|
||||
|
||||
config = {}
|
||||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 50
|
||||
config['total_evaluation_episodes'] = 10
|
||||
config['total_evaluation_episodes'] = 5
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-3
|
||||
config['target_sync_tau'] = 1e-1
|
||||
|
|
@ -65,28 +86,24 @@ config['prioritized_replay_sampling_priority'] = 0.6
|
|||
# 1 - Lower the importance of high losses
|
||||
# Should ideally start from 0 and move your way to 1 to prevent overfitting
|
||||
config['prioritized_replay_weight_importance'] = rltorch.scheduler.ExponentialScheduler(initial_value = 0.4, end_value = 1, iterations = 5000)
|
||||
|
||||
|
||||
|
||||
|
||||
def train(runner, agent, config, logger = None, logwriter = None):
|
||||
finished = False
|
||||
last_episode_num = 1
|
||||
while not finished:
|
||||
runner.run()
|
||||
runner.run(config['replay_skip'] + 1)
|
||||
agent.learn()
|
||||
runner.join()
|
||||
# When the episode number changes, log network paramters
|
||||
with runner.episode_num.get_lock():
|
||||
if logwriter is not None and last_episode_num < runner.episode_num.value:
|
||||
last_episode_num = runner.episode_num.value
|
||||
agent.net.log_named_parameters()
|
||||
if logwriter is not None:
|
||||
logwriter.write(logger)
|
||||
finished = runner.episode_num.value > config['total_training_episodes']
|
||||
|
||||
|
||||
if logwriter is not None:
|
||||
if last_episode_num < runner.episode_num:
|
||||
last_episode_num = runner.episode_num
|
||||
agent.value_net.log_named_parameters()
|
||||
agent.policy_net.log_named_parameters()
|
||||
logwriter.write(logger)
|
||||
finished = runner.episode_num > config['total_training_episodes']
|
||||
|
||||
if __name__ == "__main__":
|
||||
torch.multiprocessing.set_sharing_strategy('file_system') # To not hit file descriptor memory limit
|
||||
|
||||
# Setting up the environment
|
||||
rltorch.set_seed(config['seed'])
|
||||
print("Setting up environment...", end = " ")
|
||||
|
|
@ -104,24 +121,29 @@ if __name__ == "__main__":
|
|||
|
||||
# Setting up the networks
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() and not config['disable_cuda'] else "cpu")
|
||||
net = rn.Network(Value(state_size, action_size),
|
||||
torch.optim.Adam, config, device = device, name = "DQN")
|
||||
target_net = rn.TargetNetwork(net, device = device)
|
||||
net.model.share_memory()
|
||||
config2 = deepcopy(config)
|
||||
config2['learning_rate'] = 0.01
|
||||
policy_net = rn.ESNetwork(Policy(state_size, action_size),
|
||||
torch.optim.Adam, 500, None, config2, sigma = 0.1, device = device, name = "ES", logger = logger)
|
||||
value_net = rn.Network(Value(state_size, action_size),
|
||||
torch.optim.Adam, config, device = device, name = "DQN", logger = logger)
|
||||
|
||||
target_net = rn.TargetNetwork(value_net, device = device)
|
||||
value_net.model.share_memory()
|
||||
target_net.model.share_memory()
|
||||
|
||||
# Actor takes a net and uses it to produce actions from given states
|
||||
actor = ArgMaxSelector(net, action_size, device = device)
|
||||
actor = StochasticSelector(policy_net, action_size, device = device)
|
||||
# Memory stores experiences for later training
|
||||
memory = M.PrioritizedReplayMemory(capacity = config['memory_size'], alpha = config['prioritized_replay_sampling_priority'])
|
||||
# memory = M.ReplayMemory(capacity = config['memory_size'])
|
||||
|
||||
# Runner performs a certain number of steps in the environment
|
||||
runner = rltorch.mp.EnvironmentRun(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
|
||||
runner = rltorch.env.EnvironmentRunSync(env, actor, config, name = "Training", memory = memory, logwriter = logwriter)
|
||||
|
||||
# Agent is what performs the training
|
||||
agent = rltorch.agents.DQNAgent(net, memory, config, target_net = target_net, logger = logger)
|
||||
|
||||
# agent = TestAgent(policy_net, value_net, memory, config, target_value_net = target_net, logger = logger)
|
||||
agent = rltorch.agents.QEPAgent(policy_net, value_net, memory, config, target_value_net = target_net, logger = logger)
|
||||
|
||||
print("Training...")
|
||||
|
||||
train(runner, agent, config, logger = logger, logwriter = logwriter)
|
||||
|
|
@ -132,7 +154,6 @@ if __name__ == "__main__":
|
|||
# python -m torch.utils.bottleneck /path/to/source/script.py [args] is also a good solution...
|
||||
|
||||
print("Training Finished.")
|
||||
runner.terminate() # We don't need the extra process anymore
|
||||
|
||||
print("Evaluating...")
|
||||
rltorch.env.simulateEnvEps(env, actor, config, total_episodes = config['total_evaluation_episodes'], logger = logger, name = "Evaluation")
|
||||
|
|
@ -48,7 +48,7 @@ config = {}
|
|||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 100
|
||||
config['total_training_episodes'] = 500
|
||||
config['total_evaluation_episodes'] = 10
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-3
|
||||
|
|
|
|||
|
|
@ -46,8 +46,8 @@ config = {}
|
|||
config['seed'] = 901
|
||||
config['environment_name'] = 'Acrobot-v1'
|
||||
config['memory_size'] = 2000
|
||||
config['total_training_episodes'] = 5
|
||||
config['total_evaluation_episodes'] = 2
|
||||
config['total_training_episodes'] = 50
|
||||
config['total_evaluation_episodes'] = 5
|
||||
config['batch_size'] = 32
|
||||
config['learning_rate'] = 1e-3
|
||||
config['target_sync_tau'] = 1e-1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue