From a3e41866ea80a4cf1f9bc47ad391e3da2d545801 Mon Sep 17 00:00:00 2001 From: Prashanth Ravichandar Date: Mon, 26 Feb 2024 15:55:24 -0800 Subject: [PATCH 01/16] Added Q Learning + UCB agent and util funcs --- code/agent.py | 14 +++++++++++++- code/main.py | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/code/agent.py b/code/agent.py index 80ed021..dcd0ba9 100644 --- a/code/agent.py +++ b/code/agent.py @@ -90,4 +90,16 @@ def update(self, state, action, reward, next_state, terminated, truncated): Q_next = max(self.Q(next_state)) TD_target += self.gamma*Q_next TD_error = TD_target - Q_old - self.Q(state)[action] += self.alpha*TD_error \ No newline at end of file + self.Q(state)[action] += self.alpha*TD_error + +class UCBQLearner(QLearner): + def __init__(self, params): + super(UCBQLearner, self).__init__(params) + self.exploration_constant = params["exploration_constant"] + self.action_counts = np.array([0]*self.nr_actions) + + def policy(self, state): + Q_values = self.Q(state) + action = UCB1(Q_values, self.action_counts, exploration_constant=self.exploration_constant) + self.action_counts[action] += 1 + return action \ No newline at end of file diff --git a/code/main.py b/code/main.py index 688259d..32ec707 100644 --- a/code/main.py +++ b/code/main.py @@ -23,21 +23,52 @@ def episode(env, agent, nr_episode=0): print(nr_episode, ":", discounted_return) return discounted_return +def print_info(agent, params, training_episodes, rooms_instance): + print("Agent:", agent.__class__.__name__) + print("Environment:", rooms_instance) + print("Discount factor (gamma):", params["gamma"]) + print("Learning rate (alpha):", params["alpha"]) + print("Exploration constant:", params["exploration_constant"]) + print("Epsilon decay:", params["epsilon_decay"]) + print("Training for", training_episodes, "episodes.") + print("") + params = {} rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") params["nr_actions"] = env.action_space.n params["gamma"] = 0.99 -params["epsilon_decay"] = 0.001 +params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env +params["exploration_constant"] = 5 #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) agent = a.QLearner(params) -training_episodes = 200 +# agent = a.UCBQLearner(params) +training_episodes = 2000 +print_info(agent, params, training_episodes, rooms_instance) + returns = [episode(env, agent, i) for i in range(training_episodes)] +assert params['gamma'] == 0.99 +good_returns = [] +prev_return = [] +for i in range(len(returns)): + if returns[i] >= 0.8: + prev_return.append(returns[i]) + else: + if len(prev_return) >= 10: + good_returns.append((i - len(prev_return), prev_return)) + prev_return = [] + +if len(prev_return) >= 10: + good_returns.append((len(returns) - len(prev_return), prev_return)) + +for i in good_returns: + print("Start:", i[0], "Length:", len(i[1]), "Returns:", i[1]) + x = range(training_episodes) y = returns @@ -46,5 +77,6 @@ def episode(env, agent, nr_episode=0): plot.xlabel("Episode") plot.ylabel("Discounted Return") plot.show() +plot.savefig(f"{rooms_instance}.png") env.save_video() From 2ce1c3ac0fdc65f32a2753a265a04f025eed0161 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 13:13:23 -0700 Subject: [PATCH 02/16] add loading and saving --- code/agent.py | 18 +++++++++++++++++- code/main.py | 6 +++--- code/rooms.py | 30 +++++++++++++++--------------- code/utils.py | 15 +++++++++++++++ 4 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 code/utils.py diff --git a/code/agent.py b/code/agent.py index 80ed021..3ee1268 100644 --- a/code/agent.py +++ b/code/agent.py @@ -22,6 +22,10 @@ def policy(self, state): """ def update(self, state, action, reward, next_state, terminated, truncated): pass + + def print(self): + for attribute, value in vars(self).items(): + print(f"{attribute}: {value}") """ @@ -90,4 +94,16 @@ def update(self, state, action, reward, next_state, terminated, truncated): Q_next = max(self.Q(next_state)) TD_target += self.gamma*Q_next TD_error = TD_target - Q_old - self.Q(state)[action] += self.alpha*TD_error \ No newline at end of file + self.Q(state)[action] += self.alpha*TD_error + +class UCBQLearner(QLearner): + def __init__(self, params): + super(UCBQLearner, self).__init__(params) + self.exploration_constant = params["exploration_constant"] + self.action_counts = np.array([0]*self.nr_actions) + + def policy(self, state): + Q_values = self.Q(state) + action = UCB1(Q_values, self.action_counts, exploration_constant=self.exploration_constant) + self.action_counts[action] += 1 + return action \ No newline at end of file diff --git a/code/main.py b/code/main.py index 688259d..868d80c 100644 --- a/code/main.py +++ b/code/main.py @@ -2,11 +2,11 @@ import agent as a import matplotlib.pyplot as plot import sys +from utils import save_agent, load_agent -def episode(env, agent, nr_episode=0): +def episode(env, agent, discount_factor = 0.99, nr_episode=0): state = env.reset() discounted_return = 0 - discount_factor = 0.99 done = False time_step = 0 while not done: @@ -47,4 +47,4 @@ def episode(env, agent, nr_episode=0): plot.ylabel("Discounted Return") plot.show() -env.save_video() +# env.save_video() diff --git a/code/rooms.py b/code/rooms.py index beefa82..651839c 100644 --- a/code/rooms.py +++ b/code/rooms.py @@ -5,8 +5,8 @@ from gymnasium.utils import seeding import matplotlib.pyplot as plot import random -from moviepy.editor import VideoClip -from moviepy.video.io.bindings import mplfig_to_npimage +# from moviepy.editor import VideoClip +# from moviepy.video.io.bindings import mplfig_to_npimage MOVE_NORTH = 0 MOVE_SOUTH = 1 @@ -115,19 +115,19 @@ def state_summary(self): "score": self.undiscounted_return } - def save_video(self): - if self.movie_filename is not None: - history_of_states = self.state_history - duration = len(history_of_states) - fig, ax = plot.subplots() - def make_frame(t): - ax.clear() - ax.grid(False) - ax.imshow(numpy.swapaxes(history_of_states[int(t)], 0, 2)) - ax.tick_params(axis='both', which='both', bottom=False, top=False, left=False, right=False, labelleft=False, labelbottom=False) - return mplfig_to_npimage(fig) - animation = VideoClip(make_frame, duration=duration) - animation.write_videofile(self.movie_filename, fps=1) + # def save_video(self): + # if self.movie_filename is not None: + # history_of_states = self.state_history + # duration = len(history_of_states) + # fig, ax = plot.subplots() + # def make_frame(t): + # ax.clear() + # ax.grid(False) + # ax.imshow(numpy.swapaxes(history_of_states[int(t)], 0, 2)) + # ax.tick_params(axis='both', which='both', bottom=False, top=False, left=False, right=False, labelleft=False, labelbottom=False) + # return mplfig_to_npimage(fig) + # animation = VideoClip(make_frame, duration=duration) + # animation.write_videofile(self.movie_filename, fps=1) def read_map_file(path): file = pathlib.Path(path) diff --git a/code/utils.py b/code/utils.py new file mode 100644 index 0000000..2b141c4 --- /dev/null +++ b/code/utils.py @@ -0,0 +1,15 @@ +import pickle +from datetime import datetime + + +def save_agent(agent, filename="agent"): + current_datetime = datetime.now() + date_string = current_datetime.strftime("%Y-%m-%d %H:%M:%S") + with open(f"saved_agents/{filename}: {date_string}.pkl", 'wb') as file: + pickle.dump(agent, file) + +def load_agent(path): + with open(path, 'rb') as file: + agent = pickle.load(file) + agent.print() + return agent \ No newline at end of file From be0f672ad9a4ca121c95fe6efa3f0a1823c7260c Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 13:32:08 -0700 Subject: [PATCH 03/16] numpy seed whole app --- code/main.py | 7 ++++++- code/multi_armed_bandits.py | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/code/main.py b/code/main.py index 868d80c..a8e0ed0 100644 --- a/code/main.py +++ b/code/main.py @@ -3,6 +3,7 @@ import matplotlib.pyplot as plot import sys from utils import save_agent, load_agent +import numpy as np def episode(env, agent, discount_factor = 0.99, nr_episode=0): state = env.reset() @@ -22,7 +23,8 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0): time_step += 1 print(nr_episode, ":", discounted_return) return discounted_return - + + params = {} rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") @@ -32,9 +34,12 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0): params["alpha"] = 0.1 params["env"] = env +np.random.seed(42) + #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) agent = a.QLearner(params) +# agent = load_agent("saved_agents/agent: 2024-03-11 13:06:17.pkl") training_episodes = 200 returns = [episode(env, agent, i) for i in range(training_episodes)] diff --git a/code/multi_armed_bandits.py b/code/multi_armed_bandits.py index def6688..f0728b8 100644 --- a/code/multi_armed_bandits.py +++ b/code/multi_armed_bandits.py @@ -2,12 +2,13 @@ import numpy as np import math + def random_bandit(Q_values, action_counts): - return random.choice(range(len(Q_values))) + return np.random.choice(range(len(Q_values))) def epsilon_greedy(Q_values, action_counts, epsilon=0.1): if np.random.rand() <= epsilon: - return random.choice(range(len(Q_values))) + return np.random.choice(range(len(Q_values))) else: return np.argmax(Q_values) From 45ea1aeca899e56fc9403b20b836b5b54e24c239 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 13:36:14 -0700 Subject: [PATCH 04/16] . --- code/multi_armed_bandits.py | 1 - 1 file changed, 1 deletion(-) diff --git a/code/multi_armed_bandits.py b/code/multi_armed_bandits.py index f0728b8..6c7fadf 100644 --- a/code/multi_armed_bandits.py +++ b/code/multi_armed_bandits.py @@ -2,7 +2,6 @@ import numpy as np import math - def random_bandit(Q_values, action_counts): return np.random.choice(range(len(Q_values))) From 0d0a1051a97e0ba7e9db04c40815fb18dc63cf32 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 13:44:57 -0700 Subject: [PATCH 05/16] make epsillon a programmable parameter for evaluation --- code/agent.py | 4 +++- code/main.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/code/agent.py b/code/agent.py index 3ee1268..9856099 100644 --- a/code/agent.py +++ b/code/agent.py @@ -5,6 +5,8 @@ """ Base class of an autonomously acting and learning agent. """ + +np.random.seed(42) class Agent: def __init__(self, params): @@ -50,7 +52,7 @@ def __init__(self, params): self.Q_values = {} self.alpha = params["alpha"] self.epsilon_decay = params["epsilon_decay"] - self.epsilon = 1.0 + self.epsilon = params["epsilon"] def Q(self, state): state = np.array2string(state) diff --git a/code/main.py b/code/main.py index a8e0ed0..3f204bb 100644 --- a/code/main.py +++ b/code/main.py @@ -33,6 +33,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0): params["epsilon_decay"] = 0.001 params["alpha"] = 0.1 params["env"] = env +params["epsilon"] = 1 np.random.seed(42) From 8d373a5cb014acaec9c831657a77ea5d518beb9e Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 14:07:19 -0700 Subject: [PATCH 06/16] augment episode method for evaluation --- code/main.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/code/main.py b/code/main.py index 3f204bb..35339a9 100644 --- a/code/main.py +++ b/code/main.py @@ -5,23 +5,35 @@ from utils import save_agent, load_agent import numpy as np -def episode(env, agent, discount_factor = 0.99, nr_episode=0): +def plot_returns(x,y): + plot.plot(x,y) + plot.title("Progress") + plot.xlabel("Episode") + plot.ylabel("Discounted Return") + plot.show() + + +def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=False, verbose=True): state = env.reset() discounted_return = 0 done = False time_step = 0 + if evaluation_mode: + agent.epsilon = 0 + # agent.exploration_constant = 0 # no exploration while not done: # 1. Select action according to policy action = agent.policy(state) # 2. Execute selected action next_state, reward, terminated, truncated, _ = env.step(action) # 3. Integrate new experience into agent - agent.update(state, action, reward, next_state, terminated, truncated) + if not evaluation_mode: + agent.update(state, action, reward, next_state, terminated, truncated) state = next_state done = terminated or truncated discounted_return += (discount_factor**time_step)*reward time_step += 1 - print(nr_episode, ":", discounted_return) + if verbose: print(nr_episode, ":", discounted_return) return discounted_return @@ -42,15 +54,19 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0): agent = a.QLearner(params) # agent = load_agent("saved_agents/agent: 2024-03-11 13:06:17.pkl") training_episodes = 200 -returns = [episode(env, agent, i) for i in range(training_episodes)] +evaluation_episodes = 10 +returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] +eval_returns = [episode(env, agent, i, verbose=False, evaluation_mode=True) for i in range(evaluation_episodes)] x = range(training_episodes) y = returns -plot.plot(x,y) -plot.title("Progress") -plot.xlabel("Episode") -plot.ylabel("Discounted Return") -plot.show() +x_eval = range(evaluation_episodes) +y_eval = eval_returns + +plot_returns(x,y) +plot_returns(x_eval,y_eval) + +print(f"Evaluation discounted reward: {np.mean(eval_returns)}") # env.save_video() From 2439f705b2bf5c55a6a9b33d7d152e9eddd64048 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 14:42:20 -0700 Subject: [PATCH 07/16] refactor --- code/main.py | 17 +++++++++-------- code/utils.py | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/code/main.py b/code/main.py index 35339a9..6cc62d5 100644 --- a/code/main.py +++ b/code/main.py @@ -20,7 +20,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa time_step = 0 if evaluation_mode: agent.epsilon = 0 - # agent.exploration_constant = 0 # no exploration + agent.exploration_constant = 0 while not done: # 1. Select action according to policy action = agent.policy(state) @@ -52,19 +52,20 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) agent = a.QLearner(params) -# agent = load_agent("saved_agents/agent: 2024-03-11 13:06:17.pkl") +agent = load_agent("saved_agents/agent: 2024-03-11 13:06:17.pkl") training_episodes = 200 evaluation_episodes = 10 -returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] -eval_returns = [episode(env, agent, i, verbose=False, evaluation_mode=True) for i in range(evaluation_episodes)] -x = range(training_episodes) -y = returns +# TRAINING +# returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] +# x = range(training_episodes) +# y = returns +# plot_returns(x,y) +# EVALUATION +eval_returns = [episode(env, agent, i, verbose=False, evaluation_mode=True) for i in range(evaluation_episodes)] x_eval = range(evaluation_episodes) y_eval = eval_returns - -plot_returns(x,y) plot_returns(x_eval,y_eval) print(f"Evaluation discounted reward: {np.mean(eval_returns)}") diff --git a/code/utils.py b/code/utils.py index 2b141c4..0a50be1 100644 --- a/code/utils.py +++ b/code/utils.py @@ -8,8 +8,8 @@ def save_agent(agent, filename="agent"): with open(f"saved_agents/{filename}: {date_string}.pkl", 'wb') as file: pickle.dump(agent, file) -def load_agent(path): +def load_agent(path, verbose=False): with open(path, 'rb') as file: agent = pickle.load(file) - agent.print() + if verbose: agent.print() return agent \ No newline at end of file From 901e8a1fa190d42ae42c7fc1dae85c1051700b39 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 19:11:27 -0700 Subject: [PATCH 08/16] comments --- code/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/main.py b/code/main.py index 6cc62d5..afaf1d7 100644 --- a/code/main.py +++ b/code/main.py @@ -52,8 +52,9 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) agent = a.QLearner(params) -agent = load_agent("saved_agents/agent: 2024-03-11 13:06:17.pkl") -training_episodes = 200 +save_agent(agent) +# agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file +training_episodes = 2000 evaluation_episodes = 10 # TRAINING From 15048693693479df614f9e178218d05fbf07a0c1 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 19:32:43 -0700 Subject: [PATCH 09/16] fix seeding --- code/agent.py | 2 +- code/main.py | 2 ++ code/multi_armed_bandits.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/code/agent.py b/code/agent.py index 9856099..e5ff301 100644 --- a/code/agent.py +++ b/code/agent.py @@ -6,7 +6,7 @@ Base class of an autonomously acting and learning agent. """ -np.random.seed(42) +random.seed(42) class Agent: def __init__(self, params): diff --git a/code/main.py b/code/main.py index afaf1d7..d3f42a9 100644 --- a/code/main.py +++ b/code/main.py @@ -1,4 +1,5 @@ import rooms +import random import agent as a import matplotlib.pyplot as plot import sys @@ -48,6 +49,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa params["epsilon"] = 1 np.random.seed(42) +random.seed(42) #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) diff --git a/code/multi_armed_bandits.py b/code/multi_armed_bandits.py index 6c7fadf..0be43a2 100644 --- a/code/multi_armed_bandits.py +++ b/code/multi_armed_bandits.py @@ -3,17 +3,17 @@ import math def random_bandit(Q_values, action_counts): - return np.random.choice(range(len(Q_values))) + return random.choice(range(len(Q_values))) def epsilon_greedy(Q_values, action_counts, epsilon=0.1): if np.random.rand() <= epsilon: - return np.random.choice(range(len(Q_values))) + return random.choice(range(len(Q_values))) else: return np.argmax(Q_values) def boltzmann(Q_values, action_counts, temperature=1.0): E = np.exp(Q_values/temperature) - return np.random.choice(range(len(Q_values)), p=E/sum(E)) + return random.choice(range(len(Q_values)), p=E/sum(E)) def UCB1(Q_values, action_counts, exploration_constant=1): UCB1_values = [] From 4e0dc93924d8a0a262d1733127f1ec7ef5ae61ab Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 22:20:08 -0700 Subject: [PATCH 10/16] playing with hps --- code/main.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/code/main.py b/code/main.py index d3f42a9..7017b61 100644 --- a/code/main.py +++ b/code/main.py @@ -21,7 +21,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa time_step = 0 if evaluation_mode: agent.epsilon = 0 - agent.exploration_constant = 0 + # agent.exploration_constant = 1 while not done: # 1. Select action according to policy action = agent.policy(state) @@ -43,10 +43,11 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") params["nr_actions"] = env.action_space.n params["gamma"] = 0.99 -params["epsilon_decay"] = 0.001 +params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env -params["epsilon"] = 1 +params["exploration_constant"] = 10 +params["epsilon"] = .4 np.random.seed(42) random.seed(42) @@ -54,23 +55,24 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa #agent = a.RandomAgent(params) #agent = a.SARSALearner(params) agent = a.QLearner(params) -save_agent(agent) + # agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file -training_episodes = 2000 +training_episodes = 1000 evaluation_episodes = 10 # TRAINING -# returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] -# x = range(training_episodes) -# y = returns +returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] +x = range(training_episodes) +y = returns # plot_returns(x,y) # EVALUATION eval_returns = [episode(env, agent, i, verbose=False, evaluation_mode=True) for i in range(evaluation_episodes)] x_eval = range(evaluation_episodes) y_eval = eval_returns -plot_returns(x_eval,y_eval) +# plot_returns(x_eval,y_eval) print(f"Evaluation discounted reward: {np.mean(eval_returns)}") +# save_agent(agent) # env.save_video() From 45c5c78e9f5888542a2f4aafa1b4c7189766480a Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 22:24:40 -0700 Subject: [PATCH 11/16] . --- code/agent.py | 1 - code/main.py | 10 +++++----- code/multi_armed_bandits.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/code/agent.py b/code/agent.py index e5ff301..04b141b 100644 --- a/code/agent.py +++ b/code/agent.py @@ -6,7 +6,6 @@ Base class of an autonomously acting and learning agent. """ -random.seed(42) class Agent: def __init__(self, params): diff --git a/code/main.py b/code/main.py index 7017b61..d0a9cb9 100644 --- a/code/main.py +++ b/code/main.py @@ -42,19 +42,19 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") params["nr_actions"] = env.action_space.n -params["gamma"] = 0.99 +params["gamma"] = 0.95 params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env -params["exploration_constant"] = 10 -params["epsilon"] = .4 +params["exploration_constant"] = 5 +params["epsilon"] = 1 np.random.seed(42) random.seed(42) #agent = a.RandomAgent(params) -#agent = a.SARSALearner(params) -agent = a.QLearner(params) +agent = a.SARSALearner(params) +# agent = a.QLearner(params) # agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file training_episodes = 1000 diff --git a/code/multi_armed_bandits.py b/code/multi_armed_bandits.py index 0be43a2..def6688 100644 --- a/code/multi_armed_bandits.py +++ b/code/multi_armed_bandits.py @@ -13,7 +13,7 @@ def epsilon_greedy(Q_values, action_counts, epsilon=0.1): def boltzmann(Q_values, action_counts, temperature=1.0): E = np.exp(Q_values/temperature) - return random.choice(range(len(Q_values)), p=E/sum(E)) + return np.random.choice(range(len(Q_values)), p=E/sum(E)) def UCB1(Q_values, action_counts, exploration_constant=1): UCB1_values = [] From 8a95c286a94b7e55bb3213a038e267b8cd37734e Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 11 Mar 2024 23:12:38 -0700 Subject: [PATCH 12/16] missed discounted reward var --- code/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/main.py b/code/main.py index d0a9cb9..962cd12 100644 --- a/code/main.py +++ b/code/main.py @@ -17,6 +17,7 @@ def plot_returns(x,y): def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=False, verbose=True): state = env.reset() discounted_return = 0 + discount_factor = 0.99 done = False time_step = 0 if evaluation_mode: @@ -42,7 +43,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") params["nr_actions"] = env.action_space.n -params["gamma"] = 0.95 +params["gamma"] = 0.99 params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env @@ -57,7 +58,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa # agent = a.QLearner(params) # agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file -training_episodes = 1000 +training_episodes = 2000 evaluation_episodes = 10 # TRAINING @@ -67,7 +68,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa # plot_returns(x,y) # EVALUATION -eval_returns = [episode(env, agent, i, verbose=False, evaluation_mode=True) for i in range(evaluation_episodes)] +eval_returns = [episode(env, agent, i, verbose=True, evaluation_mode=True) for i in range(evaluation_episodes)] x_eval = range(evaluation_episodes) y_eval = eval_returns # plot_returns(x_eval,y_eval) From 67bb2c8deb47e2df15afd3c37ef6038f76a08021 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 14 Mar 2024 13:09:55 -0700 Subject: [PATCH 13/16] exploration constant should be 0 --- code/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/main.py b/code/main.py index 962cd12..4bbe927 100644 --- a/code/main.py +++ b/code/main.py @@ -22,7 +22,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa time_step = 0 if evaluation_mode: agent.epsilon = 0 - # agent.exploration_constant = 1 + agent.exploration_constant = 0 while not done: # 1. Select action according to policy action = agent.policy(state) From c55079611b4e8e831d8a12055a2f5280826a1ca8 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 14 Mar 2024 13:15:03 -0700 Subject: [PATCH 14/16] medium map score is 0.825 --- code/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/main.py b/code/main.py index 4bbe927..9297d36 100644 --- a/code/main.py +++ b/code/main.py @@ -35,7 +35,7 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa done = terminated or truncated discounted_return += (discount_factor**time_step)*reward time_step += 1 - if verbose: print(nr_episode, ":", discounted_return) + if verbose: print(nr_episode, ":", discounted_return, "steps: ", time_step) return discounted_return @@ -54,8 +54,8 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa random.seed(42) #agent = a.RandomAgent(params) -agent = a.SARSALearner(params) -# agent = a.QLearner(params) +# agent = a.SARSALearner(params) +agent = a.QLearner(params) # agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file training_episodes = 2000 From 369f80422c031123d606798fa563982ea200119d Mon Sep 17 00:00:00 2001 From: Prashanth Ravichandar Date: Sun, 17 Mar 2024 16:52:37 -0700 Subject: [PATCH 15/16] Using UCBQLearner. Discounted return > 0.8 for both medium maps --- code/agent.py | 17 ++++++++++++++--- code/main.py | 25 +++++++++++-------------- code/rooms.py | 30 +++++++++++++++--------------- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/code/agent.py b/code/agent.py index 04b141b..4b7e647 100644 --- a/code/agent.py +++ b/code/agent.py @@ -101,10 +101,21 @@ class UCBQLearner(QLearner): def __init__(self, params): super(UCBQLearner, self).__init__(params) self.exploration_constant = params["exploration_constant"] - self.action_counts = np.array([0]*self.nr_actions) + self.action_counts = {} + + def get_action_counts(self, state): + state = np.array2string(state) + if state not in self.action_counts: + self.action_counts[state] = np.zeros(self.nr_actions) + return self.action_counts[state] + + def update_action_counts(self, state, action): + state = np.array2string(state) + self.action_counts[state][action] += 1 def policy(self, state): Q_values = self.Q(state) - action = UCB1(Q_values, self.action_counts, exploration_constant=self.exploration_constant) - self.action_counts[action] += 1 + action_counts = self.get_action_counts(state) + action = UCB1(Q_values, action_counts, exploration_constant=self.exploration_constant) + self.update_action_counts(state, action) return action \ No newline at end of file diff --git a/code/main.py b/code/main.py index 9297d36..4aa9e91 100644 --- a/code/main.py +++ b/code/main.py @@ -13,8 +13,7 @@ def plot_returns(x,y): plot.ylabel("Discounted Return") plot.show() - -def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=False, verbose=True): +def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True): state = env.reset() discounted_return = 0 discount_factor = 0.99 @@ -47,33 +46,31 @@ def episode(env, agent, discount_factor = 0.99, nr_episode=0, evaluation_mode=Fa params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env -params["exploration_constant"] = 5 +params["exploration_constant"] = np.sqrt(2) params["epsilon"] = 1 -np.random.seed(42) -random.seed(42) - #agent = a.RandomAgent(params) # agent = a.SARSALearner(params) -agent = a.QLearner(params) +# agent = a.QLearner(params) +agent = a.UCBQLearner(params) # agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file -training_episodes = 2000 +training_episodes = 200 evaluation_episodes = 10 # TRAINING -returns = [episode(env, agent, i, verbose=False) for i in range(training_episodes)] +returns = [episode(env, agent, nr_episode=i, verbose=True) for i in range(training_episodes)] x = range(training_episodes) y = returns -# plot_returns(x,y) +plot_returns(x,y) # EVALUATION -eval_returns = [episode(env, agent, i, verbose=True, evaluation_mode=True) for i in range(evaluation_episodes)] +eval_returns = [episode(env, agent, nr_episode=i, verbose=True, evaluation_mode=True) for i in range(evaluation_episodes)] x_eval = range(evaluation_episodes) y_eval = eval_returns -# plot_returns(x_eval,y_eval) +plot_returns(x_eval,y_eval) -print(f"Evaluation discounted reward: {np.mean(eval_returns)}") +print(f"Average evaluation discounted return: {np.mean(eval_returns)}") # save_agent(agent) -# env.save_video() +env.save_video() diff --git a/code/rooms.py b/code/rooms.py index 651839c..beefa82 100644 --- a/code/rooms.py +++ b/code/rooms.py @@ -5,8 +5,8 @@ from gymnasium.utils import seeding import matplotlib.pyplot as plot import random -# from moviepy.editor import VideoClip -# from moviepy.video.io.bindings import mplfig_to_npimage +from moviepy.editor import VideoClip +from moviepy.video.io.bindings import mplfig_to_npimage MOVE_NORTH = 0 MOVE_SOUTH = 1 @@ -115,19 +115,19 @@ def state_summary(self): "score": self.undiscounted_return } - # def save_video(self): - # if self.movie_filename is not None: - # history_of_states = self.state_history - # duration = len(history_of_states) - # fig, ax = plot.subplots() - # def make_frame(t): - # ax.clear() - # ax.grid(False) - # ax.imshow(numpy.swapaxes(history_of_states[int(t)], 0, 2)) - # ax.tick_params(axis='both', which='both', bottom=False, top=False, left=False, right=False, labelleft=False, labelbottom=False) - # return mplfig_to_npimage(fig) - # animation = VideoClip(make_frame, duration=duration) - # animation.write_videofile(self.movie_filename, fps=1) + def save_video(self): + if self.movie_filename is not None: + history_of_states = self.state_history + duration = len(history_of_states) + fig, ax = plot.subplots() + def make_frame(t): + ax.clear() + ax.grid(False) + ax.imshow(numpy.swapaxes(history_of_states[int(t)], 0, 2)) + ax.tick_params(axis='both', which='both', bottom=False, top=False, left=False, right=False, labelleft=False, labelbottom=False) + return mplfig_to_npimage(fig) + animation = VideoClip(make_frame, duration=duration) + animation.write_videofile(self.movie_filename, fps=1) def read_map_file(path): file = pathlib.Path(path) From 3a1176e922c4f9c517ae14edc9fe2aaa11b8c2ab Mon Sep 17 00:00:00 2001 From: Prashanth Ravichandar Date: Tue, 19 Mar 2024 13:34:39 -0700 Subject: [PATCH 16/16] Add code to plot confidence intervals and remove bug while saving --- code/main.py | 40 ++++++++++++++++++++++++++++++---------- code/utils.py | 6 ++++-- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/code/main.py b/code/main.py index 4aa9e91..117bd6b 100644 --- a/code/main.py +++ b/code/main.py @@ -2,6 +2,8 @@ import random import agent as a import matplotlib.pyplot as plot +import seaborn as sns +import pandas as pd import sys from utils import save_agent, load_agent import numpy as np @@ -13,6 +15,14 @@ def plot_returns(x,y): plot.ylabel("Discounted Return") plot.show() +def plot_eval_returns(x, y): + df = pd.DataFrame(y) + df = df.melt(var_name="Episode", value_name="Discounted Return") # lineplot expects data in long format + sns.lineplot(x="Episode", y="Discounted Return", data=df, errorbar='ci', ci=95) + plot.axhline(y=0.8, color='black', linestyle='--') + plot.title("Evaluation returns") + plot.show() + def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True): state = env.reset() discounted_return = 0 @@ -37,7 +47,19 @@ def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True): if verbose: print(nr_episode, ":", discounted_return, "steps: ", time_step) return discounted_return +def train(env, agent, episodes): + returns = [episode(env, agent, nr_episode=i, verbose=True) for i in range(episodes)] + return returns + +def evaluate(env, agent, runs, episodes): + eval_returns = [] + for i in range(no_runs): + returns = [episode(env, agent, nr_episode=i, verbose=False, evaluation_mode=True) for i in range(episodes)] + eval_returns.append(returns) + return np.array(eval_returns) +np.random.seed(42) +random.seed(42) params = {} rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") @@ -54,23 +76,21 @@ def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True): # agent = a.QLearner(params) agent = a.UCBQLearner(params) -# agent = load_agent("saved_agents/agent: 2024-03-11 19:10:22.pkl") # Load agent from file training_episodes = 200 evaluation_episodes = 10 +no_runs = 100 # TRAINING -returns = [episode(env, agent, nr_episode=i, verbose=True) for i in range(training_episodes)] -x = range(training_episodes) -y = returns -plot_returns(x,y) +returns = train(env, agent, training_episodes) +plot_returns(x=range(training_episodes),y=returns) +# save_agent(agent) +# exit() # EVALUATION -eval_returns = [episode(env, agent, nr_episode=i, verbose=True, evaluation_mode=True) for i in range(evaluation_episodes)] -x_eval = range(evaluation_episodes) -y_eval = eval_returns -plot_returns(x_eval,y_eval) +# agent = load_agent("saved_agents/agent: 2024-03-19 13:01:51.pkl") +eval_returns = evaluate(env, agent, runs=no_runs, episodes=evaluation_episodes) +plot_eval_returns(x=range(evaluation_episodes), y=eval_returns) print(f"Average evaluation discounted return: {np.mean(eval_returns)}") -# save_agent(agent) env.save_video() diff --git a/code/utils.py b/code/utils.py index 0a50be1..3d5f83a 100644 --- a/code/utils.py +++ b/code/utils.py @@ -1,8 +1,9 @@ import pickle from datetime import datetime - +import os def save_agent(agent, filename="agent"): + os.makedirs("saved_agents", exist_ok=True) current_datetime = datetime.now() date_string = current_datetime.strftime("%Y-%m-%d %H:%M:%S") with open(f"saved_agents/{filename}: {date_string}.pkl", 'wb') as file: @@ -11,5 +12,6 @@ def save_agent(agent, filename="agent"): def load_agent(path, verbose=False): with open(path, 'rb') as file: agent = pickle.load(file) - if verbose: agent.print() + if verbose: + agent.print() return agent \ No newline at end of file