diff --git a/code/agent.py b/code/agent.py index 80ed021..4b7e647 100644 --- a/code/agent.py +++ b/code/agent.py @@ -5,6 +5,7 @@ """ Base class of an autonomously acting and learning agent. """ + class Agent: def __init__(self, params): @@ -22,6 +23,10 @@ def policy(self, state): """ def update(self, state, action, reward, next_state, terminated, truncated): pass + + def print(self): + for attribute, value in vars(self).items(): + print(f"{attribute}: {value}") """ @@ -46,7 +51,7 @@ def __init__(self, params): self.Q_values = {} self.alpha = params["alpha"] self.epsilon_decay = params["epsilon_decay"] - self.epsilon = 1.0 + self.epsilon = params["epsilon"] def Q(self, state): state = np.array2string(state) @@ -90,4 +95,27 @@ def update(self, state, action, reward, next_state, terminated, truncated): Q_next = max(self.Q(next_state)) TD_target += self.gamma*Q_next TD_error = TD_target - Q_old - self.Q(state)[action] += self.alpha*TD_error \ No newline at end of file + self.Q(state)[action] += self.alpha*TD_error + +class UCBQLearner(QLearner): + def __init__(self, params): + super(UCBQLearner, self).__init__(params) + self.exploration_constant = params["exploration_constant"] + self.action_counts = {} + + def get_action_counts(self, state): + state = np.array2string(state) + if state not in self.action_counts: + self.action_counts[state] = np.zeros(self.nr_actions) + return self.action_counts[state] + + def update_action_counts(self, state, action): + state = np.array2string(state) + self.action_counts[state][action] += 1 + + def policy(self, state): + Q_values = self.Q(state) + action_counts = self.get_action_counts(state) + action = UCB1(Q_values, action_counts, exploration_constant=self.exploration_constant) + self.update_action_counts(state, action) + return action \ No newline at end of file diff --git a/code/main.py b/code/main.py index 688259d..c4efa5a 100644 --- a/code/main.py +++ b/code/main.py @@ -1,50 +1,97 @@ import rooms +import random import agent as a import matplotlib.pyplot as plot +import seaborn as sns +import pandas as pd import sys +from utils import save_agent, load_agent +import numpy as np -def episode(env, agent, nr_episode=0): +def plot_returns(x,y): + plot.plot(x,y) + plot.title("Progress") + plot.xlabel("Episode") + plot.ylabel("Discounted Return") + plot.show() + +def plot_eval_returns(x, y): + df = pd.DataFrame(y) + df = df.melt(var_name="Episode", value_name="Discounted Return") # lineplot expects data in long format + sns.lineplot(x="Episode", y="Discounted Return", data=df, errorbar='ci', ci=95) + plot.axhline(y=0.8, color='black', linestyle='--') + plot.title("Evaluation returns") + plot.show() + +def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True): state = env.reset() discounted_return = 0 discount_factor = 0.99 done = False time_step = 0 + if evaluation_mode: + agent.epsilon = 0 + agent.exploration_constant = 0 while not done: # 1. Select action according to policy action = agent.policy(state) # 2. Execute selected action next_state, reward, terminated, truncated, _ = env.step(action) # 3. Integrate new experience into agent - agent.update(state, action, reward, next_state, terminated, truncated) + if not evaluation_mode: + agent.update(state, action, reward, next_state, terminated, truncated) state = next_state done = terminated or truncated discounted_return += (discount_factor**time_step)*reward time_step += 1 - print(nr_episode, ":", discounted_return) + if verbose: print(nr_episode, ":", discounted_return, "steps: ", time_step) return discounted_return - + +def train(env, agent, episodes): + returns = [episode(env, agent, nr_episode=i, verbose=True) for i in range(episodes)] + return returns + +def evaluate(env, agent, runs, episodes): + eval_returns = [] + for i in range(no_runs): + returns = [episode(env, agent, nr_episode=i, verbose=False, evaluation_mode=True) for i in range(episodes)] + eval_returns.append(returns) + return np.array(eval_returns) + +np.random.seed(42) +random.seed(42) params = {} rooms_instance = sys.argv[1] env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4") params["nr_actions"] = env.action_space.n params["gamma"] = 0.99 -params["epsilon_decay"] = 0.001 +params["epsilon_decay"] = 0.0001 params["alpha"] = 0.1 params["env"] = env +params["exploration_constant"] = np.sqrt(2) +params["epsilon"] = 1 #agent = a.RandomAgent(params) -#agent = a.SARSALearner(params) -agent = a.QLearner(params) +# agent = a.SARSALearner(params) +# agent = a.QLearner(params) +agent = a.UCBQLearner(params) + training_episodes = 200 -returns = [episode(env, agent, i) for i in range(training_episodes)] +evaluation_episodes = 10 +no_runs = 100 + +# TRAINING +returns = train(env, agent, training_episodes) +plot_returns(x=range(training_episodes),y=returns) +# save_agent(agent) +# exit() + +# EVALUATION +# agent = load_agent("saved_agents/agent: 2024-03-19 13:01:51.pkl") +eval_returns = evaluate(env, agent, runs=no_runs, episodes=evaluation_episodes) +plot_eval_returns(x=range(evaluation_episodes), y=eval_returns) -x = range(training_episodes) -y = returns +print(f"Average evaluation discounted return: {np.mean(eval_returns)}") -plot.plot(x,y) -plot.title("Progress") -plot.xlabel("Episode") -plot.ylabel("Discounted Return") -plot.show() env.save_video() diff --git a/code/utils.py b/code/utils.py new file mode 100644 index 0000000..3d5f83a --- /dev/null +++ b/code/utils.py @@ -0,0 +1,17 @@ +import pickle +from datetime import datetime +import os + +def save_agent(agent, filename="agent"): + os.makedirs("saved_agents", exist_ok=True) + current_datetime = datetime.now() + date_string = current_datetime.strftime("%Y-%m-%d %H:%M:%S") + with open(f"saved_agents/{filename}: {date_string}.pkl", 'wb') as file: + pickle.dump(agent, file) + +def load_agent(path, verbose=False): + with open(path, 'rb') as file: + agent = pickle.load(file) + if verbose: + agent.print() + return agent \ No newline at end of file