Prashanth

code/agent.py

@@ -5,6 +5,7 @@
 """
  Base class of an autonomously acting and learning agent.
 """
+
 class Agent:
 
     def __init__(self, params):
@@ -22,6 +23,10 @@ def policy(self, state):
     """
     def update(self, state, action, reward, next_state, terminated, truncated):
         pass
+
+    def print(self):
+        for attribute, value in vars(self).items():
+            print(f"{attribute}: {value}")
 
 
 """
@@ -46,7 +51,7 @@ def __init__(self, params):
         self.Q_values = {}
         self.alpha = params["alpha"]
         self.epsilon_decay = params["epsilon_decay"]
-        self.epsilon = 1.0
+        self.epsilon = params["epsilon"]
 
     def Q(self, state):
         state = np.array2string(state)
@@ -90,4 +95,27 @@ def update(self, state, action, reward, next_state, terminated, truncated):
             Q_next = max(self.Q(next_state))
             TD_target += self.gamma*Q_next
         TD_error = TD_target - Q_old
-        self.Q(state)[action] += self.alpha*TD_error
+        self.Q(state)[action] += self.alpha*TD_error
+
+class UCBQLearner(QLearner):
+    def __init__(self, params):
+        super(UCBQLearner, self).__init__(params)
+        self.exploration_constant = params["exploration_constant"]
+        self.action_counts = {}
+
+    def get_action_counts(self, state):
+        state = np.array2string(state)
+        if state not in self.action_counts:
+            self.action_counts[state] = np.zeros(self.nr_actions)
+        return self.action_counts[state]
+
+    def update_action_counts(self, state, action):
+        state = np.array2string(state)
+        self.action_counts[state][action] += 1
+
+    def policy(self, state):
+        Q_values = self.Q(state)
+        action_counts = self.get_action_counts(state)
+        action = UCB1(Q_values, action_counts, exploration_constant=self.exploration_constant)
+        self.update_action_counts(state, action)
+        return action

code/main.py

@@ -1,50 +1,97 @@
 import rooms
+import random
 import agent as a
 import matplotlib.pyplot as plot
+import seaborn as sns
+import pandas as pd
 import sys
+from utils import save_agent, load_agent
+import numpy as np
 
-def episode(env, agent, nr_episode=0):
+def plot_returns(x,y):
+    plot.plot(x,y)
+    plot.title("Progress")
+    plot.xlabel("Episode")
+    plot.ylabel("Discounted Return")
+    plot.show()
+
+def plot_eval_returns(x, y):
+    df = pd.DataFrame(y)
+    df = df.melt(var_name="Episode", value_name="Discounted Return") # lineplot expects data in long format
+    sns.lineplot(x="Episode", y="Discounted Return", data=df, errorbar='ci', ci=95)
+    plot.axhline(y=0.8, color='black', linestyle='--')
+    plot.title("Evaluation returns")
+    plot.show()
+
+def episode(env, agent, nr_episode=0, evaluation_mode=False, verbose=True):
     state = env.reset()
     discounted_return = 0
     discount_factor = 0.99
     done = False
     time_step = 0
+    if evaluation_mode:
+        agent.epsilon = 0
+        agent.exploration_constant = 0
     while not done:
         # 1. Select action according to policy
         action = agent.policy(state)
         # 2. Execute selected action
         next_state, reward, terminated, truncated, _ = env.step(action)
         # 3. Integrate new experience into agent
-        agent.update(state, action, reward, next_state, terminated, truncated)
+        if not evaluation_mode: 
+            agent.update(state, action, reward, next_state, terminated, truncated)
         state = next_state
         done = terminated or truncated
         discounted_return += (discount_factor**time_step)*reward
         time_step += 1
-    print(nr_episode, ":", discounted_return)
+    if verbose: print(nr_episode, ":", discounted_return, "steps: ", time_step)
     return discounted_return
-
+
+def train(env, agent, episodes):
+    returns = [episode(env, agent, nr_episode=i, verbose=True) for i in range(episodes)]
+    return returns
+
+def evaluate(env, agent, runs, episodes):
+    eval_returns = []
+    for i in range(no_runs):
+        returns = [episode(env, agent, nr_episode=i, verbose=False, evaluation_mode=True) for i in range(episodes)]
+        eval_returns.append(returns)
+    return np.array(eval_returns)
+
+np.random.seed(42)
+random.seed(42)
 params = {}
 rooms_instance = sys.argv[1]
 env = rooms.load_env(f"layouts/{rooms_instance}.txt", f"{rooms_instance}.mp4")
 params["nr_actions"] = env.action_space.n
 params["gamma"] = 0.99
-params["epsilon_decay"] = 0.001
+params["epsilon_decay"] = 0.0001
 params["alpha"] = 0.1
 params["env"] = env
+params["exploration_constant"] = np.sqrt(2)
+params["epsilon"] = 1
 
 #agent = a.RandomAgent(params)
-#agent = a.SARSALearner(params)
-agent = a.QLearner(params)
+# agent = a.SARSALearner(params)
+# agent = a.QLearner(params)
+agent = a.UCBQLearner(params)
+
 training_episodes = 200
-returns = [episode(env, agent, i) for i in range(training_episodes)]
+evaluation_episodes = 10
+no_runs = 100
+
+# TRAINING
+returns = train(env, agent, training_episodes)
+plot_returns(x=range(training_episodes),y=returns)
+# save_agent(agent)
+# exit()
+
+# EVALUATION
+# agent = load_agent("saved_agents/agent: 2024-03-19 13:01:51.pkl")
+eval_returns = evaluate(env, agent, runs=no_runs, episodes=evaluation_episodes)
+plot_eval_returns(x=range(evaluation_episodes), y=eval_returns)
 
-x = range(training_episodes)
-y = returns
+print(f"Average evaluation discounted return: {np.mean(eval_returns)}")
 
-plot.plot(x,y)
-plot.title("Progress")
-plot.xlabel("Episode")
-plot.ylabel("Discounted Return")
-plot.show()
 
 env.save_video()

code/utils.py

@@ -0,0 +1,17 @@
+import pickle
+from datetime import datetime
+import os
+
+def save_agent(agent, filename="agent"):
+    os.makedirs("saved_agents", exist_ok=True)
+    current_datetime = datetime.now()
+    date_string = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+    with open(f"saved_agents/{filename}: {date_string}.pkl", 'wb') as file:
+        pickle.dump(agent, file)
+
+def load_agent(path, verbose=False):
+    with open(path, 'rb') as file:
+        agent = pickle.load(file)
+    if verbose: 
+        agent.print()
+    return agent