TD3_Ant.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TD3_Ant.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.8.5 64-bit ('base': conda)"
    },
    "accelerator": "GPU",
    "interpreter": {
      "hash": "6b60d692d45e4d3c8219689ad15912284a2888809b01af1d9a5e08aba46da5bd"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WXu1r8qvSzWf"
      },
      "source": [
        "# Twin-Delayed DDPG"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xjm2onHdT-Av"
      },
      "source": [
        "## Importing the libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ikr2p0Js8iB4"
      },
      "source": [
        "import os\n",
        "import time\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pybullet_envs\n",
        "import gym\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from gym import wrappers\n",
        "import mujoco_py"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "running build_ext\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2nGdtlKVydr"
      },
      "source": [
        "## Initialize the Experience Replay memory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u5rW0IDB8nTO"
      },
      "source": [
        "class ReplayBuffer(object):\n",
        "\n",
        "  def __init__(self, max_size=1e6):\n",
        "    self.storage = []\n",
        "    self.max_size = max_size\n",
        "    self.ptr = 0\n",
        "\n",
        "  def add(self, transition):\n",
        "    if len(self.storage) == self.max_size:\n",
        "      self.storage[int(self.ptr)] = transition\n",
        "      self.ptr = (self.ptr + 1) % self.max_size\n",
        "    else:\n",
        "      self.storage.append(transition)\n",
        "\n",
        "  def sample(self, batch_size):\n",
        "    ind = np.random.randint(0, len(self.storage), size=batch_size)\n",
        "    batch_states, batch_next_states, batch_actions, batch_rewards, batch_dones = [], [], [], [], []\n",
        "    for i in ind: \n",
        "      state, next_state, action, reward, done = self.storage[i]\n",
        "      batch_states.append(np.array(state, copy=False))\n",
        "      batch_next_states.append(np.array(next_state, copy=False))\n",
        "      batch_actions.append(np.array(action, copy=False))\n",
        "      batch_rewards.append(np.array(reward, copy=False))\n",
        "      batch_dones.append(np.array(done, copy=False))\n",
        "    return np.array(batch_states), np.array(batch_next_states), np.array(batch_actions), np.array(batch_rewards).reshape(-1, 1), np.array(batch_dones).reshape(-1, 1)\n",
        "    "
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jb7TTaHxWbQD"
      },
      "source": [
        "## Build one neural network for the Actor model and one neural network for the Actor target"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CeRW4D79HL0"
      },
      "source": [
        "class Actor(nn.Module):\n",
        "  \n",
        "  def __init__(self, state_dim, action_dim, max_action):\n",
        "    super(Actor, self).__init__()\n",
        "    self.layer_1 = nn.Linear(state_dim, 400)\n",
        "    self.layer_2 = nn.Linear(400, 300)\n",
        "    self.layer_3 = nn.Linear(300, action_dim)\n",
        "    self.max_action = max_action\n",
        "\n",
        "  def forward(self, x):\n",
        "    x = F.relu(self.layer_1(x))\n",
        "    x = F.relu(self.layer_2(x))\n",
        "    x = self.max_action * torch.tanh(self.layer_3(x))\n",
        "    return x"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HRDDce8FXef7"
      },
      "source": [
        "## Build two neural networks for the two Critic models and two neural networks for the two Critic targets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCee7gwR9Jrs"
      },
      "source": [
        "class Critic(nn.Module):\n",
        "  \n",
        "  def __init__(self, state_dim, action_dim):\n",
        "    super(Critic, self).__init__()\n",
        "    # Defining the first Critic neural network\n",
        "    self.layer_1 = nn.Linear(state_dim + action_dim, 400)\n",
        "    self.layer_2 = nn.Linear(400, 300)\n",
        "    self.layer_3 = nn.Linear(300, 1)\n",
        "    # Defining the second Critic neural network\n",
        "    self.layer_4 = nn.Linear(state_dim + action_dim, 400)\n",
        "    self.layer_5 = nn.Linear(400, 300)\n",
        "    self.layer_6 = nn.Linear(300, 1)\n",
        "\n",
        "  def forward(self, x, u):\n",
        "    xu = torch.cat([x, u], 1)\n",
        "    \n",
        "    x1 = F.relu(self.layer_1(xu))\n",
        "    x1 = F.relu(self.layer_2(x1))\n",
        "    x1 = self.layer_3(x1)\n",
        "    \n",
        "    x2 = F.relu(self.layer_4(xu))\n",
        "    x2 = F.relu(self.layer_5(x2))\n",
        "    x2 = self.layer_6(x2)\n",
        "\n",
        "    return x1, x2\n",
        "\n",
        "  def Q1(self, x, u):\n",
        "    xu = torch.cat([x, u], 1)\n",
        "    x1 = F.relu(self.layer_1(xu))\n",
        "    x1 = F.relu(self.layer_2(x1))\n",
        "    x1 = self.layer_3(x1)\n",
        "    return x1\n",
        "    "
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NzIDuONodenW"
      },
      "source": [
        "\n",
        "## Training Process"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zzd0H1xukdKe"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class TD3(object):\n",
        "  \n",
        "  def __init__(self, state_dim, action_dim, max_action):\n",
        "    self.actor = Actor(state_dim, action_dim, max_action).to(device)\n",
        "    self.actor_target = Actor(state_dim, action_dim, max_action).to(device)\n",
        "    self.actor_target.load_state_dict(self.actor.state_dict())\n",
        "    self.actor_optimizer = torch.optim.Adam(self.actor.parameters())\n",
        "\n",
        "    self.critic = Critic(state_dim, action_dim).to(device)\n",
        "    self.critic_target = Critic(state_dim, action_dim).to(device)\n",
        "    self.critic_target.load_state_dict(self.critic.state_dict())\n",
        "    self.critic_optimizer = torch.optim.Adam(self.critic.parameters())\n",
        "\n",
        "    self.max_action = max_action\n",
        "\n",
        "  def select_action(self, state):\n",
        "    state = torch.Tensor(state.reshape(1, -1)).to(device)\n",
        "    return self.actor(state).cpu().data.numpy().flatten()\n",
        "\n",
        "  def train(self, replay_buffer, iterations, batch_size=100, discount=0.99, tau=0.005, policy_noise=0.2, noise_clip=0.5, policy_freq=2):\n",
        "    \n",
        "    for it in range(iterations):\n",
        "      \n",
        "      batch_states, batch_next_states, batch_actions, batch_rewards, batch_dones = replay_buffer.sample(batch_size)\n",
        "      state = torch.Tensor(batch_states).to(device)\n",
        "      next_state = torch.Tensor(batch_next_states).to(device)\n",
        "      action = torch.Tensor(batch_actions).to(device)\n",
        "      reward = torch.Tensor(batch_rewards).to(device)\n",
        "      done = torch.Tensor(batch_dones).to(device)\n",
        "      \n",
        "      next_action = self.actor_target(next_state)\n",
        "      noise = torch.Tensor(batch_actions).data.normal_(0, policy_noise).to(device)\n",
        "      noise = noise.clamp(-noise_clip, noise_clip)\n",
        "      next_action = (next_action + noise).clamp(-self.max_action, self.max_action)\n",
        "      \n",
        "      target_Q1, target_Q2 = self.critic_target(next_state, next_action)\n",
        "      target_Q = torch.min(target_Q1, target_Q2)\n",
        "      target_Q = reward + ((1 - done) * discount * target_Q).detach()\n",
        "      \n",
        "      current_Q1, current_Q2 = self.critic(state, action)\n",
        "      critic_loss = F.mse_loss(current_Q1, target_Q) + F.mse_loss(current_Q2, target_Q)\n",
        "      \n",
        "      self.critic_optimizer.zero_grad()\n",
        "      critic_loss.backward()\n",
        "      self.critic_optimizer.step()\n",
        "      \n",
        "      if it % policy_freq == 0:\n",
        "        actor_loss = -self.critic.Q1(state, self.actor(state)).mean()\n",
        "        self.actor_optimizer.zero_grad()\n",
        "        actor_loss.backward()\n",
        "        self.actor_optimizer.step()\n",
        "        \n",
        "        for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()):\n",
        "          target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)\n",
        "        \n",
        "        for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()):\n",
        "          target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)\n",
        "  \n",
        "  def save(self, filename, directory):\n",
        "    torch.save(self.actor.state_dict(), '%s/%s_actor.pth' % (directory, filename))\n",
        "    torch.save(self.critic.state_dict(), '%s/%s_critic.pth' % (directory, filename))\n",
        "  \n",
        "  def load(self, filename, directory):\n",
        "    self.actor.load_state_dict(torch.load('%s/%s_actor.pth' % (directory, filename)))\n",
        "    self.critic.load_state_dict(torch.load('%s/%s_critic.pth' % (directory, filename)))\n"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ka-ZRtQvjBex"
      },
      "source": [
        "## Make a function that evaluates the policy by calculating its average reward over 10 episodes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qabqiYdp9wDM"
      },
      "source": [
        "def evaluate_policy(policy, eval_episodes=10, isRender=False):\n",
        "  avg_reward = 0.\n",
        "  for i in range(eval_episodes):\n",
        "    obs = env.reset()\n",
        "    done = False\n",
        "    while not done:\n",
        "      if i + 1 == eval_episodes and isRender:\n",
        "        env.render()\n",
        "\n",
        "      action = policy.select_action(np.array(obs))\n",
        "      obs, reward, done, _ = env.step(action)\n",
        "      avg_reward += reward\n",
        "    \n",
        "    if isRender:\n",
        "      env.close()\n",
        "      env.reset()\n",
        "\n",
        "  avg_reward /= eval_episodes\n",
        "  print (\"---------------------------------------\")\n",
        "  print (\"Average Reward over the Evaluation Step: %f\" % (avg_reward))\n",
        "  print (\"---------------------------------------\")\n",
        "  return avg_reward\n"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gGuKmH_ijf7U"
      },
      "source": [
        "## Set the parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFj6wbAo97lk"
      },
      "source": [
        "env_name = \"Ant-v2\"\n",
        "seed = 0\n",
        "start_timesteps = 1e4\n",
        "eval_freq = 5e3\n",
        "max_timesteps = 1e6\n",
        "save_models = True\n",
        "expl_noise = 0.1\n",
        "batch_size = 100\n",
        "discount = 0.99 \n",
        "tau = 0.005\n",
        "policy_noise = 0.2\n",
        "noise_clip = 0.5\n",
        "policy_freq = 2"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hjwf2HCol3XP"
      },
      "source": [
        "## Create a file name for the two saved models: the Actor and Critic models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1fyH8N5z-o3o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ef57b717-f55c-4e1c-cfb8-f613da022dca"
      },
      "source": [
        "file_name = \"%s_%s_%s\" % (\"TD3\", env_name, str(seed))\n",
        "print (\"---------------------------------------\")\n",
        "print (\"Settings: %s\" % (file_name))\n",
        "print (\"---------------------------------------\")"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------\nSettings: TD3_Ant-v2_0\n---------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kop-C96Aml8O"
      },
      "source": [
        "## Create a folder inside which will be saved the trained models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Src07lvY-zXb"
      },
      "source": [
        "if not os.path.exists(\"./results\"):\n",
        "  os.makedirs(\"./results\")\n",
        "  \n",
        "if save_models and not os.path.exists(\"./pytorch_models\"):\n",
        "  os.makedirs(\"./pytorch_models\")\n"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qEAzOd47mv1Z"
      },
      "source": [
        "## Create the PyBullet environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CyQXJUIs-6BV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "30cd9af4-39a4-45c3-961f-4e2377432f1d"
      },
      "source": [
        "env = gym.make(env_name)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5YdPG4HXnNsh"
      },
      "source": [
        "## Set seeds and we get the necessary information on the states and actions in the chosen environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3RufYec_ADj"
      },
      "source": [
        "env.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "np.random.seed(seed)\n",
        "state_dim = env.observation_space.shape[0]\n",
        "action_dim = env.action_space.shape[0]\n",
        "max_action = float(env.action_space.high[0])"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HWEgDAQxnbem"
      },
      "source": [
        "## Create the policy network and replay buffer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wTVvG7F8_EWg"
      },
      "source": [
        "# Policy network:\n",
        "policy = TD3(state_dim, action_dim, max_action)\n",
        "\n",
        "# Replay buffer\n",
        "replay_buffer = ReplayBuffer()\n"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QYOpCyiDnw7s"
      },
      "source": [
        "## Define a list where all the evaluation results over 10 episodes are stored"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhC_5XJ__Orp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "de3e191f-b977-4e0f-adad-f03061716c24"
      },
      "source": [
        "evaluations = [evaluate_policy(policy, 2, True)]"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating window glfw\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 985.153252\n",
            "---------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xm-4b3p6rglE"
      },
      "source": [
        "## Create a new folder directory in which the final results (videos of the agent) will be populated"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MTL9uMd0ru03"
      },
      "source": [
        "def mkdir(base, name):\n",
        "    path = os.path.join(base, name)\n",
        "    if not os.path.exists(path):\n",
        "        os.makedirs(path)\n",
        "    return path\n",
        "\n",
        "work_dir = mkdir('exp', 'brs')\n",
        "monitor_dir = mkdir(work_dir, 'monitor')\n",
        "max_episode_steps = env._max_episode_steps\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q9gsjvtPqLgT"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_ouY4NH_Y0I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47c599dd-6fec-4a96-b04f-55b322442094"
      },
      "source": [
        "total_timesteps = 0\n",
        "timesteps_since_eval = 0\n",
        "episode_num = 0\n",
        "done = True\n",
        "t0 = time.time()\n",
        "\n",
        "while total_timesteps < max_timesteps:\n",
        "  \n",
        "  # If the episode is done\n",
        "  if done:\n",
        "    if total_timesteps != 0:\n",
        "      print(\"Total Timesteps: {} Episode Num: {} Reward: {}\".format(total_timesteps, episode_num, episode_reward))\n",
        "      policy.train(replay_buffer, episode_timesteps, batch_size, discount, tau, policy_noise, noise_clip, policy_freq)\n",
        "\n",
        "    if timesteps_since_eval >= eval_freq:\n",
        "      timesteps_since_eval %= eval_freq\n",
        "      evaluations.append(evaluate_policy(policy))\n",
        "      policy.save(file_name, directory=\"./pytorch_models\")\n",
        "      np.save(\"./results/%s\" % (file_name), evaluations)\n",
        "    \n",
        "    obs = env.reset()\n",
        "    done = False\n",
        "    episode_reward = 0\n",
        "    episode_timesteps = 0\n",
        "    episode_num += 1\n",
        "  \n",
        "  # Before 10000 timesteps, we play random actions\n",
        "  if total_timesteps < start_timesteps:\n",
        "    action = env.action_space.sample()\n",
        "  else:\n",
        "    action = policy.select_action(np.array(obs))\n",
        "    if expl_noise != 0:\n",
        "      action = (action + np.random.normal(0, expl_noise, size=env.action_space.shape[0])).clip(env.action_space.low, env.action_space.high)\n",
        "  \n",
        "  new_obs, reward, done, _ = env.step(action)\n",
        "\n",
        "  done_bool = 0 if episode_timesteps + 1 == env._max_episode_steps else float(done)\n",
        "  episode_reward += reward\n",
        "  replay_buffer.add((obs, new_obs, action, reward, done_bool))\n",
        "\n",
        "  obs = new_obs\n",
        "  episode_timesteps += 1\n",
        "  total_timesteps += 1\n",
        "  timesteps_since_eval += 1\n",
        "\n",
        "# Add the last policy evaluation to our list of evaluations and we save our model\n",
        "evaluations.append(evaluate_policy(policy))\n",
        "if save_models: policy.save(\"%s\" % (file_name), directory=\"./pytorch_models\")\n",
        "np.save(\"./results/%s\" % (file_name), evaluations)\n"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Timesteps: 157 Episode Num: 1 Reward: -143.54351621097098\n",
            "Total Timesteps: 236 Episode Num: 2 Reward: -55.316183680573666\n",
            "Total Timesteps: 1236 Episode Num: 3 Reward: -339.29842315590344\n",
            "Total Timesteps: 1504 Episode Num: 4 Reward: -97.77895997431226\n",
            "Total Timesteps: 2504 Episode Num: 5 Reward: -399.7608979445045\n",
            "Total Timesteps: 2559 Episode Num: 6 Reward: -31.109622138509856\n",
            "Total Timesteps: 2595 Episode Num: 7 Reward: -8.118218120428118\n",
            "Total Timesteps: 2631 Episode Num: 8 Reward: -2.60210228747282\n",
            "Total Timesteps: 3631 Episode Num: 9 Reward: -323.2135853339898\n",
            "Total Timesteps: 3656 Episode Num: 10 Reward: -0.8087687788968088\n",
            "Total Timesteps: 3764 Episode Num: 11 Reward: -24.059611080985537\n",
            "Total Timesteps: 3881 Episode Num: 12 Reward: -88.70078300295349\n",
            "Total Timesteps: 4881 Episode Num: 13 Reward: -359.119997335086\n",
            "Total Timesteps: 4909 Episode Num: 14 Reward: -26.255804837668922\n",
            "Total Timesteps: 4948 Episode Num: 15 Reward: -42.701436977400284\n",
            "Total Timesteps: 5948 Episode Num: 16 Reward: -394.1362397051564\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: -126.517111\n",
            "---------------------------------------\n",
            "Total Timesteps: 5997 Episode Num: 17 Reward: -33.005142722759906\n",
            "Total Timesteps: 6032 Episode Num: 18 Reward: -0.884352989118917\n",
            "Total Timesteps: 6112 Episode Num: 19 Reward: -59.8666176548638\n",
            "Total Timesteps: 6152 Episode Num: 20 Reward: -18.691622541116995\n",
            "Total Timesteps: 7152 Episode Num: 21 Reward: -368.26425135339815\n",
            "Total Timesteps: 7202 Episode Num: 22 Reward: 26.998747559100078\n",
            "Total Timesteps: 7219 Episode Num: 23 Reward: 2.6896819686659015\n",
            "Total Timesteps: 8219 Episode Num: 24 Reward: -367.1607116769061\n",
            "Total Timesteps: 9219 Episode Num: 25 Reward: -335.3218064123797\n",
            "Total Timesteps: 10219 Episode Num: 26 Reward: -537.5126915114646\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: -96.408313\n",
            "---------------------------------------\n",
            "Total Timesteps: 10238 Episode Num: 27 Reward: -23.09467876309784\n",
            "Total Timesteps: 10342 Episode Num: 28 Reward: -79.99280792481929\n",
            "Total Timesteps: 10396 Episode Num: 29 Reward: -23.26512631586989\n",
            "Total Timesteps: 10498 Episode Num: 30 Reward: -85.58623919439964\n",
            "Total Timesteps: 10579 Episode Num: 31 Reward: 27.214183781766287\n",
            "Total Timesteps: 10623 Episode Num: 32 Reward: 27.06366824030156\n",
            "Total Timesteps: 10716 Episode Num: 33 Reward: 3.7087046710272342\n",
            "Total Timesteps: 10857 Episode Num: 34 Reward: 26.287665872931697\n",
            "Total Timesteps: 10897 Episode Num: 35 Reward: 0.3781966461143629\n",
            "Total Timesteps: 11897 Episode Num: 36 Reward: -124.42768436545714\n",
            "Total Timesteps: 12182 Episode Num: 37 Reward: 114.0253266990461\n",
            "Total Timesteps: 12503 Episode Num: 38 Reward: 124.83876619923907\n",
            "Total Timesteps: 12992 Episode Num: 39 Reward: 171.70759436632673\n",
            "Total Timesteps: 13992 Episode Num: 40 Reward: 405.4266151619375\n",
            "Total Timesteps: 14737 Episode Num: 41 Reward: 148.2250491832474\n",
            "Total Timesteps: 15737 Episode Num: 42 Reward: 299.61267073807045\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 183.889685\n",
            "---------------------------------------\n",
            "Total Timesteps: 15829 Episode Num: 43 Reward: 46.24573744087341\n",
            "Total Timesteps: 16829 Episode Num: 44 Reward: 514.2747937185977\n",
            "Total Timesteps: 16912 Episode Num: 45 Reward: 40.223685654386145\n",
            "Total Timesteps: 16970 Episode Num: 46 Reward: 25.504581435458586\n",
            "Total Timesteps: 17358 Episode Num: 47 Reward: 206.33070088353418\n",
            "Total Timesteps: 18358 Episode Num: 48 Reward: 575.9086981985953\n",
            "Total Timesteps: 18634 Episode Num: 49 Reward: 60.863811372949016\n",
            "Total Timesteps: 19634 Episode Num: 50 Reward: 430.59077720022145\n",
            "Total Timesteps: 19927 Episode Num: 51 Reward: 154.8608543720118\n",
            "Total Timesteps: 20040 Episode Num: 52 Reward: 60.47818082727809\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 434.898444\n",
            "---------------------------------------\n",
            "Total Timesteps: 20204 Episode Num: 53 Reward: 91.91235968140546\n",
            "Total Timesteps: 21204 Episode Num: 54 Reward: 344.9032951984385\n",
            "Total Timesteps: 22204 Episode Num: 55 Reward: 471.54548231074\n",
            "Total Timesteps: 22237 Episode Num: 56 Reward: 17.98675228998209\n",
            "Total Timesteps: 22273 Episode Num: 57 Reward: 5.440898008246082\n",
            "Total Timesteps: 22362 Episode Num: 58 Reward: 55.60503937453721\n",
            "Total Timesteps: 22593 Episode Num: 59 Reward: 14.241906459552556\n",
            "Total Timesteps: 22617 Episode Num: 60 Reward: 5.029221485226709\n",
            "Total Timesteps: 22685 Episode Num: 61 Reward: 42.582684115546336\n",
            "Total Timesteps: 22700 Episode Num: 62 Reward: -0.48631957675032633\n",
            "Total Timesteps: 23392 Episode Num: 63 Reward: 281.9013450142332\n",
            "Total Timesteps: 23459 Episode Num: 64 Reward: 28.45091919765835\n",
            "Total Timesteps: 23629 Episode Num: 65 Reward: 83.60548679592443\n",
            "Total Timesteps: 23684 Episode Num: 66 Reward: 23.827440239839742\n",
            "Total Timesteps: 24684 Episode Num: 67 Reward: 624.4304810700825\n",
            "Total Timesteps: 24701 Episode Num: 68 Reward: 4.637956977857277\n",
            "Total Timesteps: 25701 Episode Num: 69 Reward: 463.9851911617258\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 61.998611\n",
            "---------------------------------------\n",
            "Total Timesteps: 26701 Episode Num: 70 Reward: 249.23131512149598\n",
            "Total Timesteps: 27174 Episode Num: 71 Reward: 178.16180434267994\n",
            "Total Timesteps: 27669 Episode Num: 72 Reward: 237.77151461914283\n",
            "Total Timesteps: 28669 Episode Num: 73 Reward: 469.7143524714049\n",
            "Total Timesteps: 28682 Episode Num: 74 Reward: -17.46436679473021\n",
            "Total Timesteps: 29682 Episode Num: 75 Reward: 359.7639238110869\n",
            "Total Timesteps: 29712 Episode Num: 76 Reward: 3.0135831409564373\n",
            "Total Timesteps: 30026 Episode Num: 77 Reward: 113.15595076726551\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 146.689872\n",
            "---------------------------------------\n",
            "Total Timesteps: 30067 Episode Num: 78 Reward: -26.298763683761656\n",
            "Total Timesteps: 30222 Episode Num: 79 Reward: 79.47803874154755\n",
            "Total Timesteps: 30346 Episode Num: 80 Reward: 63.26600268659875\n",
            "Total Timesteps: 30372 Episode Num: 81 Reward: -15.341689185711417\n",
            "Total Timesteps: 30600 Episode Num: 82 Reward: 157.41951291176525\n",
            "Total Timesteps: 31372 Episode Num: 83 Reward: 425.64280036788904\n",
            "Total Timesteps: 32372 Episode Num: 84 Reward: 471.5120593215287\n",
            "Total Timesteps: 32737 Episode Num: 85 Reward: 157.3071660054051\n",
            "Total Timesteps: 32954 Episode Num: 86 Reward: 123.66425420443018\n",
            "Total Timesteps: 33845 Episode Num: 87 Reward: 375.76668140195954\n",
            "Total Timesteps: 34264 Episode Num: 88 Reward: 227.0473140763945\n",
            "Total Timesteps: 34412 Episode Num: 89 Reward: 56.562594267927224\n",
            "Total Timesteps: 35412 Episode Num: 90 Reward: 555.0555120483358\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 280.656146\n",
            "---------------------------------------\n",
            "Total Timesteps: 36412 Episode Num: 91 Reward: 598.3612369301924\n",
            "Total Timesteps: 36701 Episode Num: 92 Reward: 163.74286218126738\n",
            "Total Timesteps: 36977 Episode Num: 93 Reward: 225.92574532366453\n",
            "Total Timesteps: 37977 Episode Num: 94 Reward: 564.4829542831635\n",
            "Total Timesteps: 38977 Episode Num: 95 Reward: 522.3372480439274\n",
            "Total Timesteps: 39014 Episode Num: 96 Reward: 21.008120531877783\n",
            "Total Timesteps: 39159 Episode Num: 97 Reward: 119.86052032722428\n",
            "Total Timesteps: 40159 Episode Num: 98 Reward: 578.6391290163087\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 214.834703\n",
            "---------------------------------------\n",
            "Total Timesteps: 40505 Episode Num: 99 Reward: 265.89953094085547\n",
            "Total Timesteps: 40584 Episode Num: 100 Reward: 61.75299958146954\n",
            "Total Timesteps: 41584 Episode Num: 101 Reward: 636.2216381588506\n",
            "Total Timesteps: 42067 Episode Num: 102 Reward: 368.27138190355737\n",
            "Total Timesteps: 42269 Episode Num: 103 Reward: 146.11830205659902\n",
            "Total Timesteps: 43269 Episode Num: 104 Reward: 676.8718992867415\n",
            "Total Timesteps: 44269 Episode Num: 105 Reward: 637.3218193371789\n",
            "Total Timesteps: 44633 Episode Num: 106 Reward: 288.7530662246135\n",
            "Total Timesteps: 45633 Episode Num: 107 Reward: 486.98332061135466\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 373.440340\n",
            "---------------------------------------\n",
            "Total Timesteps: 45889 Episode Num: 108 Reward: 196.7939008234905\n",
            "Total Timesteps: 45945 Episode Num: 109 Reward: 5.496781133603969\n",
            "Total Timesteps: 46316 Episode Num: 110 Reward: 262.06055003128796\n",
            "Total Timesteps: 46414 Episode Num: 111 Reward: 48.004216433310674\n",
            "Total Timesteps: 46493 Episode Num: 112 Reward: 35.12244172835518\n",
            "Total Timesteps: 47493 Episode Num: 113 Reward: 592.8485769149719\n",
            "Total Timesteps: 48493 Episode Num: 114 Reward: 657.2882188194232\n",
            "Total Timesteps: 49056 Episode Num: 115 Reward: 473.6614221703093\n",
            "Total Timesteps: 49075 Episode Num: 116 Reward: 3.063642129894717\n",
            "Total Timesteps: 49139 Episode Num: 117 Reward: 15.668009139136812\n",
            "Total Timesteps: 50139 Episode Num: 118 Reward: 539.1678985013158\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 290.970854\n",
            "---------------------------------------\n",
            "Total Timesteps: 50288 Episode Num: 119 Reward: 129.9302821766729\n",
            "Total Timesteps: 51288 Episode Num: 120 Reward: 474.6383675659348\n",
            "Total Timesteps: 51306 Episode Num: 121 Reward: 7.974393285571139\n",
            "Total Timesteps: 51618 Episode Num: 122 Reward: 184.7187166862897\n",
            "Total Timesteps: 52618 Episode Num: 123 Reward: 742.3248315149514\n",
            "Total Timesteps: 53279 Episode Num: 124 Reward: 626.9490182577447\n",
            "Total Timesteps: 53478 Episode Num: 125 Reward: 227.97748766646012\n",
            "Total Timesteps: 54478 Episode Num: 126 Reward: 650.2864227256505\n",
            "Total Timesteps: 54495 Episode Num: 127 Reward: 11.709245540296795\n",
            "Total Timesteps: 55495 Episode Num: 128 Reward: 708.882208200552\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 182.871471\n",
            "---------------------------------------\n",
            "Total Timesteps: 55567 Episode Num: 129 Reward: 83.71801677759136\n",
            "Total Timesteps: 56075 Episode Num: 130 Reward: 583.2777153096583\n",
            "Total Timesteps: 56165 Episode Num: 131 Reward: 84.02756398362435\n",
            "Total Timesteps: 56400 Episode Num: 132 Reward: 188.0914102900106\n",
            "Total Timesteps: 56494 Episode Num: 133 Reward: 100.47165890760336\n",
            "Total Timesteps: 56807 Episode Num: 134 Reward: 232.46677984751764\n",
            "Total Timesteps: 57807 Episode Num: 135 Reward: 376.5246774139186\n",
            "Total Timesteps: 58807 Episode Num: 136 Reward: 775.0190941500991\n",
            "Total Timesteps: 59112 Episode Num: 137 Reward: 336.09453214989526\n",
            "Total Timesteps: 59141 Episode Num: 138 Reward: 18.725807922124638\n",
            "Total Timesteps: 59176 Episode Num: 139 Reward: 31.888219825573696\n",
            "Total Timesteps: 59330 Episode Num: 140 Reward: 132.05300737467658\n",
            "Total Timesteps: 60330 Episode Num: 141 Reward: 597.4575425122615\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 276.052562\n",
            "---------------------------------------\n",
            "Total Timesteps: 61330 Episode Num: 142 Reward: 719.8672357447622\n",
            "Total Timesteps: 61741 Episode Num: 143 Reward: 333.5785866563726\n",
            "Total Timesteps: 61865 Episode Num: 144 Reward: 163.60621281752793\n",
            "Total Timesteps: 61950 Episode Num: 145 Reward: 80.7912010084652\n",
            "Total Timesteps: 62187 Episode Num: 146 Reward: 179.14030897165904\n",
            "Total Timesteps: 62264 Episode Num: 147 Reward: 84.29521876214164\n",
            "Total Timesteps: 62302 Episode Num: 148 Reward: 19.2434670720116\n",
            "Total Timesteps: 62427 Episode Num: 149 Reward: 161.26941283965954\n",
            "Total Timesteps: 62473 Episode Num: 150 Reward: 34.128906666141496\n",
            "Total Timesteps: 62521 Episode Num: 151 Reward: 41.84127624396347\n",
            "Total Timesteps: 62584 Episode Num: 152 Reward: 70.66614092937844\n",
            "Total Timesteps: 63584 Episode Num: 153 Reward: 581.7849188099087\n",
            "Total Timesteps: 63604 Episode Num: 154 Reward: 7.9026657981131825\n",
            "Total Timesteps: 63773 Episode Num: 155 Reward: 138.1610638344054\n",
            "Total Timesteps: 64066 Episode Num: 156 Reward: 398.0511582302578\n",
            "Total Timesteps: 64202 Episode Num: 157 Reward: 165.78556770210434\n",
            "Total Timesteps: 65202 Episode Num: 158 Reward: 768.5993415289897\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 302.933484\n",
            "---------------------------------------\n",
            "Total Timesteps: 66202 Episode Num: 159 Reward: 760.3838539049958\n",
            "Total Timesteps: 66426 Episode Num: 160 Reward: 204.39423473257276\n",
            "Total Timesteps: 67426 Episode Num: 161 Reward: 643.0228920001734\n",
            "Total Timesteps: 67627 Episode Num: 162 Reward: 218.92039559225856\n",
            "Total Timesteps: 67857 Episode Num: 163 Reward: 193.04397131566455\n",
            "Total Timesteps: 68857 Episode Num: 164 Reward: 519.1156797547146\n",
            "Total Timesteps: 69131 Episode Num: 165 Reward: 222.62764851165252\n",
            "Total Timesteps: 69168 Episode Num: 166 Reward: 26.522538068483044\n",
            "Total Timesteps: 69353 Episode Num: 167 Reward: 162.50238934385183\n",
            "Total Timesteps: 70353 Episode Num: 168 Reward: 897.6765479277424\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 285.333328\n",
            "---------------------------------------\n",
            "Total Timesteps: 71353 Episode Num: 169 Reward: 821.556058457758\n",
            "Total Timesteps: 71399 Episode Num: 170 Reward: 53.48940830103741\n",
            "Total Timesteps: 71458 Episode Num: 171 Reward: 63.15498262111497\n",
            "Total Timesteps: 72458 Episode Num: 172 Reward: 693.2372249554165\n",
            "Total Timesteps: 72656 Episode Num: 173 Reward: 189.82392441429832\n",
            "Total Timesteps: 72754 Episode Num: 174 Reward: 48.076740903722836\n",
            "Total Timesteps: 72840 Episode Num: 175 Reward: 79.57971120783593\n",
            "Total Timesteps: 73840 Episode Num: 176 Reward: 656.6091120888575\n",
            "Total Timesteps: 73919 Episode Num: 177 Reward: 115.0579352301433\n",
            "Total Timesteps: 74041 Episode Num: 178 Reward: 166.2285131828009\n",
            "Total Timesteps: 74168 Episode Num: 179 Reward: 169.72442249293252\n",
            "Total Timesteps: 74202 Episode Num: 180 Reward: 33.51478500786077\n",
            "Total Timesteps: 74240 Episode Num: 181 Reward: 44.419541503791656\n",
            "Total Timesteps: 74600 Episode Num: 182 Reward: 323.7778742576445\n",
            "Total Timesteps: 75600 Episode Num: 183 Reward: 747.805415466583\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 259.357103\n",
            "---------------------------------------\n",
            "Total Timesteps: 75940 Episode Num: 184 Reward: 386.0969562788493\n",
            "Total Timesteps: 76488 Episode Num: 185 Reward: 332.8410044721527\n",
            "Total Timesteps: 76925 Episode Num: 186 Reward: 381.96979687086514\n",
            "Total Timesteps: 77925 Episode Num: 187 Reward: 868.4139581853296\n",
            "Total Timesteps: 78925 Episode Num: 188 Reward: 627.9104246327611\n",
            "Total Timesteps: 79116 Episode Num: 189 Reward: 193.46937514169886\n",
            "Total Timesteps: 80116 Episode Num: 190 Reward: 1112.4170527589295\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 321.484528\n",
            "---------------------------------------\n",
            "Total Timesteps: 80185 Episode Num: 191 Reward: 22.992531442836906\n",
            "Total Timesteps: 80220 Episode Num: 192 Reward: 50.396065108354236\n",
            "Total Timesteps: 80422 Episode Num: 193 Reward: 274.19499014934553\n",
            "Total Timesteps: 81422 Episode Num: 194 Reward: 765.163248543722\n",
            "Total Timesteps: 82422 Episode Num: 195 Reward: 719.9627856682092\n",
            "Total Timesteps: 82512 Episode Num: 196 Reward: 98.32443004206044\n",
            "Total Timesteps: 82685 Episode Num: 197 Reward: 243.58654102636615\n",
            "Total Timesteps: 83685 Episode Num: 198 Reward: 721.6769532376402\n",
            "Total Timesteps: 84685 Episode Num: 199 Reward: 610.8413408023293\n",
            "Total Timesteps: 84823 Episode Num: 200 Reward: 160.348244237756\n",
            "Total Timesteps: 84928 Episode Num: 201 Reward: 88.74809121782069\n",
            "Total Timesteps: 85321 Episode Num: 202 Reward: 326.70054452188504\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 258.870641\n",
            "---------------------------------------\n",
            "Total Timesteps: 85510 Episode Num: 203 Reward: 229.4158597557703\n",
            "Total Timesteps: 86510 Episode Num: 204 Reward: 666.7341094110263\n",
            "Total Timesteps: 86557 Episode Num: 205 Reward: 67.89152514479863\n",
            "Total Timesteps: 86587 Episode Num: 206 Reward: 17.831551743010646\n",
            "Total Timesteps: 86756 Episode Num: 207 Reward: 281.47895974249684\n",
            "Total Timesteps: 86986 Episode Num: 208 Reward: 190.9855951695792\n",
            "Total Timesteps: 87013 Episode Num: 209 Reward: 26.06641158289226\n",
            "Total Timesteps: 87074 Episode Num: 210 Reward: 89.57300567120866\n",
            "Total Timesteps: 87112 Episode Num: 211 Reward: 17.693603483857903\n",
            "Total Timesteps: 87131 Episode Num: 212 Reward: 17.26845080497006\n",
            "Total Timesteps: 87512 Episode Num: 213 Reward: 543.1726882060594\n",
            "Total Timesteps: 87554 Episode Num: 214 Reward: 34.424976791285985\n",
            "Total Timesteps: 87631 Episode Num: 215 Reward: 76.62082867151409\n",
            "Total Timesteps: 87717 Episode Num: 216 Reward: 97.85226570860112\n",
            "Total Timesteps: 87847 Episode Num: 217 Reward: 146.44168068159436\n",
            "Total Timesteps: 87989 Episode Num: 218 Reward: 188.52990890134035\n",
            "Total Timesteps: 88168 Episode Num: 219 Reward: 223.89373819114442\n",
            "Total Timesteps: 88347 Episode Num: 220 Reward: 247.0320758883438\n",
            "Total Timesteps: 89347 Episode Num: 221 Reward: 686.9834243962625\n",
            "Total Timesteps: 89358 Episode Num: 222 Reward: 2.0696187395052497\n",
            "Total Timesteps: 90358 Episode Num: 223 Reward: 994.6208992062518\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 285.062018\n",
            "---------------------------------------\n",
            "Total Timesteps: 91358 Episode Num: 224 Reward: 666.7070674769373\n",
            "Total Timesteps: 92358 Episode Num: 225 Reward: 791.4595686332948\n",
            "Total Timesteps: 93013 Episode Num: 226 Reward: 846.8191118958799\n",
            "Total Timesteps: 94013 Episode Num: 227 Reward: 711.8780865439718\n",
            "Total Timesteps: 94534 Episode Num: 228 Reward: 666.3776990613361\n",
            "Total Timesteps: 94868 Episode Num: 229 Reward: 472.35358748271676\n",
            "Total Timesteps: 95868 Episode Num: 230 Reward: 722.6441200929921\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 247.829162\n",
            "---------------------------------------\n",
            "Total Timesteps: 95912 Episode Num: 231 Reward: 53.70945748035418\n",
            "Total Timesteps: 95949 Episode Num: 232 Reward: -16.833792897610785\n",
            "Total Timesteps: 96083 Episode Num: 233 Reward: 155.1445636927532\n",
            "Total Timesteps: 97083 Episode Num: 234 Reward: 782.5717311321906\n",
            "Total Timesteps: 98083 Episode Num: 235 Reward: 731.4270161673874\n",
            "Total Timesteps: 98120 Episode Num: 236 Reward: 28.416098571582722\n",
            "Total Timesteps: 98161 Episode Num: 237 Reward: 26.530469967274687\n",
            "Total Timesteps: 98448 Episode Num: 238 Reward: 316.5878950519115\n",
            "Total Timesteps: 98552 Episode Num: 239 Reward: 171.3268454603197\n",
            "Total Timesteps: 99552 Episode Num: 240 Reward: 689.933492242937\n",
            "Total Timesteps: 99710 Episode Num: 241 Reward: 192.16291405668636\n",
            "Total Timesteps: 99776 Episode Num: 242 Reward: 106.60831962808147\n",
            "Total Timesteps: 99838 Episode Num: 243 Reward: 57.57956867954877\n",
            "Total Timesteps: 99859 Episode Num: 244 Reward: 21.822206361946524\n",
            "Total Timesteps: 99978 Episode Num: 245 Reward: 106.45811903258173\n",
            "Total Timesteps: 100068 Episode Num: 246 Reward: 62.00197725847436\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 559.341383\n",
            "---------------------------------------\n",
            "Total Timesteps: 100117 Episode Num: 247 Reward: 63.94921363700931\n",
            "Total Timesteps: 100199 Episode Num: 248 Reward: 129.50738202529018\n",
            "Total Timesteps: 100383 Episode Num: 249 Reward: 158.10056312744584\n",
            "Total Timesteps: 101383 Episode Num: 250 Reward: 495.69553806998425\n",
            "Total Timesteps: 102383 Episode Num: 251 Reward: 724.8136325114626\n",
            "Total Timesteps: 102515 Episode Num: 252 Reward: 183.6865647131352\n",
            "Total Timesteps: 103515 Episode Num: 253 Reward: 838.4260592792896\n",
            "Total Timesteps: 103717 Episode Num: 254 Reward: 295.79502545237443\n",
            "Total Timesteps: 103916 Episode Num: 255 Reward: 259.4115933622637\n",
            "Total Timesteps: 104916 Episode Num: 256 Reward: 1212.4481717252331\n",
            "Total Timesteps: 105192 Episode Num: 257 Reward: -52.352002128839565\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 189.388611\n",
            "---------------------------------------\n",
            "Total Timesteps: 105239 Episode Num: 258 Reward: 59.709008942801084\n",
            "Total Timesteps: 105277 Episode Num: 259 Reward: 53.88433771960274\n",
            "Total Timesteps: 105526 Episode Num: 260 Reward: 230.5801825636776\n",
            "Total Timesteps: 105621 Episode Num: 261 Reward: 117.94328191043613\n",
            "Total Timesteps: 105759 Episode Num: 262 Reward: 168.95478139125652\n",
            "Total Timesteps: 105802 Episode Num: 263 Reward: -3.782237241287776\n",
            "Total Timesteps: 105836 Episode Num: 264 Reward: 37.181472343485886\n",
            "Total Timesteps: 105872 Episode Num: 265 Reward: 43.94719596373583\n",
            "Total Timesteps: 106316 Episode Num: 266 Reward: 625.0009588898833\n",
            "Total Timesteps: 107316 Episode Num: 267 Reward: 719.686685944606\n",
            "Total Timesteps: 107399 Episode Num: 268 Reward: 102.7320619522086\n",
            "Total Timesteps: 108399 Episode Num: 269 Reward: 410.39582133419987\n",
            "Total Timesteps: 108428 Episode Num: 270 Reward: 17.5705547823252\n",
            "Total Timesteps: 108624 Episode Num: 271 Reward: 234.64477964491192\n",
            "Total Timesteps: 109624 Episode Num: 272 Reward: 680.437419172585\n",
            "Total Timesteps: 110624 Episode Num: 273 Reward: 619.145220017703\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 620.945452\n",
            "---------------------------------------\n",
            "Total Timesteps: 110802 Episode Num: 274 Reward: 220.81439298582464\n",
            "Total Timesteps: 111802 Episode Num: 275 Reward: 1012.0334075943878\n",
            "Total Timesteps: 112114 Episode Num: 276 Reward: 440.7439156489948\n",
            "Total Timesteps: 112366 Episode Num: 277 Reward: 299.81540723228767\n",
            "Total Timesteps: 112389 Episode Num: 278 Reward: 21.044038467493017\n",
            "Total Timesteps: 112531 Episode Num: 279 Reward: 253.8868739929833\n",
            "Total Timesteps: 113085 Episode Num: 280 Reward: 826.4265313536433\n",
            "Total Timesteps: 113218 Episode Num: 281 Reward: 193.13807038940396\n",
            "Total Timesteps: 113255 Episode Num: 282 Reward: 48.23683969548801\n",
            "Total Timesteps: 113632 Episode Num: 283 Reward: 625.2485342841372\n",
            "Total Timesteps: 113706 Episode Num: 284 Reward: 123.51300584239213\n",
            "Total Timesteps: 113739 Episode Num: 285 Reward: 40.79772562343888\n",
            "Total Timesteps: 114739 Episode Num: 286 Reward: 1379.6200830268237\n",
            "Total Timesteps: 115739 Episode Num: 287 Reward: 854.3655427798986\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 711.543487\n",
            "---------------------------------------\n",
            "Total Timesteps: 115800 Episode Num: 288 Reward: 45.20983693293689\n",
            "Total Timesteps: 115848 Episode Num: 289 Reward: 61.89300266612555\n",
            "Total Timesteps: 116230 Episode Num: 290 Reward: 452.78754107127855\n",
            "Total Timesteps: 116396 Episode Num: 291 Reward: 255.82365847195817\n",
            "Total Timesteps: 116474 Episode Num: 292 Reward: 121.4714523030532\n",
            "Total Timesteps: 116595 Episode Num: 293 Reward: 249.28936085262222\n",
            "Total Timesteps: 116750 Episode Num: 294 Reward: 247.3909152368992\n",
            "Total Timesteps: 116880 Episode Num: 295 Reward: 210.35070594799035\n",
            "Total Timesteps: 117880 Episode Num: 296 Reward: 1112.7519911034901\n",
            "Total Timesteps: 118122 Episode Num: 297 Reward: 338.1716799982183\n",
            "Total Timesteps: 119122 Episode Num: 298 Reward: 737.1126244690577\n",
            "Total Timesteps: 120122 Episode Num: 299 Reward: 952.1349473405967\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 321.339410\n",
            "---------------------------------------\n",
            "Total Timesteps: 120182 Episode Num: 300 Reward: 38.98744201525877\n",
            "Total Timesteps: 121182 Episode Num: 301 Reward: 708.4430348450869\n",
            "Total Timesteps: 121759 Episode Num: 302 Reward: 951.3005295910068\n",
            "Total Timesteps: 121875 Episode Num: 303 Reward: 136.13161371941175\n",
            "Total Timesteps: 122193 Episode Num: 304 Reward: 481.4394503764476\n",
            "Total Timesteps: 122283 Episode Num: 305 Reward: 105.9185694664157\n",
            "Total Timesteps: 122421 Episode Num: 306 Reward: 168.7215987818904\n",
            "Total Timesteps: 123421 Episode Num: 307 Reward: 721.5213162454016\n",
            "Total Timesteps: 123449 Episode Num: 308 Reward: 28.239122706585878\n",
            "Total Timesteps: 123699 Episode Num: 309 Reward: 336.5756483587723\n",
            "Total Timesteps: 123794 Episode Num: 310 Reward: 141.3832136781425\n",
            "Total Timesteps: 123867 Episode Num: 311 Reward: 137.1229438840272\n",
            "Total Timesteps: 123997 Episode Num: 312 Reward: 217.1746167134833\n",
            "Total Timesteps: 124997 Episode Num: 313 Reward: 799.382287617869\n",
            "Total Timesteps: 125072 Episode Num: 314 Reward: 102.87232254149016\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 533.826190\n",
            "---------------------------------------\n",
            "Total Timesteps: 125419 Episode Num: 315 Reward: 521.2775055076083\n",
            "Total Timesteps: 125461 Episode Num: 316 Reward: 54.7312150221842\n",
            "Total Timesteps: 125472 Episode Num: 317 Reward: 2.0604048399110138\n",
            "Total Timesteps: 126472 Episode Num: 318 Reward: 720.3762616003766\n",
            "Total Timesteps: 126987 Episode Num: 319 Reward: 864.2399369932292\n",
            "Total Timesteps: 127987 Episode Num: 320 Reward: 1025.2601616044144\n",
            "Total Timesteps: 128128 Episode Num: 321 Reward: 242.95042712541553\n",
            "Total Timesteps: 128166 Episode Num: 322 Reward: 38.59522157318669\n",
            "Total Timesteps: 128707 Episode Num: 323 Reward: 933.2933406832841\n",
            "Total Timesteps: 128898 Episode Num: 324 Reward: 287.08628979720055\n",
            "Total Timesteps: 129137 Episode Num: 325 Reward: 420.8146169792366\n",
            "Total Timesteps: 130137 Episode Num: 326 Reward: 810.2275998812506\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 540.195176\n",
            "---------------------------------------\n",
            "Total Timesteps: 130438 Episode Num: 327 Reward: 538.110258520138\n",
            "Total Timesteps: 130640 Episode Num: 328 Reward: 302.9660810901478\n",
            "Total Timesteps: 131640 Episode Num: 329 Reward: 492.85840077573\n",
            "Total Timesteps: 132640 Episode Num: 330 Reward: 1048.118761348734\n",
            "Total Timesteps: 132753 Episode Num: 331 Reward: 147.89623903312878\n",
            "Total Timesteps: 133019 Episode Num: 332 Reward: 351.6636780329317\n",
            "Total Timesteps: 133161 Episode Num: 333 Reward: 231.10519847667803\n",
            "Total Timesteps: 134161 Episode Num: 334 Reward: 698.6740683853626\n",
            "Total Timesteps: 134939 Episode Num: 335 Reward: 1164.6104180145342\n",
            "Total Timesteps: 135005 Episode Num: 336 Reward: 119.14552520416906\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 435.896731\n",
            "---------------------------------------\n",
            "Total Timesteps: 135611 Episode Num: 337 Reward: 1002.7516723413441\n",
            "Total Timesteps: 135745 Episode Num: 338 Reward: 93.86841610325729\n",
            "Total Timesteps: 136745 Episode Num: 339 Reward: 530.8006821561568\n",
            "Total Timesteps: 136911 Episode Num: 340 Reward: 286.31203261618055\n",
            "Total Timesteps: 137037 Episode Num: 341 Reward: 214.41535947483388\n",
            "Total Timesteps: 137132 Episode Num: 342 Reward: 161.35012686990916\n",
            "Total Timesteps: 137383 Episode Num: 343 Reward: 377.8180563164135\n",
            "Total Timesteps: 138100 Episode Num: 344 Reward: 1155.1110773404248\n",
            "Total Timesteps: 138228 Episode Num: 345 Reward: 230.28960008885278\n",
            "Total Timesteps: 139228 Episode Num: 346 Reward: 677.1791185889584\n",
            "Total Timesteps: 140228 Episode Num: 347 Reward: 1139.5493474825018\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 741.996627\n",
            "---------------------------------------\n",
            "Total Timesteps: 140372 Episode Num: 348 Reward: 174.1892683431036\n",
            "Total Timesteps: 140597 Episode Num: 349 Reward: 276.51185744869423\n",
            "Total Timesteps: 141021 Episode Num: 350 Reward: 709.5569419066634\n",
            "Total Timesteps: 142021 Episode Num: 351 Reward: 1659.2492373826067\n",
            "Total Timesteps: 143021 Episode Num: 352 Reward: 727.0340845539208\n",
            "Total Timesteps: 144021 Episode Num: 353 Reward: 1800.6944060041458\n",
            "Total Timesteps: 144111 Episode Num: 354 Reward: 178.63287638586948\n",
            "Total Timesteps: 144356 Episode Num: 355 Reward: 371.7518161556615\n",
            "Total Timesteps: 144845 Episode Num: 356 Reward: 751.3770900467784\n",
            "Total Timesteps: 145291 Episode Num: 357 Reward: 769.5687228181235\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 725.016374\n",
            "---------------------------------------\n",
            "Total Timesteps: 145394 Episode Num: 358 Reward: 162.93474304307884\n",
            "Total Timesteps: 146394 Episode Num: 359 Reward: 606.8470709190613\n",
            "Total Timesteps: 146901 Episode Num: 360 Reward: 957.261605724958\n",
            "Total Timesteps: 147146 Episode Num: 361 Reward: 394.0499770558661\n",
            "Total Timesteps: 148146 Episode Num: 362 Reward: 900.3293731722581\n",
            "Total Timesteps: 148382 Episode Num: 363 Reward: 371.416486818682\n",
            "Total Timesteps: 149382 Episode Num: 364 Reward: 993.6907175982719\n",
            "Total Timesteps: 150382 Episode Num: 365 Reward: 896.7718609344489\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 920.783387\n",
            "---------------------------------------\n",
            "Total Timesteps: 150602 Episode Num: 366 Reward: 340.9721807081289\n",
            "Total Timesteps: 151602 Episode Num: 367 Reward: 748.8290256184957\n",
            "Total Timesteps: 152602 Episode Num: 368 Reward: 1273.7351565870342\n",
            "Total Timesteps: 152718 Episode Num: 369 Reward: 196.52039811246604\n",
            "Total Timesteps: 153718 Episode Num: 370 Reward: 1106.545486578345\n",
            "Total Timesteps: 154718 Episode Num: 371 Reward: 704.2807958477878\n",
            "Total Timesteps: 155718 Episode Num: 372 Reward: 970.1561295556204\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 853.848282\n",
            "---------------------------------------\n",
            "Total Timesteps: 156217 Episode Num: 373 Reward: 834.9299142632051\n",
            "Total Timesteps: 156662 Episode Num: 374 Reward: 934.3392981246188\n",
            "Total Timesteps: 157662 Episode Num: 375 Reward: 720.8701178092367\n",
            "Total Timesteps: 158348 Episode Num: 376 Reward: 1435.324920075207\n",
            "Total Timesteps: 159111 Episode Num: 377 Reward: 1525.825242856883\n",
            "Total Timesteps: 160111 Episode Num: 378 Reward: 950.8193876441151\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 698.940443\n",
            "---------------------------------------\n",
            "Total Timesteps: 160181 Episode Num: 379 Reward: 117.13198196022462\n",
            "Total Timesteps: 161181 Episode Num: 380 Reward: 2044.077781594017\n",
            "Total Timesteps: 161961 Episode Num: 381 Reward: 1623.5463095499674\n",
            "Total Timesteps: 162411 Episode Num: 382 Reward: 850.6071970496108\n",
            "Total Timesteps: 163411 Episode Num: 383 Reward: 826.4629526339656\n",
            "Total Timesteps: 164411 Episode Num: 384 Reward: 544.9497007708316\n",
            "Total Timesteps: 165411 Episode Num: 385 Reward: 933.865465336039\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 723.969869\n",
            "---------------------------------------\n",
            "Total Timesteps: 165791 Episode Num: 386 Reward: 773.9575486174627\n",
            "Total Timesteps: 166791 Episode Num: 387 Reward: 1524.0912754081305\n",
            "Total Timesteps: 167791 Episode Num: 388 Reward: 965.7010991449604\n",
            "Total Timesteps: 168791 Episode Num: 389 Reward: 2171.224882838424\n",
            "Total Timesteps: 169791 Episode Num: 390 Reward: 703.5259942846515\n",
            "Total Timesteps: 170078 Episode Num: 391 Reward: 562.8004018153479\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 673.348363\n",
            "---------------------------------------\n",
            "Total Timesteps: 171078 Episode Num: 392 Reward: 661.9378316777977\n",
            "Total Timesteps: 171954 Episode Num: 393 Reward: 1696.1383543888876\n",
            "Total Timesteps: 172029 Episode Num: 394 Reward: 75.33813194561614\n",
            "Total Timesteps: 172354 Episode Num: 395 Reward: 608.0002905944199\n",
            "Total Timesteps: 173354 Episode Num: 396 Reward: 594.9373356952008\n",
            "Total Timesteps: 173603 Episode Num: 397 Reward: 518.5056459922868\n",
            "Total Timesteps: 173674 Episode Num: 398 Reward: 110.08789166992814\n",
            "Total Timesteps: 174674 Episode Num: 399 Reward: 1647.833718616478\n",
            "Total Timesteps: 175173 Episode Num: 400 Reward: 1042.1099520005541\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1115.545713\n",
            "---------------------------------------\n",
            "Total Timesteps: 175219 Episode Num: 401 Reward: 79.341711648022\n",
            "Total Timesteps: 176219 Episode Num: 402 Reward: 983.5881049793544\n",
            "Total Timesteps: 176459 Episode Num: 403 Reward: 505.64181348520657\n",
            "Total Timesteps: 176608 Episode Num: 404 Reward: 306.49363264442076\n",
            "Total Timesteps: 177608 Episode Num: 405 Reward: 2220.680675833096\n",
            "Total Timesteps: 177890 Episode Num: 406 Reward: 598.2338759860743\n",
            "Total Timesteps: 178890 Episode Num: 407 Reward: 814.2476715548586\n",
            "Total Timesteps: 179890 Episode Num: 408 Reward: 663.9776774670173\n",
            "Total Timesteps: 180890 Episode Num: 409 Reward: 2352.0680856982463\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1460.972989\n",
            "---------------------------------------\n",
            "Total Timesteps: 181454 Episode Num: 410 Reward: 1060.4293990345127\n",
            "Total Timesteps: 181574 Episode Num: 411 Reward: 271.1860538417682\n",
            "Total Timesteps: 182574 Episode Num: 412 Reward: 779.9688308857661\n",
            "Total Timesteps: 183574 Episode Num: 413 Reward: 698.3012816761011\n",
            "Total Timesteps: 183776 Episode Num: 414 Reward: 541.471373039202\n",
            "Total Timesteps: 183853 Episode Num: 415 Reward: 102.60956672298661\n",
            "Total Timesteps: 184157 Episode Num: 416 Reward: 598.8589677073337\n",
            "Total Timesteps: 184527 Episode Num: 417 Reward: 726.8893297720623\n",
            "Total Timesteps: 184736 Episode Num: 418 Reward: 476.28358889855684\n",
            "Total Timesteps: 185736 Episode Num: 419 Reward: 1802.6866189294137\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1788.361286\n",
            "---------------------------------------\n",
            "Total Timesteps: 186597 Episode Num: 420 Reward: 1822.0906973712615\n",
            "Total Timesteps: 187378 Episode Num: 421 Reward: 1706.9106523382995\n",
            "Total Timesteps: 187544 Episode Num: 422 Reward: 237.53342675991735\n",
            "Total Timesteps: 188544 Episode Num: 423 Reward: 1365.4193389686743\n",
            "Total Timesteps: 188601 Episode Num: 424 Reward: 102.07899177507493\n",
            "Total Timesteps: 188667 Episode Num: 425 Reward: 87.02675627768964\n",
            "Total Timesteps: 189667 Episode Num: 426 Reward: 605.7887592268587\n",
            "Total Timesteps: 189727 Episode Num: 427 Reward: 104.14182921738467\n",
            "Total Timesteps: 189837 Episode Num: 428 Reward: 246.14528055133286\n",
            "Total Timesteps: 190125 Episode Num: 429 Reward: 722.8152444965017\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1484.344544\n",
            "---------------------------------------\n",
            "Total Timesteps: 191125 Episode Num: 430 Reward: 1136.9181089707608\n",
            "Total Timesteps: 191148 Episode Num: 431 Reward: 37.83270320707479\n",
            "Total Timesteps: 191203 Episode Num: 432 Reward: 94.7158854607708\n",
            "Total Timesteps: 192203 Episode Num: 433 Reward: 1213.3535498531025\n",
            "Total Timesteps: 192239 Episode Num: 434 Reward: 22.826430911599417\n",
            "Total Timesteps: 192654 Episode Num: 435 Reward: 905.730523431431\n",
            "Total Timesteps: 193654 Episode Num: 436 Reward: 1076.687601172063\n",
            "Total Timesteps: 194654 Episode Num: 437 Reward: 1473.2251590908922\n",
            "Total Timesteps: 195654 Episode Num: 438 Reward: 493.6224886536316\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1057.277066\n",
            "---------------------------------------\n",
            "Total Timesteps: 196654 Episode Num: 439 Reward: 1312.3567344994533\n",
            "Total Timesteps: 196937 Episode Num: 440 Reward: 534.6223963895898\n",
            "Total Timesteps: 197937 Episode Num: 441 Reward: 1958.1641223957365\n",
            "Total Timesteps: 198528 Episode Num: 442 Reward: 1359.3018192076354\n",
            "Total Timesteps: 198575 Episode Num: 443 Reward: 52.95429279370956\n",
            "Total Timesteps: 199575 Episode Num: 444 Reward: 714.7632552241872\n",
            "Total Timesteps: 200133 Episode Num: 445 Reward: 1099.2285775608368\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2045.821670\n",
            "---------------------------------------\n",
            "Total Timesteps: 200255 Episode Num: 446 Reward: 258.59699945346983\n",
            "Total Timesteps: 201255 Episode Num: 447 Reward: 1824.2601959277533\n",
            "Total Timesteps: 202255 Episode Num: 448 Reward: 2130.620488160821\n",
            "Total Timesteps: 202273 Episode Num: 449 Reward: 10.69473399054435\n",
            "Total Timesteps: 203273 Episode Num: 450 Reward: 2492.3149634461074\n",
            "Total Timesteps: 204273 Episode Num: 451 Reward: 1162.2985948129106\n",
            "Total Timesteps: 204480 Episode Num: 452 Reward: 471.9939115696225\n",
            "Total Timesteps: 205237 Episode Num: 453 Reward: 1789.3368185318432\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2142.581440\n",
            "---------------------------------------\n",
            "Total Timesteps: 206237 Episode Num: 454 Reward: 2518.441249934206\n",
            "Total Timesteps: 207237 Episode Num: 455 Reward: 2609.949719717923\n",
            "Total Timesteps: 207381 Episode Num: 456 Reward: 297.7637283517569\n",
            "Total Timesteps: 208381 Episode Num: 457 Reward: 2063.632525661864\n",
            "Total Timesteps: 208421 Episode Num: 458 Reward: 53.698211688853156\n",
            "Total Timesteps: 208683 Episode Num: 459 Reward: 567.131953637126\n",
            "Total Timesteps: 209683 Episode Num: 460 Reward: 1859.6644265465968\n",
            "Total Timesteps: 209732 Episode Num: 461 Reward: 75.07366553283997\n",
            "Total Timesteps: 210732 Episode Num: 462 Reward: 932.0347511255255\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1184.093406\n",
            "---------------------------------------\n",
            "Total Timesteps: 211732 Episode Num: 463 Reward: 824.4836326633645\n",
            "Total Timesteps: 212638 Episode Num: 464 Reward: 2380.425092811729\n",
            "Total Timesteps: 213581 Episode Num: 465 Reward: 2625.3590533274005\n",
            "Total Timesteps: 214240 Episode Num: 466 Reward: 1711.1618806725373\n",
            "Total Timesteps: 214352 Episode Num: 467 Reward: 208.58965414462676\n",
            "Total Timesteps: 214876 Episode Num: 468 Reward: 1174.4882613624857\n",
            "Total Timesteps: 215876 Episode Num: 469 Reward: 678.9172996515993\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1448.670915\n",
            "---------------------------------------\n",
            "Total Timesteps: 216488 Episode Num: 470 Reward: 1689.887281127221\n",
            "Total Timesteps: 217488 Episode Num: 471 Reward: 2580.112188795623\n",
            "Total Timesteps: 218488 Episode Num: 472 Reward: 2405.0764739507486\n",
            "Total Timesteps: 219488 Episode Num: 473 Reward: 2360.0495492139007\n",
            "Total Timesteps: 220488 Episode Num: 474 Reward: 2604.579946276527\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2219.996857\n",
            "---------------------------------------\n",
            "Total Timesteps: 221488 Episode Num: 475 Reward: 2394.0138032100226\n",
            "Total Timesteps: 222488 Episode Num: 476 Reward: 1481.2058455070928\n",
            "Total Timesteps: 223488 Episode Num: 477 Reward: 1659.965634289614\n",
            "Total Timesteps: 224488 Episode Num: 478 Reward: 865.8098749717433\n",
            "Total Timesteps: 225488 Episode Num: 479 Reward: 1444.14332773172\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2149.085972\n",
            "---------------------------------------\n",
            "Total Timesteps: 226488 Episode Num: 480 Reward: 2558.5073044179544\n",
            "Total Timesteps: 226600 Episode Num: 481 Reward: 198.7167710716642\n",
            "Total Timesteps: 227600 Episode Num: 482 Reward: 772.5028031177793\n",
            "Total Timesteps: 228003 Episode Num: 483 Reward: 1007.9481789926627\n",
            "Total Timesteps: 229003 Episode Num: 484 Reward: 2643.2468067214368\n",
            "Total Timesteps: 230003 Episode Num: 485 Reward: 582.175779452721\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2710.140415\n",
            "---------------------------------------\n",
            "Total Timesteps: 231003 Episode Num: 486 Reward: 2684.0088121461104\n",
            "Total Timesteps: 232003 Episode Num: 487 Reward: 2346.5962078614893\n",
            "Total Timesteps: 233003 Episode Num: 488 Reward: 648.3336730374077\n",
            "Total Timesteps: 233035 Episode Num: 489 Reward: 28.992527610535582\n",
            "Total Timesteps: 233124 Episode Num: 490 Reward: 173.32714967613586\n",
            "Total Timesteps: 234124 Episode Num: 491 Reward: 811.5816133235346\n",
            "Total Timesteps: 235124 Episode Num: 492 Reward: 2853.362509495338\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2805.967261\n",
            "---------------------------------------\n",
            "Total Timesteps: 235954 Episode Num: 493 Reward: 2436.956079332064\n",
            "Total Timesteps: 236954 Episode Num: 494 Reward: 2694.7175618094698\n",
            "Total Timesteps: 237085 Episode Num: 495 Reward: 362.278188725285\n",
            "Total Timesteps: 238085 Episode Num: 496 Reward: -89.8904802192162\n",
            "Total Timesteps: 239085 Episode Num: 497 Reward: 2444.3622654495503\n",
            "Total Timesteps: 240085 Episode Num: 498 Reward: 2806.0555334786586\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2839.218982\n",
            "---------------------------------------\n",
            "Total Timesteps: 241085 Episode Num: 499 Reward: 2819.3795245926244\n",
            "Total Timesteps: 242085 Episode Num: 500 Reward: 2805.013594215292\n",
            "Total Timesteps: 242471 Episode Num: 501 Reward: 1012.7668859902134\n",
            "Total Timesteps: 243471 Episode Num: 502 Reward: 2868.293190227356\n",
            "Total Timesteps: 244471 Episode Num: 503 Reward: 2938.4324147512166\n",
            "Total Timesteps: 245471 Episode Num: 504 Reward: 583.6106193475558\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2252.605008\n",
            "---------------------------------------\n",
            "Total Timesteps: 246471 Episode Num: 505 Reward: 2869.5791327796437\n",
            "Total Timesteps: 247471 Episode Num: 506 Reward: 2870.0688130249155\n",
            "Total Timesteps: 248471 Episode Num: 507 Reward: 3063.049932617701\n",
            "Total Timesteps: 249471 Episode Num: 508 Reward: 1292.3056563700327\n",
            "Total Timesteps: 250342 Episode Num: 509 Reward: 2395.3229710512533\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2124.713815\n",
            "---------------------------------------\n",
            "Total Timesteps: 250700 Episode Num: 510 Reward: 1052.2035723191075\n",
            "Total Timesteps: 251700 Episode Num: 511 Reward: 1985.6317573171502\n",
            "Total Timesteps: 252700 Episode Num: 512 Reward: 3039.0111502828568\n",
            "Total Timesteps: 253700 Episode Num: 513 Reward: 2859.6669808673487\n",
            "Total Timesteps: 254700 Episode Num: 514 Reward: 2815.01758381959\n",
            "Total Timesteps: 255700 Episode Num: 515 Reward: 2965.766881122624\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2294.299546\n",
            "---------------------------------------\n",
            "Total Timesteps: 256700 Episode Num: 516 Reward: 2702.0572439861294\n",
            "Total Timesteps: 257655 Episode Num: 517 Reward: 2903.220848285655\n",
            "Total Timesteps: 257758 Episode Num: 518 Reward: 288.92728047764587\n",
            "Total Timesteps: 258758 Episode Num: 519 Reward: 3056.3958005202016\n",
            "Total Timesteps: 259758 Episode Num: 520 Reward: 3020.439507342844\n",
            "Total Timesteps: 260758 Episode Num: 521 Reward: 3100.0323813198133\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3226.646311\n",
            "---------------------------------------\n",
            "Total Timesteps: 261758 Episode Num: 522 Reward: 2870.275415861915\n",
            "Total Timesteps: 261779 Episode Num: 523 Reward: 14.899286029509554\n",
            "Total Timesteps: 262561 Episode Num: 524 Reward: 2351.468847538043\n",
            "Total Timesteps: 262782 Episode Num: 525 Reward: 595.1560005420852\n",
            "Total Timesteps: 263782 Episode Num: 526 Reward: 3289.3167993139186\n",
            "Total Timesteps: 264782 Episode Num: 527 Reward: 3213.256343462352\n",
            "Total Timesteps: 265782 Episode Num: 528 Reward: 3102.6075456759863\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2429.912724\n",
            "---------------------------------------\n",
            "Total Timesteps: 266782 Episode Num: 529 Reward: 3192.1309780365805\n",
            "Total Timesteps: 267782 Episode Num: 530 Reward: 1860.341971759323\n",
            "Total Timesteps: 268782 Episode Num: 531 Reward: 1134.189435019632\n",
            "Total Timesteps: 269685 Episode Num: 532 Reward: 2851.111087290537\n",
            "Total Timesteps: 270685 Episode Num: 533 Reward: 3031.4323973858204\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1663.028642\n",
            "---------------------------------------\n",
            "Total Timesteps: 270918 Episode Num: 534 Reward: 589.6244061934561\n",
            "Total Timesteps: 270969 Episode Num: 535 Reward: 74.50787337400567\n",
            "Total Timesteps: 271132 Episode Num: 536 Reward: 441.6028146689228\n",
            "Total Timesteps: 272132 Episode Num: 537 Reward: 2925.0788208126023\n",
            "Total Timesteps: 272513 Episode Num: 538 Reward: 1049.1171742670176\n",
            "Total Timesteps: 273513 Episode Num: 539 Reward: 1150.1321151030484\n",
            "Total Timesteps: 273782 Episode Num: 540 Reward: 945.8342001916718\n",
            "Total Timesteps: 274782 Episode Num: 541 Reward: 3334.2986291043103\n",
            "Total Timesteps: 275782 Episode Num: 542 Reward: 3107.684772363974\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1903.346581\n",
            "---------------------------------------\n",
            "Total Timesteps: 276782 Episode Num: 543 Reward: 2901.721251329395\n",
            "Total Timesteps: 277782 Episode Num: 544 Reward: 3101.3048065584426\n",
            "Total Timesteps: 278782 Episode Num: 545 Reward: 2867.402222482396\n",
            "Total Timesteps: 279237 Episode Num: 546 Reward: 1385.4097181224515\n",
            "Total Timesteps: 279953 Episode Num: 547 Reward: 2112.1267814240573\n",
            "Total Timesteps: 280953 Episode Num: 548 Reward: 1089.7692431644668\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3109.895362\n",
            "---------------------------------------\n",
            "Total Timesteps: 281953 Episode Num: 549 Reward: 3261.2409849524834\n",
            "Total Timesteps: 282953 Episode Num: 550 Reward: 3275.0595457842724\n",
            "Total Timesteps: 283953 Episode Num: 551 Reward: 1514.1630664230593\n",
            "Total Timesteps: 284320 Episode Num: 552 Reward: 1178.8015144144474\n",
            "Total Timesteps: 285320 Episode Num: 553 Reward: 3235.761165764725\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 1597.486279\n",
            "---------------------------------------\n",
            "Total Timesteps: 286320 Episode Num: 554 Reward: 2906.8108455404617\n",
            "Total Timesteps: 287320 Episode Num: 555 Reward: 3127.407605284999\n",
            "Total Timesteps: 288320 Episode Num: 556 Reward: 3412.289774285942\n",
            "Total Timesteps: 288967 Episode Num: 557 Reward: 2133.7607372369507\n",
            "Total Timesteps: 289368 Episode Num: 558 Reward: 1224.9883225913545\n",
            "Total Timesteps: 290368 Episode Num: 559 Reward: 3126.342533305473\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3182.791559\n",
            "---------------------------------------\n",
            "Total Timesteps: 291368 Episode Num: 560 Reward: 3124.1254337144205\n",
            "Total Timesteps: 292368 Episode Num: 561 Reward: 3418.269296513599\n",
            "Total Timesteps: 293368 Episode Num: 562 Reward: 1137.442456185147\n",
            "Total Timesteps: 294368 Episode Num: 563 Reward: 3219.8573200164174\n",
            "Total Timesteps: 294401 Episode Num: 564 Reward: 18.86580746311187\n",
            "Total Timesteps: 294748 Episode Num: 565 Reward: 1043.762885120853\n",
            "Total Timesteps: 295748 Episode Num: 566 Reward: 3244.3774984780503\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2888.717201\n",
            "---------------------------------------\n",
            "Total Timesteps: 296748 Episode Num: 567 Reward: 3448.2515395583187\n",
            "Total Timesteps: 297748 Episode Num: 568 Reward: 3360.3874168751904\n",
            "Total Timesteps: 297856 Episode Num: 569 Reward: 333.5670433665391\n",
            "Total Timesteps: 298856 Episode Num: 570 Reward: 3408.531133894529\n",
            "Total Timesteps: 299856 Episode Num: 571 Reward: 3769.653324331469\n",
            "Total Timesteps: 300169 Episode Num: 572 Reward: 974.9702542677056\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2284.179561\n",
            "---------------------------------------\n",
            "Total Timesteps: 300885 Episode Num: 573 Reward: 2544.4510211739944\n",
            "Total Timesteps: 301885 Episode Num: 574 Reward: 3194.924682310004\n",
            "Total Timesteps: 302885 Episode Num: 575 Reward: 3575.052993946788\n",
            "Total Timesteps: 302898 Episode Num: 576 Reward: 1.5999894716022998\n",
            "Total Timesteps: 303898 Episode Num: 577 Reward: 3283.84552620661\n",
            "Total Timesteps: 304898 Episode Num: 578 Reward: 3385.9076863779283\n",
            "Total Timesteps: 305898 Episode Num: 579 Reward: 313.4520671061266\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3202.592703\n",
            "---------------------------------------\n",
            "Total Timesteps: 306898 Episode Num: 580 Reward: 3198.8841601644685\n",
            "Total Timesteps: 307898 Episode Num: 581 Reward: 3111.349573605524\n",
            "Total Timesteps: 308729 Episode Num: 582 Reward: 2693.0272193144706\n",
            "Total Timesteps: 309729 Episode Num: 583 Reward: 3242.0298120284083\n",
            "Total Timesteps: 309956 Episode Num: 584 Reward: 671.0602750035903\n",
            "Total Timesteps: 310890 Episode Num: 585 Reward: 3312.805860970607\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2555.665704\n",
            "---------------------------------------\n",
            "Total Timesteps: 311890 Episode Num: 586 Reward: 3603.0672048983492\n",
            "Total Timesteps: 312890 Episode Num: 587 Reward: 3415.588204615338\n",
            "Total Timesteps: 313890 Episode Num: 588 Reward: 3198.8533441147547\n",
            "Total Timesteps: 314661 Episode Num: 589 Reward: 2766.317337190691\n",
            "Total Timesteps: 315661 Episode Num: 590 Reward: 3421.4401841818303\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3334.313230\n",
            "---------------------------------------\n",
            "Total Timesteps: 316661 Episode Num: 591 Reward: 3582.608441638023\n",
            "Total Timesteps: 317661 Episode Num: 592 Reward: 3567.420470438087\n",
            "Total Timesteps: 318661 Episode Num: 593 Reward: 3316.834150020951\n",
            "Total Timesteps: 319661 Episode Num: 594 Reward: 1187.6140954764269\n",
            "Total Timesteps: 319884 Episode Num: 595 Reward: 788.5219717990735\n",
            "Total Timesteps: 320884 Episode Num: 596 Reward: 3411.5803858057807\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2192.037467\n",
            "---------------------------------------\n",
            "Total Timesteps: 321884 Episode Num: 597 Reward: 3340.5667922870157\n",
            "Total Timesteps: 322884 Episode Num: 598 Reward: 3576.874495841464\n",
            "Total Timesteps: 323884 Episode Num: 599 Reward: 3732.8843171521594\n",
            "Total Timesteps: 324884 Episode Num: 600 Reward: 3671.3036203414767\n",
            "Total Timesteps: 325884 Episode Num: 601 Reward: 3746.424053596943\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2099.770550\n",
            "---------------------------------------\n",
            "Total Timesteps: 326792 Episode Num: 602 Reward: 3507.740443927137\n",
            "Total Timesteps: 327792 Episode Num: 603 Reward: 2003.7294250580487\n",
            "Total Timesteps: 328792 Episode Num: 604 Reward: 3790.8023529560965\n",
            "Total Timesteps: 328825 Episode Num: 605 Reward: 54.110819825144816\n",
            "Total Timesteps: 329825 Episode Num: 606 Reward: 3890.348969144616\n",
            "Total Timesteps: 330825 Episode Num: 607 Reward: -110.53462040352156\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3448.314782\n",
            "---------------------------------------\n",
            "Total Timesteps: 331825 Episode Num: 608 Reward: -191.33490666799813\n",
            "Total Timesteps: 332825 Episode Num: 609 Reward: 3730.308135839195\n",
            "Total Timesteps: 332933 Episode Num: 610 Reward: 373.0520226095491\n",
            "Total Timesteps: 333933 Episode Num: 611 Reward: 1522.7790488832727\n",
            "Total Timesteps: 334409 Episode Num: 612 Reward: 1546.9487060994384\n",
            "Total Timesteps: 334603 Episode Num: 613 Reward: 630.5892349035314\n",
            "Total Timesteps: 335603 Episode Num: 614 Reward: 3815.48255073423\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2208.789837\n",
            "---------------------------------------\n",
            "Total Timesteps: 336603 Episode Num: 615 Reward: 2106.2330285146163\n",
            "Total Timesteps: 337603 Episode Num: 616 Reward: 3950.7799434420144\n",
            "Total Timesteps: 337816 Episode Num: 617 Reward: 663.3117656375174\n",
            "Total Timesteps: 338816 Episode Num: 618 Reward: 3653.6394595428255\n",
            "Total Timesteps: 338849 Episode Num: 619 Reward: 75.35472953989711\n",
            "Total Timesteps: 339849 Episode Num: 620 Reward: 2165.9746889053827\n",
            "Total Timesteps: 340849 Episode Num: 621 Reward: 3997.8783530458354\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3947.573248\n",
            "---------------------------------------\n",
            "Total Timesteps: 341266 Episode Num: 622 Reward: 1365.636751210265\n",
            "Total Timesteps: 341751 Episode Num: 623 Reward: 1660.9840860704026\n",
            "Total Timesteps: 342751 Episode Num: 624 Reward: 3611.0446215140873\n",
            "Total Timesteps: 343751 Episode Num: 625 Reward: 3712.8176886854476\n",
            "Total Timesteps: 344751 Episode Num: 626 Reward: 2787.4386715715955\n",
            "Total Timesteps: 345529 Episode Num: 627 Reward: 2599.9540563707424\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2829.911325\n",
            "---------------------------------------\n",
            "Total Timesteps: 346130 Episode Num: 628 Reward: 2193.9034391528057\n",
            "Total Timesteps: 346636 Episode Num: 629 Reward: 1967.7537205496546\n",
            "Total Timesteps: 347636 Episode Num: 630 Reward: 3744.8799336642196\n",
            "Total Timesteps: 347801 Episode Num: 631 Reward: 573.3792926037512\n",
            "Total Timesteps: 348069 Episode Num: 632 Reward: 926.0421213489299\n",
            "Total Timesteps: 349069 Episode Num: 633 Reward: 173.9984550451055\n",
            "Total Timesteps: 350069 Episode Num: 634 Reward: 3700.2083619157324\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3468.551164\n",
            "---------------------------------------\n",
            "Total Timesteps: 350772 Episode Num: 635 Reward: 2610.0148820549707\n",
            "Total Timesteps: 351449 Episode Num: 636 Reward: 2617.1676499780997\n",
            "Total Timesteps: 352449 Episode Num: 637 Reward: 4041.12145321354\n",
            "Total Timesteps: 353449 Episode Num: 638 Reward: 3943.2307386433645\n",
            "Total Timesteps: 354449 Episode Num: 639 Reward: 1696.9411045323511\n",
            "Total Timesteps: 355449 Episode Num: 640 Reward: 1657.9322208223443\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2443.601246\n",
            "---------------------------------------\n",
            "Total Timesteps: 356351 Episode Num: 641 Reward: 3555.4307254944597\n",
            "Total Timesteps: 357351 Episode Num: 642 Reward: 3834.006711496607\n",
            "Total Timesteps: 357736 Episode Num: 643 Reward: 1447.1205956891124\n",
            "Total Timesteps: 358736 Episode Num: 644 Reward: 3903.747409718932\n",
            "Total Timesteps: 359330 Episode Num: 645 Reward: 2247.144732748284\n",
            "Total Timesteps: 360163 Episode Num: 646 Reward: 3079.9497613491476\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3755.554556\n",
            "---------------------------------------\n",
            "Total Timesteps: 361163 Episode Num: 647 Reward: 4183.127457262783\n",
            "Total Timesteps: 361991 Episode Num: 648 Reward: 3088.926079006257\n",
            "Total Timesteps: 362991 Episode Num: 649 Reward: 3675.980913788455\n",
            "Total Timesteps: 363991 Episode Num: 650 Reward: 3849.619247803403\n",
            "Total Timesteps: 364991 Episode Num: 651 Reward: 4027.4982115621856\n",
            "Total Timesteps: 365991 Episode Num: 652 Reward: 4006.3042311308595\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4016.525960\n",
            "---------------------------------------\n",
            "Total Timesteps: 366991 Episode Num: 653 Reward: 4467.852316300355\n",
            "Total Timesteps: 367369 Episode Num: 654 Reward: 1429.399939944537\n",
            "Total Timesteps: 367671 Episode Num: 655 Reward: 1244.0365196983362\n",
            "Total Timesteps: 368671 Episode Num: 656 Reward: 3897.278798432189\n",
            "Total Timesteps: 369671 Episode Num: 657 Reward: 4032.596206846811\n",
            "Total Timesteps: 370671 Episode Num: 658 Reward: 3990.760842027553\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4112.581336\n",
            "---------------------------------------\n",
            "Total Timesteps: 371671 Episode Num: 659 Reward: 4235.674598086499\n",
            "Total Timesteps: 372671 Episode Num: 660 Reward: 4148.936747946671\n",
            "Total Timesteps: 373179 Episode Num: 661 Reward: 1920.2626900478765\n",
            "Total Timesteps: 373394 Episode Num: 662 Reward: 649.5705159401588\n",
            "Total Timesteps: 373966 Episode Num: 663 Reward: 2380.0305999593193\n",
            "Total Timesteps: 374966 Episode Num: 664 Reward: 4193.942110709048\n",
            "Total Timesteps: 375966 Episode Num: 665 Reward: 3853.468647226395\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4041.661588\n",
            "---------------------------------------\n",
            "Total Timesteps: 376966 Episode Num: 666 Reward: 4245.810118216826\n",
            "Total Timesteps: 377966 Episode Num: 667 Reward: 3956.9236626605793\n",
            "Total Timesteps: 378900 Episode Num: 668 Reward: 3598.740795248046\n",
            "Total Timesteps: 379900 Episode Num: 669 Reward: 3825.5704000522246\n",
            "Total Timesteps: 380900 Episode Num: 670 Reward: 4047.760665429922\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4145.629560\n",
            "---------------------------------------\n",
            "Total Timesteps: 381900 Episode Num: 671 Reward: 4182.227985022127\n",
            "Total Timesteps: 382900 Episode Num: 672 Reward: 3990.7408724589154\n",
            "Total Timesteps: 382953 Episode Num: 673 Reward: -23.47550995654828\n",
            "Total Timesteps: 383953 Episode Num: 674 Reward: 4175.988046641922\n",
            "Total Timesteps: 384953 Episode Num: 675 Reward: 4231.47088603189\n",
            "Total Timesteps: 385953 Episode Num: 676 Reward: 4268.548065367501\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3230.777375\n",
            "---------------------------------------\n",
            "Total Timesteps: 386953 Episode Num: 677 Reward: 3918.1965894820155\n",
            "Total Timesteps: 387953 Episode Num: 678 Reward: 4294.115882817031\n",
            "Total Timesteps: 388799 Episode Num: 679 Reward: 3667.3329702743667\n",
            "Total Timesteps: 389799 Episode Num: 680 Reward: 4122.375902787342\n",
            "Total Timesteps: 390799 Episode Num: 681 Reward: -394.65688397779644\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4203.182260\n",
            "---------------------------------------\n",
            "Total Timesteps: 391799 Episode Num: 682 Reward: 3982.2880461043037\n",
            "Total Timesteps: 392438 Episode Num: 683 Reward: 2700.868024188054\n",
            "Total Timesteps: 393438 Episode Num: 684 Reward: 3962.6507085485\n",
            "Total Timesteps: 394438 Episode Num: 685 Reward: 3976.3903007712306\n",
            "Total Timesteps: 394896 Episode Num: 686 Reward: 1851.971316538363\n",
            "Total Timesteps: 395896 Episode Num: 687 Reward: 4215.229073237507\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2671.370318\n",
            "---------------------------------------\n",
            "Total Timesteps: 396896 Episode Num: 688 Reward: 4194.071990380306\n",
            "Total Timesteps: 397896 Episode Num: 689 Reward: 816.2171902428005\n",
            "Total Timesteps: 398896 Episode Num: 690 Reward: 4201.345282410245\n",
            "Total Timesteps: 399896 Episode Num: 691 Reward: 4145.092323578284\n",
            "Total Timesteps: 400896 Episode Num: 692 Reward: 4242.6202471834895\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3740.872033\n",
            "---------------------------------------\n",
            "Total Timesteps: 401896 Episode Num: 693 Reward: 4101.399369830906\n",
            "Total Timesteps: 402782 Episode Num: 694 Reward: 3977.3628013957386\n",
            "Total Timesteps: 403619 Episode Num: 695 Reward: 3474.529787974667\n",
            "Total Timesteps: 403636 Episode Num: 696 Reward: 7.45429969037088\n",
            "Total Timesteps: 404636 Episode Num: 697 Reward: 3831.089360194097\n",
            "Total Timesteps: 405636 Episode Num: 698 Reward: 4133.9017180538185\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4014.096804\n",
            "---------------------------------------\n",
            "Total Timesteps: 406636 Episode Num: 699 Reward: 4213.1353726788875\n",
            "Total Timesteps: 406943 Episode Num: 700 Reward: 1339.6811049858961\n",
            "Total Timesteps: 407943 Episode Num: 701 Reward: 4044.944068509685\n",
            "Total Timesteps: 408943 Episode Num: 702 Reward: 4249.4631358212255\n",
            "Total Timesteps: 409943 Episode Num: 703 Reward: 4266.567835996441\n",
            "Total Timesteps: 410943 Episode Num: 704 Reward: 4189.988715152442\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3570.409320\n",
            "---------------------------------------\n",
            "Total Timesteps: 411943 Episode Num: 705 Reward: 4020.384936536864\n",
            "Total Timesteps: 412943 Episode Num: 706 Reward: 3817.7640628317567\n",
            "Total Timesteps: 413943 Episode Num: 707 Reward: 4171.149389845213\n",
            "Total Timesteps: 414943 Episode Num: 708 Reward: 3912.4963257957124\n",
            "Total Timesteps: 415943 Episode Num: 709 Reward: 4017.212268764609\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4415.179931\n",
            "---------------------------------------\n",
            "Total Timesteps: 415975 Episode Num: 710 Reward: -3.4895484432169224\n",
            "Total Timesteps: 415994 Episode Num: 711 Reward: -26.962086816913136\n",
            "Total Timesteps: 416232 Episode Num: 712 Reward: 907.3201680181814\n",
            "Total Timesteps: 416933 Episode Num: 713 Reward: 2961.2852271956203\n",
            "Total Timesteps: 417933 Episode Num: 714 Reward: 4447.598728868612\n",
            "Total Timesteps: 418933 Episode Num: 715 Reward: 4123.356185540288\n",
            "Total Timesteps: 419933 Episode Num: 716 Reward: 4414.085746723696\n",
            "Total Timesteps: 420933 Episode Num: 717 Reward: 1449.4390877685655\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4365.208889\n",
            "---------------------------------------\n",
            "Total Timesteps: 421933 Episode Num: 718 Reward: 4190.963910732249\n",
            "Total Timesteps: 422933 Episode Num: 719 Reward: 4206.611245526108\n",
            "Total Timesteps: 423933 Episode Num: 720 Reward: 4072.6630242492015\n",
            "Total Timesteps: 424933 Episode Num: 721 Reward: 4314.753221673468\n",
            "Total Timesteps: 425933 Episode Num: 722 Reward: 4305.770425947956\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3170.686664\n",
            "---------------------------------------\n",
            "Total Timesteps: 426703 Episode Num: 723 Reward: 2979.284331621437\n",
            "Total Timesteps: 426937 Episode Num: 724 Reward: 826.4359639778727\n",
            "Total Timesteps: 427328 Episode Num: 725 Reward: 1687.5618282662529\n",
            "Total Timesteps: 427668 Episode Num: 726 Reward: 1396.67610410419\n",
            "Total Timesteps: 427768 Episode Num: 727 Reward: 366.2654017357148\n",
            "Total Timesteps: 428768 Episode Num: 728 Reward: 4462.54064965468\n",
            "Total Timesteps: 429768 Episode Num: 729 Reward: 4407.10337231626\n",
            "Total Timesteps: 430768 Episode Num: 730 Reward: 4338.026353681555\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4005.893823\n",
            "---------------------------------------\n",
            "Total Timesteps: 430799 Episode Num: 731 Reward: 37.77623806084796\n",
            "Total Timesteps: 431364 Episode Num: 732 Reward: 2530.1691814712453\n",
            "Total Timesteps: 432364 Episode Num: 733 Reward: 4388.08702832009\n",
            "Total Timesteps: 433307 Episode Num: 734 Reward: 3964.733422335638\n",
            "Total Timesteps: 434008 Episode Num: 735 Reward: 2827.5063843975663\n",
            "Total Timesteps: 435008 Episode Num: 736 Reward: 4281.46946934248\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4373.965352\n",
            "---------------------------------------\n",
            "Total Timesteps: 436008 Episode Num: 737 Reward: 4384.546240349435\n",
            "Total Timesteps: 436414 Episode Num: 738 Reward: 1835.2157226146446\n",
            "Total Timesteps: 436664 Episode Num: 739 Reward: 1097.2747969574486\n",
            "Total Timesteps: 436879 Episode Num: 740 Reward: 877.852578497545\n",
            "Total Timesteps: 436955 Episode Num: 741 Reward: 280.8111270861985\n",
            "Total Timesteps: 437154 Episode Num: 742 Reward: 791.5903166848275\n",
            "Total Timesteps: 438154 Episode Num: 743 Reward: 4547.575496298038\n",
            "Total Timesteps: 439154 Episode Num: 744 Reward: 1558.2630744200578\n",
            "Total Timesteps: 440154 Episode Num: 745 Reward: 4281.320390699183\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4663.532327\n",
            "---------------------------------------\n",
            "Total Timesteps: 441154 Episode Num: 746 Reward: 4362.056201625171\n",
            "Total Timesteps: 442154 Episode Num: 747 Reward: 4104.7439372973395\n",
            "Total Timesteps: 442579 Episode Num: 748 Reward: 1652.8001439388393\n",
            "Total Timesteps: 443579 Episode Num: 749 Reward: 4444.938731913115\n",
            "Total Timesteps: 444579 Episode Num: 750 Reward: 4276.728586784857\n",
            "Total Timesteps: 445579 Episode Num: 751 Reward: 4086.0842471178175\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4300.583724\n",
            "---------------------------------------\n",
            "Total Timesteps: 446579 Episode Num: 752 Reward: 4435.4809826473\n",
            "Total Timesteps: 447579 Episode Num: 753 Reward: 430.03086944348155\n",
            "Total Timesteps: 448579 Episode Num: 754 Reward: 3793.18135654794\n",
            "Total Timesteps: 449579 Episode Num: 755 Reward: 4524.029455084247\n",
            "Total Timesteps: 450579 Episode Num: 756 Reward: 4265.010441549767\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3823.610416\n",
            "---------------------------------------\n",
            "Total Timesteps: 451579 Episode Num: 757 Reward: 4647.9707761264435\n",
            "Total Timesteps: 452329 Episode Num: 758 Reward: 3449.045117347157\n",
            "Total Timesteps: 452401 Episode Num: 759 Reward: 193.71155652197055\n",
            "Total Timesteps: 453401 Episode Num: 760 Reward: 4312.647636883421\n",
            "Total Timesteps: 454401 Episode Num: 761 Reward: 4458.461376468439\n",
            "Total Timesteps: 455401 Episode Num: 762 Reward: 4319.796603204582\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 2041.894173\n",
            "---------------------------------------\n",
            "Total Timesteps: 456401 Episode Num: 763 Reward: 4238.661113525548\n",
            "Total Timesteps: 457401 Episode Num: 764 Reward: 4316.1055154085625\n",
            "Total Timesteps: 458401 Episode Num: 765 Reward: 4310.6163850508465\n",
            "Total Timesteps: 458763 Episode Num: 766 Reward: 1489.1973593626985\n",
            "Total Timesteps: 459565 Episode Num: 767 Reward: 3500.1870356176005\n",
            "Total Timesteps: 460029 Episode Num: 768 Reward: 1965.599024739867\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3530.878907\n",
            "---------------------------------------\n",
            "Total Timesteps: 460767 Episode Num: 769 Reward: 3309.9118750443945\n",
            "Total Timesteps: 461767 Episode Num: 770 Reward: 4532.957114352793\n",
            "Total Timesteps: 462767 Episode Num: 771 Reward: 4274.353923577887\n",
            "Total Timesteps: 463767 Episode Num: 772 Reward: 4436.158609101652\n",
            "Total Timesteps: 464767 Episode Num: 773 Reward: 4654.646659338778\n",
            "Total Timesteps: 464988 Episode Num: 774 Reward: 1042.539481750344\n",
            "Total Timesteps: 465729 Episode Num: 775 Reward: 3212.3510114216733\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4817.321350\n",
            "---------------------------------------\n",
            "Total Timesteps: 466729 Episode Num: 776 Reward: 4463.429110469755\n",
            "Total Timesteps: 466802 Episode Num: 777 Reward: 207.63399560731702\n",
            "Total Timesteps: 467802 Episode Num: 778 Reward: 4613.294653431124\n",
            "Total Timesteps: 468802 Episode Num: 779 Reward: 4503.7678469253\n",
            "Total Timesteps: 469614 Episode Num: 780 Reward: 3135.242508054613\n",
            "Total Timesteps: 470614 Episode Num: 781 Reward: 4361.963761335152\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4650.537198\n",
            "---------------------------------------\n",
            "Total Timesteps: 471614 Episode Num: 782 Reward: 4507.457449432512\n",
            "Total Timesteps: 472614 Episode Num: 783 Reward: 4474.31274659817\n",
            "Total Timesteps: 473614 Episode Num: 784 Reward: 4687.127445658115\n",
            "Total Timesteps: 474614 Episode Num: 785 Reward: 4651.242737341898\n",
            "Total Timesteps: 475614 Episode Num: 786 Reward: 4419.084522446754\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4780.781384\n",
            "---------------------------------------\n",
            "Total Timesteps: 476614 Episode Num: 787 Reward: 4589.817203878918\n",
            "Total Timesteps: 477614 Episode Num: 788 Reward: 4091.213919415974\n",
            "Total Timesteps: 478417 Episode Num: 789 Reward: 3538.8138311429498\n",
            "Total Timesteps: 478455 Episode Num: 790 Reward: 62.73080508490522\n",
            "Total Timesteps: 479455 Episode Num: 791 Reward: 4342.663552753645\n",
            "Total Timesteps: 480455 Episode Num: 792 Reward: 4527.781403990057\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3320.367312\n",
            "---------------------------------------\n",
            "Total Timesteps: 480591 Episode Num: 793 Reward: 461.7092388655488\n",
            "Total Timesteps: 481591 Episode Num: 794 Reward: 4132.122378094028\n",
            "Total Timesteps: 482591 Episode Num: 795 Reward: 4669.15864043506\n",
            "Total Timesteps: 482900 Episode Num: 796 Reward: 1248.8569104966468\n",
            "Total Timesteps: 483900 Episode Num: 797 Reward: 4270.642810219649\n",
            "Total Timesteps: 484900 Episode Num: 798 Reward: 4631.88487158794\n",
            "Total Timesteps: 485331 Episode Num: 799 Reward: 1819.2220398544225\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4048.146324\n",
            "---------------------------------------\n",
            "Total Timesteps: 485847 Episode Num: 800 Reward: 2386.479272201608\n",
            "Total Timesteps: 486847 Episode Num: 801 Reward: 4774.655721504652\n",
            "Total Timesteps: 487847 Episode Num: 802 Reward: 4854.036502956554\n",
            "Total Timesteps: 487868 Episode Num: 803 Reward: 19.030337418418036\n",
            "Total Timesteps: 488868 Episode Num: 804 Reward: 4179.294653197334\n",
            "Total Timesteps: 488936 Episode Num: 805 Reward: 195.13201720250842\n",
            "Total Timesteps: 489610 Episode Num: 806 Reward: 3022.5617570440563\n",
            "Total Timesteps: 490610 Episode Num: 807 Reward: 4391.516741203766\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4751.316841\n",
            "---------------------------------------\n",
            "Total Timesteps: 491610 Episode Num: 808 Reward: 4565.9427062177865\n",
            "Total Timesteps: 492610 Episode Num: 809 Reward: 4588.101365820339\n",
            "Total Timesteps: 493610 Episode Num: 810 Reward: 4213.803468663977\n",
            "Total Timesteps: 494610 Episode Num: 811 Reward: 5018.588840694135\n",
            "Total Timesteps: 495610 Episode Num: 812 Reward: 4518.6265308162665\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4961.252185\n",
            "---------------------------------------\n",
            "Total Timesteps: 496610 Episode Num: 813 Reward: 4540.3055340687415\n",
            "Total Timesteps: 497610 Episode Num: 814 Reward: 4547.413104056866\n",
            "Total Timesteps: 498610 Episode Num: 815 Reward: 4230.273882548943\n",
            "Total Timesteps: 499610 Episode Num: 816 Reward: 4520.419074653165\n",
            "Total Timesteps: 500610 Episode Num: 817 Reward: 4578.893070906676\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3825.255167\n",
            "---------------------------------------\n",
            "Total Timesteps: 501610 Episode Num: 818 Reward: 4839.734099193325\n",
            "Total Timesteps: 502610 Episode Num: 819 Reward: 4444.837828628629\n",
            "Total Timesteps: 503101 Episode Num: 820 Reward: 2238.4910765072204\n",
            "Total Timesteps: 504101 Episode Num: 821 Reward: 4619.8219275777565\n",
            "Total Timesteps: 505101 Episode Num: 822 Reward: 4960.332870387688\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4615.469906\n",
            "---------------------------------------\n",
            "Total Timesteps: 506101 Episode Num: 823 Reward: 4748.456735395746\n",
            "Total Timesteps: 507101 Episode Num: 824 Reward: 4726.33710401746\n",
            "Total Timesteps: 508030 Episode Num: 825 Reward: 4536.823554182315\n",
            "Total Timesteps: 509030 Episode Num: 826 Reward: 4678.403464324955\n",
            "Total Timesteps: 510030 Episode Num: 827 Reward: 4770.49853888358\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4808.114129\n",
            "---------------------------------------\n",
            "Total Timesteps: 511030 Episode Num: 828 Reward: 4912.633290144066\n",
            "Total Timesteps: 512030 Episode Num: 829 Reward: 4650.506121831431\n",
            "Total Timesteps: 513030 Episode Num: 830 Reward: 4555.8428251263795\n",
            "Total Timesteps: 514030 Episode Num: 831 Reward: 4550.553358781386\n",
            "Total Timesteps: 515030 Episode Num: 832 Reward: 4665.995114857257\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4882.367080\n",
            "---------------------------------------\n",
            "Total Timesteps: 516030 Episode Num: 833 Reward: 4712.639178098993\n",
            "Total Timesteps: 517030 Episode Num: 834 Reward: 4568.384327961657\n",
            "Total Timesteps: 518030 Episode Num: 835 Reward: 4641.456990159136\n",
            "Total Timesteps: 519030 Episode Num: 836 Reward: 4195.024354604633\n",
            "Total Timesteps: 520030 Episode Num: 837 Reward: 4530.788057595517\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4060.018826\n",
            "---------------------------------------\n",
            "Total Timesteps: 520471 Episode Num: 838 Reward: 1912.2214519030886\n",
            "Total Timesteps: 521471 Episode Num: 839 Reward: 827.6896073805932\n",
            "Total Timesteps: 522471 Episode Num: 840 Reward: 4708.138435455393\n",
            "Total Timesteps: 523471 Episode Num: 841 Reward: 4784.260106180555\n",
            "Total Timesteps: 524471 Episode Num: 842 Reward: 956.7504308598709\n",
            "Total Timesteps: 525471 Episode Num: 843 Reward: 4565.974481752299\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3477.477777\n",
            "---------------------------------------\n",
            "Total Timesteps: 525524 Episode Num: 844 Reward: 130.31546090152221\n",
            "Total Timesteps: 526524 Episode Num: 845 Reward: 5070.883519763673\n",
            "Total Timesteps: 527524 Episode Num: 846 Reward: 4923.249206285037\n",
            "Total Timesteps: 528524 Episode Num: 847 Reward: 4320.417997049046\n",
            "Total Timesteps: 529524 Episode Num: 848 Reward: 4830.479462202523\n",
            "Total Timesteps: 529880 Episode Num: 849 Reward: 1492.1041264006485\n",
            "Total Timesteps: 530880 Episode Num: 850 Reward: 4764.816445926321\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4196.990174\n",
            "---------------------------------------\n",
            "Total Timesteps: 531197 Episode Num: 851 Reward: 1396.7365915625826\n",
            "Total Timesteps: 532197 Episode Num: 852 Reward: 4759.2600152210725\n",
            "Total Timesteps: 533197 Episode Num: 853 Reward: 4796.255545137591\n",
            "Total Timesteps: 534197 Episode Num: 854 Reward: 4872.241219004235\n",
            "Total Timesteps: 534660 Episode Num: 855 Reward: 2176.303626243888\n",
            "Total Timesteps: 535660 Episode Num: 856 Reward: 4775.385102740817\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4254.730769\n",
            "---------------------------------------\n",
            "Total Timesteps: 536660 Episode Num: 857 Reward: 4389.786731258291\n",
            "Total Timesteps: 537660 Episode Num: 858 Reward: 4798.521267138211\n",
            "Total Timesteps: 538660 Episode Num: 859 Reward: 4841.781418492543\n",
            "Total Timesteps: 539660 Episode Num: 860 Reward: 4650.526617456418\n",
            "Total Timesteps: 540660 Episode Num: 861 Reward: 4495.007859399219\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4182.732125\n",
            "---------------------------------------\n",
            "Total Timesteps: 541660 Episode Num: 862 Reward: 4861.896434940915\n",
            "Total Timesteps: 542660 Episode Num: 863 Reward: 5018.111475391705\n",
            "Total Timesteps: 543660 Episode Num: 864 Reward: 5001.816561225904\n",
            "Total Timesteps: 544660 Episode Num: 865 Reward: 4863.88019714715\n",
            "Total Timesteps: 545660 Episode Num: 866 Reward: 5001.607404845891\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4587.374118\n",
            "---------------------------------------\n",
            "Total Timesteps: 546660 Episode Num: 867 Reward: 4647.175841429148\n",
            "Total Timesteps: 547660 Episode Num: 868 Reward: 4875.964891518648\n",
            "Total Timesteps: 548660 Episode Num: 869 Reward: 4357.511554122184\n",
            "Total Timesteps: 549660 Episode Num: 870 Reward: 4636.479069369145\n",
            "Total Timesteps: 550660 Episode Num: 871 Reward: 4951.976343846972\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4414.955532\n",
            "---------------------------------------\n",
            "Total Timesteps: 551660 Episode Num: 872 Reward: 4311.010445602473\n",
            "Total Timesteps: 552660 Episode Num: 873 Reward: 4792.63175576804\n",
            "Total Timesteps: 553660 Episode Num: 874 Reward: 986.2155881453034\n",
            "Total Timesteps: 554089 Episode Num: 875 Reward: 2075.5345515547647\n",
            "Total Timesteps: 554843 Episode Num: 876 Reward: 3848.883113292295\n",
            "Total Timesteps: 555843 Episode Num: 877 Reward: 5146.545771573196\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4945.957006\n",
            "---------------------------------------\n",
            "Total Timesteps: 556058 Episode Num: 878 Reward: 884.3608348603917\n",
            "Total Timesteps: 557058 Episode Num: 879 Reward: 4865.502815105551\n",
            "Total Timesteps: 558058 Episode Num: 880 Reward: 4639.469454824699\n",
            "Total Timesteps: 559058 Episode Num: 881 Reward: 4863.117232476348\n",
            "Total Timesteps: 560058 Episode Num: 882 Reward: 4742.355534829883\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3646.498528\n",
            "---------------------------------------\n",
            "Total Timesteps: 561058 Episode Num: 883 Reward: 5015.280556607932\n",
            "Total Timesteps: 562058 Episode Num: 884 Reward: 2786.4319879207483\n",
            "Total Timesteps: 563058 Episode Num: 885 Reward: 4748.666993320123\n",
            "Total Timesteps: 564058 Episode Num: 886 Reward: 4990.552111697832\n",
            "Total Timesteps: 564267 Episode Num: 887 Reward: 959.5951727813248\n",
            "Total Timesteps: 564832 Episode Num: 888 Reward: 2732.9021846885657\n",
            "Total Timesteps: 565832 Episode Num: 889 Reward: 4905.642929074707\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4998.788864\n",
            "---------------------------------------\n",
            "Total Timesteps: 566832 Episode Num: 890 Reward: 4632.387505629919\n",
            "Total Timesteps: 567197 Episode Num: 891 Reward: 1547.9747129919074\n",
            "Total Timesteps: 568197 Episode Num: 892 Reward: 4700.096760667612\n",
            "Total Timesteps: 569193 Episode Num: 893 Reward: 4840.592458713954\n",
            "Total Timesteps: 569219 Episode Num: 894 Reward: 21.321441840126482\n",
            "Total Timesteps: 570219 Episode Num: 895 Reward: 4728.494706184684\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4917.231053\n",
            "---------------------------------------\n",
            "Total Timesteps: 571219 Episode Num: 896 Reward: 1228.1869005547699\n",
            "Total Timesteps: 572219 Episode Num: 897 Reward: 4994.119844466145\n",
            "Total Timesteps: 573219 Episode Num: 898 Reward: 2715.999114741829\n",
            "Total Timesteps: 574219 Episode Num: 899 Reward: 5071.136841420439\n",
            "Total Timesteps: 574461 Episode Num: 900 Reward: 988.817289342173\n",
            "Total Timesteps: 575461 Episode Num: 901 Reward: 5290.77572011561\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5228.649579\n",
            "---------------------------------------\n",
            "Total Timesteps: 576461 Episode Num: 902 Reward: 5092.255103201305\n",
            "Total Timesteps: 576526 Episode Num: 903 Reward: 39.22946009401084\n",
            "Total Timesteps: 577526 Episode Num: 904 Reward: 4818.689955449307\n",
            "Total Timesteps: 578526 Episode Num: 905 Reward: 5010.605134228538\n",
            "Total Timesteps: 578964 Episode Num: 906 Reward: 2121.8121376329796\n",
            "Total Timesteps: 579964 Episode Num: 907 Reward: 5032.461059778583\n",
            "Total Timesteps: 580183 Episode Num: 908 Reward: 1026.2455037539798\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4850.085659\n",
            "---------------------------------------\n",
            "Total Timesteps: 581183 Episode Num: 909 Reward: 4427.0518150379885\n",
            "Total Timesteps: 582183 Episode Num: 910 Reward: 4669.471782304833\n",
            "Total Timesteps: 583183 Episode Num: 911 Reward: 4907.19403898889\n",
            "Total Timesteps: 584183 Episode Num: 912 Reward: 4910.882157003797\n",
            "Total Timesteps: 585183 Episode Num: 913 Reward: 4545.882277284933\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4159.662473\n",
            "---------------------------------------\n",
            "Total Timesteps: 586183 Episode Num: 914 Reward: 4823.453022689907\n",
            "Total Timesteps: 587183 Episode Num: 915 Reward: 4824.0400324199645\n",
            "Total Timesteps: 588183 Episode Num: 916 Reward: 5060.539131505583\n",
            "Total Timesteps: 589183 Episode Num: 917 Reward: 5161.698591612423\n",
            "Total Timesteps: 590183 Episode Num: 918 Reward: 4912.373524839546\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4691.565712\n",
            "---------------------------------------\n",
            "Total Timesteps: 591183 Episode Num: 919 Reward: 4623.144770134352\n",
            "Total Timesteps: 592183 Episode Num: 920 Reward: 5152.778704431104\n",
            "Total Timesteps: 593183 Episode Num: 921 Reward: 4839.153052851732\n",
            "Total Timesteps: 594183 Episode Num: 922 Reward: 4692.0802360572525\n",
            "Total Timesteps: 595183 Episode Num: 923 Reward: 4845.482533668559\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4565.313505\n",
            "---------------------------------------\n",
            "Total Timesteps: 596158 Episode Num: 924 Reward: 4526.927737724335\n",
            "Total Timesteps: 597158 Episode Num: 925 Reward: 5050.901587764159\n",
            "Total Timesteps: 598158 Episode Num: 926 Reward: 5266.203402069635\n",
            "Total Timesteps: 598415 Episode Num: 927 Reward: 1264.154248930033\n",
            "Total Timesteps: 598887 Episode Num: 928 Reward: 2006.6725218200381\n",
            "Total Timesteps: 599887 Episode Num: 929 Reward: 4918.533632912012\n",
            "Total Timesteps: 600887 Episode Num: 930 Reward: 5094.866070906939\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5069.243369\n",
            "---------------------------------------\n",
            "Total Timesteps: 601887 Episode Num: 931 Reward: 4965.547717797862\n",
            "Total Timesteps: 602887 Episode Num: 932 Reward: 5200.012802530328\n",
            "Total Timesteps: 603887 Episode Num: 933 Reward: 5125.6437253308495\n",
            "Total Timesteps: 604887 Episode Num: 934 Reward: 1024.2935725176972\n",
            "Total Timesteps: 605887 Episode Num: 935 Reward: 4824.186168615376\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4252.478594\n",
            "---------------------------------------\n",
            "Total Timesteps: 606887 Episode Num: 936 Reward: 5078.45348497327\n",
            "Total Timesteps: 607887 Episode Num: 937 Reward: 4895.75244523032\n",
            "Total Timesteps: 608887 Episode Num: 938 Reward: 5012.736205805838\n",
            "Total Timesteps: 609887 Episode Num: 939 Reward: 4939.451623907448\n",
            "Total Timesteps: 610887 Episode Num: 940 Reward: 4828.00843380841\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4824.716991\n",
            "---------------------------------------\n",
            "Total Timesteps: 611887 Episode Num: 941 Reward: 4863.692199423556\n",
            "Total Timesteps: 612515 Episode Num: 942 Reward: 2978.0285950550956\n",
            "Total Timesteps: 613515 Episode Num: 943 Reward: 4989.835318551707\n",
            "Total Timesteps: 613850 Episode Num: 944 Reward: 1636.327941352019\n",
            "Total Timesteps: 614694 Episode Num: 945 Reward: 4126.00763505177\n",
            "Total Timesteps: 615694 Episode Num: 946 Reward: 4982.191261893229\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5071.367638\n",
            "---------------------------------------\n",
            "Total Timesteps: 616694 Episode Num: 947 Reward: 4568.5165975154\n",
            "Total Timesteps: 617694 Episode Num: 948 Reward: 4631.363705363164\n",
            "Total Timesteps: 617786 Episode Num: 949 Reward: 299.5741134090475\n",
            "Total Timesteps: 618786 Episode Num: 950 Reward: 4935.137598772931\n",
            "Total Timesteps: 618936 Episode Num: 951 Reward: 626.2123610080573\n",
            "Total Timesteps: 619864 Episode Num: 952 Reward: 4489.808246550505\n",
            "Total Timesteps: 620864 Episode Num: 953 Reward: 4910.215638447264\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5162.932035\n",
            "---------------------------------------\n",
            "Total Timesteps: 621864 Episode Num: 954 Reward: 5014.053496117463\n",
            "Total Timesteps: 622864 Episode Num: 955 Reward: 4932.0340046150795\n",
            "Total Timesteps: 622894 Episode Num: 956 Reward: 1.6978419369050854\n",
            "Total Timesteps: 623894 Episode Num: 957 Reward: 5247.787424288547\n",
            "Total Timesteps: 623978 Episode Num: 958 Reward: 189.63473310971273\n",
            "Total Timesteps: 624978 Episode Num: 959 Reward: 5153.8290775984215\n",
            "Total Timesteps: 625978 Episode Num: 960 Reward: 4935.258947811314\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5072.052086\n",
            "---------------------------------------\n",
            "Total Timesteps: 626978 Episode Num: 961 Reward: 4931.780115689436\n",
            "Total Timesteps: 627978 Episode Num: 962 Reward: 4989.553433433833\n",
            "Total Timesteps: 628218 Episode Num: 963 Reward: 824.8472503528968\n",
            "Total Timesteps: 629218 Episode Num: 964 Reward: 4997.845077137941\n",
            "Total Timesteps: 629863 Episode Num: 965 Reward: 3113.2824052250694\n",
            "Total Timesteps: 630863 Episode Num: 966 Reward: 4826.539496401107\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5264.052857\n",
            "---------------------------------------\n",
            "Total Timesteps: 631863 Episode Num: 967 Reward: 4720.866460392269\n",
            "Total Timesteps: 632863 Episode Num: 968 Reward: 4832.877224411953\n",
            "Total Timesteps: 633310 Episode Num: 969 Reward: 2300.905661597213\n",
            "Total Timesteps: 634310 Episode Num: 970 Reward: 4851.777464612624\n",
            "Total Timesteps: 634339 Episode Num: 971 Reward: 35.0903188342285\n",
            "Total Timesteps: 635339 Episode Num: 972 Reward: 4624.952126091261\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4559.879895\n",
            "---------------------------------------\n",
            "Total Timesteps: 636117 Episode Num: 973 Reward: 3780.0638670577446\n",
            "Total Timesteps: 636262 Episode Num: 974 Reward: 640.1968874623996\n",
            "Total Timesteps: 637262 Episode Num: 975 Reward: 4766.982436331693\n",
            "Total Timesteps: 638262 Episode Num: 976 Reward: 5162.132244374852\n",
            "Total Timesteps: 639262 Episode Num: 977 Reward: 5303.431368966097\n",
            "Total Timesteps: 640262 Episode Num: 978 Reward: 4855.057229984127\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4616.437252\n",
            "---------------------------------------\n",
            "Total Timesteps: 641262 Episode Num: 979 Reward: 4686.135874864034\n",
            "Total Timesteps: 642262 Episode Num: 980 Reward: 5142.62062116721\n",
            "Total Timesteps: 643262 Episode Num: 981 Reward: 5027.683418259386\n",
            "Total Timesteps: 643946 Episode Num: 982 Reward: 3302.4319502987128\n",
            "Total Timesteps: 644946 Episode Num: 983 Reward: 4947.373668531649\n",
            "Total Timesteps: 645946 Episode Num: 984 Reward: 4846.4137232698085\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4043.430975\n",
            "---------------------------------------\n",
            "Total Timesteps: 646946 Episode Num: 985 Reward: 4756.3567052151175\n",
            "Total Timesteps: 647221 Episode Num: 986 Reward: 1165.953433896513\n",
            "Total Timesteps: 648221 Episode Num: 987 Reward: 4453.908454601161\n",
            "Total Timesteps: 648291 Episode Num: 988 Reward: 190.74366835970451\n",
            "Total Timesteps: 649291 Episode Num: 989 Reward: 5020.652898417661\n",
            "Total Timesteps: 650291 Episode Num: 990 Reward: 4863.960767575651\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5343.183942\n",
            "---------------------------------------\n",
            "Total Timesteps: 651291 Episode Num: 991 Reward: 4921.953951349427\n",
            "Total Timesteps: 652291 Episode Num: 992 Reward: 5000.742023464005\n",
            "Total Timesteps: 653291 Episode Num: 993 Reward: 4943.902585408098\n",
            "Total Timesteps: 654291 Episode Num: 994 Reward: 5115.448134499432\n",
            "Total Timesteps: 655291 Episode Num: 995 Reward: 4816.580975913947\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4298.790170\n",
            "---------------------------------------\n",
            "Total Timesteps: 655731 Episode Num: 996 Reward: 2083.9510834062053\n",
            "Total Timesteps: 656731 Episode Num: 997 Reward: 4928.700658457851\n",
            "Total Timesteps: 657731 Episode Num: 998 Reward: 4996.619542797518\n",
            "Total Timesteps: 658731 Episode Num: 999 Reward: 4922.427746057714\n",
            "Total Timesteps: 659731 Episode Num: 1000 Reward: 5145.498999453722\n",
            "Total Timesteps: 660731 Episode Num: 1001 Reward: 4761.15669366794\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4122.200172\n",
            "---------------------------------------\n",
            "Total Timesteps: 661731 Episode Num: 1002 Reward: 4837.855336651507\n",
            "Total Timesteps: 662145 Episode Num: 1003 Reward: 1945.9837127130286\n",
            "Total Timesteps: 662645 Episode Num: 1004 Reward: 2276.4743837292212\n",
            "Total Timesteps: 663645 Episode Num: 1005 Reward: 4933.551029237981\n",
            "Total Timesteps: 664604 Episode Num: 1006 Reward: 4656.9362336157\n",
            "Total Timesteps: 665604 Episode Num: 1007 Reward: 4921.843664747959\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5062.438694\n",
            "---------------------------------------\n",
            "Total Timesteps: 666604 Episode Num: 1008 Reward: -185.88784979747973\n",
            "Total Timesteps: 667604 Episode Num: 1009 Reward: 5074.952913206702\n",
            "Total Timesteps: 668604 Episode Num: 1010 Reward: 1539.540191709631\n",
            "Total Timesteps: 669604 Episode Num: 1011 Reward: 4855.410892897113\n",
            "Total Timesteps: 670373 Episode Num: 1012 Reward: 3667.7347338045047\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5204.675115\n",
            "---------------------------------------\n",
            "Total Timesteps: 671373 Episode Num: 1013 Reward: 4948.570197018728\n",
            "Total Timesteps: 671860 Episode Num: 1014 Reward: 2186.065208605053\n",
            "Total Timesteps: 672860 Episode Num: 1015 Reward: 4713.734227886422\n",
            "Total Timesteps: 673860 Episode Num: 1016 Reward: 4933.046641995017\n",
            "Total Timesteps: 674860 Episode Num: 1017 Reward: 4990.612434763899\n",
            "Total Timesteps: 675182 Episode Num: 1018 Reward: 1530.209278088247\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5393.696463\n",
            "---------------------------------------\n",
            "Total Timesteps: 676182 Episode Num: 1019 Reward: 5128.443310213725\n",
            "Total Timesteps: 677182 Episode Num: 1020 Reward: 4705.1974850441475\n",
            "Total Timesteps: 678182 Episode Num: 1021 Reward: 5110.649425380047\n",
            "Total Timesteps: 679142 Episode Num: 1022 Reward: 4745.760851266579\n",
            "Total Timesteps: 679158 Episode Num: 1023 Reward: 13.022174452929741\n",
            "Total Timesteps: 680158 Episode Num: 1024 Reward: 5022.692439307544\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4604.429571\n",
            "---------------------------------------\n",
            "Total Timesteps: 681158 Episode Num: 1025 Reward: 4961.619647217174\n",
            "Total Timesteps: 682158 Episode Num: 1026 Reward: 4701.952340105416\n",
            "Total Timesteps: 683158 Episode Num: 1027 Reward: 4686.291455383452\n",
            "Total Timesteps: 684158 Episode Num: 1028 Reward: 4950.051093543526\n",
            "Total Timesteps: 685158 Episode Num: 1029 Reward: 5020.094949923487\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4835.804205\n",
            "---------------------------------------\n",
            "Total Timesteps: 686158 Episode Num: 1030 Reward: 4714.044219979862\n",
            "Total Timesteps: 687158 Episode Num: 1031 Reward: 4678.378199931122\n",
            "Total Timesteps: 688158 Episode Num: 1032 Reward: 4992.945129732147\n",
            "Total Timesteps: 689158 Episode Num: 1033 Reward: 2300.5224260913023\n",
            "Total Timesteps: 690158 Episode Num: 1034 Reward: 4902.824592505469\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4660.112574\n",
            "---------------------------------------\n",
            "Total Timesteps: 691158 Episode Num: 1035 Reward: 5167.521167067527\n",
            "Total Timesteps: 692158 Episode Num: 1036 Reward: 4946.951401676016\n",
            "Total Timesteps: 693158 Episode Num: 1037 Reward: 5114.897831418923\n",
            "Total Timesteps: 694158 Episode Num: 1038 Reward: 4862.787095285019\n",
            "Total Timesteps: 695158 Episode Num: 1039 Reward: 5020.124874534865\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5091.257235\n",
            "---------------------------------------\n",
            "Total Timesteps: 696158 Episode Num: 1040 Reward: 4721.960898967421\n",
            "Total Timesteps: 696205 Episode Num: 1041 Reward: 18.085307444166684\n",
            "Total Timesteps: 697205 Episode Num: 1042 Reward: 4826.699460815423\n",
            "Total Timesteps: 698205 Episode Num: 1043 Reward: 5010.892024519509\n",
            "Total Timesteps: 698595 Episode Num: 1044 Reward: 1976.7477992564152\n",
            "Total Timesteps: 699595 Episode Num: 1045 Reward: 4874.2463175778785\n",
            "Total Timesteps: 700595 Episode Num: 1046 Reward: 4616.346079111998\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5189.725153\n",
            "---------------------------------------\n",
            "Total Timesteps: 701595 Episode Num: 1047 Reward: 5205.761695260877\n",
            "Total Timesteps: 702595 Episode Num: 1048 Reward: 5346.862398536125\n",
            "Total Timesteps: 703595 Episode Num: 1049 Reward: 5135.255450757195\n",
            "Total Timesteps: 704595 Episode Num: 1050 Reward: 5025.216957190168\n",
            "Total Timesteps: 705595 Episode Num: 1051 Reward: 5166.888455977287\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5205.133922\n",
            "---------------------------------------\n",
            "Total Timesteps: 706595 Episode Num: 1052 Reward: 4970.714299094775\n",
            "Total Timesteps: 707595 Episode Num: 1053 Reward: 4819.24306761984\n",
            "Total Timesteps: 708595 Episode Num: 1054 Reward: 5021.990901130962\n",
            "Total Timesteps: 709595 Episode Num: 1055 Reward: 5011.647052860316\n",
            "Total Timesteps: 709865 Episode Num: 1056 Reward: 1157.5713047061686\n",
            "Total Timesteps: 710620 Episode Num: 1057 Reward: 3441.4364807879983\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4694.409257\n",
            "---------------------------------------\n",
            "Total Timesteps: 711620 Episode Num: 1058 Reward: 4861.3798832735265\n",
            "Total Timesteps: 712601 Episode Num: 1059 Reward: 4711.975243374401\n",
            "Total Timesteps: 713601 Episode Num: 1060 Reward: 4833.828048801091\n",
            "Total Timesteps: 714601 Episode Num: 1061 Reward: 4804.078524992751\n",
            "Total Timesteps: 715265 Episode Num: 1062 Reward: 3224.0033227456993\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4213.486614\n",
            "---------------------------------------\n",
            "Total Timesteps: 716265 Episode Num: 1063 Reward: 5173.048896354364\n",
            "Total Timesteps: 717265 Episode Num: 1064 Reward: 5376.885835604631\n",
            "Total Timesteps: 718265 Episode Num: 1065 Reward: 4904.415613337533\n",
            "Total Timesteps: 719265 Episode Num: 1066 Reward: 5019.256553556727\n",
            "Total Timesteps: 720265 Episode Num: 1067 Reward: 4876.511792633888\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3875.877622\n",
            "---------------------------------------\n",
            "Total Timesteps: 721052 Episode Num: 1068 Reward: 3871.536791948849\n",
            "Total Timesteps: 721919 Episode Num: 1069 Reward: 4296.525656784648\n",
            "Total Timesteps: 722919 Episode Num: 1070 Reward: 4813.15905625658\n",
            "Total Timesteps: 723919 Episode Num: 1071 Reward: 5163.542769838842\n",
            "Total Timesteps: 724919 Episode Num: 1072 Reward: 4885.789547020315\n",
            "Total Timesteps: 725919 Episode Num: 1073 Reward: 4812.927298424978\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5166.357574\n",
            "---------------------------------------\n",
            "Total Timesteps: 726919 Episode Num: 1074 Reward: 4783.139201445769\n",
            "Total Timesteps: 727919 Episode Num: 1075 Reward: 4992.348688367139\n",
            "Total Timesteps: 728919 Episode Num: 1076 Reward: 5027.004007733145\n",
            "Total Timesteps: 729919 Episode Num: 1077 Reward: 5088.053703047653\n",
            "Total Timesteps: 730919 Episode Num: 1078 Reward: 4949.756841466341\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5313.062774\n",
            "---------------------------------------\n",
            "Total Timesteps: 731919 Episode Num: 1079 Reward: 4766.362029939951\n",
            "Total Timesteps: 732919 Episode Num: 1080 Reward: 5146.050088728154\n",
            "Total Timesteps: 733919 Episode Num: 1081 Reward: 5087.146029580242\n",
            "Total Timesteps: 734919 Episode Num: 1082 Reward: 4699.560707360757\n",
            "Total Timesteps: 735919 Episode Num: 1083 Reward: 4935.816423529358\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5160.498672\n",
            "---------------------------------------\n",
            "Total Timesteps: 736919 Episode Num: 1084 Reward: 4968.16099592559\n",
            "Total Timesteps: 737919 Episode Num: 1085 Reward: 4690.187437036082\n",
            "Total Timesteps: 738919 Episode Num: 1086 Reward: 5146.852283969136\n",
            "Total Timesteps: 739594 Episode Num: 1087 Reward: 3346.7250479960067\n",
            "Total Timesteps: 740594 Episode Num: 1088 Reward: 4876.99468722564\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5274.741924\n",
            "---------------------------------------\n",
            "Total Timesteps: 741594 Episode Num: 1089 Reward: 5106.707396868949\n",
            "Total Timesteps: 742594 Episode Num: 1090 Reward: 4777.480104765149\n",
            "Total Timesteps: 743594 Episode Num: 1091 Reward: 4856.37730678063\n",
            "Total Timesteps: 744594 Episode Num: 1092 Reward: 4706.943472970826\n",
            "Total Timesteps: 745594 Episode Num: 1093 Reward: 5162.082303471955\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5306.883032\n",
            "---------------------------------------\n",
            "Total Timesteps: 746594 Episode Num: 1094 Reward: 5318.728101409334\n",
            "Total Timesteps: 746963 Episode Num: 1095 Reward: 1721.198128890301\n",
            "Total Timesteps: 747963 Episode Num: 1096 Reward: 5289.7779979136185\n",
            "Total Timesteps: 748963 Episode Num: 1097 Reward: 4940.255706554562\n",
            "Total Timesteps: 749963 Episode Num: 1098 Reward: 5133.494435774397\n",
            "Total Timesteps: 750929 Episode Num: 1099 Reward: 4595.796564406879\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5436.440002\n",
            "---------------------------------------\n",
            "Total Timesteps: 751131 Episode Num: 1100 Reward: 947.9729923926499\n",
            "Total Timesteps: 752131 Episode Num: 1101 Reward: 4928.603323124949\n",
            "Total Timesteps: 752392 Episode Num: 1102 Reward: 1051.9715868705891\n",
            "Total Timesteps: 753392 Episode Num: 1103 Reward: 5195.808258516594\n",
            "Total Timesteps: 754392 Episode Num: 1104 Reward: 4959.384858860344\n",
            "Total Timesteps: 755252 Episode Num: 1105 Reward: 4369.174940377746\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4596.502693\n",
            "---------------------------------------\n",
            "Total Timesteps: 756252 Episode Num: 1106 Reward: 4687.92514566256\n",
            "Total Timesteps: 757252 Episode Num: 1107 Reward: 5076.4821191019055\n",
            "Total Timesteps: 757698 Episode Num: 1108 Reward: 2124.6718024113766\n",
            "Total Timesteps: 758698 Episode Num: 1109 Reward: 4995.970670276837\n",
            "Total Timesteps: 758919 Episode Num: 1110 Reward: 1043.0552807542897\n",
            "Total Timesteps: 759062 Episode Num: 1111 Reward: 452.11974973278046\n",
            "Total Timesteps: 759192 Episode Num: 1112 Reward: 527.4517693673508\n",
            "Total Timesteps: 760192 Episode Num: 1113 Reward: 5133.93189278261\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4527.028721\n",
            "---------------------------------------\n",
            "Total Timesteps: 761192 Episode Num: 1114 Reward: 5119.239711999779\n",
            "Total Timesteps: 762192 Episode Num: 1115 Reward: 4955.28082368095\n",
            "Total Timesteps: 763192 Episode Num: 1116 Reward: 5075.702728498178\n",
            "Total Timesteps: 764192 Episode Num: 1117 Reward: 4882.808880008277\n",
            "Total Timesteps: 765192 Episode Num: 1118 Reward: 4468.16070510082\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4597.035097\n",
            "---------------------------------------\n",
            "Total Timesteps: 766192 Episode Num: 1119 Reward: 5232.456794480945\n",
            "Total Timesteps: 767192 Episode Num: 1120 Reward: 4729.948978869927\n",
            "Total Timesteps: 768192 Episode Num: 1121 Reward: 5134.481250872019\n",
            "Total Timesteps: 768953 Episode Num: 1122 Reward: 3657.295807800807\n",
            "Total Timesteps: 769953 Episode Num: 1123 Reward: 5014.197176722978\n",
            "Total Timesteps: 770724 Episode Num: 1124 Reward: 3628.1555514819606\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5258.605122\n",
            "---------------------------------------\n",
            "Total Timesteps: 771068 Episode Num: 1125 Reward: 1698.411363957658\n",
            "Total Timesteps: 772068 Episode Num: 1126 Reward: 4719.372504378734\n",
            "Total Timesteps: 773068 Episode Num: 1127 Reward: 5126.710725516965\n",
            "Total Timesteps: 774068 Episode Num: 1128 Reward: 5180.598181329478\n",
            "Total Timesteps: 775068 Episode Num: 1129 Reward: 4803.479050223421\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4932.529897\n",
            "---------------------------------------\n",
            "Total Timesteps: 776068 Episode Num: 1130 Reward: 5133.935933006976\n",
            "Total Timesteps: 777068 Episode Num: 1131 Reward: 5060.052284207254\n",
            "Total Timesteps: 778068 Episode Num: 1132 Reward: 5020.189772407038\n",
            "Total Timesteps: 779068 Episode Num: 1133 Reward: 5304.592627830006\n",
            "Total Timesteps: 780068 Episode Num: 1134 Reward: 4826.883291250017\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5306.405489\n",
            "---------------------------------------\n",
            "Total Timesteps: 780765 Episode Num: 1135 Reward: 3398.8367219832026\n",
            "Total Timesteps: 781765 Episode Num: 1136 Reward: 4817.82773119176\n",
            "Total Timesteps: 782765 Episode Num: 1137 Reward: 4993.908256914229\n",
            "Total Timesteps: 783765 Episode Num: 1138 Reward: 5065.396772317983\n",
            "Total Timesteps: 784765 Episode Num: 1139 Reward: 5022.208263096814\n",
            "Total Timesteps: 785765 Episode Num: 1140 Reward: 4965.761418108788\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4649.795350\n",
            "---------------------------------------\n",
            "Total Timesteps: 786381 Episode Num: 1141 Reward: 3004.9985088369026\n",
            "Total Timesteps: 787381 Episode Num: 1142 Reward: 2150.8342209171205\n",
            "Total Timesteps: 788381 Episode Num: 1143 Reward: 5312.096237469009\n",
            "Total Timesteps: 789381 Episode Num: 1144 Reward: 4851.277547691801\n",
            "Total Timesteps: 790381 Episode Num: 1145 Reward: 4797.445960919834\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5130.992321\n",
            "---------------------------------------\n",
            "Total Timesteps: 790499 Episode Num: 1146 Reward: 449.03971808615364\n",
            "Total Timesteps: 791156 Episode Num: 1147 Reward: 3347.0006448599397\n",
            "Total Timesteps: 792156 Episode Num: 1148 Reward: 4984.690662664802\n",
            "Total Timesteps: 793156 Episode Num: 1149 Reward: 5055.88774928143\n",
            "Total Timesteps: 794156 Episode Num: 1150 Reward: -183.34979793411344\n",
            "Total Timesteps: 795156 Episode Num: 1151 Reward: 4670.821857297757\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4699.484034\n",
            "---------------------------------------\n",
            "Total Timesteps: 796156 Episode Num: 1152 Reward: 4625.771458459062\n",
            "Total Timesteps: 797156 Episode Num: 1153 Reward: 4689.5705888392\n",
            "Total Timesteps: 798156 Episode Num: 1154 Reward: 4977.887681080201\n",
            "Total Timesteps: 799156 Episode Num: 1155 Reward: 5097.628921128616\n",
            "Total Timesteps: 800156 Episode Num: 1156 Reward: 4855.312318537408\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4963.779164\n",
            "---------------------------------------\n",
            "Total Timesteps: 801156 Episode Num: 1157 Reward: 5067.096331435576\n",
            "Total Timesteps: 802156 Episode Num: 1158 Reward: 5252.697053690721\n",
            "Total Timesteps: 803156 Episode Num: 1159 Reward: 5056.138597454406\n",
            "Total Timesteps: 804156 Episode Num: 1160 Reward: 4954.560981061959\n",
            "Total Timesteps: 805156 Episode Num: 1161 Reward: 5105.670992293972\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5482.187403\n",
            "---------------------------------------\n",
            "Total Timesteps: 806156 Episode Num: 1162 Reward: 5147.67410718303\n",
            "Total Timesteps: 807156 Episode Num: 1163 Reward: 4896.966279464033\n",
            "Total Timesteps: 808156 Episode Num: 1164 Reward: 4979.647341203612\n",
            "Total Timesteps: 809156 Episode Num: 1165 Reward: 5312.181936563486\n",
            "Total Timesteps: 810156 Episode Num: 1166 Reward: 5127.975492276848\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4348.457726\n",
            "---------------------------------------\n",
            "Total Timesteps: 811156 Episode Num: 1167 Reward: 5011.1515176397215\n",
            "Total Timesteps: 812156 Episode Num: 1168 Reward: 5092.598083678835\n",
            "Total Timesteps: 813156 Episode Num: 1169 Reward: 4995.6898378826745\n",
            "Total Timesteps: 813549 Episode Num: 1170 Reward: 1873.9273864006843\n",
            "Total Timesteps: 814549 Episode Num: 1171 Reward: 4869.983816427132\n",
            "Total Timesteps: 815549 Episode Num: 1172 Reward: 4760.316323810212\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5073.287860\n",
            "---------------------------------------\n",
            "Total Timesteps: 816549 Episode Num: 1173 Reward: 4725.291853229458\n",
            "Total Timesteps: 817549 Episode Num: 1174 Reward: 4879.432544143028\n",
            "Total Timesteps: 818549 Episode Num: 1175 Reward: 4874.300812650368\n",
            "Total Timesteps: 819549 Episode Num: 1176 Reward: 4820.707109455987\n",
            "Total Timesteps: 820549 Episode Num: 1177 Reward: 5000.951123929233\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5289.224987\n",
            "---------------------------------------\n",
            "Total Timesteps: 821549 Episode Num: 1178 Reward: 4994.574590636002\n",
            "Total Timesteps: 822549 Episode Num: 1179 Reward: 4926.310480818067\n",
            "Total Timesteps: 823549 Episode Num: 1180 Reward: 4979.445328002218\n",
            "Total Timesteps: 824549 Episode Num: 1181 Reward: 5168.579227758691\n",
            "Total Timesteps: 825549 Episode Num: 1182 Reward: 4884.971638378746\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5305.937653\n",
            "---------------------------------------\n",
            "Total Timesteps: 826549 Episode Num: 1183 Reward: 4926.3479765928305\n",
            "Total Timesteps: 827549 Episode Num: 1184 Reward: 4744.178405235626\n",
            "Total Timesteps: 828549 Episode Num: 1185 Reward: 5186.235462169231\n",
            "Total Timesteps: 829549 Episode Num: 1186 Reward: 4856.167841017052\n",
            "Total Timesteps: 830549 Episode Num: 1187 Reward: 5175.028725557375\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4608.148652\n",
            "---------------------------------------\n",
            "Total Timesteps: 830635 Episode Num: 1188 Reward: 335.9616658394623\n",
            "Total Timesteps: 831635 Episode Num: 1189 Reward: 4952.663601350669\n",
            "Total Timesteps: 832635 Episode Num: 1190 Reward: 4901.535243818123\n",
            "Total Timesteps: 833635 Episode Num: 1191 Reward: 5156.645900076415\n",
            "Total Timesteps: 834635 Episode Num: 1192 Reward: 5158.512873754067\n",
            "Total Timesteps: 834663 Episode Num: 1193 Reward: 16.616314121930948\n",
            "Total Timesteps: 835663 Episode Num: 1194 Reward: 5186.207738094964\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4542.851363\n",
            "---------------------------------------\n",
            "Total Timesteps: 836663 Episode Num: 1195 Reward: 4485.749822275445\n",
            "Total Timesteps: 837663 Episode Num: 1196 Reward: 5129.15717695217\n",
            "Total Timesteps: 838663 Episode Num: 1197 Reward: 5188.814731934305\n",
            "Total Timesteps: 839663 Episode Num: 1198 Reward: 4956.884812346644\n",
            "Total Timesteps: 840308 Episode Num: 1199 Reward: 3198.5748040485505\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5012.996400\n",
            "---------------------------------------\n",
            "Total Timesteps: 840677 Episode Num: 1200 Reward: 716.6718640777751\n",
            "Total Timesteps: 841385 Episode Num: 1201 Reward: 3319.595506311102\n",
            "Total Timesteps: 842385 Episode Num: 1202 Reward: 4915.740391310699\n",
            "Total Timesteps: 843385 Episode Num: 1203 Reward: 5079.445641662807\n",
            "Total Timesteps: 844385 Episode Num: 1204 Reward: 4959.061452829501\n",
            "Total Timesteps: 845385 Episode Num: 1205 Reward: 4699.249032667816\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5378.242225\n",
            "---------------------------------------\n",
            "Total Timesteps: 846385 Episode Num: 1206 Reward: 5289.305910779536\n",
            "Total Timesteps: 847385 Episode Num: 1207 Reward: 5334.344874340961\n",
            "Total Timesteps: 848385 Episode Num: 1208 Reward: 5321.355722031143\n",
            "Total Timesteps: 849385 Episode Num: 1209 Reward: 5188.237295219014\n",
            "Total Timesteps: 850385 Episode Num: 1210 Reward: 5103.741262522098\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5380.561001\n",
            "---------------------------------------\n",
            "Total Timesteps: 851385 Episode Num: 1211 Reward: 5153.574725492851\n",
            "Total Timesteps: 852385 Episode Num: 1212 Reward: 4729.217360348159\n",
            "Total Timesteps: 853385 Episode Num: 1213 Reward: 5066.66622040038\n",
            "Total Timesteps: 853832 Episode Num: 1214 Reward: 2143.9766174169085\n",
            "Total Timesteps: 854832 Episode Num: 1215 Reward: 5143.715560356556\n",
            "Total Timesteps: 855107 Episode Num: 1216 Reward: 1170.1384176946206\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5601.153872\n",
            "---------------------------------------\n",
            "Total Timesteps: 856107 Episode Num: 1217 Reward: 5192.3875056855695\n",
            "Total Timesteps: 857107 Episode Num: 1218 Reward: 5237.007590234928\n",
            "Total Timesteps: 858107 Episode Num: 1219 Reward: 4940.044746980041\n",
            "Total Timesteps: 859107 Episode Num: 1220 Reward: 5117.687352345006\n",
            "Total Timesteps: 860107 Episode Num: 1221 Reward: 4829.7859189669225\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5088.469152\n",
            "---------------------------------------\n",
            "Total Timesteps: 860608 Episode Num: 1222 Reward: 2475.1114365363696\n",
            "Total Timesteps: 861608 Episode Num: 1223 Reward: 5121.03476035161\n",
            "Total Timesteps: 862608 Episode Num: 1224 Reward: 5124.057554601039\n",
            "Total Timesteps: 863608 Episode Num: 1225 Reward: 4958.511918765299\n",
            "Total Timesteps: 863866 Episode Num: 1226 Reward: 1143.9074972000778\n",
            "Total Timesteps: 864866 Episode Num: 1227 Reward: 5019.835598289016\n",
            "Total Timesteps: 865866 Episode Num: 1228 Reward: 5277.1407953531625\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5346.835312\n",
            "---------------------------------------\n",
            "Total Timesteps: 866866 Episode Num: 1229 Reward: 5176.725443928189\n",
            "Total Timesteps: 867866 Episode Num: 1230 Reward: 5056.982799987815\n",
            "Total Timesteps: 868866 Episode Num: 1231 Reward: 5034.867767913887\n",
            "Total Timesteps: 869311 Episode Num: 1232 Reward: 2173.0889186816776\n",
            "Total Timesteps: 870311 Episode Num: 1233 Reward: 4975.096179003804\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4673.289579\n",
            "---------------------------------------\n",
            "Total Timesteps: 871311 Episode Num: 1234 Reward: 5050.7888236043555\n",
            "Total Timesteps: 871799 Episode Num: 1235 Reward: 2280.8535457724633\n",
            "Total Timesteps: 871948 Episode Num: 1236 Reward: 615.6418376905611\n",
            "Total Timesteps: 872948 Episode Num: 1237 Reward: 5321.706476720403\n",
            "Total Timesteps: 873948 Episode Num: 1238 Reward: 5073.20561436159\n",
            "Total Timesteps: 874948 Episode Num: 1239 Reward: 4745.054586898271\n",
            "Total Timesteps: 875948 Episode Num: 1240 Reward: 4904.479004497452\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5279.379210\n",
            "---------------------------------------\n",
            "Total Timesteps: 876772 Episode Num: 1241 Reward: 4176.663664784143\n",
            "Total Timesteps: 877772 Episode Num: 1242 Reward: 4995.56925046293\n",
            "Total Timesteps: 878772 Episode Num: 1243 Reward: 4982.412427963963\n",
            "Total Timesteps: 879772 Episode Num: 1244 Reward: 4948.661138691654\n",
            "Total Timesteps: 880772 Episode Num: 1245 Reward: 4864.4376519978705\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5461.343725\n",
            "---------------------------------------\n",
            "Total Timesteps: 881772 Episode Num: 1246 Reward: 5141.708027841668\n",
            "Total Timesteps: 882772 Episode Num: 1247 Reward: 4848.8240261393275\n",
            "Total Timesteps: 883772 Episode Num: 1248 Reward: 5210.911968905213\n",
            "Total Timesteps: 884414 Episode Num: 1249 Reward: 3248.200676365437\n",
            "Total Timesteps: 885414 Episode Num: 1250 Reward: 5020.682371493383\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5395.404394\n",
            "---------------------------------------\n",
            "Total Timesteps: 886414 Episode Num: 1251 Reward: 5071.873744188564\n",
            "Total Timesteps: 887414 Episode Num: 1252 Reward: 4782.898270647023\n",
            "Total Timesteps: 888414 Episode Num: 1253 Reward: 5332.832727301224\n",
            "Total Timesteps: 889414 Episode Num: 1254 Reward: 5186.657251012855\n",
            "Total Timesteps: 890097 Episode Num: 1255 Reward: 3688.520485097864\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5527.643750\n",
            "---------------------------------------\n",
            "Total Timesteps: 891097 Episode Num: 1256 Reward: 5296.101062038397\n",
            "Total Timesteps: 892097 Episode Num: 1257 Reward: 5197.125704401519\n",
            "Total Timesteps: 893097 Episode Num: 1258 Reward: 5309.125477108629\n",
            "Total Timesteps: 894097 Episode Num: 1259 Reward: 5016.008943131848\n",
            "Total Timesteps: 895097 Episode Num: 1260 Reward: 5054.189075495846\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5087.767397\n",
            "---------------------------------------\n",
            "Total Timesteps: 895163 Episode Num: 1261 Reward: 178.20017312482463\n",
            "Total Timesteps: 896163 Episode Num: 1262 Reward: 5338.040878368219\n",
            "Total Timesteps: 896382 Episode Num: 1263 Reward: 1100.4082162094548\n",
            "Total Timesteps: 897382 Episode Num: 1264 Reward: 5192.60154185779\n",
            "Total Timesteps: 898048 Episode Num: 1265 Reward: 3287.563796331413\n",
            "Total Timesteps: 898165 Episode Num: 1266 Reward: 480.6585573140914\n",
            "Total Timesteps: 899165 Episode Num: 1267 Reward: 5055.850663368718\n",
            "Total Timesteps: 900165 Episode Num: 1268 Reward: 4923.6935688161675\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4955.962834\n",
            "---------------------------------------\n",
            "Total Timesteps: 901165 Episode Num: 1269 Reward: 5047.899444964699\n",
            "Total Timesteps: 902165 Episode Num: 1270 Reward: 5010.8088126770745\n",
            "Total Timesteps: 903165 Episode Num: 1271 Reward: 5158.98696254704\n",
            "Total Timesteps: 903536 Episode Num: 1272 Reward: 1704.7010508061705\n",
            "Total Timesteps: 904082 Episode Num: 1273 Reward: 2643.2417601143634\n",
            "Total Timesteps: 905082 Episode Num: 1274 Reward: 5067.700125695616\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4558.328075\n",
            "---------------------------------------\n",
            "Total Timesteps: 906082 Episode Num: 1275 Reward: 5059.959133289025\n",
            "Total Timesteps: 907082 Episode Num: 1276 Reward: 5336.989598994758\n",
            "Total Timesteps: 908082 Episode Num: 1277 Reward: 5209.315955106029\n",
            "Total Timesteps: 909082 Episode Num: 1278 Reward: 5408.742692860023\n",
            "Total Timesteps: 910082 Episode Num: 1279 Reward: 5188.537181165053\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5458.079590\n",
            "---------------------------------------\n",
            "Total Timesteps: 911082 Episode Num: 1280 Reward: 5163.376109412234\n",
            "Total Timesteps: 912082 Episode Num: 1281 Reward: 5434.364779338898\n",
            "Total Timesteps: 912590 Episode Num: 1282 Reward: 2316.9660785539504\n",
            "Total Timesteps: 913590 Episode Num: 1283 Reward: 5210.3965327795695\n",
            "Total Timesteps: 914590 Episode Num: 1284 Reward: 5461.22449893766\n",
            "Total Timesteps: 914954 Episode Num: 1285 Reward: 1869.955009830631\n",
            "Total Timesteps: 915954 Episode Num: 1286 Reward: 5133.712378926596\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5399.614407\n",
            "---------------------------------------\n",
            "Total Timesteps: 916954 Episode Num: 1287 Reward: 5162.549914338086\n",
            "Total Timesteps: 917954 Episode Num: 1288 Reward: 5249.150644958422\n",
            "Total Timesteps: 918954 Episode Num: 1289 Reward: 4824.841176783291\n",
            "Total Timesteps: 919954 Episode Num: 1290 Reward: 5434.155678216018\n",
            "Total Timesteps: 920954 Episode Num: 1291 Reward: 5107.8049954353455\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3743.307126\n",
            "---------------------------------------\n",
            "Total Timesteps: 921954 Episode Num: 1292 Reward: 5252.191648921235\n",
            "Total Timesteps: 922954 Episode Num: 1293 Reward: 5094.9717898281415\n",
            "Total Timesteps: 923954 Episode Num: 1294 Reward: 5406.160775474769\n",
            "Total Timesteps: 924468 Episode Num: 1295 Reward: 2685.759937654163\n",
            "Total Timesteps: 925468 Episode Num: 1296 Reward: 5033.988863752341\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5407.769588\n",
            "---------------------------------------\n",
            "Total Timesteps: 926468 Episode Num: 1297 Reward: 5125.671036015966\n",
            "Total Timesteps: 927468 Episode Num: 1298 Reward: 4894.1148725415305\n",
            "Total Timesteps: 928468 Episode Num: 1299 Reward: 5166.1369567735865\n",
            "Total Timesteps: 929468 Episode Num: 1300 Reward: 5053.840492042587\n",
            "Total Timesteps: 930468 Episode Num: 1301 Reward: 5159.263855237399\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5153.114858\n",
            "---------------------------------------\n",
            "Total Timesteps: 930607 Episode Num: 1302 Reward: 527.0655784055086\n",
            "Total Timesteps: 931607 Episode Num: 1303 Reward: 5378.3568543436495\n",
            "Total Timesteps: 932607 Episode Num: 1304 Reward: 4931.254381653251\n",
            "Total Timesteps: 933607 Episode Num: 1305 Reward: 4937.788624141643\n",
            "Total Timesteps: 934607 Episode Num: 1306 Reward: 4997.5102811179\n",
            "Total Timesteps: 935607 Episode Num: 1307 Reward: 5027.4208755886875\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5113.195146\n",
            "---------------------------------------\n",
            "Total Timesteps: 936607 Episode Num: 1308 Reward: 5023.5542218718565\n",
            "Total Timesteps: 937607 Episode Num: 1309 Reward: 5239.096193566086\n",
            "Total Timesteps: 938607 Episode Num: 1310 Reward: 5106.013134679746\n",
            "Total Timesteps: 939607 Episode Num: 1311 Reward: 5158.475767436035\n",
            "Total Timesteps: 940607 Episode Num: 1312 Reward: 5133.601056851441\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5394.540717\n",
            "---------------------------------------\n",
            "Total Timesteps: 941607 Episode Num: 1313 Reward: 5349.678696707886\n",
            "Total Timesteps: 942607 Episode Num: 1314 Reward: 5395.397189329521\n",
            "Total Timesteps: 943607 Episode Num: 1315 Reward: 4912.503079616113\n",
            "Total Timesteps: 944607 Episode Num: 1316 Reward: 5043.833147745878\n",
            "Total Timesteps: 945607 Episode Num: 1317 Reward: 5004.999442758805\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5409.679782\n",
            "---------------------------------------\n",
            "Total Timesteps: 945806 Episode Num: 1318 Reward: 873.2081970076397\n",
            "Total Timesteps: 946806 Episode Num: 1319 Reward: 4975.663350532901\n",
            "Total Timesteps: 947806 Episode Num: 1320 Reward: 5116.635530814338\n",
            "Total Timesteps: 947904 Episode Num: 1321 Reward: 355.604478381223\n",
            "Total Timesteps: 948904 Episode Num: 1322 Reward: 5122.518540095023\n",
            "Total Timesteps: 949904 Episode Num: 1323 Reward: 5279.996667306672\n",
            "Total Timesteps: 950904 Episode Num: 1324 Reward: 5342.894592963908\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5429.371183\n",
            "---------------------------------------\n",
            "Total Timesteps: 951904 Episode Num: 1325 Reward: 5333.22284557108\n",
            "Total Timesteps: 952904 Episode Num: 1326 Reward: 5237.792815803241\n",
            "Total Timesteps: 953904 Episode Num: 1327 Reward: 5669.7113777415225\n",
            "Total Timesteps: 954904 Episode Num: 1328 Reward: 5325.400880506462\n",
            "Total Timesteps: 955904 Episode Num: 1329 Reward: 5405.793645875346\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5272.686472\n",
            "---------------------------------------\n",
            "Total Timesteps: 956904 Episode Num: 1330 Reward: 5109.168306030571\n",
            "Total Timesteps: 957904 Episode Num: 1331 Reward: 4958.447265089441\n",
            "Total Timesteps: 958588 Episode Num: 1332 Reward: 3345.3288489808447\n",
            "Total Timesteps: 959588 Episode Num: 1333 Reward: 5052.542330027661\n",
            "Total Timesteps: 960588 Episode Num: 1334 Reward: 5189.813468579865\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5100.907058\n",
            "---------------------------------------\n",
            "Total Timesteps: 960665 Episode Num: 1335 Reward: 286.49446514962966\n",
            "Total Timesteps: 961665 Episode Num: 1336 Reward: 4996.755073024454\n",
            "Total Timesteps: 962665 Episode Num: 1337 Reward: 5057.519626186949\n",
            "Total Timesteps: 963665 Episode Num: 1338 Reward: 5021.696818116603\n",
            "Total Timesteps: 964275 Episode Num: 1339 Reward: 3154.1285121491496\n",
            "Total Timesteps: 965275 Episode Num: 1340 Reward: 5189.561360039741\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5442.441757\n",
            "---------------------------------------\n",
            "Total Timesteps: 966275 Episode Num: 1341 Reward: 5431.624103515115\n",
            "Total Timesteps: 967275 Episode Num: 1342 Reward: 5227.16307019766\n",
            "Total Timesteps: 968275 Episode Num: 1343 Reward: 5024.928186457081\n",
            "Total Timesteps: 969275 Episode Num: 1344 Reward: 4844.704818706196\n",
            "Total Timesteps: 969806 Episode Num: 1345 Reward: 2515.051645056825\n",
            "Total Timesteps: 970806 Episode Num: 1346 Reward: 5215.102692921408\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5556.961962\n",
            "---------------------------------------\n",
            "Total Timesteps: 971806 Episode Num: 1347 Reward: 5360.663905775877\n",
            "Total Timesteps: 972806 Episode Num: 1348 Reward: 5178.149557064765\n",
            "Total Timesteps: 973806 Episode Num: 1349 Reward: 4990.638432129156\n",
            "Total Timesteps: 974806 Episode Num: 1350 Reward: 5119.516393722252\n",
            "Total Timesteps: 975434 Episode Num: 1351 Reward: 3284.6455593651435\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5538.998301\n",
            "---------------------------------------\n",
            "Total Timesteps: 976434 Episode Num: 1352 Reward: 5219.986429535402\n",
            "Total Timesteps: 977434 Episode Num: 1353 Reward: 5049.019769099897\n",
            "Total Timesteps: 977758 Episode Num: 1354 Reward: 1524.8915616615172\n",
            "Total Timesteps: 978046 Episode Num: 1355 Reward: 1317.130356534508\n",
            "Total Timesteps: 979046 Episode Num: 1356 Reward: 5415.763917953131\n",
            "Total Timesteps: 980046 Episode Num: 1357 Reward: 5029.030933729773\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4886.449700\n",
            "---------------------------------------\n",
            "Total Timesteps: 981046 Episode Num: 1358 Reward: 4997.690804035434\n",
            "Total Timesteps: 981755 Episode Num: 1359 Reward: 3603.9557488242367\n",
            "Total Timesteps: 982159 Episode Num: 1360 Reward: 1871.6076748118667\n",
            "Total Timesteps: 983159 Episode Num: 1361 Reward: 5162.73628188456\n",
            "Total Timesteps: 984159 Episode Num: 1362 Reward: 5039.853738290478\n",
            "Total Timesteps: 985159 Episode Num: 1363 Reward: 4958.4551875791585\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4733.726924\n",
            "---------------------------------------\n",
            "Total Timesteps: 986159 Episode Num: 1364 Reward: 5152.8023033903355\n",
            "Total Timesteps: 987159 Episode Num: 1365 Reward: 5028.4416190282445\n",
            "Total Timesteps: 988159 Episode Num: 1366 Reward: 5104.633952047465\n",
            "Total Timesteps: 989159 Episode Num: 1367 Reward: 4875.85636279026\n",
            "Total Timesteps: 990159 Episode Num: 1368 Reward: 5110.594231404758\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 3880.637049\n",
            "---------------------------------------\n",
            "Total Timesteps: 991159 Episode Num: 1369 Reward: 4813.323227539886\n",
            "Total Timesteps: 991478 Episode Num: 1370 Reward: 1559.00732310961\n",
            "Total Timesteps: 992478 Episode Num: 1371 Reward: 5583.465521074555\n",
            "Total Timesteps: 993478 Episode Num: 1372 Reward: 5030.784214237551\n",
            "Total Timesteps: 994478 Episode Num: 1373 Reward: 5235.597585755812\n",
            "Total Timesteps: 995478 Episode Num: 1374 Reward: 5030.366614262241\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 4715.011039\n",
            "---------------------------------------\n",
            "Total Timesteps: 996478 Episode Num: 1375 Reward: 5279.451010036752\n",
            "Total Timesteps: 997478 Episode Num: 1376 Reward: 5054.191698088161\n",
            "Total Timesteps: 998478 Episode Num: 1377 Reward: 5071.0161976717645\n",
            "Total Timesteps: 999478 Episode Num: 1378 Reward: 4788.943568816254\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5491.594655\n",
            "---------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wi6e2-_pu05e"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oW4d1YAMqif1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebadf7eb-6666-4cff-fd41-8494cb93b07f"
      },
      "source": [
        "file_name = \"%s_%s_%s\" % (\"TD3\", env_name, str(seed))\n",
        "print (\"---------------------------------------\")\n",
        "print (\"Settings: %s\" % (file_name))\n",
        "print (\"---------------------------------------\")\n",
        "\n",
        "eval_episodes = 10\n",
        "isRender = True\n",
        "env = gym.make(env_name)\n",
        "max_episode_steps = env._max_episode_steps\n",
        "env.reset()\n",
        "\n",
        "env.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "np.random.seed(seed)\n",
        "\n",
        "state_dim = env.observation_space.shape[0]\n",
        "action_dim = env.action_space.shape[0]\n",
        "max_action = float(env.action_space.high[0])\n",
        "\n",
        "policy = TD3(state_dim, action_dim, max_action)\n",
        "policy.load(file_name, './pytorch_models/')\n",
        "_ = evaluate_policy(policy, eval_episodes=eval_episodes, isRender=isRender)\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------\n",
            "Settings: TD3_Ant-v2_0\n",
            "---------------------------------------\n",
            "Creating window glfw\n",
            "---------------------------------------\n",
            "Average Reward over the Evaluation Step: 5298.986548\n",
            "---------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ]
}