reinforce/REINFORCE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REINFORCE\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train REINFORCE with OpenAI Gym's Cartpole environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "gym.logger.set_level(40) # suppress warnings (please remove if gives error)\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "torch.manual_seed(0) # set random seed\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define the Architecture of the Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation space: Box(4,)\n",
      "action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "class Policy(nn.Module):\n",
    "    def __init__(self, s_size=4, h_size=16, a_size=2):\n",
    "        super(Policy, self).__init__()\n",
    "        self.fc1 = nn.Linear(s_size, h_size)\n",
    "        self.fc2 = nn.Linear(h_size, a_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return F.softmax(x, dim=1)\n",
    "    \n",
    "    def act(self, state):\n",
    "        state = torch.from_numpy(state).float().unsqueeze(0).to(device)\n",
    "        probs = self.forward(state).cpu()\n",
    "        m = Categorical(probs)\n",
    "        action = m.sample()\n",
    "        return action.item(), m.log_prob(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with REINFORCE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: 34.47\n",
      "Episode 200\tAverage Score: 66.26\n",
      "Episode 300\tAverage Score: 87.82\n",
      "Episode 400\tAverage Score: 72.83\n",
      "Episode 500\tAverage Score: 172.00\n",
      "Episode 600\tAverage Score: 160.65\n",
      "Episode 700\tAverage Score: 167.15\n",
      "Environment solved in 691 episodes!\tAverage Score: 196.69\n"
     ]
    }
   ],
   "source": [
    "policy = Policy().to(device)\n",
    "optimizer = optim.Adam(policy.parameters(), lr=1e-2)\n",
    "\n",
    "def reinforce(n_episodes=1000, max_t=1000, gamma=1.0, print_every=100):\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        saved_log_probs = []\n",
    "        rewards = []\n",
    "        state = env.reset()\n",
    "        for t in range(max_t):\n",
    "            action, log_prob = policy.act(state)\n",
    "            saved_log_probs.append(log_prob)\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            rewards.append(reward)\n",
    "            if done:\n",
    "                break \n",
    "        scores_deque.append(sum(rewards))\n",
    "        scores.append(sum(rewards))\n",
    "        \n",
    "        discounts = [gamma**i for i in range(len(rewards)+1)]\n",
    "        R = sum([a*b for a,b in zip(discounts, rewards)])\n",
    "        \n",
    "        policy_loss = []\n",
    "        for log_prob in saved_log_probs:\n",
    "            policy_loss.append(-log_prob * R)\n",
    "        policy_loss = torch.cat(policy_loss).sum()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        policy_loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if i_episode % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "        if np.mean(scores_deque)>=195.0:\n",
    "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_deque)))\n",
    "            break\n",
    "        \n",
    "    return scores\n",
    "    \n",
    "scores = reinforce()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot the Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXncHEW1939nnj37vpCFJBASwmICj4RNCBB2BHFj8Spw1YiAyNVXRZTtot5cF1yvCAiKioAKCAKi7AGUQCAhhAAhG0lIyEr25VnmvH9090x3dVV3dU/3TM9MfT+f5Jneqs70dNepOufUKWJmGAwGg8Egkqu0AAaDwWDIJkZBGAwGg0GKURAGg8FgkGIUhMFgMBikGAVhMBgMBilGQRgMBoNBilEQBoPBYJBiFITBYDAYpBgFYTAYDAYpjZUWoBQGDRrEY8aMqbQYBoPBUFW8/PLLG5h5cNh5Va0gxowZgzlz5lRaDIPBYKgqiOgdnfOMiclgMBgMUoyCMBgMBoMUoyAMBoPBIMUoCIPBYDBIMQrCYDAYDFJSUxBENIqIniKiN4jodSL6sr1/ABE9RkRv23/72/uJiH5GRIuJaD4RHZKWbAaDwWAIJ80RRBeArzLz/gAOB3ApEU0CcCWAJ5h5PIAn7G0AOBXAePvfDAA3pSibwWAwGEJIbR4EM68BsMb+vI2I3gAwAsBZAKbZp90B4GkA37D3/46tNVBfIKJ+RDTcLsdQpXR253H/3Hfx8UNGIpejWGU8+OpqTJswGH1am7B43Xas37YHR+wzUHruM4vWY9ygnhg1oEcpYleERxeswZL1O9C7tREbtu2JfP3urjwAoLXR2+9jAFt3dRa2J4/uhxwR5q/aghwB3Xlg4rDeWPX+Tmze1YnerY3YsacbE4b1RlNDDh87ZAQeW7gWk0f3w+ylm3DM+MF4872tWLR2G049aDj+8MI72NnRjYNH9kVTQw7/XrIRvVsbcex+g8EAnl20Hnu68ujKM3o2N4R+j827OtGvRzO27upEW3MDOrryaGrIoX+PJrQ05rB22x405QjdzNh7YE+s2rQTALCzoxsNDYSWhhz2HtgTPZob8MaarYVyt+zqRN+2Jmy270VjLoderY0Y2qcFi97bhr49mrF5ZwfamhvQ0pAryNKYy6GlKYcm1/PrlFVJ9hvWG2ccvFeqdZRlohwRjQEwBcBsAEOdRp+Z1xDREPu0EQBWui5bZe/zKAgimgFrhIHRo0enKrehdG5+Zgl++M9FaCDCxw4dGfn6RWu34fK75uLkA4bi5k+3Y/qNzwAAls883XfuYwvX4vO/m4OGHGHJ904rWfZyc/EfXvFsUwR9Ki4t775WPHbHv7XmSBXo09qIGb9/GY05QleeMW3CYDy/eAM6uxk/fWIxNmyXK7OfP7kYU8cOwOxlm5SyiYiylkLP5gbs6OgGUXLlimVF+Y2S5oyD96p+BUFEvQDcC+AKZt5K6jsqO+D7WZn5FgC3AEB7e3uCj5MhDTZs7wBg9bjisGNPFwDgvS27A89bsXEnPv87a1Z9d776H4ufnjsZZ00eoX3+qvd34uj/fQoA8OzXj/OMoMZc+XBJsmy3f4Mu+76u2bwbnd3WZ5VycMgLLfO1H56Ei44aqzz/N88vw/V/WxgqU1MDFWT4ZPtIfPbocTj5J7MAABceOQa//ddydHYzPnv0WFx9xiT88unF+P6jb4WW6zC4dwvGDuqJF13KbcH1J6NXSyO+df9ruHP2Cpx64DDc9B+HapdZjaQaxURETbCUw53MfJ+9ey0RDbePDwewzt6/CsAo1+UjAaxOUz5D9ik0LyFdtZ2dXanLkhZvvbcNa7bs8uwb2T+aiSznuj9xTXkqxFtf0simdHEAAC2NRVNVThDI2cwzJ1ZfvZLaCIKsocJtAN5g5htdhx4EcAGAmfbfB1z7LyOiuwFMBbDF+B8MTgMT9qJTFTcFTs/XzdA+LZHK8CiIDN0KcSwXYEGwjmuW29KYgzN4ISKP0nLuRZ65oCyjPh8kkcXZduqqpHmpXKRpYjoKwKcBvEZE8+x9V8FSDH8ios8CWAHgE/axRwCcBmAxgJ0ALkpRNkPVYDUx9fAyumlqiDa4z0kayKRIUvkmJVprk3sE4W3Mnc959jfqutTb86YizSim56DuEJwgOZ8BXJqWPIbKEveFc0YQSTd6WSd6g0auzwkLUwIs2JhCR4Kawje7IrVy4gjCrS3J8ycSKtOaozCredSqi5lJbcg0TvMS3rCkLUl5iaoQUx1BJFicrgIIo8H1ha2PLgXpOs+5F5EVbsATVyirxp45GUZBGDJNkmGP1UTURt7dYDakrC2jNPJ+H0RY2ZoyBMjjGU3pFadZZx1oBAGjIAypIpoY4l4f1mDW2qsbtZF3N4q6yuW+S46MVIebuI7wpBpZ0Snt3pZ9jlpv0PerowGEURCGaOzq6C7rPIO8ro2pxqCIb6bH7K55bf8ezdEqcaHrRA+awCcjzs8sOqll5rYkBlXF6CXy/K1ljIIwRGL/ax7Ft+5/rWz1sRPFFHJerb2r0X0Q0UcQcW8ZAWjWVRAJ1ekr11VwLkfKEVSNPRZlxygIgzaOueful1aGnOkn9ovqzIOoszc9qgknzjyIUu5pY0O8i0OVVwyhSBHm6i4vam+fiELNUvXwSBoFYdCmEhksiham8KlytUTUEYRsolhSdfidwFFMTClNpXbh+x4SZRmnWvZn+vEUXw+dFqMgDNo4eXXK+WJw3Y4g4kcxlWPOiK6C6OqOOA8ihiw5EhWku7zk7kW9PYOAURCGCFQi5JTrdCZ1lk1MRN6JakF05fPCtaX/kF84dpw/iglyBZmL2dvPBXy94kS52scoCIM2hRFEhGtK1SnFXEwh9uAae1uzNFFORpOmD6IrH3UmdXiZ+w/rI6TcFsJcJeUlcUcKikGIZqpljIIwaFOJtBcFH0Ttv4seypFqQ7eBk52la7rxmZhS8UEI2zn3vaifkNQ0MArCoE1lfBB6Y5Bae/1LadBKafj1rtO/UpwzEz7hMbxssQhxopzs3CRSbYijkVp75mQYBWHQphRzUdwGz6lTbFg+/PPnkHc1PqaHmE06u0UfRPJ1iCMIr7kp+QpNLiaDQUJxBKH/ZpTq2GbXqGXuivcL+197dwt2d3UXtuvgXU0cfVOUf58qBFQk6qz7ODJZPgjFRLlEfRD1h1EQBm3Y7gxW4kXZuacbZ//yX5599Zg8rRx867T9MWV0P+XxKKMAn5M6oSGEZyY1eZ8Er8O+UHHkOlTPV8GvUQfPX+prUhtqh0rOg3hx+SbfMVWCNoMeqgZu6rgBaG1uwNwVmxXX6Y8Mu0QTU6hM4YhKxj8PwjWCKCEkVRwlFRWDsx2j0CojtREEEd1OROuIaIFr3z1ENM/+t9xZaY6IxhDRLtexX6UlV71zz0sr8PD8eCu5luaDiHdd3Nnb67bujndhHaF07AqqQ6ZIdH8WcQSRRgScOA/CTT3Nek6DNE1MvwVwinsHM5/DzJOZeTKAewHc5zq8xDnGzBenKFdd8417X8Olf3wl1rV5zdTbMuL6IoKimILKvPbB1+NVaLByGyU0OvObmMLrDkM8xZ9pQxLmmoA5iIQP9aBz0lxydBYRjZEdI+tX+ySA49Oq35A8cSbKlYquXqnXhYVKQfU7WsnvAn5lIu3wY9FJncaz41tyVDFpzhCdSjmpPwRgLTO/7do3lojmEtEzRPShCsllCKCYFymGwy/mmxrUDulG0hgUBJmYXMdkv0HcOx++HkSceRDer5JEqo2gesUZ1bVMpZzU5wG4y7W9BsBoZt5IRIcC+CsRHcDMW8ULiWgGgBkAMHr06LIIa7CoRC/90QVqf4lbnqhLWxrU5HLeBjef6A+fgg9C0BBiCGzStYqKopYp+wiCiBoBfBTAPc4+Zt7DzBvtzy8DWAJgP9n1zHwLM7czc/vgwYPLIbLBphJRTH+dt1p5zIwfSkPp2BVHEL7jkp26dSYQxiTKLa7d4HGwJzqCqH2FIFIJE9N0AG8y8ypnBxENJqIG+/M4AOMBLK2AbIYA4iXrS68Zd9vBRZt4tfTuFq/bVrG6Ve1dTvBBlLquuLfsBJzFEhOT93g6TupCmQo5apE0w1zvAvBvABOIaBURfdY+dC685iUAOAbAfCJ6FcBfAFzMzP7Ad0NFKckHkbAstcL0G2dVWgQfRPD8YIn6IEo8LkN0Ukszu8Yo2ITOphvFdJ5i/4WSfffCCns1ZJisRQqx4jMAo5E0UN8ib9Mo+iCI4o8qEjH1CNuik9pjbiqhPhMEYVJtGCJQnAdRYUFsPE7q9Fe2rDlUI0FrZrLbxCRcV1Kd8WQKu0aV7jyXhpO6jvK5GgVh0CZOsr6kkFZpOnipIP6+siim+Cam4GdH58mSpvtWlFH0FyTog6gjE5NREAZtKtkey/WD2shUjxEnUVFOlBOOyX73uObGtNJ9K30QdTTrOQ2MgjBowxWYSe1QDw3+jZ/8QKrlX3jkGM+2OopJWIBHNDGluJiRXtHek4Iio4oryumUq0f9GJiMgjBEIB8jiikpx7bM71FrPojGhnRfxwZN55GYi0luYorppI51VUiZQliuak5EohXCmJgMBg8lTZQr8W2SKaWgJqoaX960nf9i8UFhnO5jcTPqqsou5bjsnBz5wphc59ZPY54GRkEYtClHmOuOPV3YvqfLt1/qg3BPlEtRpnKR1uS+A0f0scrXLJ5InEntT7gX2weRRqoN0STmOZZ8vUUTU+1rHbNgkEGbcmRz/cD1/0RXnrF85ume/bIGwOOirgETU1ojiD9/4Uhs39OFW58VkhMo6hN3SyfKpeSk1krWJ2z750H464szggibKFcP8ySMgjBo4zQKOSLs7uxGS2Mu1B8R9RUS1w9wkDkiO7ry6M6ztm09S+zp6kZjzjuAT8sR39bcgLbmBt/+YCe1ex5EtDUdgkgjJ1LQdimpPepBAYRhTEwGbZwRxLbdnZh49aP42ROLy1a37DU/cuaTuPROa/Ej2fKQ+SSN5wnAzAWZJnz7UVxy58ue42nbyXWLF036yeZyTSKKyYs4sU9WRKLzIJx031U5To2GURAGbZyO5I6ObgDA/XNXBZydLKoX/NHX35Pun7dyM8Zd9Qj+tXhDmmJF4vxbZ2PcVY8Utv/x+lrP8TSW4wxCOQ9CjGLyLfqjv2CQrOxS8ZuY1E11GqOyejIxGQVh0KaUdQFKfU3DrhdFW7ZhBwDgmbfXl1hzcvx76cbA42lbyvxrZihs7PCmz872ahD2+hWKiXJFJ7UhDkZBGLQRLTblnLxWS2GKqt53Vr6jKIfMUhd/RbnSv6RYhG89CLeySGH1t3qKYjIKwqBNkusCRCUX0r1WiZbFl7izW6Ug0pXVPw9Cjhg26vvdSxAzbJQU5x740n1LlEWiYa7GxGQw+MlaLqZqpSufl+4vuw8iIMzV46QuY5hrnDKClE6Sa1LXIybM1RDIA/PexXUPvo6XvjU9VlRQcqk2wsJps222cdPZpZC1xHLvv+TIQCWu+1P4RhCSK+P3nkOimGKUKD4b3s00nNQZfKhSwigIQyBX/3UBtu62Zjf7fBBllCPsnczaYkZBdHSnM4KYMrp/pPOVZhcq/AegvOtBaJUBv0JQjoaMk7ok0lxy9HYiWkdEC1z7riOid4lonv3vNNexbxLRYiJ6i4hOTksuQzSc3hKz3xa9dMMO3P3iirLKEfm6hOWIw5adnTjhR08XttUmJuB3/3kY7vr84anI4fNBBDSq3mR9/uPxU22EHI81D4KUyq6wYFAKD0IW/VtJk6YP4rcATpHs/zEzT7b/PQIARDQJ1lrVB9jX/JKI/FM/DWWn6JCTmyiuvO+1SOXEpQonSxd45u31WLJ+R2FbZWICAcfsNxhH7DOwTJLJEecV6IY3tzaFNyeJmGd8PghxTWrZpDnjpI5DagqCmWcB2KR5+lkA7mbmPcy8DMBiAIelJZtBH+e1YuaS5kGULkfMEUQGFIsoQmeFnNRBv97HDx1Z+EwIb8hlZV0xfb9QGUKjmGLmYlIdLyUXk6EyUUyXEdF82wTlGE1HAFjpOmeVvc9QYdwNRSUzV1SzD0KUvVPhg6hkG/bVk4qNu28EIZ1JHa+eVLK55kiqFIB016Q2JqbkuQnAPgAmA1gD4Ef2ftmdlj6CRDSDiOYQ0Zz167MzS7bWYZQ2k7pU4vaus/ASi7KrTExhcz1KJa4PQvdX15E+mfUgvCeJyQUjCxURY2JKCWZey8zdzJwHcCuKZqRVAEa5Th0JYLWijFuYuZ2Z2wcPHpyuwAaXiQkxJ0Ik8xK9u3lXSC3ZfVn1TUzpy+JGtQobICgI6a3Nzv32p/sm1zHHSV35jkI1UlYFQUTDXZtnA3AinB4EcC4RtRDRWADjAbxYTtkMcrzRLCWsKFfHiI2TOoFg5XwQbsQRjzhyVEUxxVkNznc8vAh5sj6Pk9p/brImJudv7b8Iqc2DIKK7AEwDMIiIVgG4FsA0IpoM61ldDuALAMDMrxPRnwAsBNAF4FJm7k5LNkN0GFxckxpZ6j9aKFNtZOAdFmX44T8XSc8r+whC1aiSt/HTNzGFf4E0HPFBRabhpK4nE1NqCoKZz5Psvi3g/O8C+G5a8hjiQcW3oaRcTGn3trL8qup+87SjmHw+CMVn3xrPshFEXBkS8UF4t0UfhO+7aJZr8GNyMRkCcd6rPFcmiqnU1eKy0C7oNvxZacQI3vum+7trNe5pRDEFpNpIwxxkopgMBhvnZcszF0YQ5XT46Y5aKplpNgzd21XueRAka0nhRDG5w5vLuWCQzjwIMYopoAwTxVQSRkEYtLDCXK3P3RGGEqW22yW/ghnolmd1BKGK/CFhHoTsN0gnVV/MMgNuXOHeV/4xqEqMgjAE4jQV+TyX1GOK2/jpKpgs9+V0v3uWTBay6LXwazR6/6n4INTH04hiKpadnd8rLYyCqHPmr9qMDdv3KI8XhtMl+iAqZQHKwiusajibG72vXy7ltzFoopwooqfxSzJZXwLDJFmYq+q4U1+SZtFCAstMd0uSwSiIOufMXzyPM372XOh5DM60nT/DoqmVlCBzRX0QvmPq6wC5zyfOHIY4ZQDA+VNHFz4H3TezJnVpGAVhwHtbdyuPeaOYymtiyrJCioK2DyJlOaLU7YliSjB8Lawnr9XTJ+Cio8a6rlGXkU6ab+dv7asdoyAMgTgvmxXFVN66o9WX3RXlVDKIJopKpoPw1e3xQfjPl91tzbY9ccQcVnITU3L1mSgmQ2bZsacLawN6/ElT9EFw2edBRKkuy4MNlWzi/rRnUsctXleRaZmYwpzUOmWEhLmmPQ+injAKoso46/+ex9TvPaE8vjokqV1cOKaJqZSGO4qJ6f2dndL9WWgYVD3NKD6BZORQ4zcxFfdIf4aYv2safpagMlMZQRT+Vv7ZShujIKqMxeu2K489umANjpz5JGYtSi4NenGiXKmpNqITpbbP/25OjBrKg2rkJd7PSq6a57fjFz/Lfne5iSmBCKUYYa5B22k4qespM6xREDXEa+9uAWCFriZFYR4EMxTr3KRGEmajLLzLKsUq7k07iikKQRPlSpEyja/oD3Ml/+fs3NqqwiiIGqLRDqTv6E7eIM8MdCvWMUiLWnECqr5Fuf0mQW2kfz2I4rbMtCgNc9Xq/YdFMWmUIWz7FGvA/I4kME5qQ1XiTLxSLWkZB3cupq4YXupSXqJERhClF1Eyuqa5tFeUC5SCgJH924qbMaKYdEgliimg0GKmjSw8CdWHURAVYuvuToy58mHcOmtpYmU2NVgvQVeSCsL+a40gaq/HNObKh3Hdg6+nWoeuoqukDwIA/nbZ0Xj0ig8BEExMutlcdc4JjWIKL0Uchfi2JeWlMx+i9pWOURAVYv02K73FXS+uSKzMpgZnBJH8xKa4I4hSXqJymWB++6/lqZavnS67gg0OEdC/ZzMmDutT2C6it6KcTiucTrpvUQz/RLk07qwxMRlSI43Gz1EQHWmMIBBvBFGSiSmBFzCozSrXTG1tE1OmOqQuH0Q5XU8xopiCU204I4hM3dyqITUFQUS3E9E6Ilrg2vcDInqTiOYT0f1E1M/eP4aIdhHRPPvfr9KSKzu41u9MiGZnBNGV4Bvt9kGk4PwOIu32u1wWM+1qKhnmKm67w1zFEYRkn6yMtAhzUntnUqcpR+0rnTRHEL8FcIqw7zEABzLzwQAWAfim69gSZp5s/7s4RbkyRZKPWKPtg0jUSW3/ZWZ0Z3m6soKgnmO5fCr6I4jspNoI8kEQUWrKO84dCJoHQYpzLjpqTIyavBgTUwkw8ywAm4R9/2TmLnvzBQAj06o/66RpYtL1QUQxsZQa5horWV/s2vQoJflgFPSd1NnpkXrDXHWvSUmYkHoC50E4JiahjJMmDUtDtJqjkj6I/wTwd9f2WCKaS0TPENGHKiVUuUnSNtoUcQSh03AVndSIF+ZaplQb8cpPtfhiPZrnVVI9BNV97yurfPukPmqhlLamhuhyxHgftMJcQ2Zfx8GYmFKCiL4FoAvAnfauNQBGM/MUAF8B8Eci6qO4dgYRzSGiOevXJ5dSojaIqCC0S7RnUpfbB5Fy+eUymemvyJayIBHqDp5UB60fZ0iflhIkCsIrnW8EQbJjahNaXIyJKQWI6AIAZwD4FNtdRGbew8wb7c8vA1gCYD/Z9cx8CzO3M3P74MGDyyV24qTzaFmlpmViijOCKIX0ndTZMjFlqUcaR1lFiS5SlhG92kDlpvJBmKgmPRrLWRkRnQLgGwCOZeadrv2DAWxi5m4iGgdgPIDkZpDF4I+zV6CjqxsXuhYmSYMkH1OnIdINc9UaQThpBZgjOXUfX7gWr6x43ydbJFJuv7lM4ZtuRTRmYA8s37gz4OzK4Eu1EfJk6vSe02qCozT2qkPGxKRHagqCiO4CMA3AICJaBeBaWFFLLQAes3/UF+yIpWMA/DcRdQHoBnAxM2+SFlwmrrr/NQBIXUEkifPKdmiGueo02rmYPojP2dlVP3HoSI9sUUh7HoRjYipnZ7IhW5MdlMQaQYTuSKfeIEFyCie1MTHpkZqCYObzJLtvU5x7L4B705Ili6Rp3ejSjDZyP+Abtu/BoF5qm7GVzbXMyfrKZGJKu8l2fw8n0kxKhnwQYedqBTjEF6ekcmU9e396jgQFqmHMTOoK4TTOST6ozkurO6HN/ZL/998WBp8LlJTuO05jXythrr95flnhszNXRUaWGq1gWUhryVGZD2JI72DHdRJmG888CGWqjQTqqQMTk1EQFSbJh8xROnEmgMkien7z/DIs27ADQPwRRClNcBJhrkH31/k6aTssX121pfDZSckuo1dzWV2CgcR5Lv0pw73HLzpqTCIZa6P8XsVUG+L+ksWoC4yCqBBpdF6dMnUVRJAM3XnG9X9biD22P4NjJusr1BVDVZRrBNGdZ+zq6E68fGbGr5/1xlo0KUYQXzlxv9TTfQcRZZ6AZWLScVLHUDIal4SbmPzlBa13YVBjFESFEZ/T3Z3d+PWzS2MmxrPQbciDGm2xAUgj3fe23Z34zfPL1CuuKar7/WcPE85TyyVrB5x77DbF/fzJt8MFjsib723Ddx5+w7Mvq07qsN6/SJQIuHKvyeDJ5qqoM5u/QvbQHtMS0dEAxjPzb+yw1F7MvCzsOkM0fvrE27jp6SXo36MZHzs0WiYS5mgmJk+7KlwimpzizqSW1mVz3YMLce8rq7DvkF740Hj9OS3iSx91NPaTx9/Gr55Zgh17iqOGnQmOIGYv3YgezY1SH0egkzpDJOMLKEYQMZILjQ1dU8ITxSS/xgwg9NB6WonoWljzF5zkek0A/pCWUPWAqlHbsqsTALCzM36Dpa0ggo4JB/MR50HosHlnBwBgT6fct6FqUMSXO0gqWTuwbXen52/SnHPLC/jwL56TKojGrI4gIoolXw8icLMyqOZBZEO6zKPbnTkbwJkAdgAAM68G0DstoeoZ58WL8/gWopi0RxDq88TGrXQfRHIXifcm6HsERdu4v08aeZ9kJTZmdAThmycQ5INQHRC+cNHE5IwkEnAwaJTjPqpyUpsRhB66T2uHnRaDAYCIeqYnkgGI9wAXo5hKn0kt6gJmIF/uZH2qA8K9eXfzrkj1Ow2MWwmm4RCXKR2VkzprBEmp60fxj/TKE1YsTfdtRgyx0FUQfyKimwH0I6LPA3gcwK3piVX7qF+W+L3/6CMI9TFxBGH5IMo7EUL3kmN/8LS6DNnCNtIRRBTJ9JCV2RAQ5lpJokwk0+28qGYxB8qhcXa0SX0KJ7XRF1poOamZ+YdEdCKArQAmALiGmR9LVbIap2BKEp5UZ/+sRetx1D6DMGaQfLDGLOmhRQxzfXXlZrV8gi4o1QeRZKqNKL1B+QjCIs6IKAqy4lUDiOytxaS+x/rJB8XtcrXKxXqUTuoEZDGpNgAQUQOAfzDzdFgrwhkSxG9Pt/7+4/W1eH31Vjz3jeOl18kezShhris27sRnbn/Rda04YhB8EJrlJomqISq19+co5bS/j8xJnaVFgdxE8UEoFYSikELCR50opgRuj9fEZHwQpRA63mXmbgA7iahvGeQxuFj1fpBtXWZi0g9z3byrI/C4zEmtKnfFxp2hE82STLUR5d2W3SencXB/nzR6g7LvrJoMV+kGK8p6EMoUJaKTuvBX/8slcRtIueHanYQiqgO/hu48iN0AXiOix2BHMgEAM1+eilR1QCFRXIKOPOfK7jxj884O9OvRHLssURfkmZU5no75wVM4et9B+MPnpmqVvWLjTuzVrzX0PFVkUZRZsEFO6rTXpJbJr/LvZs3EFHSPdUUVv2tSDWqUxl11v5MYyRkTU5GH7X+GhFB2wjSfOelprp2T//sxLJ95ulC2/gMtnrtxewd2BczNeG7xBq3ydnd245gfPIXp+w/RkEFD0LAyAKzduhtD+xQVUmEE4Y5iSsNJLdmXWROTzxcWEDqs64MQMuXpmZg0nNRhYa7umdQqJ3VoLQZAM4qJme8AcBeAl+1/f7T3GWJSanske0nDXsAojaDYuf7Ow28UEvdFQZTJye30+BvrIpflEKWN/cvLqzD1e09grmvxIqdX6V5CNY2+oMwUUy05gIKoRhCsAAAgAElEQVQnUerdrbS/6cCe8hEyKT57zjEmJi20RhBENA3AHQCWw7rno4joAmaelZ5otY3axKSHTBmEvbfuw+K5Xd2M3Z3daLUXmk86FbZTmjtySPzuYlSR0kkdod4Vm6zV295etx1TRve36y2Pk1rqg6iSNiXo59e9a2Kq7aRCWJ1znvraNOwO8X2pR2zGxKSDblD2jwCcxMzHMvMxAE4G8OP0xKp9SjYxSUcQ8fnnwrWYePWjhe201kqQpRV3+PYDCzzbuqk23Nz6mXbpfndDUQhzdcmSzyefSqSaopj8BJmYNEcQotkq4Qa1T2sThvTx+7Jk60EEnWNQo6sgmpj5LWeDmRfByscUCBHdTkTriGiBa98AInqMiN62//a39xMR/YyIFhPRfCI6JOqXqS7sEUSS60GEjSAizBxOevE4p2rZ3ANnzx9nr9AsTX3PhkkaDEDouUsmyt390krsc9UjmvXrIVMQfdtCX5tMEDyJUnFAEQkVaWKbzjkhJ2mNVPTEKbmeakdXQcwhotuIaJr971ZYvogwfgvgFGHflQCeYObxAJ6wtwHgVADj7X8zANykKVtV4rxkcaOY3C/wl++eizFXPhzug4gkXzomJu8IIvgFiyOCqvHwjiDsVBsJjxgeXbAGY64sxnLIVuAbNaAN937xCFz74UmJ1p00gT4IzQNpTE7TwZvNVRVWXPuNexLoKogvAngdwOUAvgxgIYCLwy6yfRSbhN1nwfJnwP77Edf+37HFC7DSegzXlK/qUDZ+2lFMxRMfmLc6uExp/cEnx1EQY658GHf8a3ngOVHMOMqOaox3W5YCuqTUIRL+One1Z1uWE4tAOHTvAQVfj0PW7NmBPghtJzV5/iY1US7SvAqViUm7hPpGV0E0AvgpM3+Umc8G8DMADSHXqBjKzGsAwP7rxDuOALDSdd4qe19NovuSdXTlMebKh/H7F94RrpeUGVqnpnAIMCOEcNPTSxR1WwW628wwJaScBxFDLs8IQjJRLgnENEvSNbwFx21WCV5MSq8M535UsrOuHkGUWZAqRVdBPAGgzbXdBithX5LIfjLfo0hEM4hoDhHNWb9+fcIilI+CiUnYL35hZ+7B1X8VHbgSwhpc11XhyiRe4+nrlQvFuE1MHV3BPXiZBH+5+IhA80AUE1PSCkLs2cpGKCq7fDXZs3VHO75V6vQ8DOFnhPkgPKk2VLVUz/2uJLoKopWZtzsb9uceMetc65iO7L9OQPwqAKNc540EsFq4Fsx8CzO3M3P74MH6q5BlDdVL5muYldFOamevss4yjCDERl8sxt0o7+mKnp5jwrDeMUcQxc/FiXL+8y6/ay4u++MrMWqArzWKMg+iukxMigOi0hNGS2VL9w3/aNF3jtEPWugqiB3uqCIiagegThQUzIMALrA/XwDgAdf+z9jRTIcD2OKYorLGl++eixseWlhaIfa78uqqLVi3dbfyNJUZRrY3Sb9yXCd1p9DqOorsOw+/gSvvne8pN2gE4Vp+xEOYc1G5BrFkdq3MR/Dgq6vx0Px4j51ozpCZmKLMCagksRREAs+fng8iSnnGxFQKugriCgB/JqJniWgWgLsBXBZ2ERHdBeDfACYQ0Soi+iyAmQBOJKK3AZxobwPAIwCWAlgMa62JSyJ9kzLywLzVuO25ZSWV4e6hP/VWcVax+I4p30WZDyJBDRFXQYhmFff3vPullcIIwjpXtbaFKlV3nJc7JzE7qHJLubn49y9j0jWPanUIRLGkTuqihsg0Qb193WejuJqb11ldKlFMTOpzMv4DZITAmdRE9EEAK5n5JSKaCOALAD4K4FEAoS0kM5+nOHSC5FwGcGmoxDWCxx/get/Ed083c6Zil/d4pCgn/XPd+EYQwnG3gnBGEDJz1v/8/Q1s2uFfM5oouKEJ8kH86aWVePmd9zHcThSo09A9+vp7AIDbnluGq88IDk0V65aOIKrESR00oU/30YgTwp3I/ISERyH1TNgI4mYATl7oIwBcBeD/ALwP4JYU5ap5dNt99XnyXndgnQqlJCOpeRDi6MC9GTSCuPXZZbj3lVXSMuOsdpbLAV+/dz7umbOyoGCSTrWhNYKAt1edVSYO640rpo+XH9S8bekNlkovMeO3PzOEKYgGZnbmMZwD4BZmvpeZrwawb7qi1Ta6DbDamS07V6gjsAEMmwcRLJcuvhGES3BHQQSl3xBJwkzhNA5JT5QTG31ZlFS1jCCICFdM3096TPfZ9S1jqhWhlMDvm/m7Wz2EKggicsxQJwB40nVMN1W4QeDZt9crfRhBPW7Pfo16xB6yu6ywtjGtEYS70XR62FHa6bhpFtyd+eJEuXRHEEHlV3MPVtvEJHxIKoopER+EUSJahDXydwF4hog2wIpaehYAiGhfAFtSlq1m+fRtL3q2WfEZCErqJzMxefd15fNodvUB3EfDes+l9K53B6wb4VY8jrKI6lyP07i66y1GManrZWbc+Ngiz753Nu7APS+txNdOniDt6Yr7ZAqi4LAV7fPZinINRPl7hYS56pCIDyKBMgwWgSMIZv4ugK/Cyql0NBefjByAL6UrmgGIFuYqEtSDVZl1CjOeS2iwnn6rOIFRrMbdKDsfo4xW4jqpZd9HVBB9Wov9pXXb9uDnTy72HP/sHXPwy6eX4J2NVgrxzu48Zv79TWzZ1SmtWxbGWwuNl/LZCPkZE4tiSqKMWvghykComcjOiyTuWyQ715AAHLgZfKnYGCvmJMjOde8nKs3E5M5YKpaT9ygIWxlFSIlEoFgvN3tGEHLZtu7uKnz+00srIeKMjHJEmLVoPf788ir87dXV2La7E989+yDfWg+dkjCmYq+apPurgSgeIyCaXyGRhXyq6F5mHeNHyBCPL1zrWfkMUJt6dFaU68yrZzWrzCvdzMiBSlIQ7obRp7QkJqaodQVGMSn2u79uTsPE9KPH/H0gR8xcDvjM7UUzofN9xUa/Q6YgUMMmJoG0GupwhaPhDE9AjqzNfk8DoyAygPO+fe53c0LOc40ANMJcxQbQ66SWP9zO/lIaLLdpJWgeRJy64jY6Hh9EYV+8MsQ5AjmFX0E2Ea8WerdRn41o60HUwA2qIXRnUhsqhPtldDeuf3t1DTZs3+M9V7g2aKawUkHkg4/r8NLyYob3oIl/cUYQhLg+iHATUxiOvKKCKJiNfAoiKFlf9TaEyp6z6KROqf7Q8YNOxUmYsupAmRkFkXHcjZjb6XzDQwsxQxhxiA3erLeFbLceZRNcXylO6ptnLS1W6Qtzddfl/asDUZgPQn7QLUbcBYNUizyppoTJo5iCpKwOdPVqrCgmrTQZIcd16kngF6gHE5NREBnH/QiKDc7arcIIQnhev3X/Ajz1pjvPk7/3LlJUEMk8/GI5snqjjyBKk6OYzTXad2SFSSynHEHIypebo6qJoNt2/tTRhc9JzRqfOKw3pu8/VPv8ah6dZQ2jIDKOxyQjNDjNjd6fT9Yjvui3L2HlJissU8sHYffwk0r8J5YiqzfJeRBRwlyjjpK6FcrTMTn5opgCkvVVs3kiKD/Y984+CP17NGH6/kPk58Tg0SuOwa8vaC9sx7l3e/Vr82wnEi1Vxb+hLkZBSEgyK6pWfZqrd4mZUpsahNQOCrl3dHT59oWOIBJajfOVd7xRWfIRhH55STipC/siaghnRCBelVM0+tJUG5FqzCZhr8fca07Cry/4YNnNaQN7NmPS8D7S+vq0NmH5zNPLJEntYKKYJCScgaEk3MpKbHA6uvJ4872thW2V3IUZy659YVFMSZmY3HMLVOVGmygXbGRSHXHfx7jfsUORXFA1O1oexaQIc40kSWXRdVLHoRQfxMtXnwjAmvEeWkYUoeoYM4KQkFTjqEtQdUE+iOUbd+KUnzzrKkffbKSq0xmFvBewiFEplDqCAOKm2ih+7ipET0Urw5nXoLp3YnSTdKKc8Lca2b7bPyKVoYruMlQPRkFIyNKkJVWYq4ygyW+65zqnXvPA63oCRkRWb0dXHss3hPf6HILaG5WD0q30S83i6ndSy+sMzOZappDQNFi9Ra/zEMdGn0i23jJlja0HjIKQUO4RRBBuWWQ9Uu+5wWW4D6v8FVt2dWLzzg7psSSQ3dv/ffRNTPvh09plxHm5ZSOIuIgmFtE57bBcYuooiu69KDtPnJcHLztK/2TllzCpNqqVsvsgiGgCgHtcu8YBuAZAPwCfB+AE71/FzI+UWTwA1TuCWL9tj3S/02P2RDEpyjrpx7OiCRiREB1XMjo+iB179EwkKsRb5zRI4khiyXqJgqiyMNd+bc3xL04t1UYCZZReRF1QdgXBzG8BmAwARNQA4F0A9wO4CMCPmfmH5ZZJJEsTYFQT5WSoVmDrzjP2dHV70nBHnQOQFEnUG2xiku93K8Rbn11WUv2ir0eVakNKlU2Ua2yIL2lx1niEa6rlxtQJlY5iOgHAEmZ+J0s2waB2OI0Q2K7uvNJ8FGUEoaKbGUf8z5PYtKNoOqpUpFYSq7iV6qSWcU77KNwzx5/BVYZYVCEySeNa55zsdEGCKUlBRLgvccpVH9cpIyFhapxK+yDOhbUokcNlRDSfiG4nov6VEirIB5FGx/u6vy3E4d97QnqsO4IPQkV3nj3KAUh+uc0ospRKYC4m1YpyIT9cv55NgcfdiEUVTEwqZ4Tn3OpqmZpyEZqIjEw+q7Z7nGUqpiCIqBnAmQD+bO+6CcA+sMxPawD8SHHdDCKaQ0Rz1q9fLzulZDigHU7Lgb1xh9wx7G5Q49Ysi8dPoqGOQ1wl5ybeehAhZUZomHzzIIS/wfVUF+U2MUUpN+5x65xq+yUqQyVHEKcCeIWZ1wIAM69l5m5mzgO4FcBhsouY+RZmbmfm9sGDB6ciWJAPoty2e7dCimvekjXKlYrUkq2yliQ62VyjXAcADcLIwD+TWt+xUMnO7V59WyNf09QQv4mI813LZh4y+kGLSiqI8+AyLxHRcNexswEsKLtENsE+iPLJAXh7/39/7b14ZUi+UKVGELKFeMpB2NcN+l0PGd0v8NyC/tC4pcqeaxkerM8cMQa3fqY9/EQXjRpmswLKCdblbY2NhSk5KqIgiKgHgBMB3Ofa/X0ieo2I5gM4DsB/VUI2ILi3qdPz/veSjRhz5cN4d/OuRGX53QvvxCpDPoIIvmZw75ZYdZWDWCamCAa6kf3bPIkQxRGEqFwdm7dODZVsvHI5womT9LOiAv7vHoU4VyZhHtKbKKcpUJ1TEQXBzDuZeSAzb3Ht+zQzH8TMBzPzmcy8phKyWbKoj+n0vO9+aQUA4KVlm0LODMfd+4/rWP7y3fN8+8IUXVtTQ6y6ykEcJ2SUDvqpBw7zbIuNpJg0sbAAkcbvU23tUqR7Lc4OjxL+a8gklY5iyiRBtv5yW2bcjU6SfoOwxixLs8lFSp0HEUaOvH3QBiGSp1Nw+ju3SquGOmosY40gEvBBaJWhJ04gWZovlRZGQUgo9zyIILxrOCdYbsj3qFQYbKmoczEFX+d+2cVV60Q7vGiyc54JHaXqqJ4M69/EKbdOjFvfCGHNCINREFKCXvRyO3fTipoKUwCdGVYQ8SbKRRlBiNvBPghnU6eKmja3+GYQ2n8ifWkN/0HI8ZbGcPOoTKaJw3qHXueVo5Z/TAujICQEvec67abTUCTRGKTVkw9TPGmHo+o6P784bR/fvuCJckV+dt4U/PniI0AUbeRnmZiKJYWNIJx7qVNH7TcpRSrVgLY1Z9d/Vm0YBSEhqFGO0tAk0fkvNfOoirD5amkrCN3gmLOnjPDt01W8Z35gL3xwzADkiCKvWueuQ1Rmog8iz4wf/uMtPPxaeFxFTc/yTSET63UfnhS5HnEpXmk9EWSqZyqdiymTBEYxVXCiXDnL7dCc8Wz1zqPXb5ltwi8cP6SXv84QeX567mQ0uyZ45Sj4+1774Ul4z7XGAfmc1EIUk88HAfziqcUBUnnlqxeS+K6nHjQ8/CRDapgRhISg6ASdnmiSjYAsTUYSuDO7ytD1tagWywlDx8T0n0eNlfa4w3rhZ00e4WlYKGQEcdFRYz3bomiiiWnTTnVeq96tjThq34HKuupIP8TL5hqybe1LIl9TyUXUBUZBSAhqTMod3ZPWiGWb5rKRKn567mRcddpENMR80+IolhnHjAMQrFxkjUdHVx43z1oiP19SlCibWN+KTTs92+7fqK2pAT/4+AfU8tVRw5TWd01mPQhvIS2NOVx35gE4p31U6YXXEEZBCPzqmSWelcBEn0O55wekpZC27e6Mdd1PzpmMC48cg7Mmj8CMY/ZBlGSfbuJM0L3qtP0BIJZSUv1sspJy5B2liApi1SbvDHl32URhoyP5sezGjJVOtPUgxNl2ycqi4hfnH4JRA3rgkuP8QRH1jPFBuNi4fQ9m/v1Nzz5m7wNe7ujPtEYQW3dFG0H84vwpeGDeanxkygh8xOU4TtPEpDL1NQRkGI0qjixNhuiDENN4v7fVuyazu9OQIwoe4RQO1aBKENOgp9S6J1Gq+JyUe35TtWAUhAvZIyLuK/c8iLRGEFt2RRtBnHHwXjjj4L18+90K4kvH74ufP6nnrNVSEIqvHjSCiNp4yMQQixd9EO+La2uICiJB+bJA79bGWCbJYpJb/W/t90HEu2OXTNsn8jNu8GNMTC5kj6LYs4jS0/jj7BXY2VGarV8MqUyK93fK15+IirstjCJrlHDP/zh8ND78gaJyimvWksoh+dVzRJ6HQVRm4tod7qCmrbs7AxcOqsYw15+fN0XvRNE6lFIuJp17+PVTJuK7Zx+kXaYZP8gxCsKF7METH5woE+VeXL4JNzz0Bu55aYUvNFKXtEYsm3cm07tyN57def3vGKXN+M5HDvI0Uo1BGiJqYyR1Unu3xRGE2DN1dxq27e6Sjo4G9WqOJV4WmDZhCP73Y/qNrUOcpSR8LohqvGE1hFEQLmSjA3FX1Ab7rhdX4Bv3vobf/mt5LJk6IzS6UdCd5xCG25xy5gf8k9pkTBrepyRfTlQH99dOnqA85hTFgpkoyAchIgYuiGcvn3k6BvWy0qfXU4PnPBvH7mct7PWRKX4TZRzS8UFYf4f0thZVuvyE8QnUUv0YBeFC5hAWHaVxo5hUS4qGkdY8CJGoeWgcnFHX/ZcciYNG9tW65mfnTUYpg/ogE4PSZARrRbUzDh4uPSaW74liCmnVRWXXEjCTt1rz98SaDGkr1r0H9sTymafj4JH9Qq7w359y36225gYsn3k6Pn7oyDLXnE3qXkFs3tmBV1a8D0D+Eoj7yr6iXJmc4vd84YhY1zmd6yjRTGET1xySiixxTB2defbJ6Wy6qwozMYmIo8rGhhxmX3WC9FzVbcp6EE0c8Zx7nXSgRRrzIIwXQk7dK4jzbp2Nj/7yXwD0Rgdxw07jNgBxfRdRaYq5OH0uhiPSyo2UzgsZNPGtO89+G7esDKEccT0IwJvvR7ZiXz2ZklQ4vpgov7XfB1GeGxnlcWzfewAAYHrE1fmqkYopCCJabi8xOo+I5tj7BhDRY0T0tv23f9pyvLFmKwCrt6rT0YnbsMVdXCTOCGL5zNMjXxPo+A3AaQSimE5yVN4Z6Y6Mnd15yQhCYpLKeb9No0R5trg8sDIzoGpEpWrvalGhFFbaS/inrnSqjUl79cHymacXfCu1TKVHEMcx82RmdlZSvxLAE8w8HsAT9nZZyLO80fKbmDRSOkvjZePJVS4fROwRhP0ERR1B6HyrON9cJoajILrzrMz1I06UcyNr7JtcIwhxCVKVHNZ+e2JeBU2XfduaIjeQceRzfDeljBalYiZiYvJiDExyKq0gRM4CcIf9+Q4AHylXxd15lj7IYs+/TBafAovXb491ncoGriLuUD6WiSlHvgYnzM6vi3REYO/r6mbfcblJyluOTDa3Qu2IMILIAi9+6wS8ecMpka6JMwJ2m/YM1UklFQQD+CcRvUxEM+x9Q5l5DQDYf4eIFxHRDCKaQ0Rz1q9fn5gweYWJSWzI4vaG3tu6O9aLMmtRvO/Yr0dTrOui4jQCURpEWfrtx75yLG7+9KGefUn1qp0RRFc+r2Xj1glzdfsgtkpm7GZYP6ClsaGw6lqay2zmctEVhM48iESc1EIhWQ8SqBSVTLVxFDOvJqIhAB4jojdDrwDAzLcAuAUA2tvbE/tZ1SMIL3EVxAPzVmNon9ZCwrm0aY4wS+nOz02NXU8hnUJUE5NwG3s2N6B3i/dxjNNrlZqYCqYOf4SSagThRjaCcN/feSs3++XIsoZw8dhXjtFaHErrsRfOce57lGi0arlv9ULFRhDMvNr+uw7A/QAOA7CWiIYDgP13Xbnk6WZWTJQT5kEo3qXte4opNVTvw+ML18aWLypRXrSR/eP3Ip1GIFK+HckIojHOtFtN3CMAVZy9NyMrBa4oB4Sve5yQxSx1ejQ3ol+P5tDz4oW5Wn9LSTgpe64SmSgnbMcNIql1KqIgiKgnEfV2PgM4CcACAA8CuMA+7QIAD5RLpnye0dEVbwSxaO02HHjtP3DfK6sCe2NJZmaVrbQWlYnDeuPPFx+BvQf2jF1GnFBG2QiiIUfJOB8lZbh1jxisJZ0o57vef05rU/CrU+s94U8cOhKfmjrau1P4yrlcceSmS23fteqjUiOIoQCeI6JXAbwI4GFmfhTATAAnEtHbAE60t8tCd55x2s+e9e13GjK2RxiyhtAJlf3+o29hv2//HS8s3aisIymc1A06TN/fG6/tmEz26teGD44ZUJIcRPEUhG8EkSNfbzEpfepVAjpOavKcV8sjCG2EH2P/4X1C/U6JTJST+iCSD3M1Pgg5FfFBMPNSAL5lt5h5I4Bo4TcJoezd27tn/v1N3DxrKW67oN1/in2Os07Aum17pEXl85yYkpDF5qsYO6iHZ7tXayM27+zUDm19MSAiymkIo7xgMid1lO8ThMwk4W7gxYZbNs+kISeYmCQNUkvICCJqFFO1mTisSK/gc5yRWylOakNlyVqYa8VQ6wfrwM2zlgKA1ISk+3J35RlrhcVm4hIlLFTscfWyncGi3f9Hn5AvlTmkT6uybFUo44VHjlFfk/PPg2jM5fy9OmUJ0XA38GIdnfbv6f4NfetBSJRX3ImFDtWlDvw05MK9ToURRCkDCNkIIn5xrnKF0WoCZdYiRkHYqHo5zMDld80tbMuyoOr2nvPMWPX+rvATNZClf1AhvlC9W60Q2CZByUzaq09kOXIKH8She6snwct8EDIXRKxhv8xk5BlBeE+QrWEhNn4yE5PKp37jJz8grcdB1ZnIehI/UWoxoaGMXAzzY6Xug1lRTo5ZUc5GqSAAPPjq6sL2HtcIgtmaeKX7bHXnGas3J6MgdEYQD152FNqaGnDvK+969vdutX72tmavHT3O8N4RQ2wEglaMk6/iFt7g6CB1UrtHEMIxR+G7xW8Qopj6tPrnlKhGEI6S9YfTOiGfcrmzbmIS5db5qeIEMIjIqjFmqPJhFISN6iEWexYdHgVhPay6j393nj3hsKWgY7N30iuLL5Qz36C1yasgSkun4N0fZIPPEWF431as2ZKMuS0Mt7LSUULixLgBPf1hoCoFWAj7FWdsh9ZafYTdykKYa8I+iKyPtmoJY2KyCRpBuPEoCOevZsua5+Sys+qs6ezQJNhDnBFE2DoHOjiNpziiCRrhEAF/+eKRGktZht/Xj0z2LkIjq9UzD0Kn50veiKr+URSEvT/s56k2i4b4jBP8UWciccJcRWo9XDjrGAVhox5BeLfdJqbHFr5nnaNZR3eetbOzBq2CBkRr3NuEkYITohm2UpoO3//4wbj2w5MweZR3MZhgExNhRL82zzrTQEwTl8Z38JqYNM4XypTNSg9TELXWsMWbKBc9zFXnrqVxa6tNYZcLoyBsVB170Tbszv3/X/e86pykWQdrLfU5akAbPhGyolWUBqhNCMlsaHB6uaW/af16NOOio8b6M6CGKAiHhy8/uuDYjeOk1kvf7T5fo0whzFV2jUpBRxnZuRFHeVlD5oMIDXN1ItySzuZqKBvGB2GjMjG9tmqLZ9ttYnIaA10HYzezdvrusJ5xlHZIdEY3aZpBSiFohOOu94C9+uKAvaylSuPoK51LgsJcVec7p33jlIlSpdOg8AHpKghHH5x8wFCMHdQLXzhmH63rsoT4TUVlXTAxVcFU6qwHCVQKoyBsVCYmd4grAOzp6i58jvosd+dZ2wcRZkKK0lMVndFOiKxq+c0kCIpcUY9+0mkdvBPldJzURQXw4Q8MlypSlY9F1/Q3ff+h+NzRY/HFaftgYIRZ8VmBUHxeLj1uH+zqyONMwWToKMGSopjMEKKiGAVhoxpBdAr7PRPl7Ic3yvMvlqcibAQRxcQkKojGgolJu4jI6GQIFZk8qh8uPHIMerU04hdPLda6rzrmC8+91BxB/ObCw/CnOSsxol8btu72R56pFI2u4m5syOHbZ0zSOjcLyO6y8wz2aW3C1072j4DiTJTT8REZH0T5MArCRtXLEXv8bh+C8wJEebY6NRpOAil7qJNH9cO4wT3x/07aD3e9uEKrTv8IwlYQIY3Zby78IN5au02rDpE9MRREQ45w3ZkH4G77ewUN+3967mRs39OFF5dtCi+XwkcQ7iidXI6w75BehdTskUYQNZeEyUIWqed8U5UCoER8ELV5P6sFoyBsVApCfPjdDd+WXZ14YenGSL0P3SgmVUPT1tSAGz85Wb9C+KOYPv+hcViybjsuOGJM4HXHTRyC4yb61mzSYndnd/hJJXDW5BEAgBeWehWErCHzzIPQKNtnS5f5IOpMQYgI+QylxFkPQqvuFJSGGUHIyXboRBnRnZ4gmk7OveWFSA6uTs2Koi56H8TBI/viQ+MHFbYH9GzGLZ9p98X3jxvUC4eM7ideHovdXfEVRJTvqOMAzWmMINyIjbzskk+0j9K6tlZxz4NQPf9xkvX56qmP25lZjIKw0X2Ipcn6oowgNKOYklqjGbBMTL//bMqdHPYAABJQSURBVPiqcc2NOfzx84eXVNf/nX8Ijt53EE45YFhJ5ejSJazgJLu77nN0o5jc+EcUwJA+csdylteiThRXmKvq+S8mcky46jR8EMkXWRMYBWGj27OXJutLuB4Ge/wDlx+/b+HzV0/aL0JtXr579oE4bGzw+g+lvnynHzwcf/jc1EQic3QUr45e3+ZyMquT6BUR0yyJl+SI0KTIxVSrI4g4Jpg4yfoqhUnWJ8coCBttBSFzvkZ4uO6b+y56NgcvNiPylZOKs6oP3Tv+Aj+fmro3/vSFIwLPSSL9hkPcUVDRdBGOjonpAFeWWtFhL0Ns5GXhwO5z+vdowg1nHYBhfVqV3/lrJ09Ar5ZGjB0Uf/W+SiIzIznfVNW4Hrp3f7Q25XDxscnO8ahNFZxN6lpBvO2K0ClFQUS1se7oiG6fHzeoJ744Tf2iXWOHTMrSQkQhSRNJ7N50hMucCJlTDhiGxhyhh0QB9OvRjJMmWavqtTSG358wExORN8ps7jUn4dNHjMELV52gDD8+buIQLLj+ZPRsqY24EPc8CFX/qH/PZrx5w6mho9bAeiS3M400Jmb8IKfsCoKIRhHRU0T0BhG9TkRftvdfR0TvEtE8+99pacty4o9nFT53aPoGZCYmzUtL4sn/Nw3fOGWi8rgzW7pZowEMIon8TA5J+lFUOMr5vKmjsfh7p/kWQXJwdLhqJTh3IyfeA/FbkOSceiSVaKIMNdX/edTYSotQcSoxgugC8FVm3h/A4QAuJSJnxtCPmXmy/e+RcgqlMz8BUI0gonvhoqwpraK5MVfIpnrUPlaU0hXTx5dcblKIjej5U0dH8nHo+SCsk8Kzp1rnha0lDfhHEKLM9eKHduPPxVTMV5Vmky5TQkndfk8ad8mXOOmAYVg+8/SEaqtOyq4gmHkNM79if94G4A0AI8oth8iidXoTwuQKInp9c749PfQcMdupyKLvnIpXrj4Ry2eejtEDe2D5zNPxuQ+Niy5MSjgjiLOnWD/v984+CMv+J/yFi9IAOCOIMHNWvqAg5I/8qQcWo678Ya5qk1PQ0qq1hG9FObgnymWn1x+FV64+MTQpZr1TUR8EEY0BMAXAbHvXZUQ0n4huJyLpmpVENIOI5hDRnPXr1ycmy83PLNU6b5c9AWzc4KKzMWwEcfzEIbjyVLV5SNVL+fl5U6q6B+PkfPrmqROx7H+iWwx1zA0FBRHSrS+YmBQK4sh9B6HVNj+FL4RjnbB85um47swDQmWsWcowlJL7IJKvJ0umrSxRMQVBRL0A3AvgCmbeCuAmAPsAmAxgDYAfya5j5luYuZ2Z2wcPHlw2eR2ckEn3MyobQYzo14bRA3oAAHq2NJbFHp81ChOl7KVZdTlopJXd9XiNWdxOw687ggjy0eiWVc8mppH92wAAB4zo44piKo8M/3H46JLLEEflzjN24Ii+JZddi1REQRBREyzlcCcz3wcAzLyWmbuZOQ/gVgCHVUK2MJwlQ90NnizXTI/mBvSwHce9Who9+f4f+tLRAIDhfVvTFLXinD3FGr73ihi5M3FYH7x5wyk44+BgExtQHEGEKSDWaPwdP4VqNHL0vpafpw71Q4HTDx6ON284BROH9SmTD6LIf595IN684ZSSoph+es5kvPWdUwrbpx5kfR8n5bzBS9lj7sj6dW8D8AYz3+jaP5yZ19ibZwNYUG7ZouCO/ZaZmE47aDgeW7gWANCrpcHT63QUx9Nfm4YY/u2ycMS4gSWX8fWTJ+DS4/ZB79amyNfqzFcAgOMmDMG8lZuxV79gZVt0ZqsbF2cEIYtQWnD9ydjd2Y327zxec6vF6eA2wfh+mwSHEEGRUbkcoTUXbQ6RrIwWoQzdZ60eqURQ9lEAPg3gNSKaZ++7CsB5RDQZVodkOYAvVEA2bdyvxOadnZ5jL31rOgb2bMbDr1n6rk9rE3bsKc59cMwcOhE1lWDeNSf6FhmKQy5HsZRDFL50/L44f+poDO4dHBXmKIigtj1oBNGrpbEQoBDFWjhqQJv+yRnG0QHuBjzKhMa41KMyzhJlVxDM/Bzko/SyhrWWkkAMgOetuHO2N+2201gtXrcdADB+aG8seLe4Ml3YZDbHlFEp+vVoDj8pI+RyFKocANfoIOYIwjquZ85yePnb02uud+r+6mET5RKpL72iDRrU5Uzq3Z3dWLh6a2F7wtDekcvQeSfGD+kFADhwRJ+C7wKQO0qH2snfnvnaNFx/5oGR5TEEM8XOUqujTFR+CqexP2IfPfPbwF4tNTNzWoZpvGuf2n16A3hjzVac/ct/FbYH9W7GW2uTr+f2Cz+ItVt3Y2T/HhhlRzQBcgXxzyuOxdbdnZ7zDMnxtZMm4COTR2Cfwb1Cz1U5qXu1NOLxrxyDkf3r9zdy35mikzq9IYSxMFWWuhxBiFE17hC3cz8oz/M/oGcz/v3N4wvbOtkfRw3ogfYxVh6ai1wTqmQmpr49moxySJHGhhz2H24l7XvxqhPw7NePU56rSNQKANh3SO+aMxvpwBIfjmNqq9J5cgYN6lNBtBYVxKemjsZHpxRnUwbFQw/v24Y+9rUqO/Q/rjhGut9t11blCzKUhyF9WgOVcZIZbWsZJ0x7eL/0HPHGSV1Z6rKlctuFp08aWphBC6hHEE4P6vGvHov7LznSE1Y5qFfRqTthmNqfMUTD/m2oPHWz6E8EZKOEs6eMwG0XtONTh5U+gc2QTerSB9Gzufi1ezQ1eMJNxd59S2MOe7ryGD3QSq0xpHcrhvRuxdSxA/H84o0AgFMOHIY/vOCNZJLx0JeOxtINO5L4CoYUMZla/Tj6wRPmSoQT9h+aaD1NDdm793+77GiP1aGeqMtv7Y5SaWtuUObnuf3Cdkwc1gevr97qW6v5kmn74I5/LcfGHR1o1ZzPMKRPK4b0qe3Z09XMoF4t2LB9T6XFyCT9ejR5/qZXTzNu+fSh+PLd8wp5zyqNk/qlHqlLBeGmR3ODMj/PtP2GIJcj7CWxsTY25PDJD47CTU8vqUunZS3ywGVHeearGIp8aureaMzl8Mn29LOfnnTAMDz5/47Fqys3p16XIZi6VxCtTf4RxICezdi0o0M7xE5nlTJD9vjTF47w/MYj+rVhRIoO12qmIUc4f2r5fA3D+7ZheF/zW1SaulUQzQ05dHTn0aO5seB3cBqL+y85ErOXbgqNoOiy07g2ZNBuaginlKUwDYZ6oG4VxONfORZ/m78a/W2b6nfPPhAftOcs7D2wJ/YeGL64/Hh7BvZ7W3anJ6jBYDBUiLpVEKMH9sClx+1b2P7U1L0jl3H2lBF4Z+MOnHfYaBw+bmAiCe4MBoMhK9StgkiCpoYcvnaytVJcPadfMBgMtYnxrhoMBoNBilEQBoPBYJBiFITBYDAYpBgFYTAYDAYpmVMQRHQKEb1FRIuJ6MpKy2MwGAz1SqYUBBE1APg/AKcCmARrnepJlZXKYDAY6pNMKQgAhwFYzMxLmbkDwN0AzqqwTAaDwVCXZE1BjACw0rW9yt5XgIhmENEcIpqzfv36sgpnMBgM9UTWJsrJkhp5liph5lsA3AIARLSeiN6JWdcgABtiXps2WZUtq3IB2ZUtq3IB2ZUtq3IB2ZUtqlxaqSOypiBWAXAv6TYSwGrVycw8OG5FRDSHmdvjXp8mWZUtq3IB2ZUtq3IB2ZUtq3IB2ZUtLbmyZmJ6CcB4IhpLRM0AzgXwYIVlMhgMhrokUyMIZu4iossA/ANAA4Dbmfn1CotlMBgMdUmmFAQAMPMjAB4pQ1W3lKGOuGRVtqzKBWRXtqzKBWRXtqzKBWRXtlTkImYOP8tgMBgMdUfWfBAGg8FgyAh1qSAqmc6DiG4nonVEtMC1bwARPUZEb9t/+9v7iYh+Zss5n4gOSVm2UUT0FBG9QUSvE9GXsyAfEbUS0YtE9Kot1/X2/rFENNuW6x47sAFE1GJvL7aPj0lDLpd8DUQ0l4geyphcy4noNSKaR0Rz7H1Zedb6EdFfiOhN+3k7otKyEdEE+145/7YS0RWVlssl33/Zz/8CIrrLfi/SfdaYua7+wXJ+LwEwDkAzgFcBTCpj/ccAOATAAte+7wO40v58JYD/tT+fBuDvsOaHHA5gdsqyDQdwiP25N4BFsFKeVFQ+u/xe9ucmALPt+v4E4Fx7/68AfNH+fAmAX9mfzwVwT8r37SsA/gjgIXs7K3ItBzBI2JeVZ+0OAJ+zPzcD6JcV2ew6GwC8B2u+QMXlgjVheBmANtczdmHaz1qqNzmL/wAcAeAfru1vAvhmmWUYA6+CeAvAcPvzcABv2Z9vBnCe7LwyyfkAgBOzJB+AHgBeATAV1sSgRvF3hRUFd4T9udE+j1KSZySAJwAcD+Ahu7GouFx2HcvhVxAV/y0B9LEbO8qabK46TgLwfFbkQjHLxAD72XkIwMlpP2v1aGIKTedRAYYy8xoAsP8OsfdXTFZ7SDoFVm+94vLZZpx5ANYBeAzWKHAzM3dJ6i7IZR/fAmBgGnIB+AmArwPI29sDMyIXYGUh+CcRvUxEM+x9Ff8tYY3e1wP4jW2a+zUR9cyIbA7nArjL/lxxuZj5XQA/BLACwBpYz87LSPlZq0cFEZrOI0NURFYi6gXgXgBXMPPWoFMl+1KRj5m7mXkyrB77YQD2D6i7LHIR0RkA1jHzy+7dlZbLxVHMfAis7MiXEtExAeeWU7ZGWGbWm5h5CoAdsEw3Ksp632w7/pkA/hx2qmRfKnLZfo+zAIwFsBeAnrB+V1X9ichWjwoiUjqPMrGWiIYDgP13nb2/7LISURMs5XAnM9+XNfmYeTOAp2HZfPsRkTOXx113QS77eF8Am1IQ5ygAZxLRcliZh4+HNaKotFwAAGZebf9dB+B+WIo1C7/lKgCrmHm2vf0XWAojC7IBVsP7CjOvtbezINd0AMuYeT0zdwK4D8CRSPlZq0cFkcV0Hg8CuMD+fAEs27+z/zN2tMThALY4Q900ICICcBuAN5j5xqzIR0SDiaif/bkN1svyBoCnAHxcIZcj78cBPMm2MTZJmPmbzDySmcfAeo6eZOZPVVouACCinkTU2/kMy6a+ABl41pj5PQAriWiCvesEAAuzIJvNeSial5z6Ky3XCgCHE1EP+z117lm6z1qajp6s/oMVfbAIlh37W2Wu+y5YNsROWFr+s7Bsg08AeNv+O8A+l2AtoLQEwGsA2lOW7WhYw9D5AObZ/06rtHwADgYw15ZrAYBr7P3jALwIYDEsc0CLvb/V3l5sHx9Xht91GopRTBWXy5bhVfvf685zXunf0iXfZABz7N/0rwD6Z0E2WEEQGwH0de2ruFx2fdcDeNN+B34PoCXtZ83MpDYYDAaDlHo0MRkMBoNBA6MgDAaDwSDFKAiDwWAwSDEKwmAwGAxSjIIwGAwGgxSjIAx1CRF1C5k7A7P6EtHFRPSZBOpdTkSDYlx3MhFdR0T9iagcC2oZDNlbUc5gKBO72ErdoQUz/ypNYTT4EKxJUccAeL7CshjqBKMgDAYXdtqMewAcZ+86n5kXE9F1ALYz8w+J6HIAFwPoArCQmc8logEAboc1cWkngBnMPJ+IBsKaHDkY1oQlctX1HwAuh5XuejaAS5i5W5DnHFgZh8fBysUzFMBWIprKzGemcQ8MBgdjYjLUK22Ciekc17GtzHwYgF/Ayq0kciWAKcx8MCxFAVizXOfa+64C8Dt7/7UAnmMrKd2DAEYDABHtD+AcWAn1JgPoBvApsSJmvgfF9UMOgjWLdopRDoZyYEYQhnolyMR0l+vvjyXH5wO4k4j+CitNBGClKfkYADDzk0Q0kIj6wjIJfdTe/zARvW+ffwKAQwG8ZKXWQRuKSeBExsNK5wAAPZh5m8b3MxhKxigIg8EPKz47nA6r4T8TwNVEdACC0yvLyiAAdzDzN4MEIWup0EEAGoloIYDh9roYX2LmZ4O/hsFQGsbEZDD4Ocf199/uA0SUAzCKmZ+CtVBQPwC9AMyCbSIiomkANrC1loZ7/6mwktIBVtK3jxPREPvYACLaWxSEmdsBPAzL//B9WEn3JhvlYCgHZgRhqFfa7J64w6PM7IS6thDRbFgdqPOE6xoA/ME2HxGAHzPzZtuJ/Rsimg/LSe2kWr4ewF1E9AqAZ2ClbQYzLySib8Na8S0HK7vvpQDekch6CCxn9iUAbpQcNxhSwWRzNRhc2FFM7cy8odKyGAyVxpiYDAaDwSDFjCAMBoPBIMWMIAwGg8EgxSgIg8FgMEgxCsJgMBgMUoyCMBgMBoMUoyAMBoPBIMUoCIPBYDBI+f/nLl5WlqjMUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10d0f46a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "state = env.reset()\n",
    "for t in range(1000):\n",
    "    action, _ = policy.act(state)\n",
    "    env.render()\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        break \n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}