hill-climbing/Hill_Climbing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hill Climbing\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train hill climbing with adaptive noise scaling with OpenAI Gym's Cartpole environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define the Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "observation space: Box(4,)\n",
      "action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "\n",
    "class Policy():\n",
    "    def __init__(self, s_size=4, a_size=2):\n",
    "        self.w = 1e-4*np.random.rand(s_size, a_size)  # weights for simple linear policy: state_space x action_space\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = np.dot(state, self.w)\n",
    "        return np.exp(x)/sum(np.exp(x))\n",
    "    \n",
    "    def act(self, state):\n",
    "        probs = self.forward(state)\n",
    "        #action = np.random.choice(2, p=probs) # option 1: stochastic policy\n",
    "        action = np.argmax(probs)              # option 2: deterministic policy\n",
    "        return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with Stochastic Policy Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Episode 100\tAverage Score: 175.24\n",
      "Environment solved in 13 episodes!\tAverage Score: 196.21\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "policy = Policy()\n",
    "\n",
    "def hill_climbing(n_episodes=1000, max_t=1000, gamma=1.0, print_every=100, noise_scale=1e-2):\n",
    "    \"\"\"Implementation of hill climbing with adaptive noise scaling.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        noise_scale (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_R = -np.Inf\n",
    "    best_w = policy.w\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        rewards = []\n",
    "        state = env.reset()\n",
    "        for t in range(max_t):\n",
    "            action = policy.act(state)\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            rewards.append(reward)\n",
    "            if done:\n",
    "                break \n",
    "        scores_deque.append(sum(rewards))\n",
    "        scores.append(sum(rewards))\n",
    "\n",
    "        discounts = [gamma**i for i in range(len(rewards)+1)]\n",
    "        R = sum([a*b for a,b in zip(discounts, rewards)])\n",
    "\n",
    "        if R >= best_R: # found better weights\n",
    "            best_R = R\n",
    "            best_w = policy.w\n",
    "            noise_scale = max(1e-3, noise_scale / 2)\n",
    "            policy.w += noise_scale * np.random.rand(*policy.w.shape) \n",
    "        else: # did not find better weights\n",
    "            noise_scale = min(2, noise_scale * 2)\n",
    "            policy.w = best_w + noise_scale * np.random.rand(*policy.w.shape)\n",
    "\n",
    "        if i_episode % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "        if np.mean(scores_deque)>=195.0:\n",
    "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_deque)))\n",
    "            policy.w = best_w\n",
    "            break\n",
    "        \n",
    "    return scores\n",
    "            \n",
    "scores = hill_climbing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot the Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmUXGd55/HvU0sv2mWrJcuyhGywDWaJIB1ChiUGE4IJgwOTwTgkdhwmggzrDDlgyJwAOYdzMoQl4SRjENhjMyEGglg8iUNwDINhiB1k7JGNN2xjxxItqSXZWlpSd1XdZ/6491bfrq5Wd7V8l+77+5zTR1W3qrrfcvm8Tz3v8y7m7oiIiHSq5N0AEREpJgUIERHpSgFCRES6UoAQEZGuFCBERKQrBQgREelKAUJERLpSgBARka4UIEREpKta3g04FWvWrPHNmzfn3QwRkQXljjvu2O/uQ7M9b0EHiM2bN7Njx468myEisqCY2WNzeZ6GmEREpCsFCBER6UoBQkREulKAEBGRrhQgRESkq9QChJltNLPvmtm9ZvYTM3t3dP00M7vZzH4a/bs6um5m9mkze8jMdprZC9Jqm4iIzC7NDKIJvNfdLwBeBLzdzC4ArgJucfdzgVui+wAXA+dGP1uBq1Nsm4iIzCK1dRDuPgKMRLePmNl9wAbgEuDC6GnXA/8HeH90/QsenoF6m5mtMrP10e9Z1H740H7OWDnAOUPL5vyaYxNNrvvho5yYaKXYMhEpqvPOWM5rn3dmqn8jk4VyZrYZeD5wO7Au0envAdZFtzcAjydetiu6NiVAmNlWwgyDTZs2pdbmLL1v+07+3dNP52O/9Qtzfs3tjxzkY996AACztFomIkX12uedufADhJktA7YD73H3w5bozdzdzcx7+X3uvg3YBjA8PNzTa4tqvBlwvBH0+Jowc7jpXS/lgjNXpNEsESm5VGcxmVmdMDh80d2/Fl3ea2bro8fXA/ui67uBjYmXnxVdW/RagdNo9hYgJlphbKxXlT6ISDrSnMVkwDXAfe7+ycRDNwJXRLevAL6ZuH55NJvpRcChMtQfABqtgIlWbwGiGT2/VtVMZRFJR5pDTC8Gfhe428zuiq59EPgz4Ctm9hbgMeCN0WM3Aa8BHgKOAVem2LZCaQVOo+cAEWYQtYoyCBFJR5qzmH4AzNR7XdTl+Q68Pa32FFkzcMZ7HGJqBOHz68ogRCQl6l0K4JQyCNUgRCQlpQwQh080uPXBUQ4cHc+7Kbg7rcCZ6DWDiAJKvVLKj1BEMlDK3uWR0TEuv/Zf+X+7nsy7KTSDMBPoOYMIlEGISLpKGSDiwm6jlf8yilY7QPTWlngWk2oQIpKWUvYucafaLECAiDOH3oeYtA5CRNJVygBRjTKIZtBbp5yGOIPodR1EoxVQrRimfTZEJCWlDBDxt+4iZBBxLaHXDKIZuNZAiEiqShkg4tXHRcogei1SN1qB6g8ikqpS9jD1AhWp51uDaLZcM5hEJFWlDBDtDKLHb+1piDOIZuAEwdwDVjMIqGkNhIikqJQ9TPzNu9lDh5yWZBsaPQx5NVquGUwikqpSBoh49XERhpiShfJehpmaqkGISMpK2cO0M4gCDDElC+W9BKxGoBqEiKSrnAEiLlIXYIipFcwvg2g0A+3DJCKpKmUPY2bUKlaQDCJRg+ihPU1lECKSslIGCAhXUxeiSJ0YVurlTIhGK9BpciKSqjSPHL3WzPaZ2T2Ja182s7uin0fjk+bMbLOZHU889pm02hWrVysFWUmdrEH0UqT29noOEZE0pHnk6HXAXwFfiC+4+6XxbTP7BHAo8fyH3X1Liu2Zola1Qq2khh5nMQWBhphEJFWpZRDufitwsNtjFu4w90bghrT+/mxqlUoxprnOswYRroPQEJOIpCevHualwF53/2ni2tlmdqeZfc/MXpp2A+rVghSpk+sgeipSax2EiKQrzSGmk7mMqdnDCLDJ3Q+Y2S8C3zCzZ7v74c4XmtlWYCvApk2b5t2AcIgp/wyilRjm6m2hnHZzFZF0Zf4V1MxqwBuAL8fX3H3c3Q9Et+8AHgbO6/Z6d9/m7sPuPjw0NDTvdtQrlZ53UE3D1CGmHhbKaSW1iKQsjx7mlcD97r4rvmBmQ2ZWjW6fA5wLPJJmI2pVK8YspnlutdHQbq4ikrI0p7neAPwLcL6Z7TKzt0QPvYnpxemXATujaa9fBd7m7l0L3E+VWqVSiFlM814o19JuriKSrtRqEO5+2QzXf6/Lte3A9rTa0k29aoWYxTTfGkQj0G6uIpKu0n4FDVdSFyuD6GkWU0vrIEQkXaUNELWirKRuzXeIyTXEJCKpKm0PUy/INNfmfHdzDQINMYlIqkobIGqVSiEWyrVOZS8mTXMVkRSVtocpSpG6MY9pru4ebfdd2o9PRDJQ2h6mKNNc4836qhVjYo4BKw4q2s1VRNJU3gBRlIVyUYAYrFfnnEHEgU0ZhIikqbQ9TL1aoVGIDCKgVjH6anPf+qOdQahILSIpKm2ACI8cLUAG0XKqFaNetblnEFEg0WZ9IpKm8gaIanHOg+g1g4iHpTTEJCJpKm0PUyvISupWEGcQlTmvpI4DiYaYRCRN5Q0QVaNVgAwi3ra7r1rpYYgprkGU9uMTkQyUtocpTpE6zCB6G2LSLCYRSV9pe5jCFKmjGkRvQ0xaByEi6StvgKhWaAaOe75BohWtiO6rVmg0u7flRKPFiUarfT/ONJRBiEiaStvDxN++896wr9EK10HUaxXGZ8gg3r99J++84c7Ea+JZTMogRCQ9qR0YVHTxt+9w07v82tGuQVQrNGYoUj924NiUGVfxOoi6tvsWkRSleeTotWa2z8zuSVz7sJntNrO7op/XJB77gJk9ZGYPmNmvp9WuWDxFNO9CdbNdpLYZi9Rj403GG8GU14AyCBFJV5pfQa8DXt3l+qfcfUv0cxOAmV1AeFb1s6PX/A8zS/V7fbwKOe9CdSvwyWmuJwsQzenbgmsdhIikKbUA4e63Agfn+PRLgC+5+7i7/wx4CHhhWm2D5BBTvhlEoxW0F8rNNMQ0NtGaskZC6yBEJAt59DDvMLOd0RDU6ujaBuDxxHN2RddSE2cQjZyL1K14mmutewbh7lEGMTmLqb0OQjUIEUlR1j3M1cDTgS3ACPCJXn+BmW01sx1mtmN0dHTeDYkziLxXU4cH/9iMK6nHmwHNwDuGmLSbq4ikL9MA4e573b3l7gHwOSaHkXYDGxNPPSu61u13bHP3YXcfHhoamndbClOkbgXUKhX6ZsggxsabAF1rEFoHISJpyrSHMbP1ibuvB+IZTjcCbzKzfjM7GzgX+Nc02xIPzxShSN2e5tqlLWPjrfbz4npJ3GZt9y0iaUptHYSZ3QBcCKwxs13Ah4ALzWwL4MCjwFsB3P0nZvYV4F6gCbzd3Vvdfu9TJZ4iOtf9j9KS3GqjFXg7YMSORhkEwEQrCLcpD+JZTMogRCQ9qQUId7+sy+VrTvL8jwIfTas9neIhprxXUrfiGkQt7OzDWU2TM3zHJiYDxHgjYElfIoNQDUJEUlTar6CTQ0z5T3OtVSrtgNVZh0hmEHEdoqGV1CKSgdL2MJNDTPlnEPF238C0mUxjySGm6DGtpBaRLJQ2QMTj93mfKpec5grTayJjUzKIsCzT3otJNQgRSVFpe5iibLXRbE0WqWF6BnF0fLJWPznEpHUQIpK+0gaI+gzf2LMWbtZXmVKkTuqaQQTh9hxmChAikp7SBoh4Kmkr91lMwZQMYvwkNYhkBqE1ECKSttIGiMmV1MXYaqO/nUFMbc9Ms5hUfxCRtJW2lynKNNfOGsRJh5gakyupNYNJRNJW3gBRLUaRuhXVINrrILoUqeP6RLIGoZ1cRSRtpe1l2t/Yc5/mGtYg2usgumQQpy/tCx9L1CA0g0lE0lbaAFGEaa5B4AQeZjMzTXMdm2iyekkYIOIaRFM1CBHJQGl7mVoBprm2V0RXkkXq6VttnL5saoBoBKpBiEj6ShsgirBZXzzFNqxBzLzVxmlL4wAxuZJa+zCJSNpK28sUYRZTs71tt51koVyrPcSUrEEogxCRtJU4QOS/WV9c/6gmt9pItMfdGZtosnygRl+1MmUdhE6TE5G0lbaXqVSMiuW7kjpZg+i2m+uxiRbusLS/Rn+tMmUdRF0rqUUkZaUNEMCU09nykKxBdNvNNV4kt7S/Rn+9MnUdhIaYRCRlqQUIM7vWzPaZ2T2Ja39uZveb2U4z+7qZrYqubzaz42Z2V/TzmbTalVSvWK7TXOMaRDjNdfpCuXibjWX91Y4hJtc0VxFJXZq9zHXAqzuu3Qw8x92fBzwIfCDx2MPuviX6eVuK7WqrVSv5Fqlbk0NMtWqFinVmEGHGsLSvRn+9mjgwSOsgRCR9qfUy7n4rcLDj2rfdPd5c6DbgrLT+/lzUq5brZn3NYLJIHbanMkMGEdUg2tNctZuriKQvz6+hvw/8Y+L+2WZ2p5l9z8xeOtOLzGyrme0wsx2jo6On1IBaJd8MIq5BxNlAX60yZauNKTWIWkW7uYpIpnLpZczsj4Em8MXo0giwyd2fD/xX4G/NbEW317r7NncfdvfhoaGhU2pHrZpvDSIeTooziL6ODGJsIhkgqpOzmLSSWkQykHmAMLPfA14LvNndHcDdx939QHT7DuBh4Ly021KvVnIdYmolprlCmEEkaxDJIaZkdtFoajdXEUlfpr2Mmb0aeB/wOnc/lrg+ZGbV6PY5wLnAI2m3p1axnFdST69BJBfuTQ4xVafUIBqBdnMVkfTV0vrFZnYDcCGwxsx2AR8inLXUD9wcnad8WzRj6WXAn5pZAwiAt7n7wa6/+ClU6+iQs9ZZg6hXraNInZzFlFwop3UQIpK+1AKEu1/W5fI1Mzx3O7A9rbbMpFYxWjkulGt21iBq1WlF6iV9VSoVC2sQzcmV1BpiEpG0lbqXqVUt191cm501iI4MYmy8ydL+WvTYZAG7EQQaYhKR1JU6QNQrlVzPg2gXqRPTXDuL1MuiADFlqw2tpBaRDJS6lynKNNfaDAvlwgyiCtBeB+Hu0TTXUn90IpKBUvcytYJMc63OMM11bLzF0r4og4hqEPGwlHZzFZG0zTlAmNlLzOzK6PaQmZ2dXrOyUS/INNcpGUQiozk6Hp4FAWHwaAXOiUY4zKQMQkTSNqdexsw+BLyfyc316sDfpNWorOQ9xDStBlGtMBHVGSBcSR0XqeMzq+MN/FSkFpG0zfVr6OuB1wFjAO7+c2B5Wo3KSt7nQXTWIMIhpqkL5ToDRLy6Wpv1iUja5hogJqJtMRzAzJam16Ts5H0eRGcNYvpCueQsprBYfSzan0lDTCKStrn2Ml8xs88Cq8zsD4B/Bj6XXrOykft5EO0hpulF6mYr4EQjSBSpp2YQGmISkbTNaSW1u3/czH4NOAycD/yJu9+cassyUKvku1BucrO+eKuNyWmuYxPRNhvRNNe+aTUIZRAikq5ZA0S0id4/u/vLCU+EWzTyXkk9bbvvxI6tY4mdXCGc5pq8riEmEUnbrL2Mu7eAwMxWZtCeTNWKspI6eR5EK1wMlzwsCLoMMalILSIpm+tmfUeBu83sZqKZTADu/q5UWpWRes7TXKfVIKoV3MPAcXRaBhEPMSmDEJFszDVAfC36WVRq1QrNXHdz7ahBREGg0fJ2raG9Wd+0AKEMQkTSNdci9fVm1sfkKW8PuHsjvWZlo14xGi3H3YnOp8hUvNV4PFoUF54nmkE7g5jciyn8Nz4joq7tvkUkZXMKEGZ2IXA98ChgwEYzu8Ldb02vaemLh2laOZ3x3AycWsXawSnOEiZawfQidV0ZhIhka65DTJ8AXuXuDwCY2XnADcAvptWwLMSdbLg7avZ/vzMw9UW3J1oBYxPdi9Txda2DEJG0zXWcoh4HBwB3f5BwP6aTMrNrzWyfmd2TuHaamd1sZj+N/l0dXTcz+7SZPWRmO83sBb2+mV7FwzR5zWRqdJwMF2cQjWbA3bsOsby/xsrB+pTH2hmEhphEJGVz7WV2mNnnzezC6OdzwI45vO464NUd164CbnH3c4FbovsAFwPnRj9bgavn2LZ5a2cQOc1kagVBew0ETNYgxiaafPvevbzygnXta5PrILRQTkSyMdde5g+Be4F3RT/3RtdOKqpRHOy4fAlhPYPo399MXP+Ch24j3NZj/RzbNy/x+oO8Fss1A58yVNQXdfq3PrifQ8cbXPycM9qPaasNEcnaXGsQNeAv3f2T0F5d3T/Pv7nO3Uei23uAddHtDcDjieftiq6NJK5hZlsJMww2bdo0zyaE4iJ1XlNdmy2fmkFEQeCbd+1maV+Vl5031H6sswahdRAikra59jK3AIOJ+4OEG/adkuQOsT28Zpu7D7v78NDQ0OwvOIl2BpHTEFM4iylRg4g6/fv3HOGiZ61joD5ZOTcz+qqVRA1CGYSIpGuuAWLA3Y/Gd6LbS+b5N/fGQ0fRv/ui67uBjYnnnRVdS008jp9XkbqzBhEXogFe89zpo2v9tUpiiEkZhIika669zFhyVpGZDQPH5/k3bwSuiG5fAXwzcf3yaDbTi4BDiaGoVCSnueah2THNNe70l/RVufD86dlRf73CiUZ0yJBqECKSsrnWIN4D/J2Z/Ty6vx64dLYXmdkNwIXAGjPbBXwI+DPC8yXeAjwGvDF6+k3Aa4CHgGPAlXNs27zVcp7m2mz5lKGieIjpFc9cO2V4KdafWKyhldQikraTBggz+yXgcXf/kZk9E3gr8AbgW8DPZvvl7n7ZDA9d1OW5Drx91hY/heo5T3NtBk410dGvXdHPQL3Cfxze2PX5/YkhKGUQIpK22b6GfhaYiG7/CvBB4K+BJ4BtKbYrE3nPYmoFwZTpqmuW9XPPh3+dXz2ve/E9WaNQDUJE0jbbEFPV3eN1DJcC29x9O7DdzO5Kt2npi89UaOSaQUzNBE42fbV/SoBQBiEi6Zrta2jVzOIgchHwncRjc61fFFY7g8grQHTUIGYT1yCqiQ3+RETSMlsnfwPwPTPbTzhr6fsAZvYM4FDKbUtdtb2SOq8hpukZxMnEO7pqDYSIZOGkAcLdP2pmtxDOWvp2VEiGMPN4Z9qNS1v+ReqApfW5J2LxLCfVH0QkC7P2TtG+SJ3XHkynOdmKp7nmttXGfDMI1R9EJAOl/ioaZxC5FannWYPQVt8ikoVS9zT5T3P1njr7eBaTZjCJSBbKHSByn+YaUO2hs++rqQYhItkpdU9Tz3uaa9DrEJNqECKSnVIHiMnN+opxHsRs4hqE9mESkSyUuqeZPJM6ryNHvafOXhmEiGSp1AFi8kzqMIO4avtO/vR/35vZ359vDUKnyYlIFhb8dhmnotpxJvXtPzvI0PL5nqTau/nWIOpaSS0iGSj1V9HOIvXokXFONFqZ/f1WrzWI6IwIDTGJSBZKHSDCTe/CoZ5jE02OjjczDRDNwHuastqvaa4ikqHMh5jM7Hzgy4lL5wB/AqwC/gAYja5/0N1vSrs99UqFRsvZfyQ89uJ4pgEimNcsJm3WJyJZyDxAuPsDwBYAM6sCu4GvEx4x+il3/3iW7alVjWYrYPToCYD2mc9Z6LUGoYVyIpKlvHuai4CH3f2xvBpQqxjNwBk9Mg7AiYlsMoggcNx721dJQ0wikqW8e5o3EZ45EXuHme00s2vNbHUWDahXKzRawWSAaGYTIBrR4rxeCs5aByEiWcotQJhZH/A64O+iS1cDTyccfhoBPjHD67aa2Q4z2zE6OtrtKT0Jh5icfVGAaLS8vS4iTa1oau28ZjFpJbWIZCDPnuZi4MfuvhfA3fe6e8vdA+BzwAu7vcjdt7n7sLsPDw0NnXIjapUKjWAygwA40Uw/QMRrL3qqQbQPDFIGISLpyzNAXEZieMnM1iceez1wTxaNqEcZRDJAHM+gDtFq9R4gdGCQiGQpl5XUZrYU+DXgrYnLHzOzLYADj3Y8lppqxWgFzujRRAaRwVTXuAZRncc6CA0xiUgWcgkQ7j4GnN5x7XfzaEuySD1Yr3K80cokQLTmMcTU3s1VGYSIZKD0X0VrVaPRCth/dJyNpw0C2SyWa85jiEnrIEQkS6XvaWqVCvuPTtBoOZtOWwJks1iunUHMa5pr6T82EclA6XuaetUYOXQcgI1RgMgkg4hrED0ulHvVBesYflomS0REpORKvd03TGYQABtXxxlEFgGi9yEmM2Pb5cNpNUlEZIrSZxDJIZ7JIaZi1iBERLJU+gCRLPhuzDJAzKMGISKSpdIHiPgb/EC9wpplfUBGC+XmUYMQEclS6XunOIMYWt7Pkr6wJJPJVhvREJOODxWRoip9gIg3yxta1t+eRppNBtH7Zn0iIlkqfYCIawBrlw9QqRj9tUomW343VIMQkYIrfYCoVyaHmAAG6tVMDg1SDUJEiq70vVP8DT4OEIP1aiYrqTXNVUSKrvQBIlmkhnA2UzYrqTXEJCLFVvoAUUsUqSEaYiroSmoRkSwpQEzLIKqZZBBxDUJnO4hIUZW+d6p3qUGMZ1iD0DRXESmq0geIlYN1ButV1ixTDUJEJCm33VzN7FHgCNACmu4+bGanAV8GNhMeO/pGd38izXa8+ZefxiueubZ9GM9gX7Y1CGUQIlJUeWcQL3f3Le4e72F9FXCLu58L3BLdT9VgX5Vzhpa17w/UMqpBtMJhrLpqECJSUEXrnS4Bro9uXw/8ZtYNGOjLaB1EnEFoiElECirPAOHAt83sDjPbGl1b5+4j0e09wLqsGzVQ0zRXERHI90S5l7j7bjNbC9xsZvcnH3R3NzPvfFEUTLYCbNq06Slv1GBfJZMAoc36RKTocssg3H139O8+4OvAC4G9ZrYeIPp3X5fXbXP3YXcfHhoaesrbNVCr0gycRivdYabJ7b6LNsonIhLKpXcys6Vmtjy+DbwKuAe4EbgietoVwDezbttgXxVI/1S5ZhBgBhVlECJSUHkNMa0Dvm5mcRv+1t2/ZWY/Ar5iZm8BHgPemHXD+uthgDjeaLF8oJ7a32kGrvqDiBRaLgHC3R8BfqHL9QPARdm3aNJgFCDSXk3dClzbbIhIoamH6jBQj06VS3uIqaUMQkSKTQGiQ5xBZFGD0BoIESkyBYgOA3ENIuVT5VSDEJGiU4DoEAeIE810axBHTjRZ1p/nMhQRkZNTgOjQrkGknEEcODrO6dEOsiIiRaQA0aE9i6mZdoCY4PSlfan+DRGRU6EA0SGrGsSBMWUQIlJsChAdBhML5dISBM7BsQnWLFMGISLFpQDRoV2kTnGh3JPHGwQOp2mISUQKTAGiQ38t/YVyB46OA2iISUQKTQGiQ6Vi9NcqjJ9igLj1wVFufXC062P7j04AsEYZhIgUmCbidzHYd+rHjv75Pz1AxeBl503fkvzgWBgglEGISJEpg+hisN79VLkH9hzhO/fvndPvGDl0gr2Hx7s+dmAsHmJSBiEixaUA0cVAvcrxLkXqv/ruQ/zR3+2c9fUTzYD9R8cZPTrePjkuaf/RCcxg9RIFCBEpLgWILgZmyCBGnjzOwbEJjk00T/r6vYdPAOGW3nG2kHTg6Dirl/TpuFERKTQFiC4G6t3PpR45FHb8P3/y+ElfvycKEAD7ugwzHRzTKmoRKT4FiC661SBagbczg11PnDxAxIEEJrOJpANHJ1R/EJHCyzxAmNlGM/uumd1rZj8xs3dH1z9sZrvN7K7o5zVZty0W1iCmBogDR8dpRvWE3bNlEIcmH+9WqN4/Ns7pSzWDSUSKLY9prk3gve7+YzNbDtxhZjdHj33K3T+eQ5umCDOIqUXqZFYw2xDTz588Ef6OZksZhIgsWJkHCHcfAUai20fM7D5gQ9btOJn+emXaZn3JALF7liGmPYdOsGH1IE8ea7DvyNQA0WgFHDreUAYhIoWXaw3CzDYDzwdujy69w8x2mtm1ZrZ6htdsNbMdZrZjdLT7SuVTNVivTtvuOx42esbaZbMOMY0cPsH6lQOsW9E/bYjpifYiOWUQIlJsuQUIM1sGbAfe4+6HgauBpwNbCDOMT3R7nbtvc/dhdx8eGpq+SvmpMFCvTs8gDp+gr1rhuRtW8vMnpw8bJe05dJwzVgywbsXAtCGm9jYbChAiUnC5BAgzqxMGhy+6+9cA3H2vu7fcPQA+B7wwj7ZBVINoBrhPLnLbc+gEZ6wc4KzVg+w5fIJmq/tur41WwL4j4zNmEPG6iNM0xCQiBZfHLCYDrgHuc/dPJq6vTzzt9cA9WbctNlCv0AqcRmsyQIw8GQaIDasGaQU+Za1D0uiRcdxh/apB1i4f4MDYOI1EMDlwVENMIrIw5DGL6cXA7wJ3m9ld0bUPApeZ2RbAgUeBt+bQNiBxJkSzRV+0/ffI4eO8YNNqzlw1CISF6rNWL5n22riYfcbKAdzBHfYfHWf9yvB1B8binVyVQYhIseUxi+kHQLc9Jm7Kui0zaQeIiRYrBuoEgbP3UNjJb1gddvQ/P9S9UL0nChDrVw4QROsm9h5OBIij49QqxopBbaQrIsWmXqqLwY5T5Q4em2CiFbA+GmKCmae6jkSBY/2KQZqtOEBMDkfFayDCkTYRkeJSgOhioONc6j2JYaOBepU1y/pmnOo6cihcJLdisMbaVjiMtC8ZIMbGVaAWkQVBezF1MdgX/meJ92MaSQwbAZy5apDdM0x13XPoBOtXDWBmnL60n2rFpsxk2n90QlNcRWRBUIDoYqA2NYOIh43OiALEhlWD7H7iWNfXjhw63g4k1YoxtKx/yhCTdnIVkYVCAaKLgb64BjGZQdQq1p55tGHVILufPD5lnURsz6ETnLFisH1/3Yp+9h6ZzCAOHB3XUaMisiAoQHQRZxAnEjWIdSsGqEQH/GxYPciJRtA+WzrWCpy90SK52NoVA+0axPGJFmMTLa2BEJEFQQGii8G+qbOYRg4d58xVk51+vBaic8uN/dERo2ckAkS4mjp8Xvssag0xicgCoADRxUA9/M+SnMV0xsrJYaP2VNcnp9Yh4mJ2MpisWz7AE8cajDdb7YxDO7mKyEKgANHFkno4+/ffDh7D3Rk5dGLKsNFZ0WK5zpPl4h1fp9YgwtftOzzO/SNHAG2zISILg9ZBdLFySZ1XPmsdn/3ew5yzZinjzYA6Lh5CAAAIKUlEQVQzVkwGiJWDdZb2Vfnaj3ezYdUgL3/mWn706EGu/+FjAB01iDBb+Of79vKxbz3Aczas4Nlnrsz2DYmIzIMCxAw+fdkW3rTtNt63fScwtdM3M977qvP57K0P84df/DHVitEKnJWDdf7wwqezOlFjiDOIP/37e9mwapBrf++X2vs7iYgUmQLEDJb01bjmil/iDVf/Xx4/eHxK4Rng919yNpf/ytP4/kP7+d4DowxvXs0rn7WuvQo7FgeIFQN1rrvyhaxdPvX3iIgUlQLESQwt7+cLv//L/K9/eYwLzlwx7fFatcLLz1/Ly89fO+PvWL2kzjtf8QwuetY6nrF2WZrNFRF5Slm3xV4LxfDwsO/YsSPvZoiILChmdoe7D8/2PA2Gi4hIVwoQIiLSVeEChJm92sweMLOHzOyqvNsjIlJWhQoQZlYF/hq4GLiA8BjSC/JtlYhIORUqQAAvBB5y90fcfQL4EnBJzm0SESmlogWIDcDjifu7omsiIpKxogWIWZnZVjPbYWY7RkdH826OiMiiVbQAsRvYmLh/VnStzd23ufuwuw8PDQ1l2jgRkTIp1EI5M6sBDwIXEQaGHwG/7e4/meH5o8BjPf6ZNcD+U2lnQS3W9wWL973pfS08i+W9Pc3dZ/2GXaitNty9aWbvAP4JqALXzhQcouf3nEKY2Y65rCBcaBbr+4LF+970vhaexfzeuilUgABw95uAm/Juh4hI2RWtBiEiIgVRxgCxLe8GpGSxvi9YvO9N72vhWczvbZpCFalFRKQ4yphBiIjIHJQmQCymTQDNbKOZfdfM7jWzn5jZu6Prp5nZzWb20+jf1Xm3dT7MrGpmd5rZ30f3zzaz26PP7stm1jfb7ygaM1tlZl81s/vN7D4z+5VF9Hn9l+j/w3vM7AYzG1iIn5mZXWtm+8zsnsS1rp+RhT4dvb+dZvaC/FqenlIEiEW4CWATeK+7XwC8CHh79H6uAm5x93OBW6L7C9G7gfsS9/878Cl3fwbwBPCWXFp1av4S+Ja7PxP4BcL3t+A/LzPbALwLGHb35xBOT38TC/Mzuw54dce1mT6ji4Fzo5+twNUZtTFTpQgQLLJNAN19xN1/HN0+QtjZbCB8T9dHT7se+M18Wjh/ZnYW8BvA56P7BrwC+Gr0lAX3vsxsJfAy4BoAd59w9ydZBJ9XpAYMRgtdlwAjLMDPzN1vBQ52XJ7pM7oE+IKHbgNWmdn6bFqanbIEiEW7CaCZbQaeD9wOrHP3keihPcC6nJp1Kv4CeB8QRPdPB55092Z0fyF+dmcDo8D/jIbOPm9mS1kEn5e77wY+DvwbYWA4BNzBwv/MYjN9Rou2T0kqS4BYlMxsGbAdeI+7H04+5uH0tAU1Rc3MXgvsc/c78m7LU6wGvAC42t2fD4zRMZy0ED8vgGhM/hLCIHgmsJTpwzSLwkL9jE5FWQLErJsALjRmVicMDl90969Fl/fGaW7077682jdPLwZeZ2aPEg4DvoJw7H5VNHwBC/Oz2wXscvfbo/tfJQwYC/3zAngl8DN3H3X3BvA1ws9xoX9msZk+o0XXp3RTlgDxI+DcaGZFH2ER7cac2zRv0bj8NcB97v7JxEM3AldEt68Avpl1206Fu3/A3c9y982En9F33P3NwHeB34qethDf1x7gcTM7P7p0EXAvC/zzivwb8CIzWxL9fxm/twX9mSXM9BndCFwezWZ6EXAoMRS1aJRmoZyZvYZwfDveBPCjOTdp3szsJcD3gbuZHKv/IGEd4ivAJsJdbt/o7p1FtwXBzC4E/sjdX2tm5xBmFKcBdwK/4+7jebavV2a2hbDw3gc8AlxJ+AVtwX9eZvYR4FLC2XV3Av+JcDx+QX1mZnYDcCHhjq17gQ8B36DLZxQFw78iHE47Blzp7jvyaHeaShMgRESkN2UZYhIRkR4pQIiISFcKECIi0pUChIiIdKUAISIiXSlASCmZWcvM7kr8nHSjPDN7m5ld/hT83UfNbM08XvfrZvaRaHfRfzzVdojMReHOpBbJyHF33zLXJ7v7Z9JszBy8lHDx2UuBH+TcFikJZRAiCdE3/I+Z2d1m9q9m9ozo+ofN7I+i2++KzuLYaWZfiq6dZmbfiK7dZmbPi66fbmbfjs5L+Dxgib/1O9HfuMvMPhttS9/ZnkvN7C7CLbX/AvgccKWZLdidAGThUICQshrsGGK6NPHYIXd/LuFK2b/o8tqrgOe7+/OAt0XXPgLcGV37IPCF6PqHgB+4+7OBrxOuyMXMnkW4+vjFUSbTAt7c+Yfc/cuEu/XeE7Xp7uhvv+5U3rzIXGiIScrqZENMNyT+/VSXx3cCXzSzbxBuxQDwEuA/ALj7d6LMYQXhORBviK7/g5k9ET3/IuAXgR+FuzYwyMyb9Z1HuD0HwNLoDBCR1ClAiEznM9yO/QZhx//vgT82s+fO428YcL27f+CkTzLbQbg3UM3M7gXWR0NO73T378/j74rMmYaYRKa7NPHvvyQfMLMKsNHdvwu8H1gJLCPcPPHN0XMuBPZHZ3TcCvx2dP1iID53+hbgt8xsbfTYaWb2tM6GuPsw8A+EZy58DPhjd9+i4CBZUAYhZTUYfROPfcvd46muq81sJzAOXNbxuirwN9ExogZ82t2fNLMPA9dGrzvG5BbRHwFuMLOfAD8k3B4bd7/XzP4b8O0o6DSAtxPuGNrpBYRF6v8MfLLL4yKp0G6uIgnRYUXD7r4/77aI5E1DTCIi0pUyCBER6UoZhIiIdKUAISIiXSlAiIhIVwoQIiLSlQKEiIh0pQAhIiJd/X9rg5xASb1ARAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "state = env.reset()\n",
    "for t in range(200):\n",
    "    action = policy.act(state)\n",
    "    env.render()\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        break \n",
    "\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}