DL4D_17_Reinforcement_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing a Game with Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simulating game environments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "Ouy6OwzAkgnp",
    "outputId": "3b0beb9a-1874-4d44-d0d5-6961347e0196"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: h5py in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (2.8.0)\n",
      "Requirement already satisfied: numpy>=1.7 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from h5py) (1.15.1)\n",
      "Requirement already satisfied: six in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from h5py) (1.11.0)\n",
      "Requirement already satisfied: gym in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (0.11.0)\n",
      "Requirement already satisfied: numpy>=1.10.4 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from gym) (1.15.1)\n",
      "Requirement already satisfied: pyglet>=1.2.0 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from gym) (1.3.2)\n",
      "Requirement already satisfied: requests>=2.0 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from gym) (2.19.1)\n",
      "Requirement already satisfied: scipy in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from gym) (1.1.0)\n",
      "Requirement already satisfied: six in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from gym) (1.11.0)\n",
      "Requirement already satisfied: future in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from pyglet>=1.2.0->gym) (0.17.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from requests>=2.0->gym) (2018.8.24)\n",
      "Requirement already satisfied: idna<2.8,>=2.5 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from requests>=2.0->gym) (2.7)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from requests>=2.0->gym) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.24,>=1.21.1 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from requests>=2.0->gym) (1.23)\n",
      "Collecting package metadata: ...working... done\n",
      "Solving environment: ...working... done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!pip install h5py\n",
    "!pip install gym\n",
    "!conda install -c menpo ffmpeg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Hp6Qi5XakpWF",
    "outputId": "9c34096b-e4b5-4c6a-a387-f00f18973ebd"
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AUwyhqRgl6Pm"
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "np.random.seed(42), env.seed(42) \n",
    "nb_actions = env.action_space.n\n",
    "input_shape = (1, env.observation_space.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode concluded after 12 timesteps\n"
     ]
    }
   ],
   "source": [
    "observation = env.reset()\n",
    "for t in range(200):\n",
    "    env.render()\n",
    "    act = env.action_space.sample()\n",
    "    obs, rwrd, done, info = env.step(act)\n",
    "    if done:\n",
    "        print(\"Episode concluded after %i timesteps\" % (t+1))\n",
    "        break\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Presenting Q-learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras-rl in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (0.4.2)\n",
      "Requirement already satisfied: keras>=2.0.7 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras-rl) (2.2.4)\n",
      "Requirement already satisfied: numpy>=1.9.1 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (1.15.1)\n",
      "Requirement already satisfied: scipy>=0.14 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (1.1.0)\n",
      "Requirement already satisfied: six>=1.9.0 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (1.11.0)\n",
      "Requirement already satisfied: pyyaml in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (3.13)\n",
      "Requirement already satisfied: h5py in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (2.8.0)\n",
      "Requirement already satisfied: keras_applications>=1.0.6 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (1.0.6)\n",
      "Requirement already satisfied: keras_preprocessing>=1.0.5 in c:\\users\\john\\anaconda3\\envs\\dl4denv\\lib\\site-packages (from keras>=2.0.7->keras-rl) (1.0.5)\n"
     ]
    }
   ],
   "source": [
    "!pip install keras-rl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential \n",
    "from keras.layers import Dense, Activation \n",
    "from keras.layers import Flatten, Dropout \n",
    "from keras.optimizers import Adam\n",
    "from rl.agents.dqn import DQNAgent\n",
    "from rl.policy import EpsGreedyQPolicy\n",
    "from rl.memory import SequentialMemory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 394
    },
    "colab_type": "code",
    "id": "JDLC7tLsmEif",
    "outputId": "a7d90c14-b6ad-4cf3-aa2c-79a372ba238b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 12)                60        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 26        \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 86\n",
      "Trainable params: 86\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Flatten(input_shape=input_shape))\n",
    "model.add(Dense(12))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(nb_actions))\n",
    "model.add(Activation('linear')) \n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1307
    },
    "colab_type": "code",
    "id": "18vnnQc9mLTq",
    "outputId": "da1378dd-ca9b-4a86-8d2e-130864dacdb6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 30000 steps ...\n",
      "Interval 1 (0 steps performed)\n",
      "10000/10000 [==============================] - 21s 2ms/step - reward: 1.0000\n",
      "219 episodes - episode_reward: 45.128 [8.000, 200.000] - loss: 3.995 - mean_q: 21.590\n",
      "\n",
      "Interval 2 (10000 steps performed)\n",
      "10000/10000 [==============================] - 20s 2ms/step - reward: 1.0000: 0s - reward: 1.\n",
      "51 episodes - episode_reward: 195.588 [93.000, 200.000] - loss: 10.790 - mean_q: 52.841\n",
      "\n",
      "Interval 3 (20000 steps performed)\n",
      "10000/10000 [==============================] - 19s 2ms/step - reward: 1.0000\n",
      "done, took 59.886 seconds\n"
     ]
    }
   ],
   "source": [
    "policy = EpsGreedyQPolicy(eps=0.3)\n",
    "memory = SequentialMemory(limit=50000, \n",
    "                          window_length=1)\n",
    "\n",
    "dqn = DQNAgent(model=model, \n",
    "               nb_actions=nb_actions, \n",
    "               memory=memory, \n",
    "               nb_steps_warmup=50, \n",
    "               target_model_update=0.01, \n",
    "               policy=policy)\n",
    "\n",
    "dqn.compile(Adam(lr=0.001))\n",
    "\n",
    "training = dqn.fit(env, nb_steps=30000, \n",
    "                   visualize=False, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hHWjixFkoXtL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 1 episodes ...\n",
      "Episode 1: reward: 200.000, steps: 200\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "mon = gym.wrappers.Monitor(env, \n",
    "                           \"./gym-results\",\n",
    "                           force=True)\n",
    "mon.reset()\n",
    "dqn.test(mon, nb_episodes=1, visualize=True)\n",
    "mon.close()\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<video width=\"520\" height=\"auto\" alt=\"test\" controls>\n",
       "<source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAKZttZGF0AAACrwYF//+r3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE0NiByMjUzOCAxMjEzOTZjIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNSAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTEyIGxvb2thaGVhZF90aHJlYWRzPTIgc2xpY2VkX3RocmVhZHM9MCBucj0wIGRlY2ltYXRlPTEgaW50ZXJsYWNlZD0wIGJsdXJheV9jb21wYXQ9MCBjb25zdHJhaW5lZF9pbnRyYT0wIGJmcmFtZXM9MyBiX3B5cmFtaWQ9MiBiX2FkYXB0PTEgYl9iaWFzPTAgZGlyZWN0PTEgd2VpZ2h0Yj0xIG9wZW5fZ29wPTAgd2VpZ2h0cD0yIGtleWludD0yNTAga2V5aW50X21pbj0yNSBzY2VuZWN1dD00MCBpbnRyYV9yZWZyZXNoPTAgcmNfbG9va2FoZWFkPTQwIHJjPWNyZiBtYnRyZWU9MSBjcmY9MjMuMCBxY29tcD0wLjYwIHFwbWluPTAgcXBtYXg9NjkgcXBzdGVwPTQgaXBfcmF0aW89MS40MCBhcT0xOjEuMDAAgAAAAipliIQAK//+9nN8CmtHM5UuBXb3ZqPl0JLl+xBg+tAADU+QAAADABU6vx16OQENUgAAAwALEAHSElDyDzETFWKgSvFPBNxdwAXQBfBz5WaCw+jJAmckBCknXGyM5fPTmZZOE9b0scN9/N/xfPDiLd2gegxszh/P1piBveVxj21sTHzcWdQV7TOFN9671K8SayjKNowHfP6vpJ6igqFkbpo3WS8bCTceiHeS1P6rIbMqGpr7CPBPwQOFAVq0UI6VKrmt/+fK+bSIlD1cNwjwJtzvyKdYIUgB6mpd7egziag2P24FVm7pvJ1005FVUKHAFNfmvhxeJ+mFCEJUMKsoprp1KRwUqhsN7QrgWj2IsRkAaq/viHLwH364e9T+9XQFnSASTRHqwCe+AAKJySz96f75tEfOXYXkabpdt8VNCZgRh1DBBKeHYgrwi1XTtEHF1bzw72zy9yhmHFyYVB4BD4pOut7MLHbbMCwGGUqVvEoLMAAQx8n4nH4v4F62/+xkxgZGa8sEcMMywn5O7VfjJNtgaZ+x8//gSjDWKnH8Cz7roBB0HOjI/DC9zWvgv/qCBTCQQsLx+EQE9S6EL8OHm76hT8O36y4GN+SL0Kl1pnIbm3ERfxNF6HbsT/9K6ORfO8V0846ghgAAAwDdWhKQgydgtpijV/srASrGwSuee4SKjgvNOXaf/TcN6krDRt1Jn0HJlqiywMM0LlJwiIAfKAAAAwAAAwAAAwACewAAAL5BmiRsQn/98QAAAwKf4zwH8XAlKa4Pdpbo2SHTXT7Jy8E65/NPSYcGZzbkoJaEO8357fsbw0ZC/qYtKiIRqS/KPhAcArP9Uf0gw0UI+3E1mENi14IS2oRbjJoDToRdHRvsJ8PV6QnpuYqorj3db68ewO5F7xhA2/YlXXPxSFlNWtygAJlgUl09B5a/xAqqiAKbN4OwF0ZVAE0eYv9p6AThdWMha6YjvQalCe/qmezMNIAnapmtfSzeiuHhWipgAAAASkGeQniEfwAACG2E/0ZhZL5YEZzeebWeBeh4+40FbgpIbM40xQwS78zRPz+9WzFXImjaAAYAKVEj2vvDN+GwhbiJlAZFvWqZYAFbAAAAMgGeYXRH/wAADX35KDwE54DsMkePFXlFhJFmdXi7sUPoAAADAAADAAYv+AEEIRSqAAVsAAAALgGeY2pH/wAADYJOuVSHsLn8X/ugc4sumOIrkRBqZLAWNmAAAAMAAAcrJrYQHHEAAAB+QZpoSahBaJlMCFf//jhAAAENyhFW84CgO9Gc7+FINlERYOKMacpjSI8iL05bzwjR0gSeP4102qetgG7pILIxmYKtyjNdEOmYrGN2/Iw7kPKY6Chg+NcHL6rogcsG5/59oNHMqwd+uLLpbVyRIAAAAwABme5IaR9ZvEWvzzuXAAAAOEGehkURLCP/AAAWtVhPm3cqCvpT/Dt9EUqU0JsqjSdzzuNa+aWSpCbqAAADAA8wGFmKNLwPhAlZAAAANwGepXRH/wAAI6w5XcwcbaY4OtKiJgAWQjcIqXd/BiBgFK/BsEjj+ajxq1OAAAADAAOf2izwgdcAAAA3AZ6nakf/AAAFDz16JeLHGABEgdf1PrRmn9NvKFANO/Ct/ubh4i9CDWpMAAADAAADADT+tGWBUwAAAH5BmqxJqEFsmUwIT//98QAAAwKfAwmQHwr8Hg/Ic3N6wXwchda9NJPdGpeg23S65PvC3pA4nnIHjHXHaFo8U/s4mQ4Ac189ee1J1Woj/FTWEpA/TPTyd7dgyPbTDwpLb1xAi8bzjACVepWJQqM9WcNAAAADAAAnMHtPHlr/WB4AAABKQZ7KRRUsI/8AABYsULbwEUK5pSGQWgqz7ACatY0AVtfNUz37Sp/bjJVyEt6tuu62Pk/shadn142xBQ2IAAADAADJVdC+fKqAfkEAAAA0AZ7pdEf/AAAjw0Cql+EhcVMAEtU8UAmZe6Ti3T8TjI0I6sp7N2XeF2rNoAAPtBIH8UAh4AAAADEBnutqR/8AACOxzAGNJyn1yBUD9XBLAAXE/rl+5aFtkYmDBGboyYNTnZQxgByf4ebQAAAAgEGa8EmoQWyZTAhP//3xAAADAqEAx1qLgGZOnBI5LinDisipH7okkBq4PUTgVNfTj4atRc6fKC21/HS1iJSjplmIttO7zH5zdfyBhWtQMweuHxleY1vsmvvWb3xCexAGicGIDok83hoqjY1tAuMLEVp/mvOU6/ATL/Cd+zGBHECBAAAAH0GfDkUVLCP/AAAWtTmSPnlnVzS8pKr/+pBEk72C0/0AAAA3AZ8tdEf/AAAjw0JXukUEYcd/1VfO87d4ATVi/NcGFbeW4/ikSnMXGjFXuTm72rCLQjFqUwAyVQAAABQBny9qR/8AACPHv4zwHEP9MxvaqAAAAExBmzRJqEFsmUwIT//98QAAAwKfAsJJFmIuPBkkhhmVsrEPn8FGaC4iTepKRAL9EKIp3SWqtEug955XKtY2mkn/O4YoLEQAJxj93DC6AAAAHUGfUkUVLCP/AAAWtSs3lR8YvFNXX8F6ABWX5VsxAAAAPwGfcXRH/wAAI5jmprTgAWAAyrTucsItqMVigPnwLfa8vJXma+GOfYBSHypNwwx9zemm+n38KSCvkm1Z/uURAQAAACMBn3NqR/8AACOaWt8mAG5mBrmclGM4FXzK/cvY1W8KTwKg4AAAAEtBm3hJqEFsmUwIT//98QAAAwKf7eDaAdHa4os84jvWueBgbe04WHWFuWuu/3VdKmhyxhdZQxOebL0F4EFxPKlyHCG9v1Ad5i7JIcEAAAAqQZ+WRRUsI/8AABa8TAfgy0Wx2ABO20PWAxxBP4CHSY66poOXo+qrjlbMAAAAHAGftXRH/wAADYXdL2yS8F+AJUW6eP41AynJEgMAAAAWAZ+3akf/AAAjsiruvNvWv8Hzxzf5gQAAADNBm7xJqEFsmUwIT//98QAAAwKfAsSYZTyU2ApXpWJFSDGDSDqLgiZ8WBKEsDbbyyiOtcwAAAAvQZ/aRRUsI/8AABa8TmzCyWOIAE5EpUJfMXN4J03M4S+IOmanPvN8ilOk3EQKfs0AAAAdAZ/5dEf/AAADAeavjyv01q1iwgRFO9v3Wzq4mzAAAAAUAZ/7akf/AAAjsisHEy1/C4UjwXkAAABdQZvgSahBbJlMCFf//jhAAAENFOxAUM7kDGpJ9fVihtEf/dIZNjdf8u3t9cJYE42BL8kQGQS48BBe3Kad3uy8Q6t87zTq4KTrMnYK7in0sfCS453JgUZYjAX7qBDBAAAAK0GeHkUVLCP/AAAWvE5swthLCxawZAAftoOejkZ3PN07uISEtKp8WQkMGxUAAAAbAZ49dEf/AAADAEWGLtGC8snD4ASvWup1NihMAAAAIgGeP2pH/wAAIz3UzElQWx8gAsWW+8GV8AA4KwEfL5fELKEAAABGQZokSahBbJlMCE///fEAAAMCnwNA4BmgrHdBo3E+Yx/086MT++1aZ76+h1NLWEFwu2ggfwo3SVG8vbg9R8/XkfcF0pKaoAAAAC9BnkJFFSwj/wAAF0xG395q3WRLHyG5BeMACdT5XuOZ92A8hcFY7iZQ9m2hXEjrwQAAABkBnmF0R/8AACOsOQZRr8g2OHnZ0w8RJUWAAAAAJAGeY2pH/wAAJL8EK/lGhR5EZ4AEg74Ac46Py6MS6NefJKH7cQAAAFVBmmhJqEFsmUwIT//98QAAAwKfAwmQHeyTTY7wVal8NWANdaBTZy/ubhe0LWkW0CfK8DA3rRXm9rsnAPJybeYiFAkGbeTiDZ8+9e/rS1NwH18lUR3JAAAAOkGehkUVLCP/AAAXUJwdtmbWU/COaLS36cuH8cu1s/96nUT8MgBaUA2Q31SIpM8YQcuX08zbCBLsB/kAAAApAZ6ldEf/AAAkrEHjxu6PgAWjPnZq8hX/b98gqqFJ9BVJ7oL9wFWdIsEAAAAhAZ6nakf/AAAkvx8/mB+Ld+9LIL4uoz4VeaOlFFbv1wpIAAAAWEGarEmoQWyZTAhH//3hAAAEO91YCIaY/TbLh2LRpHcon1gKlbsNgarFXJ7kufZg9+YmxYiWdTd0rqfqXoRpfq+FSbiraQzrNHkyzUL34BtQmLN/CgiX9iQAAABVQZ7KRRUsI/8AABdFXA3sBCf2cGk0RKxElc4U1nj0u+S6g0mgdq2b6ljxUq8txVUWr2KqiF27TPg6FOxqBWWlZ653fVpuopmpN7GjHGoRQKEXGxdqQQAAAC0Bnul0R/8AACSsPRlk5s58AC0Z23neQvdCDyWqR5ZzH8OA4igt8iOtDUTukWAAAAAfAZ7rakf/AAAFHkb3Q0/oYXJfubae245rcIjYBnw2YAAAAHxBmvBJqEFsmUwIV//+OEAAAQ2YILEA1HebvnXw1aQg0mheRLXzRXEWHyixNXje3jG8g8CO8f081g/An4U/HFgRSZ6LhLI4IvvE4D4zaWF1aE3Sne7spP2/DA56D786NVoVWGOg0aA6ID/71jyLl1viS3MYATnm3oQ8TMHBAAAAR0GfDkUVLCP/AAAIb+U8igAG1eRmmJgVBT9Ozm3kgDsaC2SsUa4vey+K1M7BITrGkbRYav1L7KZOgfmCHspx12Vxbaj5ibtTAAAAKwGfLXRH/wAADXrTslYLFyFMMFcGRDMK/NqmDwik3gd6ylJqPMtY3W67O90AAAAfAZ8vakf/AAAjsibo6pyAC9cEA0GprLXjpLdlTppswAAAAG9BmzRJqEFsmUwIR//94QAABBX/ZEAezcjXDLuSi8uzp/gLmqAvAZ8Uh02B700eujifsfc9WYikK5DL4NhZMv1uiJem+86WpS+g+Up9axhPRq44ktM6nxLgQPmf16GtN0UE+iysKFmM2fSa3abZLOAAAABDQZ9SRRUsI/8AABa8Thu7f1qgZ5mA+FhB0Ud6JWsEchGwAjtiSYc6b/27Wqzfg1w/SaXeeKiFCZwegMyhKCGaPp7akQAAACQBn3F0R/8AACOe0JquKmlxvyfbTDq25zfMFLSGdhVzEVBz1IAAAAAiAZ9zakf/AAAjmTaPSz6yEdpJhrN+7U3kTlXPvZ1nAyDpgAAAAEVBm3hJqEFsmUwIT//98QAAAwKfA13cBQSt+sLq1Yf8L1S0xugLmMRILsS+KfgmUewJLUsykvbBemhSNqn1STMo/C/+JekAAAAkQZ+WRRUsI/8AABa1XCM8D3M25ieAjVcUKm+QtjYT2C1ZBxkWAAAAHgGftXRH/wAAIzg7759R58SfEKIQqEVI8JZSaghSsQAAAB4Bn7dqR/8AAAVCTniXPfL4E2EY2Q3NcnS9xTl44IEAAAA2QZu8SahBbJlMCE///fEAAAMAYd4i1BQZcMFm7ur8LB5VH6R2jGaC4AHaCHomUty2uLCkH6WUAAAAIkGf2kUVLCP/AAADA0zPF4KBoKcNISDXP/eh1CGH5+jN5FkAAAAVAZ/5dEf/AAAFQ8vuv50iJ5rJ+PmAAAAAHQGf+2pH/wAABUJBbSn85dPkAD/oeVb7fARMX9qRAAAAM0Gb4EmoQWyZTAhf//6MsAAAGowEMZf17mfagEf34tABdRtGmkSH3M4+j5WUwbVx1ABNwQAAADdBnh5FFSwj/wAAAwNDlAAFCOzr80Ev0cC2XVpXSSspGzvXNClywnq8SWrByJwt2f4xcwT4+t2pAAAAGwGePXRH/wAABUSLrd1n5y6n66s6ixHnJLIrUgAAAB0Bnj9qR/8AAAVBMA+JgEJpfK3jO7+N9PWAOmpakQAAAF9BmiRJqEFsmUwIV//+OEAAAQ0U7EAcw7krQaSqA+e56/uZePhUJMp/TkLkaHZoNiPCAbfcfntpt7AQ+cOY5LC8kFW1v0Re04W0jzaJMu92XU3reA0UAa6SxFnC5OLZJAAAAC9BnkJFFSwj/wAAFrVbApZJI5qAAjTxqOU8HftgrBSjhl4vBKwO3yyE3JWks1FIsQAAABYBnmF0R/8AACPDQLJK632aBjrJiB/gAAAAKAGeY2pH/wAAAwC80zEA0b7glRAmo0H2qdwGUxKzv+z1fXoa2p2VLvcAAABGQZpoSahBbJlMCE///fEAAAMCobO5/joZwAgw388ECpe0Vy6LOGwJxbeCLdFRuMPXnt3woKtknj9tMzF/GjUuzQgmEbyKCQAAAChBnoZFFSwj/wAAFr7EXAAi9nAv93EUZWS3r3dpY216hVDF+1ET8aOVAAAAMwGepXRH/wAAI8M/tXc0CzouF4DZkt6oARVaYMmF3U7T7GSKvl7zePgoKoB4fFUPlzHe4QAAABQBnqdqR/8AACPHxxUORplkQivZeQAAACNBmqxJqEFsmUwIX//+jLAAAEYUdOHYLEOkvY61rMLzJtrVsAAAABZBnspFFSwj/wAAFrVZDGEZhFGBRzwfAAAAEgGe6XRH/wAAI8M/tU+DmLYNvQAAABsBnutqR/8AAA1/1viNwAWjO287yF7oQejQA7oAAAAvQZrwSahBbJlMCFf//jhAAAEOCNnw+M0Q8BKBSugRZg15AD6vKOTuQFBNrUvRoEkAAAAbQZ8ORRUsI/8AABa1Wv4YeQlymrvjt2hrIcRZAAAAJQGfLXRH/wAAI8NAqpmb2es/zMcAFz3uqlgncGHIybQdAvPdPUkAAAAeAZ8vakf/AAAjaReRCgcU8UertSACZZqojT/EIDAgAAAAO0GbNEmoQWyZTAhP//3xAAADAqG3y4TScALWcl+izTDqLnXQewwEONDTx95PAgEv2IE7Lcc001SsIoOAAAAAFEGfUkUVLCP/AAAWvqa4HrthgQwJAAAAEAGfcXRH/wAAI8M/seuj8QcAAAARAZ9zakf/AAAjvxvY9bL8QcAAAABJQZt4SahBbJlMCF///oywAABGEQBWcAOkA1CNPTOdF1VMnhlpPKRX1c3z42aVBS71Z31sqi/7rRrAee2D7KNnBNJRdPL5KF/yqQAAACRBn5ZFFSwj/wAAFr6mpq35CP00jiEDRQg4U0jZZVvio2tKUWAAAAAaAZ+1dEf/AAAjt/YFEXozNCHe9iRT2gpmpsEAAAApAZ+3akf/AAAjvxrOe6q2CAEqgf+9wKHjqIt03BrLGvx/VwhTLJ0spsEAAABBQZu8SahBbJlMCFf//jhAAAEN+uej4zRfn4a3RaCRRu37bjvCeA+X2bhoGGh9xpBzpkd4j1QGc4xEMl22BfzedeAAAAAdQZ/aRRUsI/8AABa8Tx67fk4VGzXY0XzdLYRgFFkAAAAaAZ/5dEf/AAADAeWKCUIh+UrJwhwINhnpx7gAAAATAZ/7akf/AAAjvxvahYSfwBXSQQAAAD5Bm+BJqEFsmUwIT//98QAAAwKhbqAOeATQEPGe17HTTrQbuLX9j06MoWzoRkNwpkoGCRCqWvdnAWwk9QH9IQAAABZBnh5FFSwj/wAAFrxDz4RvNQQLdPtQAAAAGwGePXRH/wAAI0VqJgwgA4YbnmSn7qEcRgGbUAAAABIBnj9qR/8AACO/BCvAco0e1UEAAAA+QZokSahBbJlMCFf//jhAAAEOc0qf8jW0m76iAITSzMAFEQBjmMwNnswsGcq+QeQX2zVYt9aNL16sVXk6AMAAAAAiQZ5CRRUsI/8AABaoZNNG5UnzcVzogyRTKvXBKul2u3vtgQAAACABnmF0R/8AACM4O+B58xxDioALn92+sVXFA6+tMADbgAAAABsBnmNqR/8AACO/HAg5aL8ITsZSr7Ut6XYuwlUAAAAgQZpoSahBbJlMCE///fEAAAMCn8pU57R+gGHnPgktVwUAAAATQZ6GRRUsI/8AABa8TuB64Cag4QAAABIBnqV0R/8AACOUd0lGQ3MHUHEAAAARAZ6nakf/AAAjvxvY9bL8QcAAAABMQZqsSahBbJlMCF///oywAABGEcB2ECDMsaAG4S2szON6WcnT/gIaXw4I0BtKsLG2RS7f4H0AKQVjCGga6X0JxMSXhNyi2CNM7OGxUAAAAB5BnspFFSwj/wAAFrxObMLYSwtKnjTvYWNhjx+2EqEAAAAOAZ7pdEf/AAADAAADAakAAAApAZ7rakf/AAAjvZaAraACWP2DhuyNy0U9eQmcjr11Pu4Ci8SJG/UBr+kAAABdQZrwSahBbJlMCFf//jhAAAEM+vKABFU6gitAo0d0D1aT1ChAAZi+/D8eJ1Q7x+iAHvHXjsKN+6RnfFCvVzLQ8EP8vxFwmykty7qTpeJmXm/hjfmhnTJsgRkJCtOxAAAAGUGfDkUVLCP/AAAWtVpTkRG0cZhHMUvZO28AAAAcAZ8tdEf/AAAjmM5v/Y8oiuqbwAk0Fjf8G4iDgQAAABYBny9qR/8AAAMACj5hIJrXKtALEf+LAAAAPEGbNEmoQWyZTAhP//3xAAADAAC+yhqtoBW+OAlhR0Uryew4dtAqVuabeh2UdfNwlP4266bvVeW8DSHHzAAAAC5Bn1JFFSwj/wAAFqhk06fQAjAJUbteQoypNLk3TNKz+XvQttG2ly4ZHo+wjUZfAAAAFwGfcXRH/wAAI8M/tVO2hGgYrm+bYZNmAAAAEQGfc2pH/wAAI78b2PWy/EHAAAAAPkGbeEmoQWyZTAhX//44QAABDToM2cAcmMn2j4OZtXh2djWJaF5RW5ozxiQq3sIfKOZHIjipn63CnP0TdDNrAAAALkGflkUVLCP/AAAWvE5subR4AJxisVsA6qxgSs5GLrc1L0olmE/yApfgZ8OHEBAAAAAnAZ+1dEf/AAAjlG28ooujQVpKbSCGQnAQnOLaAB4SViXG5PfNfEBBAAAAHAGft2pH/wAAI72WgAY5i7Sby8N/d9Yt27PKSYEAAAAxQZu8SahBbJlMCE///fEAAAMCn74NRXe6pAS7v+zjoj5yG7LxBNlMe6jWX4/VkunTcAAAAB5Bn9pFFSwj/wAAFqpsx+QxaEVGfFlUJPYZ09G1SycAAAAcAZ/5dEf/AAAirDxbU3nqhydT1e0CfBR0+dJJgAAAABMBn/tqR/8AACPHv4zwHKKA5XYNAAAASUGb4EmoQWyZTAhf//6MsAAARjPulLmuIArC0hiUmOGbTjwdSRzjJEnjz9ZfDjAHSZad85YW+/ei1sbp0rcip84axlgjAwfeLFkAAAAvQZ4eRRUsI/8AABa8RxX+/h5hOI5jNmF4KekVx4cSZivxm0H9DUe2tdcyIMbz4sAAAAArAZ49dEf/AAAjn5uMxABxU25F2ua5Cctjildee0wXOoQNrggokXgQM3RbMAAAACYBnj9qR/8AACO/BCcYUloHgcfuqp+l8soA6FTKoOvKKFJ+U6AtgwAAAENBmiRJqEFsmUwIV//+OEAAAQ06CWoAunNhJtoGmvn4CAq2QV2g58D08uTM35QhZvb+7ExSkCrxI4BVZgbWWe6wbAnAAAAAKUGeQkUVLCP/AAAWvEcWiJdBw+YzZheCnpFcWSa57SFy7x0+Q3s6f/wZAAAAOAGeYXRH/wAAIqw9Is3KbTCN9I0vllACmkiAEoOTtMr5YgwtD9JjOXi6p1uPuWoRIWeclMlwZfqgAAAAEwGeY2pH/wAAI78EK8Byi9RHYMEAAAAfQZpoSahBbJlMCE///fEAAAMCn8pZ4IY6IMC6IGdzQQAAACBBnoZFFSwj/wAAFrxO4VQGRqsNgFQoGXZmN8QNdxb9UQAAABcBnqV0R/8AAAMACj+XxM3iXCr7CIrLgQAAABsBnqdqR/8AACOeTESZla8obFUwCzGBkThCrBgAAAA1QZqsSahBbJlMCFf//jhAAAENOgAIoMUVAoPvBstjwgvbI6VE/71EeTqr0AEJEU1RblOxB1oAAAAjQZ7KRRUsI/8AABa7lEneBNxdpWXB1NRLDhje/lc2w16ltmEAAAAaAZ7pdEf/AAAirEJUJPn2x6sNRTlp32Io9WwAAAAYAZ7rakf/AAAjvwQrwTnQdXGdJeoqsDtwAAAAKEGa8EmoQWyZTAhP//3xAAADAp/IaQULDGTm7ADXuAKT6xY2vfIBfD0AAAAbQZ8ORRUsI/8AABa1K0MivCScxFxsFkIii6wZAAAAFwGfLXRH/wAAI8MXdVOK8mGnvAQu8S25AAAAFgGfL2pH/wAAAwAKPmEgmtcq0AsR/4sAAABpQZs0SahBbJlMCFf//jhAAAENT6hP8aBUNzFh9atU+fvXjnyMjUUBXWy5Dz4Y5djCYXQiN2RyrdhJ9gwZq940ewew6djaW97nrYwJ61uTTk24vfNjR25psukMloJFKkWOFl/xeEAQjyNWAAAAI0GfUkUVLCP/AAAWvEPNtxXMrM7AKDMp+ZDWOcZch+1fvnJBAAAADgGfcXRH/wAAAwAAAwGpAAAALQGfc2pH/wAAI72V42M5+c3S2ksABcT6JpeGR21QQ4hIlrbzEZHgQYu9AZxyQAAAACxBm3hJqEFsmUwIT//98QAAAwKfvh5kAyHZEpS4uJQa86Oly63JEFU3ljCvIQAAABVBn5ZFFSwj/wAAFrxDz4Qrk5vEEfAAAAAiAZ+1dEf/AAAjx40IAON2Pa27QLcBy5EfpnB7FEuw8XudgwAAABsBn7dqR/8AACPVIQAcTen4YXX2kqKg9SYOwYEAAABBQZu8SahBbJlMCF///oywAABGMhmcsAX40phk2PeIoX1uvvZ2xNQqs0iOw6PtClrZJE+LuUy4URTPa7KFv2lVx4AAAAAhQZ/aRRUsI/8AABZuTy6pOcxJtVl6+8SvmoL8RlqK/TkhAAAADwGf+XRH/wAAI8M/sADbgAAAACABn/tqR/8AACO9lgKJmA99j5CfgEzb99CmZWMiOa1hyQAAAC1Bm+BJqEFsmUwIV//+OEAAAQ0lcsAT9Vn3r0hW0kaVDAltU7RT5zfDORzVmLEAAAARQZ4eRRUsI/8AABa1WlAAlYAAAAAoAZ49dEf/AAAjsKXEwA2ztOg743AeM5IWEcscLI3RI8+HjGFSjvOwYAAAAA4Bnj9qR/8AAAMAAAMBqQAAAEBBmiRJqEFsmUwIT//98QAAAwKfl6MqAjL0Ap6ULLtzEmh5O1gt9dwYcXEpeq6RCirvUCsY9pVHUubsFRbTJmV5AAAAFEGeQkUVLCP/AAAWtStDIPrT94EfAAAAEgGeYXRH/wAAI8MXdVFQ0STCggAAABEBnmNqR/8AACOvMSkbrAAPmQAAAGBBmmhJqEFsmUwIV//+OEAAAQ1PqrcAnysQJPFkQa3lM0hXP8KVs91cAa78wOZoTc1q+DyNF40hWCs6MuMgNnB8QvcCzOQLkCC6bOwICg3ve5rAhMuyACUwMGeqB7+wkOEAAAAdQZ6GRRUsI/8AABa1KzdrWl64ooPCMLA53MeahyUAAAAuAZ6ldEf/AAAjwLhOPRY0DUS4CHQhACVJIGFofpMi/Rlq0djtvfqZTUzYeLikmQAAABwBnqdqR/8AACK5zoHFdrDgAnE3H3s7lC0SQeuAAAAAOkGarEmoQWyZTAhP//3xAAADAqEqYIKwBRyUWVmVRg1dm6ybZjiyk5VEOk8XEoG/27BYWjmOBifr4eAAAAAWQZ7KRRUsI/8AABa+eA+bCFb/gqD1wQAAABQBnul0R/8AACOJJPrVOsI6PJ67BgAAAA4BnutqR/8AAAMAAAMBqQAAADdBmvBJqEFsmUwIV//+OEAAAQ1PqFAJEAlboal287lU4Ano4TWRkm2xCGBYFq7rPy/3sJezCRCnAAAAHUGfDkUVLCP/AAAWtSs3a1peuKKDwjCwOdzHmoclAAAAHwGfLXRH/wAAI8C4Tj0WNA1E5mF/neyZnuhKd7A7OSEAAAAfAZ8vakf/AAAjRBGvh7FhHj/oJAAnGaqI0/w3Ops8cAAAACBBmzRJqEFsmUwIT//98QAAAwKfl3BsS2yHlaBHGAr4eAAAACxBn1JFFSwj/wAAFrxDz4R0loYcJ2AEsPJFGrtpqTzf/ov3DrO2yhYJMYEuLQAAABYBn3F0R/8AAAMACj+XxM3iZpmIYVigAAAAGAGfc2pH/wAAI78EK8E50HVxnSXqKrA7cAAAAF1Bm3hJqEFsmUwIV//+OEAAAQ06B0wECS8TthZEx8dkO/9dlk5zM0bLOThaBppkGHJf3IhnOIhOv6owkpq0pBJEzjeYWRSieX20rkq3oGewdIs+mNP6OObma2hCXFkAAAAvQZ+WRRUsI/8AABa5M6pTYSdBJeVa6otEeocAJphATHNDfVz3uZvFMefAteYQkmAAAAAkAZ+1dEf/AAAjrDj+89mDAAF0AGiNjTVRVBTETzPkTz76hqFtAAAAHAGft2pH/wAAI77thNc1nxG1ZPjhGy5UXvoqduEAAAAjQZu8SahBbJlMCE///fEAAAMCn8hpBQsMXCCanykAQsYMOOAAAAAVQZ/aRRUsI/8AABa8Q8+EK5ObxBHxAAAAGgGf+XRH/wAAIshsPMSipckAEtO59EW5TN+AAAAAEwGf+2pH/wAAI78EK8Byi9RHYMEAAABEQZvgSahBbJlMCFf//jhAAAENOgAUrH6ACwThPEQmK/8fKh/VDT9W/9BlRskw+vlmb3N5jLLA1pJdAQhwAbu/bSZk7+EAAAAyQZ4eRRUsI/8AABa5M30Da15pdSS7n8wATt1LAlbD8meaMkbYLHPfXAXp/QnrA2hq2YAAAAAUAZ49dEf/AAAjq/hbeBfVTV2BAZUAAAAoAZ4/akf/AAAjvwQr+on47gcAC6iVnu3qJrmRms7oN8iF0BRJQL9pAQAAAEJBmiRJqEFsmUwIT//98QAAAwKNl7m6AEYFb9b33jkPCMy5kw+Vhxi7kubNfLzB60IUO1xihjmVWeGgE7OioB8W2qwAAAAcQZ5CRRUsI/8AABYlWlmG8u5KHTp3euTr28QHJQAAACMBnmF0R/8AACPIYbLqQAbmYOMybQ5D7Ayy3/UskK7T9vCggAAAABoBnmNqR/8AACLHxxbXfGsysgCeYKpSSkc5IQAAADlBmmhJqEFsmUwI//yEAAAP1D/DUhU+gAcWDFSWWzl7lFxfAPsgt6AdTmXhPRhCirlCd4jRIKXfy58AAAA1QZ6GRRUsI/8AABa1KzdttoAS2qwF9y7eqQNm43xFMu6wd+ulMOORAOG57lhEZORJGsUQR28AAAAsAZ6ldEf/AAAjwxdoqaVT1RIQktAA3XLt6lWFiRE2zTUh3OKMdBFuIASKqMEAAAApAZ6nakf/AAADAeXJUPwBkADjdCn+4PvaSbL1KESkBCdoBE3biFjqW3AAAAx/bW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAD7QAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAC6l0cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAD7QAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAlgAAAGQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAA+0AAACAAABAAAAAAshbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAyAAAAyQBVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAKzG1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAACoxzdGJsAAAAmHN0c2QAAAAAAAAAAQAAAIhhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAlgBkABIAAAASAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAMmF2Y0MBZAAf/+EAGWdkAB+s2UCYM+XhAAADAAEAAAMAZA8YMZYBAAZo6+PLIsAAAAAYc3R0cwAAAAAAAAABAAAAyQAAAQAAAAAUc3RzcwAAAAAAAAABAAAAAQAABlhjdHRzAAAAAAAAAMkAAAABAAACAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAMkAAAABAAADOHN0c3oAAAAAAAAAAAAAAMkAAAThAAAAwgAAAE4AAAA2AAAAMgAAAIIAAAA8AAAAOwAAADsAAACCAAAATgAAADgAAAA1AAAAhAAAACMAAAA7AAAAGAAAAFAAAAAhAAAAQwAAACcAAABPAAAALgAAACAAAAAaAAAANwAAADMAAAAhAAAAGAAAAGEAAAAvAAAAHwAAACYAAABKAAAAMwAAAB0AAAAoAAAAWQAAAD4AAAAtAAAAJQAAAFwAAABZAAAAMQAAACMAAACAAAAASwAAAC8AAAAjAAAAcwAAAEcAAAAoAAAAJgAAAEkAAAAoAAAAIgAAACIAAAA6AAAAJgAAABkAAAAhAAAANwAAADsAAAAfAAAAIQAAAGMAAAAzAAAAGgAAACwAAABKAAAALAAAADcAAAAYAAAAJwAAABoAAAAWAAAAHwAAADMAAAAfAAAAKQAAACIAAAA/AAAAGAAAABQAAAAVAAAATQAAACgAAAAeAAAALQAAAEUAAAAhAAAAHgAAABcAAABCAAAAGgAAAB8AAAAWAAAAQgAAACYAAAAkAAAAHwAAACQAAAAXAAAAFgAAABUAAABQAAAAIgAAABIAAAAtAAAAYQAAAB0AAAAgAAAAGgAAAEAAAAAyAAAAGwAAABUAAABCAAAAMgAAACsAAAAgAAAANQAAACIAAAAgAAAAFwAAAE0AAAAzAAAALwAAACoAAABHAAAALQAAADwAAAAXAAAAIwAAACQAAAAbAAAAHwAAADkAAAAnAAAAHgAAABwAAAAsAAAAHwAAABsAAAAaAAAAbQAAACcAAAASAAAAMQAAADAAAAAZAAAAJgAAAB8AAABFAAAAJQAAABMAAAAkAAAAMQAAABUAAAAsAAAAEgAAAEQAAAAYAAAAFgAAABUAAABkAAAAIQAAADIAAAAgAAAAPgAAABoAAAAYAAAAEgAAADsAAAAhAAAAIwAAACMAAAAkAAAAMAAAABoAAAAcAAAAYQAAADMAAAAoAAAAIAAAACcAAAAZAAAAHgAAABcAAABIAAAANgAAABgAAAAsAAAARgAAACAAAAAnAAAAHgAAAD0AAAA5AAAAMAAAAC0AAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTYuMzYuMTAw\"\n",
       " type=\"video/mp4\" />\n",
       "</video>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import io\n",
    "import base64\n",
    "from IPython.display import HTML\n",
    "\n",
    "template = './gym-results/openaigym.video.%s.video000001.mp4'\n",
    "video = io.open(template % mon.file_infix, 'r+b').read()\n",
    "encoded = base64.b64encode(video)\n",
    "HTML(data='''\n",
    "<video width=\"520\" height=\"auto\" alt=\"test\" controls>\n",
    "<source src=\"data:video/mp4;base64,{0}\"\n",
    " type=\"video/mp4\" />\n",
    "</video>'''.format(encoded.decode('ascii')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "DL4D_17_Reinforcement_Learning.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}