OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open AI Atari Games. Reinforcement Learning with PyTorch, deep Learning\n",
    "## By Nasrudin Bin Salim\n",
    "### Requirements: Python 2.7. Linux Environment/UNIX Environment\n",
    "    Please Install Pytorch\n",
    "    OpenAI Gym\n",
    "    Open AI Universe\n",
    "    cv2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import numpy as np\n",
    "\n",
    "import json\n",
    "import logging\n",
    "from cv2 import resize\n",
    "from skimage.color import rgb2gray\n",
    "import os\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"1\" #should be set to 1 to prevent conflicts\n",
    "import time\n",
    "\n",
    "\n",
    "\n",
    "from torch.autograd import Variable\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import OpenAI Universe environment and gym\n",
    "### Import Pytorch for model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "from universe import vectorized\n",
    "from universe.wrappers import Unvectorize, Vectorize\n",
    "\n",
    "from gym.spaces.box import Box\n",
    "from gym.configuration import undo_logger_setup\n",
    "\n",
    "import torch\n",
    "from torch.multiprocessing import Process\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "import torch.optim as optim\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions Section"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logging function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def setup_logger(logger_name, log_file, level=logging.INFO):\n",
    "    \n",
    "    ''' Makes use of the logging module'''\n",
    "    #Instantiates the logging class\n",
    "    l = logging.getLogger(logger_name)\n",
    "    \n",
    "    #Formatter\n",
    "    formatter = logging.Formatter('%(asctime)s : %(message)s')\n",
    "    \n",
    "    #file handler    \n",
    "    fileHandler = logging.FileHandler(log_file, mode='w')\n",
    "    fileHandler.setFormatter(formatter)\n",
    "    \n",
    "    #streamhandler\n",
    "    streamHandler = logging.StreamHandler()\n",
    "    streamHandler.setFormatter(formatter)\n",
    "    \n",
    "    #add the above handles to the logger instance\n",
    "    l.setLevel(level)\n",
    "    l.addHandler(fileHandler)\n",
    "    l.addHandler(streamHandler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Json Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_config(file_path):\n",
    "    \"\"\"Read JSON config.\"\"\"\n",
    "    #use the context manager\n",
    "    with open(file_path, 'r') as f:\n",
    "        json_object = json.load(f)\n",
    "        \n",
    "    return json_object"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Share grads between 2 models\n",
    "#### More on this later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ensure_shared_grads(model, shared_model):\n",
    "    for param, shared_param in zip(model.parameters(),\n",
    "                                   shared_model.parameters()):\n",
    "        if shared_param.grad is not None:\n",
    "            return\n",
    "        shared_param._grad = param.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Environment, setting up the openAI and Universe "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the atari environment function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def atari_env(env_id, env_conf):\n",
    "    env = gym.make(env_id)\n",
    "    if len(env.observation_space.shape) > 1:\n",
    "        env = Vectorize(env)\n",
    "        env = AtariRescale(env, env_conf)\n",
    "        env = NormalizedEnv(env)\n",
    "        env = Unvectorize(env)\n",
    "        \n",
    "    return env"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a frame for environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _process_frame(frame, conf):\n",
    "    frame = frame[conf[\"crop1\"]:conf[\"crop2\"] + 160, :160]\n",
    "    frame = resize(rgb2gray(frame), (80, conf[\"dimension2\"]))\n",
    "    frame = resize(frame, (80, 80))\n",
    "    frame = np.reshape(frame, [1, 80, 80])\n",
    "    return frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Atari rescale class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AtariRescale(vectorized.ObservationWrapper):\n",
    "    def __init__(self, env, env_conf):\n",
    "        super(AtariRescale, self).__init__(env)\n",
    "        self.observation_space = Box(0.0, 1.0, [1, 80, 80])\n",
    "        self.conf = env_conf\n",
    "\n",
    "    def _observation(self, observation_n):\n",
    "        return [\n",
    "            _process_frame(observation, self.conf)\n",
    "            for observation in observation_n\n",
    "        ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalized environment class, where we can move from one state and observation to another"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NormalizedEnv(vectorized.ObservationWrapper):\n",
    "    def __init__(self, env=None):\n",
    "        super(NormalizedEnv, self).__init__(env)\n",
    "        self.state_mean = 0\n",
    "        self.state_std = 0\n",
    "        self.alpha = 0.9999\n",
    "        self.num_steps = 0\n",
    "\n",
    "    def _observation(self, observation_n):\n",
    "        for observation in observation_n:\n",
    "            self.num_steps += 1\n",
    "            self.state_mean = self.state_mean * self.alpha + \\\n",
    "                observation.mean() * (1 - self.alpha)\n",
    "            self.state_std = self.state_std * self.alpha + \\\n",
    "                observation.std() * (1 - self.alpha)\n",
    "\n",
    "        unbiased_mean = self.state_mean / (1 - pow(self.alpha, self.num_steps))\n",
    "        unbiased_std = self.state_std / (1 - pow(self.alpha, self.num_steps))\n",
    "\n",
    "        return [(observation - unbiased_mean) / (unbiased_std + 1e-8)\n",
    "                for observation in observation_n]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "\n",
    "Using Google DeepMind's Idea. \n",
    "\n",
    "    Research Paper: https://arxiv.org/pdf/1602.01783.pdf\n",
    "    Asynchronous Advantage Actor-Critic (A3C)\n",
    "\n",
    "\n",
    "The A3C algorithm was released by Google’s DeepMind group earlier this year, and it made a splash by… essentially obsoleting DQN. It was faster, simpler, more robust, and able to achieve much better scores on the standard battery of Deep RL tasks. On top of all that it could work in continuous as well as discrete action spaces. Given this, it has become the go-to Deep RL algorithm for new challenging problems with complex state and action spaces\n",
    "\n",
    "\n",
    "    \n",
    "<a href= \"https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-8-asynchronous-actor-critic-agents-a3c-c88f72a5e9f2\" >Medium Article explaining A3c reinforcement learning </a>\n",
    "\n",
    "## The Actor-Critic Structure\n",
    "<img src = \"img/A3CStructure.png\">\n",
    "\n",
    "## Many workers training and learning concurrently, and then updates global network with gradients\n",
    "### Process Flow\n",
    "<img src = \"img/A3CProcessFlow.png\">\n",
    "    \n",
    "### Long Short Term Memory Recurrent Neural Nets\n",
    "    \n",
    "## Implementing LSTM and A3C with Pytorch\n",
    "\n",
    "Created as a module and then imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from A3CModel import A3Clstm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The player Agent\n",
    "## (Reinforcement Learning agent to interact with the env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Agent(object):\n",
    "    def __init__(self, model, env, args, state):\n",
    "        self.model = model\n",
    "        self.env = env\n",
    "        self.current_life = 0\n",
    "        self.state = state\n",
    "        self.hx = None\n",
    "        self.cx = None\n",
    "        self.eps_len = 0\n",
    "        self.args = args\n",
    "        self.values = []\n",
    "        self.log_probs = []\n",
    "        self.rewards = []\n",
    "        self.entropies = []\n",
    "        self.done = True\n",
    "        self.info = None\n",
    "        self.reward = 0\n",
    "\n",
    "    def action_train(self):\n",
    "        if self.done:\n",
    "            self.cx = Variable(torch.zeros(1, 512))\n",
    "            self.hx = Variable(torch.zeros(1, 512))\n",
    "        else:\n",
    "            self.cx = Variable(self.cx.data)\n",
    "            self.hx = Variable(self.hx.data)\n",
    "        value, logit, (self.hx, self.cx) = self.model((Variable(self.state.unsqueeze(0)), (self.hx, self.cx)))\n",
    "        prob = F.softmax(logit)\n",
    "        log_prob = F.log_softmax(logit)\n",
    "        entropy = -(log_prob * prob).sum(1)\n",
    "        self.entropies.append(entropy)\n",
    "        action = prob.multinomial().data\n",
    "        log_prob = log_prob.gather(1, Variable(action))\n",
    "        state, self.reward, self.done, self.info = self.env.step(action.numpy())\n",
    "        self.state = torch.from_numpy(state).float()\n",
    "        self.eps_len += 1\n",
    "        self.done = self.done or self.eps_len >= self.args['M']\n",
    "        self.reward = max(min(self.reward, 1), -1)\n",
    "        self.values.append(value)\n",
    "        self.log_probs.append(log_prob)\n",
    "        self.rewards.append(self.reward)\n",
    "        return self\n",
    "\n",
    "    def action_test(self):\n",
    "        if self.done:\n",
    "            self.cx = Variable(torch.zeros(1, 512), volatile=True)\n",
    "            self.hx = Variable(torch.zeros(1, 512), volatile=True)\n",
    "        else:\n",
    "            self.cx = Variable(self.cx.data, volatile=True)\n",
    "            self.hx = Variable(self.hx.data, volatile=True)\n",
    "        value, logit, (self.hx, self.cx) = self.model((Variable(self.state.unsqueeze(0), volatile=True), (self.hx, self.cx)))\n",
    "        prob = F.softmax(logit)\n",
    "        action = prob.max(1)[1].data.numpy()\n",
    "        state, self.reward, self.done, self.info = self.env.step(action[0])\n",
    "        self.state = torch.from_numpy(state).float()\n",
    "        self.eps_len += 1\n",
    "        self.done = self.done or self.eps_len >= self.args['M']\n",
    "        return self\n",
    "\n",
    "    def check_state(self):\n",
    "        if self.current_life > self.info['ale.lives']:\n",
    "            self.done = True\n",
    "        self.current_life = self.info['ale.lives']\n",
    "        return self\n",
    "\n",
    "    def clear_actions(self):\n",
    "        self.values = []\n",
    "        self.log_probs = []\n",
    "        self.rewards = []\n",
    "        self.entropies = []\n",
    "        return self\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shared Memory and optimization algorithims\n",
    "## As Part of the A3C Network, multiple workers will be working together to update a global network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RMSprop\n",
    "\n",
    "RMSprop is an unpublished, adaptive learning rate method proposed by Geoff Hinton in Lecture 6e of his Coursera Class.\n",
    "\n",
    "RMSprop and Adadelta have both been developed independently around the same time stemming from the need to resolve Adagrad's radically diminishing learning rates. RMSprop in fact is identical to the first update vector of Adadelta \n",
    "\n",
    "RMSprop as well divides the learning rate by an exponentially decaying average of squared gradients. Hinton suggests γ\n",
    "to be set to 0.9, while a good default value for the learning rate η is 0.001."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from SharedOptimizers import SharedRMSprop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adaptive Moment Estimation (Adam) \n",
    "is another method that computes adaptive learning rates for each parameter. In addition to storing an exponentially decaying average of past squared gradients vt like Adadelta and RMSprop, Adam also keeps an exponentially decaying average of past gradients mt, similar to momentum:\n",
    "\n",
    "Adam (short for Adaptive Moment Estimation) is an update to the RMSProp optimizer. In this optimization algorithm, running averages of both the gradients and the second moments of the gradients are used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from SharedOptimizers import SharedAdam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adam but only with shared Lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from SharedOptimizers import SharedLrSchedAdam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions to run the model on the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test\n",
    "## Function To test the model on a game/environ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test(args, shared_model, env_conf,render=False):\n",
    "    log = {}\n",
    "    setup_logger('{}_log'.format(args['ENV']),\n",
    "                 r'{0}{1}_log'.format(args['LG'], args['ENV']))\n",
    "    log['{}_log'.format(args['ENV'])] = logging.getLogger(\n",
    "        '{}_log'.format(args['ENV']))\n",
    "    d_args = args\n",
    "    for k in d_args.keys():\n",
    "        log['{}_log'.format(args['ENV'])].info('{0}: {1}'.format(k, d_args[k]))\n",
    "\n",
    "    torch.manual_seed(args['seed'])\n",
    "    env = atari_env(args['ENV'], env_conf)\n",
    "    reward_sum = 0\n",
    "    start_time = time.time()\n",
    "    num_tests = 0\n",
    "    reward_total_sum = 0\n",
    "    player = Agent(None, env, args, None)\n",
    "    player.model = A3Clstm(\n",
    "        player.env.observation_space.shape[0], player.env.action_space)\n",
    "    player.state = player.env.reset()\n",
    "    player.state = torch.from_numpy(player.state).float()\n",
    "    player.model.eval()\n",
    "\n",
    "    while True:\n",
    "        if player.done:\n",
    "            player.model.load_state_dict(shared_model.state_dict())\n",
    "        if render:\n",
    "            env.render()\n",
    "        player.action_test()\n",
    "        reward_sum += player.reward\n",
    "\n",
    "        if player.done:\n",
    "            num_tests += 1\n",
    "            player.current_life = 0\n",
    "            reward_total_sum += reward_sum\n",
    "            reward_mean = reward_total_sum / num_tests\n",
    "            log['{}_log'.format(args['ENV'])].info(\n",
    "                \"Time {0}, episode reward {1}, episode length {2}, reward mean {3:.4f}\".\n",
    "                format(\n",
    "                    time.strftime(\"%Hh %Mm %Ss\",\n",
    "                                  time.gmtime(time.time() - start_time)),\n",
    "                    reward_sum, player.eps_len, reward_mean))\n",
    "\n",
    "            if reward_sum > args['SSL']:\n",
    "                player.model.load_state_dict(shared_model.state_dict())\n",
    "                state_to_save = player.model.state_dict()\n",
    "                torch.save(state_to_save, '{0}{1}.dat'.format(\n",
    "                    args['SMD'], args['ENV']))\n",
    "\n",
    "            reward_sum = 0\n",
    "            player.eps_len = 0\n",
    "            state = player.env.reset()\n",
    "            time.sleep(60)\n",
    "            player.state = torch.from_numpy(state).float()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train\n",
    "## Function to Train the model with an optimizer algorithim on an environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(rank, args, shared_model, optimizer, env_conf):\n",
    "\n",
    "    torch.manual_seed(args['seed'] + rank)\n",
    "    env = atari_env(args['ENV'], env_conf)\n",
    "    if optimizer is None:\n",
    "        if args['OPT'] == 'RMSprop':\n",
    "            optimizer = optim.RMSprop(shared_model.parameters(), lr=args['LR'])\n",
    "        if args['OPT'] == 'Adam':\n",
    "            optimizer = optim.Adam(shared_model.parameters(), lr=args['LR'])\n",
    "\n",
    "    env.seed(args['seed'] + rank)\n",
    "    player = Agent(None, env, args, None)\n",
    "    player.model = A3Clstm(\n",
    "        player.env.observation_space.shape[0], player.env.action_space)\n",
    "    player.state = player.env.reset()\n",
    "    player.state = torch.from_numpy(player.state).float()\n",
    "    player.model.train()\n",
    "\n",
    "    while True:\n",
    "        player.model.load_state_dict(shared_model.state_dict())\n",
    "        for step in range(args['NS']):\n",
    "            player.action_train()\n",
    "            if args['CL']:\n",
    "                player.check_state()\n",
    "            if player.done:\n",
    "                break\n",
    "\n",
    "        if player.done:\n",
    "            player.eps_len = 0\n",
    "            player.current_life = 0\n",
    "            state = player.env.reset()\n",
    "            player.state = torch.from_numpy(state).float()\n",
    "\n",
    "        R = torch.zeros(1, 1)\n",
    "        if not player.done:\n",
    "            value, _, _ = player.model(\n",
    "                (Variable(player.state.unsqueeze(0)), (player.hx, player.cx)))\n",
    "            R = value.data\n",
    "\n",
    "        player.values.append(Variable(R))\n",
    "        policy_loss = 0\n",
    "        value_loss = 0\n",
    "        R = Variable(R)\n",
    "        gae = torch.zeros(1, 1)\n",
    "        for i in reversed(range(len(player.rewards))):\n",
    "            R = args['G'] * R + player.rewards[i]\n",
    "            advantage = R - player.values[i]\n",
    "            value_loss = value_loss + 0.5 * advantage.pow(2)\n",
    "\n",
    "            # Generalized Advantage Estimataion\n",
    "            delta_t = player.rewards[i] + args['G'] * \\\n",
    "                player.values[i + 1].data - player.values[i].data\n",
    "            gae = gae * args['G'] * args['T'] + delta_t\n",
    "\n",
    "            policy_loss = policy_loss - \\\n",
    "                player.log_probs[i] * \\\n",
    "                Variable(gae) - 0.01 * player.entropies[i]\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        (policy_loss + 0.5 * value_loss).backward()\n",
    "        torch.nn.utils.clip_grad_norm(player.model.parameters(), 40)\n",
    "        ensure_shared_grads(player.model, shared_model)\n",
    "        optimizer.step()\n",
    "        player.clear_actions()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Putting it altogether"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List of Games, pick one here and then edit the environment accordingly\n",
    "    Choose an Atari game and it has to be a 4D Tensor Game \n",
    "    Or if you don't know what that means, just guess and check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[EnvSpec(flashgames.UrbanMicroRacers-v0),\n",
       " EnvSpec(flashgames.PlopPlopLite-v0),\n",
       " EnvSpec(DoubleDunk-ramDeterministic-v4),\n",
       " EnvSpec(flashgames.Sieger2LevelPack-v0),\n",
       " EnvSpec(DoubleDunk-ramDeterministic-v0),\n",
       " EnvSpec(gym-core.Krull-v0),\n",
       " EnvSpec(gym-core.Krull-v3),\n",
       " EnvSpec(Pooyan-ram-v4),\n",
       " EnvSpec(Pooyan-ram-v0),\n",
       " EnvSpec(flashgames.GonAndMon-v0),\n",
       " EnvSpec(flashgames.NeonRaceLvl4-v0),\n",
       " EnvSpec(flashgames.Hash-v0),\n",
       " EnvSpec(gym-core.JourneyEscapeSlow-v3),\n",
       " EnvSpec(gym-core.JourneyEscapeSlow-v0),\n",
       " EnvSpec(flashgames.FlashBombs-v0),\n",
       " EnvSpec(gym-core.JamesbondDeterministicSlow-v0),\n",
       " EnvSpec(VentureNoFrameskip-v0),\n",
       " EnvSpec(Centipede-v0),\n",
       " EnvSpec(Centipede-v4),\n",
       " EnvSpec(flashgames.Crumbs2-v0),\n",
       " EnvSpec(flashgames.CosmoGravity2-v0),\n",
       " EnvSpec(gym-core.Zaxxon30FPS-v0),\n",
       " EnvSpec(gym-core.Zaxxon30FPS-v3),\n",
       " EnvSpec(flashgames.ZombiesAndDonuts-v0),\n",
       " EnvSpec(Frostbite-ramNoFrameskip-v0),\n",
       " EnvSpec(Frostbite-ramNoFrameskip-v4),\n",
       " EnvSpec(IceHockey-ramNoFrameskip-v4),\n",
       " EnvSpec(flashgames.SpacePunkRacerLvl8-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl8-v0),\n",
       " EnvSpec(flashgames.GalacticGems2NewFrontiers-v0),\n",
       " EnvSpec(flashgames.DisasterWillStrikeDefender-v0),\n",
       " EnvSpec(flashgames.BikeTrial2-v0),\n",
       " EnvSpec(gym-core.SolarisDeterministic-v3),\n",
       " EnvSpec(gym-core.SolarisDeterministic-v0),\n",
       " EnvSpec(AirRaidNoFrameskip-v4),\n",
       " EnvSpec(gym-core.BerzerkSlow-v3),\n",
       " EnvSpec(AirRaidNoFrameskip-v0),\n",
       " EnvSpec(gym-core.BankHeistSync-v3),\n",
       " EnvSpec(flashgames.NinjaTrainingWorlds-v0),\n",
       " EnvSpec(flashgames.3dFlashRacer-v0),\n",
       " EnvSpec(KrullDeterministic-v0),\n",
       " EnvSpec(SemisuperPendulumRandom-v0),\n",
       " EnvSpec(KrullDeterministic-v4),\n",
       " EnvSpec(wob.real.Quizlet-Planet-Test-v0),\n",
       " EnvSpec(Go19x19-v0),\n",
       " EnvSpec(gym-core.CrazyClimber30FPS-v0),\n",
       " EnvSpec(flashgames.GravityBall-v0),\n",
       " EnvSpec(gym-core.CrazyClimber30FPS-v3),\n",
       " EnvSpec(gym-core.ChopperCommandDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.ChopperCommandDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.CentipedeDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.Zaxxon-v0),\n",
       " EnvSpec(gym-core.CentipedeDeterministicSlow-v3),\n",
       " EnvSpec(FishingDerbyNoFrameskip-v4),\n",
       " EnvSpec(FishingDerbyNoFrameskip-v0),\n",
       " EnvSpec(flashgames.FormulaXspeed3d-v0),\n",
       " EnvSpec(flashgames.BobbyNutcaseMotoJumping-v0),\n",
       " EnvSpec(flashgames.RollerRider-v0),\n",
       " EnvSpec(gym-core.YarsRevengeNoFrameskip-v0),\n",
       " EnvSpec(gym-core.Assault30FPS-v0),\n",
       " EnvSpec(gym-core.Assault30FPS-v3),\n",
       " EnvSpec(Qbert-v4),\n",
       " EnvSpec(Qbert-v0),\n",
       " EnvSpec(flashgames.BusinessmanSimulator-v0),\n",
       " EnvSpec(gym-core.MontezumaRevenge30FPS-v3),\n",
       " EnvSpec(flashgames.PunchBallJump-v0),\n",
       " EnvSpec(Robotank-ram-v4),\n",
       " EnvSpec(flashgames.SpectrumRunner-v0),\n",
       " EnvSpec(flashgames.LooneyAndJohny-v0),\n",
       " EnvSpec(gym-core.TennisDeterministicSync-v3),\n",
       " EnvSpec(flashgames.TheThreeTowers-v0),\n",
       " EnvSpec(gym-core.TennisDeterministicSync-v0),\n",
       " EnvSpec(flashgames.DriveToWreck-v0),\n",
       " EnvSpec(flashgames.SkiSim-v0),\n",
       " EnvSpec(MontezumaRevenge-ramNoFrameskip-v4),\n",
       " EnvSpec(MontezumaRevenge-ramNoFrameskip-v0),\n",
       " EnvSpec(gym-core.BerzerkDeterministicSync-v0),\n",
       " EnvSpec(flashgames.FlashRacer-v0),\n",
       " EnvSpec(flashgames.Sundrops-v0),\n",
       " EnvSpec(gym-core.TimePilot30FPS-v0),\n",
       " EnvSpec(gym-core.TimePilot30FPS-v3),\n",
       " EnvSpec(flashgames.Rocketeer-v0),\n",
       " EnvSpec(Go9x9-v0),\n",
       " EnvSpec(flashgames.MineDrop-v0),\n",
       " EnvSpec(flashgames.RollingHills-v0),\n",
       " EnvSpec(flashgames.DrawGems-v0),\n",
       " EnvSpec(MountainCarContinuous-v0),\n",
       " EnvSpec(gym-core.CrazyClimberDeterministicSync-v3),\n",
       " EnvSpec(gym-core.CrazyClimberDeterministicSync-v0),\n",
       " EnvSpec(gym-core.EnduroSync-v0),\n",
       " EnvSpec(flashgames.WastelandSiege-v0),\n",
       " EnvSpec(gym-core.EnduroSync-v3),\n",
       " EnvSpec(gym-core.Zaxxon-v3),\n",
       " EnvSpec(Pong-v4),\n",
       " EnvSpec(flashgames.PapaLouie3WhenSundaesAttack-v0),\n",
       " EnvSpec(Pong-v0),\n",
       " EnvSpec(flashgames.StormRage-v0),\n",
       " EnvSpec(flashgames.GalleonFight-v0),\n",
       " EnvSpec(gym-core.BattleZone30FPS-v0),\n",
       " EnvSpec(gym-core.BattleZone30FPS-v3),\n",
       " EnvSpec(flashgames.GalacticGems2LevelPack-v0),\n",
       " EnvSpec(flashgames.MagicSafari-v0),\n",
       " EnvSpec(gym-core.CentipedeDeterministicSync-v3),\n",
       " EnvSpec(gym-core.CentipedeDeterministicSync-v0),\n",
       " EnvSpec(gym-core.UpNDownSync-v0),\n",
       " EnvSpec(gym-core.UpNDownSync-v3),\n",
       " EnvSpec(flashgames.RhythmSnake-v0),\n",
       " EnvSpec(flashgames.SuperK9-v0),\n",
       " EnvSpec(gym-core.KrullNoFrameskip-v3),\n",
       " EnvSpec(flashgames.AmericanRacingLvl2-v0),\n",
       " EnvSpec(flashgames.GroundBattles-v0),\n",
       " EnvSpec(BattleZoneDeterministic-v4),\n",
       " EnvSpec(BattleZoneDeterministic-v0),\n",
       " EnvSpec(gym-core.Asterix-v0),\n",
       " EnvSpec(gym-core.Asterix-v3),\n",
       " EnvSpec(flashgames.AmericanRacingLvl19-v0),\n",
       " EnvSpec(flashgames.MysteriousPirateJewels-v0),\n",
       " EnvSpec(flashgames.GemPop-v0),\n",
       " EnvSpec(gym-core.AirRaid-v3),\n",
       " EnvSpec(flashgames.JumpOverTheRings-v0),\n",
       " EnvSpec(gym-core.SeaquestDeterministic-v3),\n",
       " EnvSpec(gym-core.SeaquestDeterministic-v0),\n",
       " EnvSpec(ConvergenceControl-v0),\n",
       " EnvSpec(flashgames.MonkeyManic-v0),\n",
       " EnvSpec(flashgames.SurvivalLab-v0),\n",
       " EnvSpec(flashgames.MeerkatMission-v0),\n",
       " EnvSpec(gym-core.AlienSync-v0),\n",
       " EnvSpec(gym-core.AlienSync-v3),\n",
       " EnvSpec(gym-core.SkiingSlow-v3),\n",
       " EnvSpec(gym-core.SkiingSlow-v0),\n",
       " EnvSpec(BattleZoneNoFrameskip-v0),\n",
       " EnvSpec(gym-core.SpaceInvaders30FPS-v3),\n",
       " EnvSpec(flashgames.BulletHeaven-v0),\n",
       " EnvSpec(BattleZoneNoFrameskip-v4),\n",
       " EnvSpec(gym-core.DemonAttackNoFrameskip-v3),\n",
       " EnvSpec(gym-core.DemonAttackNoFrameskip-v0),\n",
       " EnvSpec(flashgames.MexicoRex-v0),\n",
       " EnvSpec(gym-core.CentipedeSlow-v0),\n",
       " EnvSpec(gym-core.CentipedeSlow-v3),\n",
       " EnvSpec(flashgames.Pyro-v0),\n",
       " EnvSpec(TutankhamDeterministic-v0),\n",
       " EnvSpec(UpNDown-ramDeterministic-v4),\n",
       " EnvSpec(TutankhamDeterministic-v4),\n",
       " EnvSpec(UpNDown-ramDeterministic-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministic-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministic-v3),\n",
       " EnvSpec(flashgames.ShimmyChute-v0),\n",
       " EnvSpec(Jamesbond-ramNoFrameskip-v4),\n",
       " EnvSpec(flashgames.SuperbikeRacer-v0),\n",
       " EnvSpec(Jamesbond-ramNoFrameskip-v0),\n",
       " EnvSpec(gym-core.DoubleDunkDeterministic-v3),\n",
       " EnvSpec(gym-core.DoubleDunkDeterministic-v0),\n",
       " EnvSpec(flashgames.Krome-v0),\n",
       " EnvSpec(gym-core.KrullSlow-v0),\n",
       " EnvSpec(gym-core.KrullSlow-v3),\n",
       " EnvSpec(wob.mini.ScrollText2-v0),\n",
       " EnvSpec(gym-core.IceHockey-v3),\n",
       " EnvSpec(gym-core.IceHockey-v0),\n",
       " EnvSpec(gym-core.AsterixDeterministic-v3),\n",
       " EnvSpec(gym-core.AsterixDeterministic-v0),\n",
       " EnvSpec(flashgames.KeeperOfTheGrove3-v0),\n",
       " EnvSpec(YarsRevengeDeterministic-v0),\n",
       " EnvSpec(YarsRevengeDeterministic-v4),\n",
       " EnvSpec(gym-core.StarGunnerDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.Tutankham-v3),\n",
       " EnvSpec(gym-core.Tutankham-v0),\n",
       " EnvSpec(flashgames.CatchTheStar-v0),\n",
       " EnvSpec(Bowling-ramNoFrameskip-v0),\n",
       " EnvSpec(Bowling-ramNoFrameskip-v4),\n",
       " EnvSpec(TwoRoundNondeterministicReward-v0),\n",
       " EnvSpec(flashgames.FlashRace-v0),\n",
       " EnvSpec(flashgames.TattooArtist-v0),\n",
       " EnvSpec(flashgames.FormulaRacer2012Lvl8-v0),\n",
       " EnvSpec(flashgames.DumperRush-v0),\n",
       " EnvSpec(flashgames.LaserCannon3LevelsPack-v0),\n",
       " EnvSpec(flashgames.SlipSlideSloth-v0),\n",
       " EnvSpec(flashgames.PaulVaulting-v0),\n",
       " EnvSpec(flashgames.CoasterRacer2Lvl2-v0),\n",
       " EnvSpec(flashgames.DigToChina-v0),\n",
       " EnvSpec(gym-core.Bowling-v0),\n",
       " EnvSpec(flashgames.ZombieTdReborn-v0),\n",
       " EnvSpec(CrazyClimber-ram-v0),\n",
       " EnvSpec(gym-core.KungFuMaster30FPS-v0),\n",
       " EnvSpec(CrazyClimber-ram-v4),\n",
       " EnvSpec(flashgames.WoollyBearJigsawPuzzle-v0),\n",
       " EnvSpec(gym-core.Pooyan30FPS-v3),\n",
       " EnvSpec(RoadRunner-ramNoFrameskip-v4),\n",
       " EnvSpec(RoadRunner-ramNoFrameskip-v0),\n",
       " EnvSpec(flashgames.SuperRallyExtreme-v0),\n",
       " EnvSpec(Jamesbond-ramDeterministic-v4),\n",
       " EnvSpec(gym-core.Pooyan-v3),\n",
       " EnvSpec(BreakoutDeterministic-v4),\n",
       " EnvSpec(gym-core.Pooyan30FPS-v0),\n",
       " EnvSpec(Jamesbond-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.AmericanRacingLvl14-v0),\n",
       " EnvSpec(BreakoutDeterministic-v0),\n",
       " EnvSpec(flashgames.CoasterRacer2Lvl3-v0),\n",
       " EnvSpec(flashgames.HeatRushFutureLvl14-v0),\n",
       " EnvSpec(flashgames.HeatRushFutureLvl15-v0),\n",
       " EnvSpec(flashgames.BottleCaps-v0),\n",
       " EnvSpec(gym-core.MontezumaRevengeSlow-v3),\n",
       " EnvSpec(SeaquestNoFrameskip-v0),\n",
       " EnvSpec(SeaquestNoFrameskip-v4),\n",
       " EnvSpec(flashgames.LearnToFlyIdle-v0),\n",
       " EnvSpec(gym-core.Asteroids30FPS-v0),\n",
       " EnvSpec(flashgames.JollySwipeLevelPack-v0),\n",
       " EnvSpec(gym-core.Asteroids30FPS-v3),\n",
       " EnvSpec(flashgames.Kinetikz3-v0),\n",
       " EnvSpec(flashgames.PiratesAndCannons-v0),\n",
       " EnvSpec(wob.real.Signup-14-v0),\n",
       " EnvSpec(flashgames.TaxiInc-v0),\n",
       " EnvSpec(flashgames.TankStorm2-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministicSync-v3),\n",
       " EnvSpec(gym-core.TennisSlow-v3),\n",
       " EnvSpec(gym-core.TennisSlow-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministicSync-v0),\n",
       " EnvSpec(flashgames.JetpackJackride-v0),\n",
       " EnvSpec(AirRaid-ram-v0),\n",
       " EnvSpec(Carnival-ram-v0),\n",
       " EnvSpec(AirRaid-ram-v4),\n",
       " EnvSpec(gym-core.AsteroidsDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.AsteroidsDeterministicSlow-v3),\n",
       " EnvSpec(Carnival-ram-v4),\n",
       " EnvSpec(flashgames.MatchStars-v0),\n",
       " EnvSpec(flashgames.GunpowderAndFeathers-v0),\n",
       " EnvSpec(gym-core.VideoPinball-v0),\n",
       " EnvSpec(BeamRiderDeterministic-v0),\n",
       " EnvSpec(flashgames.TheTowerman-v0),\n",
       " EnvSpec(BeamRiderDeterministic-v4),\n",
       " EnvSpec(flashgames.FormulaRacerLvl5-v0),\n",
       " EnvSpec(gym-core.Centipede-v0),\n",
       " EnvSpec(gym-core.Centipede-v3),\n",
       " EnvSpec(Phoenix-ramDeterministic-v0),\n",
       " EnvSpec(gym-core.KrullSync-v3),\n",
       " EnvSpec(gym-core.KrullSync-v0),\n",
       " EnvSpec(gym-core.VentureSync-v0),\n",
       " EnvSpec(Phoenix-ramDeterministic-v4),\n",
       " EnvSpec(flashgames.BubbleShooterChallenge-v0),\n",
       " EnvSpec(flashgames.BubbleBlubbs-v0),\n",
       " EnvSpec(gym-core.KungFuMasterSync-v0),\n",
       " EnvSpec(gym-core.KungFuMasterSync-v3),\n",
       " EnvSpec(gym-core.AmidarSync-v3),\n",
       " EnvSpec(Boxing-ramDeterministic-v4),\n",
       " EnvSpec(gym-core.AmidarSync-v0),\n",
       " EnvSpec(Boxing-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.WaveLucha-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl11-v0),\n",
       " EnvSpec(gym-core.ChopperCommandSync-v0),\n",
       " EnvSpec(gym-core.ElevatorActionDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.ElevatorActionDeterministicSlow-v3),\n",
       " EnvSpec(flashgames.SpaceMadness-v0),\n",
       " EnvSpec(flashgames.KingRolla-v0),\n",
       " EnvSpec(gym-core.BerzerkSlow-v0),\n",
       " EnvSpec(Thrower-v0),\n",
       " EnvSpec(flashgames.BikeTrial3-v0),\n",
       " EnvSpec(flashgames.ChickCannont-v0),\n",
       " EnvSpec(flashgames.BearInSuperActionAdventure-v0),\n",
       " EnvSpec(gym-core.VideoPinballSync-v3),\n",
       " EnvSpec(gym-core.VideoPinballSync-v0),\n",
       " EnvSpec(flashgames.Thaw-v0),\n",
       " EnvSpec(flashgames.NewSiberianSupercarsRacing-v0),\n",
       " EnvSpec(flashgames.MinedigJourneyToHollowEarth-v0),\n",
       " EnvSpec(gym-core.BankHeistSync-v0),\n",
       " EnvSpec(gym-core.SpaceInvaders-v3),\n",
       " EnvSpec(flashgames.NeonRace2Lvl9-v0),\n",
       " EnvSpec(gym-core.NameThisGameDeterministicSync-v3),\n",
       " EnvSpec(gym-core.NameThisGameDeterministicSync-v0),\n",
       " EnvSpec(flashgames.Foosball2Player-v0),\n",
       " EnvSpec(gym-core.BattleZoneNoFrameskip-v0),\n",
       " EnvSpec(flashgames.SuperShinyheadHarderThanFlappyBird-v0),\n",
       " EnvSpec(gym-core.RoadRunnerNoFrameskip-v3),\n",
       " EnvSpec(gym-core.RoadRunnerNoFrameskip-v0),\n",
       " EnvSpec(gym-core.BreakoutSync-v0),\n",
       " EnvSpec(gym-core.BreakoutSync-v3),\n",
       " EnvSpec(flashgames.BullfrogJigsawPuzzle-v0),\n",
       " EnvSpec(flashgames.TheSilentPlanet-v0),\n",
       " EnvSpec(BerzerkDeterministic-v4),\n",
       " EnvSpec(flashgames.EuroKicks2016-v0),\n",
       " EnvSpec(wob.real.Quizlet-Solar-System-Learn-v0),\n",
       " EnvSpec(BerzerkDeterministic-v0),\n",
       " EnvSpec(AssaultNoFrameskip-v0),\n",
       " EnvSpec(flashgames.TouchTheSky-v0),\n",
       " EnvSpec(AssaultNoFrameskip-v4),\n",
       " EnvSpec(PhoenixNoFrameskip-v4),\n",
       " EnvSpec(gym-core.DoubleDunkSlow-v3),\n",
       " EnvSpec(PhoenixNoFrameskip-v0),\n",
       " EnvSpec(flashgames.IdleChop-v0),\n",
       " EnvSpec(gym-core.YarsRevenge-v3),\n",
       " EnvSpec(gym-core.AlienNoFrameskip-v0),\n",
       " EnvSpec(Humanoid-v1),\n",
       " EnvSpec(gym-core.AsteroidsSync-v3),\n",
       " EnvSpec(wob.mini.FindMidpoint-v0),\n",
       " EnvSpec(flashgames.DartsSim-v0),\n",
       " EnvSpec(flashgames.SmileyShowdown-v0),\n",
       " EnvSpec(flashgames.NeonRace2Lvl8-v0),\n",
       " EnvSpec(flashgames.SneakyScubaEscape-v0),\n",
       " EnvSpec(flashgames.SuperDash-v0),\n",
       " EnvSpec(flashgames.MummyMadness-v0),\n",
       " EnvSpec(gym-core.RobotankSlow-v0),\n",
       " EnvSpec(flashgames.HoldTheFort-v0),\n",
       " EnvSpec(KungFuMasterNoFrameskip-v0),\n",
       " EnvSpec(Frostbite-ramDeterministic-v4),\n",
       " EnvSpec(Frostbite-ramDeterministic-v0),\n",
       " EnvSpec(gym-core.VentureDeterministicSync-v3),\n",
       " EnvSpec(gym-core.VentureDeterministicSync-v0),\n",
       " EnvSpec(flashgames.FairyDefense-v0),\n",
       " EnvSpec(gym-core.RobotankSync-v0),\n",
       " EnvSpec(gym-core.RobotankSync-v3),\n",
       " EnvSpec(Qbert-ramNoFrameskip-v0),\n",
       " EnvSpec(Ant-v1),\n",
       " EnvSpec(Qbert-ramNoFrameskip-v4),\n",
       " EnvSpec(gym-core.Seaquest-v0),\n",
       " EnvSpec(gym-core.Seaquest-v3),\n",
       " EnvSpec(YarsRevenge-ram-v0),\n",
       " EnvSpec(YarsRevenge-ram-v4),\n",
       " EnvSpec(flashgames.IceBlock-v0),\n",
       " EnvSpec(FishingDerby-ram-v0),\n",
       " EnvSpec(Enduro-ramNoFrameskip-v4),\n",
       " EnvSpec(FrostbiteNoFrameskip-v0),\n",
       " EnvSpec(FishingDerby-ram-v4),\n",
       " EnvSpec(Enduro-ramNoFrameskip-v0),\n",
       " EnvSpec(FrostbiteNoFrameskip-v4),\n",
       " EnvSpec(gym-core.MsPacmanDeterministic-v3),\n",
       " EnvSpec(wob.mini.ClickColor-v0),\n",
       " EnvSpec(gym-core.MsPacmanDeterministic-v0),\n",
       " EnvSpec(flashgames.Dots-v0),\n",
       " EnvSpec(flashgames.NeonRace2Lvl6-v0),\n",
       " EnvSpec(gym-core.JamesbondDeterministicSlow-v3),\n",
       " EnvSpec(flashgames.NeonRace2Lvl12-v0),\n",
       " EnvSpec(gym-core.ElevatorActionDeterministicSync-v3),\n",
       " EnvSpec(gym-core.ElevatorActionDeterministicSync-v0),\n",
       " EnvSpec(flashgames.PixelPurge-v0),\n",
       " EnvSpec(flashgames.ReleaseTheMooks-v0),\n",
       " EnvSpec(gym-core.StarGunnerSlow-v3),\n",
       " EnvSpec(gym-core.StarGunnerSlow-v0),\n",
       " EnvSpec(flashgames.SapphireClix-v0),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministicSync-v0),\n",
       " EnvSpec(flashgames.SlingBaby-v0),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministicSync-v3),\n",
       " EnvSpec(gym-core.KungFuMasterDeterministicSlow-v3),\n",
       " EnvSpec(StarGunnerDeterministic-v0),\n",
       " EnvSpec(StarGunnerDeterministic-v4),\n",
       " EnvSpec(flashgames.SandcastleShowdown-v0),\n",
       " EnvSpec(flashgames.CharlieTheDuck-v0),\n",
       " EnvSpec(CNNClassifierTraining-v0),\n",
       " EnvSpec(wob.mini.EnterText-v0),\n",
       " EnvSpec(gym-core.BankHeistDeterministic-v3),\n",
       " EnvSpec(flashgames.CanyonValleyRally-v0),\n",
       " EnvSpec(gym-core.VentureDeterministic-v0),\n",
       " EnvSpec(Boxing-ram-v0),\n",
       " EnvSpec(Boxing-ram-v4),\n",
       " EnvSpec(gym-core.VentureDeterministic-v3),\n",
       " EnvSpec(flashgames.WackyStrike-v0),\n",
       " EnvSpec(flashgames.EasterEggSlider-v0),\n",
       " EnvSpec(flashgames.MindImpulse-v0),\n",
       " EnvSpec(flashgames.NeonRace2Lvl13-v0),\n",
       " EnvSpec(flashgames.FormulaRacer-v0),\n",
       " EnvSpec(flashgames.Hamsterball-v0),\n",
       " EnvSpec(gym-core.KungFuMasterDeterministicSync-v0),\n",
       " EnvSpec(Assault-ram-v0),\n",
       " EnvSpec(wob.mini.NumberCheckboxes-v0),\n",
       " EnvSpec(Assault-ram-v4),\n",
       " EnvSpec(gym-core.BreakoutDeterministic-v0),\n",
       " EnvSpec(gym-core.BreakoutDeterministic-v3),\n",
       " EnvSpec(flashgames.Offroaders2-v0),\n",
       " EnvSpec(gym-core.JamesbondDeterministicSync-v3),\n",
       " EnvSpec(gym-core.JamesbondDeterministicSync-v0),\n",
       " EnvSpec(flashgames.SnowQueen4-v0),\n",
       " EnvSpec(flashgames.ToyWarAngryRobotDog-v0),\n",
       " EnvSpec(flashgames.HighwayRevenge-v0),\n",
       " EnvSpec(flashgames.CarrotFantasyExtreme3-v0),\n",
       " EnvSpec(Solaris-ramDeterministic-v4),\n",
       " EnvSpec(ElevatorActionDeterministic-v4),\n",
       " EnvSpec(Solaris-ramDeterministic-v0),\n",
       " EnvSpec(ElevatorActionDeterministic-v0),\n",
       " EnvSpec(Solaris-v4),\n",
       " EnvSpec(Solaris-v0),\n",
       " EnvSpec(flashgames.BlockysEscape-v0),\n",
       " EnvSpec(gym-core.StarGunnerDeterministic-v3),\n",
       " EnvSpec(flashgames.HeroesOfMangaraTheFrostCrown-v0),\n",
       " EnvSpec(SolarisNoFrameskip-v0),\n",
       " EnvSpec(gym-core.PongDeterministic-v0),\n",
       " EnvSpec(SolarisNoFrameskip-v4),\n",
       " EnvSpec(RoadRunner-v4),\n",
       " EnvSpec(flashgames.ShortCircuit-v0),\n",
       " EnvSpec(RoadRunner-v0),\n",
       " EnvSpec(gym-core.AssaultDeterministicSlow-v0),\n",
       " EnvSpec(flashgames.SupercarDomination-v0),\n",
       " EnvSpec(gym-core.AssaultDeterministicSlow-v3),\n",
       " EnvSpec(flashgames.Xmatch2016-v0),\n",
       " EnvSpec(RoadRunner-ram-v0),\n",
       " EnvSpec(flashgames.CrystalCurse-v0),\n",
       " EnvSpec(RoadRunner-ram-v4),\n",
       " EnvSpec(gym-core.AsteroidsSlow-v3),\n",
       " EnvSpec(gym-core.AsteroidsSlow-v0),\n",
       " EnvSpec(Skiing-ramDeterministic-v0),\n",
       " EnvSpec(Skiing-ramDeterministic-v4),\n",
       " EnvSpec(flashgames.BlackRacerJigsawPuzzle-v0),\n",
       " EnvSpec(VideoPinball-ram-v0),\n",
       " EnvSpec(VideoPinball-ram-v4),\n",
       " EnvSpec(flashgames.DaymareInvaders-v0),\n",
       " EnvSpec(gym-core.GravitarNoFrameskip-v0),\n",
       " EnvSpec(FreewayNoFrameskip-v4),\n",
       " EnvSpec(WizardOfWorNoFrameskip-v0),\n",
       " EnvSpec(FreewayNoFrameskip-v0),\n",
       " EnvSpec(WizardOfWorNoFrameskip-v4),\n",
       " EnvSpec(flashgames.UnderwaterSecrets-v0),\n",
       " EnvSpec(gym-core.ElevatorAction30FPS-v0),\n",
       " EnvSpec(gym-core.ElevatorAction30FPS-v3),\n",
       " EnvSpec(flashgames.OkParking-v0),\n",
       " EnvSpec(flashgames.HeatRushFuture-v0),\n",
       " EnvSpec(gym-core.Venture-v0),\n",
       " EnvSpec(DoubleDunk-v4),\n",
       " EnvSpec(gym-core.MsPacmanDeterministicSync-v0),\n",
       " EnvSpec(gym-core.MsPacmanDeterministicSync-v3),\n",
       " EnvSpec(flashgames.WorldsGuard2-v0),\n",
       " EnvSpec(gym-core.BattleZoneDeterministicSync-v3),\n",
       " EnvSpec(gym-core.BattleZoneDeterministicSync-v0),\n",
       " EnvSpec(gym-core.VideoPinballNoFrameskip-v0),\n",
       " EnvSpec(gym-core.VideoPinballNoFrameskip-v3),\n",
       " EnvSpec(gym-core.JourneyEscapeNoFrameskip-v0),\n",
       " EnvSpec(HeroDeterministic-v4),\n",
       " EnvSpec(gym-core.JourneyEscapeNoFrameskip-v3),\n",
       " EnvSpec(HeroDeterministic-v0),\n",
       " EnvSpec(ZaxxonDeterministic-v0),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministic-v0),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministic-v3),\n",
       " EnvSpec(flashgames.Colorwars-v0),\n",
       " EnvSpec(gym-core.MsPacmanNoFrameskip-v0),\n",
       " EnvSpec(gym-core.JamesbondSlow-v0),\n",
       " EnvSpec(gym-core.JamesbondSlow-v3),\n",
       " EnvSpec(gym-core.MsPacmanNoFrameskip-v3),\n",
       " EnvSpec(gym-core.RiverraidSlow-v3),\n",
       " EnvSpec(gym-core.CartPole-v0),\n",
       " EnvSpec(flashgames.GalaxyMission-v0),\n",
       " EnvSpec(BeamRider-v4),\n",
       " EnvSpec(flashgames.SuperBoxotron2000-v0),\n",
       " EnvSpec(FishingDerby-ramDeterministic-v0),\n",
       " EnvSpec(FishingDerby-ramDeterministic-v4),\n",
       " EnvSpec(flashgames.Stratega-v0),\n",
       " EnvSpec(gym-core.AsteroidsNoFrameskip-v3),\n",
       " EnvSpec(CrazyClimber-v4),\n",
       " EnvSpec(flashgames.ParticleWarsExtreme-v0),\n",
       " EnvSpec(gym-core.AsteroidsNoFrameskip-v0),\n",
       " EnvSpec(CrazyClimber-v0),\n",
       " EnvSpec(flashgames.HexBattles-v0),\n",
       " EnvSpec(gym-core.BoxingNoFrameskip-v3),\n",
       " EnvSpec(Pitfall-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.BumbleTumble-v0),\n",
       " EnvSpec(gym-core.BoxingNoFrameskip-v0),\n",
       " EnvSpec(Pitfall-ramDeterministic-v4),\n",
       " EnvSpec(wob.real.Quizlet-Geography-Test-v0),\n",
       " EnvSpec(gym-core.SkiingSync-v0),\n",
       " EnvSpec(gym-core.SkiingSync-v3),\n",
       " EnvSpec(GravitarDeterministic-v4),\n",
       " EnvSpec(flashgames.CoasterRacerLvl5-v0),\n",
       " EnvSpec(GravitarDeterministic-v0),\n",
       " EnvSpec(gym-core.BowlingNoFrameskip-v3),\n",
       " EnvSpec(gym-core.BowlingNoFrameskip-v0),\n",
       " EnvSpec(flashgames.EvilSun-v0),\n",
       " EnvSpec(flashgames.HalloweenJam-v0),\n",
       " EnvSpec(flashgames.LlamasInDistress-v0),\n",
       " EnvSpec(gym-core.PooyanDeterministic-v0),\n",
       " EnvSpec(flashgames.WreckRoad-v0),\n",
       " EnvSpec(wob.real.Signup-6-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl9-v0),\n",
       " EnvSpec(gym-core.ElevatorActionSlow-v0),\n",
       " EnvSpec(gym-core.ElevatorActionSlow-v3),\n",
       " EnvSpec(wob.real.Signup-7-v0),\n",
       " EnvSpec(flashgames.SistersOfNoMercy-v0),\n",
       " EnvSpec(Tennis-v0),\n",
       " EnvSpec(flashgames.CoasterRacerLvl4-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl4-v0),\n",
       " EnvSpec(gym-core.RoadRunner-v0),\n",
       " EnvSpec(gym-core.RoadRunner-v3),\n",
       " EnvSpec(Carnival-ramDeterministic-v4),\n",
       " EnvSpec(gym-core.Carnival-v0),\n",
       " EnvSpec(flashgames.CursedTreasureDontTouchMyGems-v0),\n",
       " EnvSpec(gym-core.Carnival-v3),\n",
       " EnvSpec(flashgames.FlappyBat-v0),\n",
       " EnvSpec(wob.mini.ResizeTextarea-v0),\n",
       " EnvSpec(gym-core.SpaceInvadersNoFrameskip-v3),\n",
       " EnvSpec(TennisNoFrameskip-v0),\n",
       " EnvSpec(gym-core.PhoenixNoFrameskip-v3),\n",
       " EnvSpec(gym-core.PhoenixNoFrameskip-v0),\n",
       " EnvSpec(gym-core.Alien30FPS-v3),\n",
       " EnvSpec(gym-core.Alien30FPS-v0),\n",
       " EnvSpec(flashgames.TheOneForkRestaurantDx-v0),\n",
       " EnvSpec(DemonAttack-ram-v0),\n",
       " EnvSpec(DemonAttack-ram-v4),\n",
       " EnvSpec(gym-core.KungFuMasterNoFrameskip-v0),\n",
       " EnvSpec(gym-core.KungFuMasterNoFrameskip-v3),\n",
       " EnvSpec(flashgames.SpacePunkRacer-v0),\n",
       " EnvSpec(TennisNoFrameskip-v4),\n",
       " EnvSpec(flashgames.FishAndDestroy-v0),\n",
       " EnvSpec(flashgames.KartRacing-v0),\n",
       " EnvSpec(flashgames.JellySnake-v0),\n",
       " EnvSpec(gym-core.AsterixSlow-v0),\n",
       " EnvSpec(gym-core.AsterixSlow-v3),\n",
       " EnvSpec(flashgames.FishEatFish-v0),\n",
       " EnvSpec(flashgames.Jumprunner-v0),\n",
       " EnvSpec(flashgames.HoleInOne-v0),\n",
       " EnvSpec(flashgames.AwesomeRun2-v0),\n",
       " EnvSpec(gym-core.BattleZoneSlow-v0),\n",
       " EnvSpec(gym-core.BattleZoneSlow-v3),\n",
       " EnvSpec(flashgames.IntoSpace-v0),\n",
       " EnvSpec(flashgames.CarsVsRobots-v0),\n",
       " EnvSpec(flashgames.BubbleHitPonyParade-v0),\n",
       " EnvSpec(flashgames.AWeekendAtTweetys-v0),\n",
       " EnvSpec(gym-core.DoubleDunkDeterministicSync-v0),\n",
       " EnvSpec(gym-core.DoubleDunkDeterministicSync-v3),\n",
       " EnvSpec(Amidar-v4),\n",
       " EnvSpec(flashgames.ModelCarRacing-v0),\n",
       " EnvSpec(gym-core.ChopperCommandDeterministicSync-v3),\n",
       " EnvSpec(gym-core.ChopperCommandDeterministicSync-v0),\n",
       " EnvSpec(flashgames.PirateRunAway-v0),\n",
       " EnvSpec(flashgames.MonsterLabFeedThemAll-v0),\n",
       " EnvSpec(Gravitar-ramDeterministic-v0),\n",
       " EnvSpec(gym-core.DemonAttack30FPS-v0),\n",
       " EnvSpec(flashgames.ViewtifulFightClub2-v0),\n",
       " EnvSpec(Gravitar-ramDeterministic-v4),\n",
       " EnvSpec(BattleZone-ram-v4),\n",
       " EnvSpec(BattleZone-ram-v0),\n",
       " EnvSpec(IceHockey-ram-v0),\n",
       " EnvSpec(IceHockey-ram-v4),\n",
       " EnvSpec(flashgames.DragonChronicles-v0),\n",
       " EnvSpec(gym-core.PitfallDeterministic-v0),\n",
       " EnvSpec(gym-core.PitfallDeterministic-v3),\n",
       " EnvSpec(flashgames.SnowPrincessMakeup-v0),\n",
       " EnvSpec(flashgames.QubeyTheCube-v0),\n",
       " EnvSpec(gym-core.Alien-v3),\n",
       " EnvSpec(gym-core.Alien-v0),\n",
       " EnvSpec(gym-core.AtlantisSync-v0),\n",
       " EnvSpec(gym-core.AtlantisSync-v3),\n",
       " EnvSpec(flashgames.TowerMoon-v0),\n",
       " EnvSpec(flashgames.MotherLoad-v0),\n",
       " EnvSpec(wob.mini.EnterTextDynamic-v0),\n",
       " EnvSpec(flashgames.BubbleGlee-v0),\n",
       " EnvSpec(flashgames.FirefighterCannon-v0),\n",
       " EnvSpec(gym-core.BerzerkDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.BerzerkDeterministicSlow-v3),\n",
       " EnvSpec(DoubleDunk-ramNoFrameskip-v0),\n",
       " EnvSpec(gym-core.IceHockeySlow-v3),\n",
       " EnvSpec(gym-core.IceHockeySlow-v0),\n",
       " EnvSpec(AssaultDeterministic-v0),\n",
       " EnvSpec(DoubleDunk-ramNoFrameskip-v4),\n",
       " EnvSpec(MsPacman-v0),\n",
       " EnvSpec(flashgames.Offroaders-v0),\n",
       " EnvSpec(MsPacman-v4),\n",
       " EnvSpec(flashgames.HiredHeroes-v0),\n",
       " EnvSpec(flashgames.WolfSpiderJigsawPuzzle-v0),\n",
       " EnvSpec(AssaultDeterministic-v4),\n",
       " EnvSpec(gym-core.FreewayDeterministic-v3),\n",
       " EnvSpec(flashgames.ToonEscapeMaze-v0),\n",
       " EnvSpec(wob.mini.ClickCheckboxes-v0),\n",
       " EnvSpec(gym-core.FreewayDeterministic-v0),\n",
       " EnvSpec(Seaquest-ramNoFrameskip-v0),\n",
       " EnvSpec(Seaquest-ramNoFrameskip-v4),\n",
       " EnvSpec(Blackjack-v0),\n",
       " EnvSpec(TennisDeterministic-v0),\n",
       " EnvSpec(TennisDeterministic-v4),\n",
       " EnvSpec(Atlantis-v4),\n",
       " EnvSpec(Atlantis-v0),\n",
       " EnvSpec(UpNDownDeterministic-v0),\n",
       " EnvSpec(flashgames.WarBerlinIdle-v0),\n",
       " EnvSpec(gym-core.BattleZoneSync-v3),\n",
       " EnvSpec(gym-core.Centipede30FPS-v0),\n",
       " EnvSpec(gym-core.Centipede30FPS-v3),\n",
       " EnvSpec(gym-core.BattleZoneSync-v0),\n",
       " EnvSpec(gym-core.FishingDerby-v0),\n",
       " EnvSpec(Asteroids-v0),\n",
       " EnvSpec(Asteroids-v4),\n",
       " EnvSpec(gym-core.SpaceInvaders30FPS-v0),\n",
       " EnvSpec(UpNDownDeterministic-v4),\n",
       " EnvSpec(IceHockeyDeterministic-v4),\n",
       " EnvSpec(flashgames.SpectrumHeist-v0),\n",
       " EnvSpec(gym-core.BattleZoneNoFrameskip-v3),\n",
       " EnvSpec(IceHockeyDeterministic-v0),\n",
       " EnvSpec(gym-core.EnduroNoFrameskip-v3),\n",
       " EnvSpec(gym-core.EnduroNoFrameskip-v0),\n",
       " EnvSpec(flashgames.TankStorm3-v0),\n",
       " EnvSpec(gym-core.BeamRiderDeterministic-v0),\n",
       " EnvSpec(gym-core.BeamRiderDeterministic-v3),\n",
       " EnvSpec(flashgames.Infinitix-v0),\n",
       " EnvSpec(flashgames.PoliceInterceptor-v0),\n",
       " EnvSpec(gym-core.CrazyClimber-v3),\n",
       " EnvSpec(wob.real.ClickButton-Airfrance-v0),\n",
       " EnvSpec(gym-core.CrazyClimber-v0),\n",
       " EnvSpec(flashgames.Autoattack-v0),\n",
       " EnvSpec(flashgames.CircuitSuperCarsRacing-v0),\n",
       " EnvSpec(flashgames.HeatRushUsaLvl8-v0),\n",
       " EnvSpec(flashgames.Blix-v0),\n",
       " EnvSpec(WizardOfWorDeterministic-v4),\n",
       " EnvSpec(gym-core.RoadRunnerDeterministic-v0),\n",
       " EnvSpec(gym-core.RoadRunnerDeterministic-v3),\n",
       " EnvSpec(WizardOfWorDeterministic-v0),\n",
       " EnvSpec(gym-core.IceHockeyNoFrameskip-v3),\n",
       " EnvSpec(gym-core.IceHockeyNoFrameskip-v0),\n",
       " EnvSpec(flashgames.ElClassico-v0),\n",
       " EnvSpec(DoubleDunk-v0),\n",
       " EnvSpec(LunarLander-v2),\n",
       " EnvSpec(MsPacman-ramDeterministic-v4),\n",
       " EnvSpec(flashgames.Neopods-v0),\n",
       " EnvSpec(MsPacman-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.TutiFruti-v0),\n",
       " EnvSpec(flashgames.WhatsInsideTheBox-v0),\n",
       " EnvSpec(BoxingDeterministic-v0),\n",
       " EnvSpec(BoxingDeterministic-v4),\n",
       " EnvSpec(gym-core.PooyanDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.PooyanDeterministicSlow-v0),\n",
       " EnvSpec(flashgames.3dMuscleCarRacer-v0),\n",
       " EnvSpec(flashgames.ColorZapper-v0),\n",
       " EnvSpec(Robotank-v4),\n",
       " EnvSpec(Robotank-v0),\n",
       " EnvSpec(flashgames.HeroSimulator-v0),\n",
       " EnvSpec(wob.mini.ClickButton-v0),\n",
       " EnvSpec(wob.mini.SimpleArithmetic-v0),\n",
       " EnvSpec(flashgames.FormulaRacer2012Lvl11-v0),\n",
       " EnvSpec(flashgames.SuperRallyChallenge2-v0),\n",
       " EnvSpec(flashgames.AmericanRacing2-v0),\n",
       " EnvSpec(gym-core.SolarisSync-v0),\n",
       " EnvSpec(gym-core.SolarisSync-v3),\n",
       " EnvSpec(Gravitar-ram-v0),\n",
       " EnvSpec(Frostbite-v4),\n",
       " EnvSpec(Gravitar-ram-v4),\n",
       " EnvSpec(Frostbite-v0),\n",
       " EnvSpec(Acrobot-v1),\n",
       " EnvSpec(gym-core.FrostbiteDeterministic-v3),\n",
       " EnvSpec(gym-core.FrostbiteDeterministic-v0),\n",
       " EnvSpec(wob.mini.ClickDialog-v0),\n",
       " EnvSpec(flashgames.Wheelers-v0),\n",
       " EnvSpec(starcraft.TerranAstralBalance-v0),\n",
       " EnvSpec(ZaxxonNoFrameskip-v0),\n",
       " EnvSpec(HeroNoFrameskip-v4),\n",
       " EnvSpec(flashgames.BubbleSlasher-v0),\n",
       " EnvSpec(ZaxxonNoFrameskip-v4),\n",
       " EnvSpec(flashgames.AmericanRacingLvl3-v0),\n",
       " EnvSpec(HeroNoFrameskip-v0),\n",
       " EnvSpec(NameThisGame-v0),\n",
       " EnvSpec(flashgames.FormulaRacer2012Lvl10-v0),\n",
       " EnvSpec(flashgames.CoverOrangeJourneyGangsters-v0),\n",
       " EnvSpec(NameThisGame-v4),\n",
       " EnvSpec(flashgames.PaintWars-v0),\n",
       " EnvSpec(gym-core.StarGunner-v0),\n",
       " EnvSpec(flashgames.AchilliaTheGame-v0),\n",
       " EnvSpec(gym-core.StarGunner-v3),\n",
       " EnvSpec(flashgames.KnightsOfRock-v0),\n",
       " EnvSpec(gym-core.Jamesbond30FPS-v0),\n",
       " EnvSpec(gym-core.Jamesbond30FPS-v3),\n",
       " EnvSpec(flashgames.GalacticCats-v0),\n",
       " EnvSpec(Krull-ram-v0),\n",
       " EnvSpec(Krull-ram-v4),\n",
       " EnvSpec(flashgames.DeathDiceOverdose-v0),\n",
       " EnvSpec(flashgames.BubbleRubble-v0),\n",
       " EnvSpec(BowlingNoFrameskip-v0),\n",
       " EnvSpec(flashgames.ImitationNationSnakeGame-v0),\n",
       " EnvSpec(gym-core.CentipedeSync-v3),\n",
       " EnvSpec(BowlingNoFrameskip-v4),\n",
       " EnvSpec(gym-core.CentipedeSync-v0),\n",
       " EnvSpec(flashgames.IceRun-v0),\n",
       " EnvSpec(flashgames.Madburger3-v0),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministic-v3),\n",
       " EnvSpec(flashgames.GsSoccerWorldCup-v0),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministic-v0),\n",
       " EnvSpec(flashgames.NeonRaceLvl3-v0),\n",
       " EnvSpec(Assault-ramNoFrameskip-v0),\n",
       " EnvSpec(Assault-ramNoFrameskip-v4),\n",
       " EnvSpec(BankHeist-ram-v0),\n",
       " EnvSpec(flashgames.HungryPiranha-v0),\n",
       " EnvSpec(BankHeist-ram-v4),\n",
       " EnvSpec(flashgames.DoodleGod2Walkthrough-v0),\n",
       " EnvSpec(SpaceInvaders-ram-v0),\n",
       " EnvSpec(SpaceInvaders-ram-v4),\n",
       " EnvSpec(FishingDerby-v4),\n",
       " EnvSpec(FishingDerby-v0),\n",
       " EnvSpec(flashgames.TheBoomlandsWorldWars-v0),\n",
       " EnvSpec(gym-core.GravitarDeterministicSync-v0),\n",
       " EnvSpec(Freeway-ramDeterministic-v4),\n",
       " EnvSpec(gym-core.GravitarDeterministicSync-v3),\n",
       " EnvSpec(DoubleDunkNoFrameskip-v0),\n",
       " EnvSpec(flashgames.Helixteus-v0),\n",
       " EnvSpec(DoubleDunkNoFrameskip-v4),\n",
       " EnvSpec(flashgames.SuperPuzzlePlatformer-v0),\n",
       " EnvSpec(flashgames.MatchAndCrash-v0),\n",
       " EnvSpec(AsterixNoFrameskip-v0),\n",
       " EnvSpec(AsterixNoFrameskip-v4),\n",
       " EnvSpec(flashgames.MatchCraft-v0),\n",
       " EnvSpec(SpaceInvaders-ramDeterministic-v4),\n",
       " EnvSpec(SpaceInvaders-ramDeterministic-v0),\n",
       " EnvSpec(JourneyEscapeDeterministic-v0),\n",
       " EnvSpec(flashgames.HandsOff-v0),\n",
       " EnvSpec(flashgames.Zevil2-v0),\n",
       " EnvSpec(JourneyEscapeDeterministic-v4),\n",
       " EnvSpec(flashgames.Paintwars-v0),\n",
       " EnvSpec(gym-core.PooyanSync-v0),\n",
       " EnvSpec(gym-core.PooyanSync-v3),\n",
       " EnvSpec(flashgames.MasterDifference-v0),\n",
       " EnvSpec(BeamRider-ram-v0),\n",
       " EnvSpec(BeamRider-ram-v4),\n",
       " EnvSpec(gym-core.ChopperCommandSlow-v0),\n",
       " EnvSpec(gym-core.ChopperCommandSlow-v3),\n",
       " EnvSpec(gym-core.Bowling30FPS-v0),\n",
       " EnvSpec(gym-core.Bowling30FPS-v3),\n",
       " EnvSpec(flashgames.Overheat-v0),\n",
       " EnvSpec(flashgames.GravityThruster-v0),\n",
       " EnvSpec(flashgames.NeonRaceLvl2-v0),\n",
       " EnvSpec(gtav.Speed-v0),\n",
       " EnvSpec(flashgames.TowerEmpire-v0),\n",
       " EnvSpec(Freeway-ramNoFrameskip-v4),\n",
       " EnvSpec(Freeway-ramNoFrameskip-v0),\n",
       " EnvSpec(PitfallDeterministic-v4),\n",
       " EnvSpec(flashgames.FormulaRacerLvl6-v0),\n",
       " EnvSpec(flashgames.HappyBallz-v0),\n",
       " EnvSpec(PitfallDeterministic-v0),\n",
       " EnvSpec(flashgames.Flagman-v0),\n",
       " EnvSpec(flashgames.PiggysCupcakeQuest-v0),\n",
       " EnvSpec(gym-core.KungFuMaster-v3),\n",
       " EnvSpec(gym-core.KungFuMaster-v0),\n",
       " EnvSpec(flashgames.AmericanRacingLvl21-v0),\n",
       " EnvSpec(flashgames.CoasterRacer-v0),\n",
       " EnvSpec(gym-core.PooyanNoFrameskip-v0),\n",
       " EnvSpec(flashgames.JungleCrash-v0),\n",
       " EnvSpec(gym-core.PooyanNoFrameskip-v3),\n",
       " EnvSpec(gym-core.RiverraidSync-v0),\n",
       " EnvSpec(gym-core.RiverraidSync-v3),\n",
       " EnvSpec(flashgames.ToyRacers-v0),\n",
       " EnvSpec(gym-core.JamesbondNoFrameskip-v0),\n",
       " EnvSpec(flashgames.DrinkBeerNeglectFamily-v0),\n",
       " EnvSpec(gym-core.ZaxxonDeterministic-v0),\n",
       " EnvSpec(gym-core.ZaxxonDeterministic-v3),\n",
       " EnvSpec(gym-core.Seaquest30FPS-v3),\n",
       " EnvSpec(gym-core.Seaquest30FPS-v0),\n",
       " EnvSpec(flashgames.FormulaRacer2012Lvl6-v0),\n",
       " EnvSpec(flashgames.TheCubicMonkeyAdventures2-v0),\n",
       " EnvSpec(flashgames.PlaneRace2-v0),\n",
       " EnvSpec(flashgames.Cruisin-v0),\n",
       " EnvSpec(DuplicatedInput-v0),\n",
       " EnvSpec(flashgames.WarOfTheShard-v0),\n",
       " EnvSpec(flashgames.UnfreezeMe3-v0),\n",
       " EnvSpec(flashgames.AnotherLife2-v0),\n",
       " EnvSpec(flashgames.HeatRushUsaLvl2-v0),\n",
       " EnvSpec(wob.mini.EnterTime-v0),\n",
       " EnvSpec(DemonAttack-ramNoFrameskip-v0),\n",
       " EnvSpec(gym-core.StarGunnerNoFrameskip-v3),\n",
       " EnvSpec(DemonAttack-ramNoFrameskip-v4),\n",
       " EnvSpec(flashgames.ExperimentalShooter2-v0),\n",
       " EnvSpec(flashgames.DodgeAndCrash-v0),\n",
       " EnvSpec(flashgames.BoxRacers-v0),\n",
       " EnvSpec(flashgames.IndependenceDaySlacking2015-v0),\n",
       " EnvSpec(flashgames.UltimateLegend-v0),\n",
       " EnvSpec(gym-core.CarnivalSlow-v0),\n",
       " EnvSpec(gym-core.CarnivalSlow-v3),\n",
       " EnvSpec(flashgames.RainbowDrops-v0),\n",
       " EnvSpec(gym-core.KungFuMasterDeterministic-v3),\n",
       " EnvSpec(gym-core.FreewaySync-v0),\n",
       " EnvSpec(gym-core.FreewaySync-v3),\n",
       " EnvSpec(gym-core.KungFuMasterDeterministic-v0),\n",
       " EnvSpec(flashgames.HyperTravel-v0),\n",
       " EnvSpec(flashgames.RiseOfChampions-v0),\n",
       " EnvSpec(flashgames.NadiasRage-v0),\n",
       " EnvSpec(flashgames.TrickyRick-v0),\n",
       " EnvSpec(flashgames.FredFigglehorn-v0),\n",
       " EnvSpec(flashgames.FormulaRacer2012Lvl7-v0),\n",
       " EnvSpec(flashgames.FlyingKiwi-v0),\n",
       " EnvSpec(flashgames.PickUpTruckRacing-v0),\n",
       " EnvSpec(flashgames.RhythmBlasterV2-v0),\n",
       " EnvSpec(flashgames.HeavyLegion2-v0),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.AlienDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.AlienDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.BattleZoneDeterministicSlow-v0),\n",
       " EnvSpec(flashgames.MushyMishy-v0),\n",
       " EnvSpec(flashgames.BombIt4-v0),\n",
       " EnvSpec(flashgames.MonkeyGems-v0),\n",
       " EnvSpec(gym-core.YarsRevengeDeterministicSync-v0),\n",
       " EnvSpec(flashgames.TechnoMania-v0),\n",
       " EnvSpec(gym-core.YarsRevengeDeterministicSync-v3),\n",
       " EnvSpec(flashgames.Mushbooms-v0),\n",
       " EnvSpec(RiverraidDeterministic-v4),\n",
       " EnvSpec(gym-core.BattleZoneDeterministicSlow-v3),\n",
       " EnvSpec(RiverraidDeterministic-v0),\n",
       " EnvSpec(flashgames.TumbleTiles-v0),\n",
       " EnvSpec(wob.mini.ChooseList-v0),\n",
       " EnvSpec(flashgames.Basement-v0),\n",
       " EnvSpec(flashgames.BikeTrial4-v0),\n",
       " EnvSpec(gym-core.CarnivalDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.CarnivalDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.CrazyClimberNoFrameskip-v3),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministicSync-v3),\n",
       " EnvSpec(gym-core.PrivateEyeDeterministicSync-v0),\n",
       " EnvSpec(flashgames.HeavenAndHell-v0),\n",
       " EnvSpec(flashgames.JamesTheSpaceZebra-v0),\n",
       " EnvSpec(flashgames.GoGreenGo-v0),\n",
       " EnvSpec(KellyCoinflipGeneralized-v0),\n",
       " EnvSpec(gym-core.UpNDown-v0),\n",
       " EnvSpec(flashgames.CaptainNutty-v0),\n",
       " EnvSpec(flashgames.TheProfessionals3-v0),\n",
       " EnvSpec(gym-core.SolarisNoFrameskip-v0),\n",
       " EnvSpec(gym-core.SolarisNoFrameskip-v3),\n",
       " EnvSpec(flashgames.PickAndDig2-v0),\n",
       " EnvSpec(flashgames.EasterBunnyCollectCarrots-v0),\n",
       " EnvSpec(Gopher-ram-v0),\n",
       " EnvSpec(flashgames.Tosuta-v0),\n",
       " EnvSpec(Gopher-ram-v4),\n",
       " EnvSpec(flashgames.StickyNinjaMissions-v0),\n",
       " EnvSpec(CarRacing-v0),\n",
       " EnvSpec(flashgames.DiscoverEurope-v0),\n",
       " EnvSpec(gym-core.VideoPinballDeterministicSync-v3),\n",
       " EnvSpec(gym-core.BerzerkSync-v3),\n",
       " EnvSpec(flashgames.4x4Monster3-v0),\n",
       " EnvSpec(wob.mini.CopyPaste2-v0),\n",
       " EnvSpec(gym-core.BerzerkSync-v0),\n",
       " EnvSpec(AmidarNoFrameskip-v0),\n",
       " EnvSpec(flashgames.BubbleTanksTd15-v0),\n",
       " EnvSpec(AmidarNoFrameskip-v4),\n",
       " EnvSpec(gym-core.CrazyClimberDeterministic-v3),\n",
       " EnvSpec(flashgames.LuxUltimate-v0),\n",
       " EnvSpec(flashgames.ZodiacMatch-v0),\n",
       " EnvSpec(gym-core.Riverraid30FPS-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.AirRaidDeterministicSlow-v3),\n",
       " EnvSpec(flashgames.Cloud9-v0),\n",
       " EnvSpec(gym-core.WizardOfWorDeterministic-v3),\n",
       " EnvSpec(gym-core.WizardOfWorDeterministic-v0),\n",
       " EnvSpec(Asteroids-ram-v0),\n",
       " EnvSpec(gym-core.MsPacmanSlow-v0),\n",
       " EnvSpec(Asteroids-ram-v4),\n",
       " EnvSpec(gym-core.ZaxxonNoFrameskip-v3),\n",
       " EnvSpec(gym-core.Enduro-v3),\n",
       " EnvSpec(gym-core.Enduro-v0),\n",
       " EnvSpec(gym-core.ZaxxonNoFrameskip-v0),\n",
       " EnvSpec(flashgames.NewSplitterPals-v0),\n",
       " EnvSpec(flashgames.21Balloons-v0),\n",
       " EnvSpec(flashgames.Match3Adventure-v0),\n",
       " EnvSpec(flashgames.DragonVsMonster-v0),\n",
       " EnvSpec(flashgames.HeatRushFutureLvl5-v0),\n",
       " EnvSpec(Solaris-ram-v0),\n",
       " EnvSpec(Solaris-ram-v4),\n",
       " EnvSpec(flashgames.EpicDefender-v0),\n",
       " EnvSpec(flashgames.DragonChain-v0),\n",
       " EnvSpec(flashgames.FootballHeads201314Ligue1-v0),\n",
       " EnvSpec(flashgames.LonelyEscapeAsylum-v0),\n",
       " EnvSpec(MontezumaRevengeDeterministic-v0),\n",
       " EnvSpec(flashgames.MonkeyBlast-v0),\n",
       " EnvSpec(wob.real.BookFlight-Delta-v0),\n",
       " EnvSpec(ReversedAddition-v0),\n",
       " EnvSpec(gym-core.YarsRevengeSlow-v3),\n",
       " EnvSpec(gym-core.YarsRevengeSlow-v0),\n",
       " EnvSpec(wob.mini.EmailInbox-v0),\n",
       " EnvSpec(flashgames.VectorRunner-v0),\n",
       " EnvSpec(RiverraidNoFrameskip-v4),\n",
       " EnvSpec(RiverraidNoFrameskip-v0),\n",
       " EnvSpec(flashgames.AmericanRacingLvl13-v0),\n",
       " EnvSpec(BipedalWalkerHardcore-v2),\n",
       " EnvSpec(flashgames.AmericanRacingLvl22-v0),\n",
       " EnvSpec(gym-core.PrivateEye-v0),\n",
       " EnvSpec(gym-core.PrivateEye-v3),\n",
       " EnvSpec(flashgames.CoasterCars2Megacross-v0),\n",
       " EnvSpec(flashgames.SkyIsland-v0),\n",
       " EnvSpec(flashgames.SuperbikeExtreme-v0),\n",
       " EnvSpec(gym-core.KangarooSync-v3),\n",
       " EnvSpec(flashgames.SpacePunkRacerLvl5-v0),\n",
       " EnvSpec(flashgames.MarshmallowsEscape-v0),\n",
       " EnvSpec(gym-core.KangarooSync-v0),\n",
       " EnvSpec(gym-core.AmidarDeterministicSync-v3),\n",
       " EnvSpec(gym-core.AmidarDeterministicSync-v0),\n",
       " EnvSpec(Reacher-v1),\n",
       " EnvSpec(gym-core.BankHeistNoFrameskip-v0),\n",
       " EnvSpec(gym-core.BankHeistNoFrameskip-v3),\n",
       " EnvSpec(flashgames.FinalSiege-v0),\n",
       " EnvSpec(wob.real.Quizlet-Planet-Learn-v0),\n",
       " EnvSpec(flashgames.ClimbingSanta-v0),\n",
       " EnvSpec(gym-core.ChopperCommand-v0),\n",
       " EnvSpec(Tutankham-ramDeterministic-v4),\n",
       " EnvSpec(gym-core.ChopperCommand-v3),\n",
       " EnvSpec(Tutankham-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.HeatRushFutureLvl4-v0),\n",
       " EnvSpec(gym-core.FrostbiteSlow-v0),\n",
       " EnvSpec(flashgames.CowboyVsUfos-v0),\n",
       " EnvSpec(gym-core.FrostbiteSlow-v3),\n",
       " EnvSpec(ElevatorAction-ramNoFrameskip-v0),\n",
       " EnvSpec(ElevatorAction-ramNoFrameskip-v4),\n",
       " EnvSpec(flashgames.CoasterCars2Contact-v0),\n",
       " EnvSpec(flashgames.JungleEagle-v0),\n",
       " EnvSpec(flashgames.CoasterRacerLvl2-v0),\n",
       " EnvSpec(flashgames.HighSpeedChase-v0),\n",
       " EnvSpec(gym-core.FreewayDeterministicSync-v0),\n",
       " EnvSpec(flashgames.HungerHunter-v0),\n",
       " EnvSpec(gym-core.FreewayDeterministicSync-v3),\n",
       " EnvSpec(gym-core.StarGunnerDeterministicSync-v3),\n",
       " EnvSpec(Bowling-v0),\n",
       " EnvSpec(PrivateEye-ramNoFrameskip-v0),\n",
       " EnvSpec(flashgames.RunRunRan-v0),\n",
       " EnvSpec(Bowling-v4),\n",
       " EnvSpec(PrivateEye-ramNoFrameskip-v4),\n",
       " EnvSpec(PitfallNoFrameskip-v0),\n",
       " EnvSpec(wob.real.Signup-12-v0),\n",
       " EnvSpec(flashgames.DriftRunners2-v0),\n",
       " EnvSpec(PitfallNoFrameskip-v4),\n",
       " EnvSpec(wob.real.Signup-4-v0),\n",
       " EnvSpec(CentipedeNoFrameskip-v0),\n",
       " EnvSpec(flashgames.MedievalShark-v0),\n",
       " EnvSpec(CentipedeNoFrameskip-v4),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.MontezumaRevengeDeterministicSlow-v0),\n",
       " EnvSpec(gym-core.WizardOfWorDeterministicSync-v0),\n",
       " EnvSpec(gym-core.WizardOfWorDeterministicSync-v3),\n",
       " EnvSpec(flashgames.DinoBubble-v0),\n",
       " EnvSpec(flashgames.BubbleMover-v0),\n",
       " EnvSpec(flashgames.CoasterRacerLvl3-v0),\n",
       " EnvSpec(Riverraid-ramNoFrameskip-v4),\n",
       " EnvSpec(gym-core.DemonAttackDeterministic-v3),\n",
       " EnvSpec(Riverraid-ramNoFrameskip-v0),\n",
       " EnvSpec(wob.MiniWorldOfBits-v0),\n",
       " EnvSpec(wob.real.Signup-2-v0),\n",
       " EnvSpec(Alien-ramNoFrameskip-v4),\n",
       " EnvSpec(PrivateEye-v0),\n",
       " EnvSpec(flashgames.RedBeard-v0),\n",
       " EnvSpec(FrozenLake8x8-v0),\n",
       " EnvSpec(Alien-ramNoFrameskip-v0),\n",
       " EnvSpec(PrivateEye-v4),\n",
       " EnvSpec(gym-core.CartPoleLowDSync-v0),\n",
       " EnvSpec(flashgames.DnaLabRush-v0),\n",
       " EnvSpec(gym-core.Berzerk30FPS-v0),\n",
       " EnvSpec(gym-core.Berzerk30FPS-v3),\n",
       " EnvSpec(LunarLanderContinuous-v2),\n",
       " EnvSpec(flashgames.HeatRushFutureLvl12-v0),\n",
       " EnvSpec(Asterix-ram-v0),\n",
       " EnvSpec(flashgames.NinjaPainter-v0),\n",
       " EnvSpec(Asterix-ram-v4),\n",
       " EnvSpec(flashgames.GSwitch-v0),\n",
       " EnvSpec(flashgames.30Seconds-v0),\n",
       " EnvSpec(wob.real.Quizlet-Geography-Learn-v0),\n",
       " EnvSpec(wob.real.Quizlet-Comet-Test-v0),\n",
       " EnvSpec(QbertDeterministic-v0),\n",
       " EnvSpec(QbertDeterministic-v4),\n",
       " EnvSpec(flashgames.SpinSprint-v0),\n",
       " EnvSpec(flashgames.SmileyPuzzle-v0),\n",
       " EnvSpec(SolarisDeterministic-v0),\n",
       " EnvSpec(gym-core.Riverraid-v0),\n",
       " EnvSpec(gym-core.TennisNoFrameskip-v3),\n",
       " EnvSpec(SolarisDeterministic-v4),\n",
       " EnvSpec(gym-core.TennisNoFrameskip-v0),\n",
       " EnvSpec(flashgames.DriftRunners3d-v0),\n",
       " EnvSpec(JamesbondNoFrameskip-v4),\n",
       " EnvSpec(flashgames.BugsGotGuns-v0),\n",
       " EnvSpec(TwoRoundDeterministicReward-v0),\n",
       " EnvSpec(flashgames.NeonRace2-v0),\n",
       " EnvSpec(gym-core.StarGunnerDeterministicSlow-v0),\n",
       " EnvSpec(flashgames.BunnyCannon-v0),\n",
       " EnvSpec(gym-core.Frostbite-v0),\n",
       " EnvSpec(gym-core.Frostbite-v3),\n",
       " EnvSpec(PhoenixDeterministic-v4),\n",
       " EnvSpec(flashgames.Mrbirdie-v0),\n",
       " EnvSpec(PhoenixDeterministic-v0),\n",
       " EnvSpec(StarGunner-ramDeterministic-v4),\n",
       " EnvSpec(PredictActionsCartpole-v0),\n",
       " EnvSpec(flashgames.FlashDrive-v0),\n",
       " EnvSpec(StarGunner-ramDeterministic-v0),\n",
       " EnvSpec(BeamRider-ramDeterministic-v4),\n",
       " EnvSpec(BeamRider-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.SmashTheSwine-v0),\n",
       " EnvSpec(gym-core.SpaceInvadersDeterministic-v3),\n",
       " EnvSpec(gym-core.SpaceInvadersDeterministic-v0),\n",
       " EnvSpec(flashgames.CemeteryRoad-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl5-v0),\n",
       " EnvSpec(wob.mini.ClickTab2-v0),\n",
       " EnvSpec(gym-core.BoxingDeterministicSync-v3),\n",
       " EnvSpec(flashgames.JollySwipe-v0),\n",
       " EnvSpec(gym-core.KangarooDeterministicSlow-v3),\n",
       " EnvSpec(NameThisGame-ramNoFrameskip-v4),\n",
       " EnvSpec(wob.mini.UseColorwheel-v0),\n",
       " EnvSpec(PongDeterministic-v4),\n",
       " EnvSpec(Pong-ramNoFrameskip-v0),\n",
       " EnvSpec(Pong-ramDeterministic-v0),\n",
       " EnvSpec(flashgames.Devilment-v0),\n",
       " EnvSpec(Pong-ramDeterministic-v4),\n",
       " EnvSpec(wob.mini.HighlightText2-v0),\n",
       " EnvSpec(gym-core.BankHeist30FPS-v3),\n",
       " EnvSpec(gym-core.BankHeist30FPS-v0),\n",
       " EnvSpec(gym-core.NameThisGameNoFrameskip-v3),\n",
       " EnvSpec(NameThisGame-ramNoFrameskip-v0),\n",
       " EnvSpec(gym-core.NameThisGameNoFrameskip-v0),\n",
       " EnvSpec(gym-core.PongSlow-v0),\n",
       " EnvSpec(gym-core.PongSlow-v3),\n",
       " EnvSpec(flashgames.ProjectMonochrome-v0),\n",
       " EnvSpec(gym-core.PooyanSlow-v3),\n",
       " EnvSpec(TutankhamNoFrameskip-v4),\n",
       " EnvSpec(flashgames.VirtualRacer-v0),\n",
       " EnvSpec(TutankhamNoFrameskip-v0),\n",
       " EnvSpec(flashgames.DotGrowth-v0),\n",
       " EnvSpec(flashgames.VanguardWars-v0),\n",
       " EnvSpec(gym-core.UpNDownSlow-v0),\n",
       " EnvSpec(wob.mini.EnterPassword-v0),\n",
       " EnvSpec(flashgames.MiniMachines-v0),\n",
       " EnvSpec(wob.mini.Terminal-v0),\n",
       " EnvSpec(gym-core.JourneyEscapeDeterministicSlow-v3),\n",
       " EnvSpec(gym-core.JourneyEscapeDeterministicSlow-v0),\n",
       " EnvSpec(flashgames.EvolutionRacingLvl16-v0),\n",
       " ...]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gym.envs.registry.all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to load arguments into play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadarguments():\n",
    "    global env_conf\n",
    "    global env\n",
    "    global setup_json\n",
    "    global shared_model\n",
    "    global saved_state\n",
    "    global optimizer\n",
    "    global torch\n",
    "    \n",
    "    \n",
    "    undo_logger_setup()\n",
    "\n",
    "    torch.set_default_tensor_type('torch.FloatTensor')\n",
    "    torch.manual_seed(args['seed'])\n",
    "    \n",
    "    setup_json = read_config(args['EC'])\n",
    "\n",
    "    env_conf = setup_json[args['config']]\n",
    "\n",
    "    for i in setup_json.keys():\n",
    "        if i in args['ENV']:\n",
    "            env_conf = setup_json[i]\n",
    "    env = atari_env(args['ENV'], env_conf)\n",
    "\n",
    "    shared_model = A3Clstm(env.observation_space.shape[0], env.action_space)\n",
    "    if args['L']:\n",
    "        saved_state = torch.load(\n",
    "            '{0}{1}.dat'.format(args['LMD'], args['ENV']))\n",
    "        shared_model.load_state_dict(saved_state)\n",
    "    shared_model.share_memory()\n",
    "\n",
    "\n",
    "\n",
    "    if args['SO']:\n",
    "        if args['OPT'] == 'RMSprop':\n",
    "            optimizer = SharedRMSprop(shared_model.parameters(), lr=args['LR'])\n",
    "        if args['OPT'] == 'Adam':\n",
    "            optimizer = SharedAdam(shared_model.parameters(), lr=args['LR'])\n",
    "        if args['OPT'] == 'LrSchedAdam':\n",
    "            optimizer = SharedLrSchedAdam(\n",
    "                shared_model.parameters(), lr=args['LR'])\n",
    "        optimizer.share_memory()\n",
    "    else:\n",
    "        optimizer = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input Desription"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Parameter: LR\n",
    "    Type: float\n",
    "    Description: Learning Rate\n",
    "##### Parameter: G\n",
    "    Type=float,\n",
    "    Description: discount factor for rewards (default: 0.99)\n",
    "##### Parameter: T\n",
    "    Type=float,\n",
    "    Description: parameter for GAE (default: 1.00)\n",
    "##### Parameter:seed\n",
    "    Type: int\n",
    "    Descrition: random seed (default: 42)\n",
    "##### Parameter:W\n",
    "    Type=int,\n",
    "    Description: how many training processes to use (default: 5)\n",
    "##### Parameter: NS\n",
    "    Type=int,\n",
    "    Description: number of forward steps in A3C (default: 20)\n",
    "##### Parameter: M\n",
    "    Type=int,\n",
    "    Description: maximum length of an episode (default: 10000)\n",
    "##### Parameter: ENV\n",
    "    Description: environment to train on (default: Pong-v0)\n",
    "##### Parameter: EC\n",
    "    Description: environment to crop and resize info (default: settings.json)\n",
    "##### Parameter: SO\n",
    "    Description: use an optimizer without shared statistics.(default: True)\n",
    "##### Parameter: L\n",
    "    Description: load a trained model, (default: False)\n",
    "##### Parameter: SSL\n",
    "    Type=int,\n",
    "    Description: reward score test evaluation must get higher than to save model (default:20)\n",
    "##### Parameter: OPT\n",
    "    Description: shares optimizer choice of Adam, LrSchedAdam or RMSprop (default: Adam)\n",
    "##### Parameter: CL\n",
    "    Description: end of life is end of training episode.(default: False)\n",
    "##### Parameter: LMD\n",
    "    Description: folder to load trained models from (default: '/modeldata/')\n",
    "##### Parameter: SMD\n",
    "    Description: folder to save trained models (default: '/modeldata/')\n",
    "##### Parameter: LG\n",
    "    Description: folder to save log (default: '/log/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running an Environment, Training and simulating(more below)\n",
    "    1. L(Load)  is set to False because I have no training data for that particular game.\n",
    "    Once trained, a training data is provided, then set L to True.\n",
    "    2. Set SO to True so that it can accumualative learn among all workers.\n",
    "### Interrupt the Kernal to Stop training or stop testing.\n",
    "    \n",
    "## Note: Important to run all cells above but don't run everything below this\n",
    "### The cells below are in sections, choose 1 section to run. E.g if you want to train, just run the Training section. or if you want to play pacman, just run the cells in the PacMan Section ( From input to render)\n",
    "    \n",
    "### Training Notes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*It is important to limit number of worker threads to number of cpu cores available\n",
    "More than one thread per cpu core available is detrimental in training speed and effectiveness*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Section"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = {'LR': 0.0001, \"G\":0.99, \"T\":1.00,\"W\":8,\"NS\":100,\"M\":10000,\"ENV\":'MsPacman-v0',\n",
    "         \"EC\":'./settings.json',\"SO\":True,\"L\":True,\"SSL\":20, \"OPT\":\"Adam\",\"CL\":False,\n",
    "         \"LMD\":'./modeldata/',\"SMD\":\"./modeldata/\",\"LG\":'./log/', \"seed\":42,\"config\":\"Default\"\n",
    "        }\n",
    "\n",
    "loadarguments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run This to Train\n",
    "    Also logs it into a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "processes = []\n",
    "\n",
    "p = Process(target=test, args=(args, shared_model, env_conf))\n",
    "p.start()\n",
    "processes.append(p)\n",
    "\n",
    "time.sleep(0.1)\n",
    "for rank in range(0, args['W']):\n",
    "    p = Process(\n",
    "        target=train, args=(rank, args, shared_model, optimizer, env_conf))\n",
    "    p.start()\n",
    "    processes.append(p)\n",
    "for p in processes:\n",
    "    p.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing the Atari Games Section\n",
    "## The best part\n",
    "\n",
    "#### If it gives tensor errors, just run the cells again and somehow it works the 2nd time. This is because we don't have the full Share optimizer data generated yet\n",
    "### Load model is disabled on default so that you can observe how it learns through iteration,  Set L to True if you want to load the trained models and see how well it performs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playing PacMan (10000 episodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = {'LR': 0.0001, \"G\":0.99, \"T\":1.00,\"W\":8,\"NS\":20,\"M\":1000000,\"ENV\":'MsPacman-v0',\n",
    "         \"EC\":'./settings.json',\"SO\":True,\"L\":True,\"SSL\":20, \"OPT\":\"Adam\",\"CL\":False,\n",
    "         \"LMD\":'./modeldata/',\"SMD\":\"./modeldata/\",\"LG\":'./log/', \"seed\":42,\"config\":\"MsPacman\"\n",
    "        }\n",
    "\n",
    "\n",
    "loadarguments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this to Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-29 23:00:51,534 : OPT: Adam\n",
      "2017-11-29 23:00:51,535 : LG: ./log/\n",
      "2017-11-29 23:00:51,536 : SMD: ./modeldata/\n",
      "2017-11-29 23:00:51,537 : ENV: MsPacman-v0\n",
      "2017-11-29 23:00:51,538 : G: 0.99\n",
      "2017-11-29 23:00:51,539 : CL: False\n",
      "2017-11-29 23:00:51,540 : config: MsPacman\n",
      "2017-11-29 23:00:51,541 : M: 1000000\n",
      "2017-11-29 23:00:51,542 : L: True\n",
      "2017-11-29 23:00:51,543 : EC: ./settings.json\n",
      "2017-11-29 23:00:51,543 : SSL: 20\n",
      "2017-11-29 23:00:51,544 : seed: 42\n",
      "2017-11-29 23:00:51,545 : LR: 0.0001\n",
      "2017-11-29 23:00:51,545 : T: 1.0\n",
      "2017-11-29 23:00:51,546 : W: 8\n",
      "2017-11-29 23:00:51,547 : SO: True\n",
      "2017-11-29 23:00:51,548 : NS: 20\n",
      "2017-11-29 23:00:51,548 : LMD: ./modeldata/\n",
      "2017-11-29 23:01:09,454 : Time 00h 00m 17s, episode reward 5040.0, episode length 2059, reward mean 5040.0000\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-40b60a3004a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshared_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv_conf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-15-12a1f9002f83>\u001b[0m in \u001b[0;36mtest\u001b[0;34m(args, shared_model, env_conf, render)\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0mplayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meps_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m             \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mplayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m60\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m             \u001b[0mplayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test(args, shared_model, env_conf,render=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playing BeamRider (4000 episodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = {'LR': 0.0001, \"G\":0.99, \"T\":1.00,\"W\":8,\"NS\":20,\"M\":4000,\"ENV\":'BeamRider-v0',\n",
    "         \"EC\":'./settings.json',\"SO\":True,\"L\":True,\"SSL\":20, \"OPT\":\"Adam\",\"CL\":False,\n",
    "         \"LMD\":'./modeldata/',\"SMD\":\"./modeldata/\",\"LG\":'./log/', \"seed\":42,\"config\":\"BeamRider\"\n",
    "        }\n",
    "\n",
    "\n",
    "loadarguments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this to Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test(args, shared_model, env_conf,render=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playing Breakout (3000 episodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "While copying the parameter named actor_linear.weight, whose dimensions in the model are torch.Size([4, 512]) and whose dimensions in the checkpoint are torch.Size([6, 512]), ...\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "inconsistent tensor size, expected tensor [4 x 512] and src [6 x 512] to have the same number of elements, but got 2048 and 3072 elements respectively at /opt/conda/conda-bld/pytorch_1503966894950/work/torch/lib/TH/generic/THTensorCopy.c:86",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-3d158bb5c32d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mloadarguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-21-aca11d89d648>\u001b[0m in \u001b[0;36mloadarguments\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m         saved_state = torch.load(\n\u001b[1;32m     28\u001b[0m             '{0}{1}.dat'.format(args['LMD'], args['ENV']))\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mshared_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msaved_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m     \u001b[0mshared_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshare_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/torch/nn/modules/module.pyc\u001b[0m in \u001b[0;36mload_state_dict\u001b[0;34m(self, state_dict)\u001b[0m\n\u001b[1;32m    358\u001b[0m                 \u001b[0mparam\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 360\u001b[0;31m                 \u001b[0mown_state\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    361\u001b[0m             \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m                 print('While copying the parameter named {}, whose dimensions in the model are'\n",
      "\u001b[0;31mRuntimeError\u001b[0m: inconsistent tensor size, expected tensor [4 x 512] and src [6 x 512] to have the same number of elements, but got 2048 and 3072 elements respectively at /opt/conda/conda-bld/pytorch_1503966894950/work/torch/lib/TH/generic/THTensorCopy.c:86"
     ]
    }
   ],
   "source": [
    "args = {'LR': 0.0001, \"G\":0.99, \"T\":1.00, \"S\":1,\"W\":8,\"NS\":20,\"M\":3000,\"ENV\":'Breakout-v0',\n",
    "         \"EC\":'./settings.json',\"SO\":True,\"L\":True,\"SSL\":20, \"OPT\":\"Adam\",\"CL\":False,\n",
    "         \"LMD\":'./modeldata/',\"SMD\":\"./modeldata/\",\"LG\":'./log/', \"seed\":42,\"config\":\"Breakout\"\n",
    "        }\n",
    "\n",
    "\n",
    "loadarguments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this to Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-24 14:52:44,553 : OPT: Adam\n",
      "2017-11-24 14:52:44,555 : LG: ./log/\n",
      "2017-11-24 14:52:44,556 : SMD: ./modeldata/\n",
      "2017-11-24 14:52:44,557 : ENV: Breakout-v0\n",
      "2017-11-24 14:52:44,559 : G: 0.99\n",
      "2017-11-24 14:52:44,561 : CL: False\n",
      "2017-11-24 14:52:44,563 : config: Breakout\n",
      "2017-11-24 14:52:44,565 : M: 3000\n",
      "2017-11-24 14:52:44,567 : L: True\n",
      "2017-11-24 14:52:44,570 : EC: ./settings.json\n",
      "2017-11-24 14:52:44,572 : SSL: 20\n",
      "2017-11-24 14:52:44,574 : S: 1\n",
      "2017-11-24 14:52:44,576 : seed: 42\n",
      "2017-11-24 14:52:44,578 : LR: 0.0001\n",
      "2017-11-24 14:52:44,580 : T: 1.0\n",
      "2017-11-24 14:52:44,583 : W: 8\n",
      "2017-11-24 14:52:44,585 : SO: True\n",
      "2017-11-24 14:52:44,587 : NS: 20\n",
      "2017-11-24 14:52:44,588 : LMD: ./modeldata/\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-40b60a3004a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshared_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv_conf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-18-12a1f9002f83>\u001b[0m in \u001b[0;36mtest\u001b[0;34m(args, shared_model, env_conf, render)\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrender\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mplayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0mreward_sum\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mplayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-04b357b58ab4>\u001b[0m in \u001b[0;36maction_test\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     48\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvolatile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvolatile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mVariable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvolatile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0mprob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/torch/nn/modules/module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_pre_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m             \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    225\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-bc8f4f957771>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaxp1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaxp2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaxp3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaxp4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/torch/nn/modules/module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    222\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_pre_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    223\u001b[0m             \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    225\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    226\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/torch/nn/modules/conv.pyc\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    252\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    253\u001b[0m         return F.conv2d(input, self.weight, self.bias, self.stride,\n\u001b[0;32m--> 254\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    255\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    256\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/torch/nn/functional.pyc\u001b[0m in \u001b[0;36mconv2d\u001b[0;34m(input, weight, bias, stride, padding, dilation, groups)\u001b[0m\n\u001b[1;32m     50\u001b[0m     f = ConvNd(_pair(stride), _pair(padding), _pair(dilation), False,\n\u001b[1;32m     51\u001b[0m                _pair(0), groups, torch.backends.cudnn.benchmark, torch.backends.cudnn.enabled)\n\u001b[0;32m---> 52\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test(args, shared_model, env_conf,render=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Playing SpaceInvader (10000 episodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = {'LR': 0.001, \"G\":0.99, \"T\":1.00, \"S\":1,\"W\":8,\"NS\":20,\"M\":1000000,\"ENV\":'SpaceInvaders-v0',\n",
    "         \"EC\":'./settings.json',\"SO\":True,\"L\":True,\"SSL\":20, \"OPT\":\"Adam\",\"CL\":False,\n",
    "         \"LMD\":'./modeldata/',\"SMD\":\"./modeldata/\",\"LG\":'./log/', \"seed\":42,\"config\":\"SpaceInvaders\"\n",
    "        }\n",
    "\n",
    "\n",
    "loadarguments()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run this to Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-11-24 16:45:17,533 : OPT: Adam\n",
      "2017-11-24 16:45:17,534 : LG: ./log/\n",
      "2017-11-24 16:45:17,535 : SMD: ./modeldata/\n",
      "2017-11-24 16:45:17,536 : ENV: SpaceInvaders-v0\n",
      "2017-11-24 16:45:17,537 : G: 0.99\n",
      "2017-11-24 16:45:17,538 : CL: False\n",
      "2017-11-24 16:45:17,539 : config: SpaceInvaders\n",
      "2017-11-24 16:45:17,539 : M: 1000000\n",
      "2017-11-24 16:45:17,541 : L: True\n",
      "2017-11-24 16:45:17,542 : EC: ./settings.json\n",
      "2017-11-24 16:45:17,543 : SSL: 20\n",
      "2017-11-24 16:45:17,544 : S: 1\n",
      "2017-11-24 16:45:17,545 : seed: 42\n",
      "2017-11-24 16:45:17,546 : LR: 0.001\n",
      "2017-11-24 16:45:17,547 : T: 1.0\n",
      "2017-11-24 16:45:17,547 : W: 8\n",
      "2017-11-24 16:45:17,548 : SO: True\n",
      "2017-11-24 16:45:17,549 : NS: 20\n",
      "2017-11-24 16:45:17,551 : LMD: ./modeldata/\n",
      "2017-11-24 16:45:49,833 : Time 00h 00m 32s, episode reward 2895.0, episode length 3753, reward mean 2895.0000\n",
      "2017-11-24 16:47:10,696 : Time 00h 01m 52s, episode reward 2340.0, episode length 2343, reward mean 2617.5000\n",
      "2017-11-24 16:49:32,367 : Time 00h 04m 14s, episode reward 11355.0, episode length 8869, reward mean 5530.0000\n",
      "2017-11-24 16:51:09,094 : Time 00h 05m 51s, episode reward 5270.0, episode length 4445, reward mean 5465.0000\n",
      "2017-11-24 16:52:45,807 : Time 00h 07m 28s, episode reward 5010.0, episode length 4336, reward mean 5374.0000\n",
      "2017-11-24 16:54:12,251 : Time 00h 08m 54s, episode reward 3320.0, episode length 3126, reward mean 5031.6667\n",
      "2017-11-24 16:56:03,930 : Time 00h 10m 46s, episode reward 7445.0, episode length 6303, reward mean 5376.4286\n",
      "2017-11-24 16:57:41,828 : Time 00h 12m 24s, episode reward 5270.0, episode length 4685, reward mean 5363.1250\n",
      "2017-11-24 16:59:05,473 : Time 00h 13m 47s, episode reward 2750.0, episode length 2639, reward mean 5072.7778\n",
      "2017-11-24 17:00:51,077 : Time 00h 15m 33s, episode reward 6550.0, episode length 5415, reward mean 5220.5000\n",
      "2017-11-24 17:02:27,071 : Time 00h 17m 09s, episode reward 4925.0, episode length 4348, reward mean 5193.6364\n",
      "2017-11-24 17:03:45,683 : Time 00h 18m 27s, episode reward 2085.0, episode length 2270, reward mean 4934.5833\n",
      "2017-11-24 17:06:06,522 : Time 00h 20m 48s, episode reward 12540.0, episode length 10000, reward mean 5519.6154\n",
      "2017-11-24 17:07:46,573 : Time 00h 22m 28s, episode reward 5310.0, episode length 4782, reward mean 5504.6429\n",
      "2017-11-24 17:09:26,329 : Time 00h 24m 08s, episode reward 5900.0, episode length 4818, reward mean 5531.0000\n",
      "2017-11-24 17:11:13,932 : Time 00h 25m 56s, episode reward 6985.0, episode length 5752, reward mean 5621.8750\n",
      "2017-11-24 17:12:33,536 : Time 00h 27m 15s, episode reward 2445.0, episode length 2344, reward mean 5435.0000\n",
      "2017-11-24 17:14:08,068 : Time 00h 28m 50s, episode reward 4930.0, episode length 4200, reward mean 5406.9444\n",
      "2017-11-24 17:15:48,918 : Time 00h 30m 31s, episode reward 5885.0, episode length 5002, reward mean 5432.1053\n",
      "2017-11-24 17:17:28,062 : Time 00h 32m 10s, episode reward 5420.0, episode length 4680, reward mean 5431.5000\n",
      "2017-11-24 17:18:56,378 : Time 00h 33m 38s, episode reward 3920.0, episode length 3476, reward mean 5359.5238\n",
      "2017-11-24 17:20:44,839 : Time 00h 35m 27s, episode reward 6870.0, episode length 5807, reward mean 5428.1818\n",
      "2017-11-24 17:22:14,225 : Time 00h 36m 56s, episode reward 4035.0, episode length 3541, reward mean 5367.6087\n",
      "2017-11-24 17:23:33,875 : Time 00h 38m 16s, episode reward 2385.0, episode length 2365, reward mean 5243.3333\n",
      "2017-11-24 17:24:56,261 : Time 00h 39m 38s, episode reward 2520.0, episode length 2723, reward mean 5134.4000\n",
      "2017-11-24 17:26:03,281 : Time 00h 40m 45s, episode reward 540.0, episode length 812, reward mean 4957.6923\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.py\", line 1132, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.py\", line 313, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.py\", line 358, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/inspect.py\", line 1049, in getinnerframes\n",
      "    framelist.append((tb.tb_frame,) + getframeinfo(tb, context))\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/inspect.py\", line 1013, in getframeinfo\n",
      "    lines, lnum = findsource(frame)\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.py\", line 170, in findsource\n",
      "    file = getsourcefile(object) or getfile(object)\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/inspect.py\", line 454, in getsourcefile\n",
      "    if hasattr(getmodule(object, filename), '__loader__'):\n",
      "  File \"/home/nasdin/anaconda3/envs/py27/lib/python2.7/inspect.py\", line 490, in getmodule\n",
      "    for modname, module in sys.modules.items():\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "string index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_code\u001b[0;34m(self, code_obj, result)\u001b[0m\n\u001b[1;32m   2897\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m                 \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_in_exec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshowtraceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m             \u001b[0moutflag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only)\u001b[0m\n\u001b[1;32m   1824\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1825\u001b[0m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0;32m-> 1826\u001b[0;31m                                             value, tb, tb_offset=tb_offset)\n\u001b[0m\u001b[1;32m   1827\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1828\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_showtraceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.pyc\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1410\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m         return FormattedTB.structured_traceback(\n\u001b[0;32m-> 1412\u001b[0;31m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[0m\u001b[1;32m   1413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1414\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.pyc\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1318\u001b[0m             \u001b[0;31m# Verbose modes need a full traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m             return VerboseTB.structured_traceback(\n\u001b[0;32m-> 1320\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1321\u001b[0m             )\n\u001b[1;32m   1322\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nasdin/anaconda3/envs/py27/lib/python2.7/site-packages/IPython/core/ultratb.pyc\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1202\u001b[0m                 \u001b[0mstructured_traceback_parts\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mformatted_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1203\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1204\u001b[0;31m             \u001b[0mstructured_traceback_parts\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mformatted_exception\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1206\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mstructured_traceback_parts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: string index out of range"
     ]
    }
   ],
   "source": [
    "test(args, shared_model, env_conf,render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}