2-Layer-MLP-AlignMNiST-wDropout0.1.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP w/AlignMNIST\n",
    "\n",
    "75 epochs of RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.913994",
     "start_time": "2016-06-10T09:28:37.005382"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.920446",
     "start_time": "2016-06-10T09:28:38.915657"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.924838",
     "start_time": "2016-06-10T09:28:38.921882"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001\n",
    "dropout_prob = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.931851",
     "start_time": "2016-06-10T09:28:38.926312"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 1000 # 1st layer num features\n",
    "n_hidden_2 = 500 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0 = 1.0/np.sqrt(n_input)\n",
    "std_h1 = 1.0/np.sqrt(n_hidden_1)\n",
    "std_h2 = 1.0/np.sqrt(n_hidden_2)\n",
    "\n",
    "logfile = \"2-layer-mlp-alignmnist-wdropout\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.939360",
     "start_time": "2016-06-10T09:28:38.933301"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.944196",
     "start_time": "2016-06-10T09:28:38.940575"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases, _dropout_prob):\n",
    "    layer_1 =  tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) , _dropout_prob)\n",
    "    layer_2 =  tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) , _dropout_prob)\n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:38.982602",
     "start_time": "2016-06-10T09:28:38.946073"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:39.015398",
     "start_time": "2016-06-10T09:28:38.983857"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases, dropout_prob)\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases, dropout_prob )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:39.030220",
     "start_time": "2016-06-10T09:28:39.016937"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:43.173035",
     "start_time": "2016-06-10T09:28:43.016250"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr, optimizer=\"RMSProp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:49.922484",
     "start_time": "2016-06-10T09:28:49.903169"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_2:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with tf.name_scope(\"training accuracy\"):\n",
    "pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"training accuracy\", train_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:50.357029",
     "start_time": "2016-06-10T09:28:50.346199"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_3:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"test 0 accuracy\", test_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:51.138373",
     "start_time": "2016-06-10T09:28:50.901342"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-temp\u001b[m\u001b[m          \u001b[1m\u001b[36m5-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-watch\u001b[m\u001b[m         \u001b[1m\u001b[36m5-layer-mlp-infimnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-modern-mlp-alignmnist\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!rm -rf ./logs/{logfile}\n",
    "!ls logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:52.986292",
     "start_time": "2016-06-10T09:28:52.116712"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX_0, trY_0 = mnist.train.images, mnist.train.labels\n",
    "teX_0, teY_0 = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T09:28:53.299997",
     "start_time": "2016-06-10T09:28:53.297337"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "alignmnist = AlignMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T11:25:04.131884",
     "start_time": "2016-06-10T09:28:54.240044"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "with tf.Session() as sess:\n",
    "    # create a log writer. run 'tensorboard --logdir=./logs/{logfile}'\n",
    "    writer = tf.train.SummaryWriter(\"./logs/{0}\".format(logfile), sess.graph) # for 0.8\n",
    "    merged = tf.merge_all_summaries()\n",
    " \n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        trX, trY = alignmnist.next_epoch()\n",
    "      \n",
    "        for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "            sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "                 \n",
    "        summary, trn_acc, tst_acc = sess.run([merged, train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX_0, yt: teY_0})\n",
    "        writer.add_summary(summary, epoch)  \n",
    "        \n",
    "        #  print(epoch, trn_acc, tst_acc)\n",
    "        train_accuracies.append(trn_acc)\n",
    "        test_accuracies.append(tst_acc)\n",
    "\n",
    "        writer.flush()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T11:25:06.172416",
     "start_time": "2016-06-10T11:25:04.140423"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T12:19:30.881324",
     "start_time": "2016-06-10T12:19:30.725322"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11a357490>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VGX2P/DPQbDQBaRLUxQsoKARG0ZBRRTBsiq6dl1w\nVUTlK0URBRdcxZ/sCrqyICAWLChNRFAIqChFpYhAQDqEgICBQAJJ5vz+ODObSTJJps+dmc/79eJF\n5s4tZ2bunOe557lzr6gqiIgoOVSIdQBERBQ9TPpEREmESZ+IKIkw6RMRJREmfSKiJMKkT0SURPxK\n+iLSRUTWiUi6iPT38XxNEflMRFaKyI8icpa/yxIRUfRIeefpi0gFAOkAOgHYBWAZgDtUdZ3XPK8A\nOKSqw0TkTABjVLWzP8sSEVH0+NPTTwGwQVW3qmoegCkAuheb5ywA8wFAVdcDaCYip/i5LBERRYk/\nSb8RgO1ej3e4p3lbCeBmABCRFABNADT2c1kiIoqScA3kvgzgZBH5GcCjAH4BUBCmdRMRUZhU9GOe\nnbCeu0dj97T/UdVDAB7wPBaRzQA2Aahc3rJey/AiQEREAVJVCWR+f3r6ywCcLiJNReR4AHcAmOE9\ng4jUEJFK7r8fBrBQVbP9WbZY8I7+N2TIkJjHwDgZJ+NknJ5/wSi3p6+qBSLyGIC5sEZivKquFZFe\n9rSOBdAawCQRcQFYA+DBspYNKlIiIgqZP+UdqOocAGcWm/a2198/Fn++rGWJiCg2+IvcAKSmpsY6\nBL8wzvBinOHFOGOr3B9nRYuIqFNiISKKByICjcBALhERJQgmfSKiJMKkT9ExaRIwbVqsoyBKekz6\nFHm7dwNPPw08/DCwcmWso4msNWuABx4ADh6MdSREPjHpU+QNHmyJ8N//Bm6+GThwINYRhV9BAfDq\nq0BqKrB2LTBsWKwjIvLJr/P0iYK2ciUwcyawbh1QsyawZAnw17/atAoJ0ufYuBG47z6gYkVg2TLg\npJOAc86xhq5161hHR1REgnzryJFUrawzeLAlfMB6w4cOAUOHxja2cNm4EejQAbj1VmD+fKBZM6Be\nPeDZZ4E+few9IHIQJn0KPTEdPQoMHAj89FPR6bNnAzt3An/7W+G0SpWAjz8Gxo0DJkwA8vL8j/HJ\nJ4Ft20KLtbi0NP9j8GXcOOvR9+1b9Mjl0UeBjAzg889DDpEonJj0E43LBWRn+zdvfr4l3mbNgGee\nCS75u1zA/fcD338P3HADcO+9lujz8oB+/YCRIy3Re6tf35LhuHFAo0aWIBcvLnv7M2YAo0YBb7wR\neIylmTcP6NQJuOsuey8ClZ8PvPuuvf7iKlWyWJ96CjhyJPRYi+MRBAWJSb80+fnA5s2xjgLIyvLv\nC+5yAZ99Bpx/viXSsnqYqnb6ZJs2wMSJwNixwFdfAUOGBB7f88/b+/TVV0B6OtC4sa23e3f7u2tX\n38tdeKE1FD/+CDRoADz4oJVJcnJKzpufDwwYAIwebY1UOJJoVpZtc9o0+/uBB+w9DMScOdZglla3\nv/JKICUF+Oc/Qw63iD17gDPOAFavDu96KTnE+tKgXpcIVUd5/XXVqlVVV60K73pdLv/nzclRbdxY\ntUsX1YyM0tf32Weqbduqtm+vOnOm6tKlqk2aqD73nGpBQeG8BQWqM2aoXnSRaps2ql98URhPZqZq\n69aqw4f7H9/48aotWqju2VN0+pYtqr17q65Z4/+6XC7V229Xffzxks+NHat65ZU2zw03qI4b5/96\nS3P//aq9etnfhw+rpqaqPvxwYJ/PTTep/ve/Zc+zdatqrVqq27cHH6s3l0u1Wzf7fB97LDzrpLjl\nzpuB5dpAF4jUP0cl/bw81aZNVZ96SrV5c9W9e8Oz3lGjVHv08H/+N99UvfZa1cGDVevXt4TtkZNj\nCefss1XPP191+vSiCSszU7VjR9WuXS3+995TPecc1fPOU/3oo6KNgcfOnaqnn24NXnnmzVOtW1d1\n3Tr/X095Dhyw933WrMJp2dmqDRuqLltmj+fMsdcQSHIubuZM+1wPHiycdvCg6sUXW6Ozd6/qjh2q\nGzdaw5WbW3Idu3er1qxZdB2luf121YkTg4/X29tv2+ednq5au7btB4HYtcsazR49rJOwdm144kpk\nX3+tesEFqsuXxzqSEpj0w2XKFNXLLrO/+/dXveIK1WPHQlvn5s32Ja1Z03rC5Tl61Hpzixfb42+/\nVW3WzHrQQ4ZYwu3a1XbI0hLgsWOWxCpWVL38ctUvvyw/WW7datspqwf766+qp5yiunBh+a8jUIsW\nWQO3a5c9HjbMkqZHQYFqy5aq331Xctkff1R98UXVDz6wL2hWVsl59u2zRmTBgpLPHThgn3utWqoN\nGljD0LSpHWXk5RWdd+RI1fvu8+81vf66fW6hWr9etU4d1d9+s8dXX22vtTyZmRbvBRfY/nf77aqT\nJ1unon79wgbVW0GB6uefq86d61/DFk9++ska/vJkZqredZd9D3v2VP3rXyMfW4CY9MPB5VK98ELb\n4VVV8/OtpFD8S7txo/W6Ro+2ntPkyapTp6oeOuR7nV27Wunk739XfeGF8uMYN061c+ei0/78U/Vv\nf7N/ni++P3bv9n9eVdUNG1Tr1bNGorg9eywZvvtuYOsMxPPPq15zjcVdu7a9195GjVK9446i01au\ntIboySdVb7vNjgYqV7bkfc01qv36qU6apHrLLb5LSKXJz7fyWt++hdNcLtWzzrIGyh8//GDxhOLY\nMdsvR48unPbRR6pXXeV7/rw8S2w33WSJ/r77VL/5pmTnZfp0a0jmzSuclpam2q6dNRKXXWbvY/v2\n9h6MHWvL/PCD6qZN1jkJRGZmaEdpoZoyxV5vnTq+OwUe77xj+1O/fvad3rdPtUaNkqXMGGPSD4dF\ni6zEkZ9fOC0ry77kgwbZTtCqlSXFu+9WfeQRqw/feaeVUy680HYQb598YmWYo0etl9G0qe/yikde\nnuppp0WmJ+2v776znX7FisJpOTmql1yi+uyzkd12Xp6VWlq2VO3Tp+TzBw5YIvMcDWzdamMfH35Y\ndL6CAjuqmjnTGtw77rCyRnZ2YPHs32/7xKRJ9vjHH+2xv8krN9cSZ2nb3bVLdfXqstcxeLA1Pt7b\nzM215PX770XndbnstbZrZ0ds5fXU09Lssx492hqJpk3tvfRsKyfHjjSHD1d94AHV66+3/bxJE9WT\nT7Zp8+eXvU+rWiepUiXVESPKni8SCgrsPWza1PbpO+8sffxq0SLbn7z3fVX7nsci9jIw6YdD9+52\n2Fvcxo22sz//vA2U+trBXS7Vp5+22rln4PXPP62c4F2OOO88O2wuzbvvWgMSax99pHrqqVbfdrns\n8PbWW8v/cofDpk024Fxaz6p3bzti2r/fGuSRIyMbz6+/WoJdtswGfAMZ8FZV7dDBkqsvffva86Xx\nDAZ7GjlvTzxhA/beJkywffDIEf/j++UX69mPGBHYctu3q776qu3TjRqpPvNMySOzo0dt0LxVKzva\naNhQdfZs/7cRquxsO8K75JLCo941a6yhK35kXlBgjaWvstny5dZoeHcIY4xJP1Tr19uOcPhw8Otw\nuayu3LKlfVn//ncrx3gbPdpKEL7k56ueeWbRw+1YevllOzNo0CDr3YXy3oTT6tVWuunY0ZJmNEoG\nU6daI3jyyTboHYi+fe299KVVK9UTTrDBWV+GDbMjSl9WrbJk60lEGzZY41TekUMkrFljR8J16qhe\nd50dYe3YYcm2R4/CcornKLK01xsql8vKfa+9ZnFUq2ZHI8UH5P/yF2uwvL3zjh1llrY/paQUPaEi\nxpj0Q9W7d8leU7D+3/+zHk2DBtYb9bZ/v9UH//ij5HJTppS900Wby2WNVuPGvnuasXTVVfbFjcaR\nh8eLL9o2AzVliu8ztzZvtkH5Pn3sKLI4l8tKfUuWlL7ulBQ7/fboUeute9f9Y+HIETtbKSXFTiIY\nOrTkZ/TWW3aEFu5B4m3brJNy2mn2ff7005LlVo+VK20g23Nkc/CgfV+XLi19/ZMm2Rl1DsGkH4q9\ne61OXNr58MF4/33Vr77y/dydd9qApLf9++1c+Wge+vqjoMCZZ3AcPhzdhO8RTIO8ZYuNAxVf9q23\nbGxo+XL7zUPx57/91vaJsrb59tuqN9+sOmCAnXTglA6Dqo2/+OJyWZmsRw9LyitW2ADxG28UnrEW\nqFWr7Ehs5Ej/34Pu3VX/9S/7e+BA1XvuKXv+nBw7StmwIbgYw4xJP1Cew8Dnn1c944zSD6EjYf58\nq7t6ds5ff7XBwWiVKii6XC7rVW7eXHR69+7WOXC5LLkXPxX1wQdV//nPstedlWVHjg0bOu7skjLl\n5trp0NWq2Xfh+uvtqLJuXdWPPw5sXQsW2HL+nMLqbflyK4+tXWvjJjt2lL/MM8/Yb3gcgEnfX4cP\n20Bcy5Y2MPP009a7iGavsaDAenZLltghaJ06kT0NkmKvR4+iZxgdPapavXphoh4+vPBXwqo2AFmz\npn/jByNH+v7tgdP56uCsWGEN2FtvFZ2ek2Ov89xz7UymPn2slDVqlPW+v/kmuBiuu84a5KFD/Zt/\n0yY7ldgB41tM+t6++KLkqWwFBZZYGze2gdRly2Lbq37pJevdN2niyF/7UZi9/LKdbeOxYIENjnt4\nztLxDDhOnmwJKRlt3GidoqFD7RTe8eOtdNOjh51SOXOmDdT26qV64412xB6sJUvs7KNAzlrq0cMS\nf61adjruccdZ5zHKgkn6YsvFnoho2GJZuRK44gq7mcVJJwGdO9sFvt5+GzjuOOD114FLLgnPtkKx\nezfw3HPA8OFA3bqxjoYibeFCoH9/u8gcYBeRO/74ovcWuPJK4PHH7Q5jnTvbZalvuy028cZaRgbQ\npUvhBeZefhm4+OJYR2WOHQP27gVOOAE48UT7+4ILgF27bFqUiAhUVQJaJiGTft++QLVq9mX67Te7\nhO7ixcBNNwG33544d2yi+HL4sDXu+/dbYjjvPODNN4t2QN55x+4qNmoU0L49sGOHJZVklZVld11L\nSQEkoNwWfR072uXEb7wxaptk0gesBW7cGPjhB+C000JfH1E4tWtnib5JE7ul4p49dptFj6wse+7B\nB+3mNGPGxC5WCsxbbwGLFgEffhi1TQaT9BPvHrmzZtn1zZnwyYk6dLDyzrp1Vr6pWOwrWKOGlTRe\nfx1YujQ2MVJwbr3V7iB3+DBQpUqsoylV4tU5JkzwfScjIifo0MGOQufMseTuy0MPWWnngguiGxuF\n5pRTbMxhxoxYR1KmxCrv7N5tvfzt24GqVcMTGFE4pafbLRqPHAFWrbK7nPlSUGAnHVB8efdd4NNP\no5b4WdN/9VU7bB4/PjxBEYWbKlC7to07rVoV62go3A4eBE491W4hWqtWxDcXTNJPnPKOKks75Hwi\nVuIprbRD8a16deDqq4GpU2MdSakSJ+kvXWo30L700lhHQlS2ESOAJ56IdRQUKXfeGdUzeAKVOOWd\n3r3tVLdBg8IXFBFRoHJzgQYNgDVrgIYNI7qp5C3v5OQAn3wC3HNPrCMhomR34olA9+7Axx/HOhKf\nEiPpT5tml1lo3DjWkRARAT17OvbUzcQo71x7LfDAA3aJBSKiWCsosKsDnHRSRDeTnKdsbt9u1zDZ\nuTO5r1FCREknOWv6kyfbVQiZ8ImIyhXf195RBSZOtMRPRETliu+e/g8/2GWSU1JiHQkRUVyI76Q/\ncSJw333Ov842EZFDxO9A7pEjdorm6tWlX7SKiCiBJddA7rRpVtZhwici8lv8Jn1PaYeIiPwWn+Wd\n7duBtm3t3PwI//iBiMipkqe88+GHdmsyJnwiooDEZ9KfOhX4y19iHQURUdyJv/LO9u3A+ecDGRlA\npUqRD4yIyKGSo7zz2WdAt25M+EREQYi/pD91KnDLLbGOgogoLsVXeWf3bqB1a/v/hBOiExgRkUMl\nfnln2jTguuuY8ImIguRX0heRLiKyTkTSRaS/j+eri8gMEVkhIqtF5D6v57aIyEoR+UVEloYULUs7\nREQhKbe8IyIVAKQD6ARgF4BlAO5Q1XVe8wwEUF1VB4pIHQDrAdRT1XwR2QSgvaoeKGc7ZZd39u0D\nWrQAdu0CqlTx79URESWwSJV3UgBsUNWtqpoHYAqA7sXmUQDV3H9XA7BPVfM9cfm5nbLNmAF07syE\nT0QUAn+ScSMA270e73BP8zYawFkisgvASgBPeD2nAOaJyDIReTjoSFnaISIKWbjunHUtgF9U9SoR\nOQ2W5NuoajaAS1U1Q0ROcU9fq6rfBbT2rCxg0SLggw/CFC4RUXLyJ+nvBNDE63Fj9zRv9wMYAQCq\n+ruIbAbQCsByVc1wT98rIp/DykU+k/4LL7zwv79TU1ORmppqD2bNAjp2BKpX9yNcIqLElJaWhrS0\ntJDW4c9A7nGwgdlOADIALAXQU1XXes0zBsAeVX1RROoBWA6gLYBcABVUNVtEqgCYC+BFVZ3rYzul\nD+Q+/DBw3nnAo48G8RKJiBJTMAO55fb0VbVARB6DJewKAMar6loR6WVP61gALwGYKCKr3Is9o6r7\nRaQ5gM9FRN3bet9Xwi9XejrQs2fAixERUVHx8YvcBg2AZcvs9ohERAQgUX+Re/Cg/WvYMNaREBHF\nPecn/Q0bgJYtgQrOD5WIyOmcn0nT0y3pExFRyJyf9DdsAM44I9ZREBElBOcn/fR0Jn0iojBh0ici\nSiLOTvqqTPpERGHk7KS/dy9w3HFA7dqxjoSIKCE4O+mzl09EFFZM+kRESYRJn4goiTDpExElEWcn\nfc8lGIiIKCyce5VNl8vuh7t3L1C1auwCIyJyqMS6yub27XaqJhM+EVHYODfps55PRBR2TPpEREmE\nSZ+IKIkw6RMRJREmfSKiJOLMUzaPHQOqVwcOHQIqVYptYEREDpU4p2xu2gQ0acKET0QUZs5M+rwv\nLhFRRDg36bOeT0QUdkz6RCHYuxcYMgTIy4t1JET+cWbS37QJOO20WEdBVK5XXgFGjQLuvRcoKIh1\nNETlc2bSP3AAqFUr1lEkJJcL6N8f+PXX6G1z3z6gRw9g6NDobTMa9uwBxo8HfvoJ2L0b6NXL3t9I\n27gRePFFICsr8tuixOPMpH/woJ2ySWE3fjzw6afAVVcB8+cHt46cHGD4cGDQIGDgQGtEBg8GVq4s\nOe8PPwDt2gFNm9q2p04NLf5Ie+01YPly/+e94w7g9NOBGTOA334DnnoKiMRZ0KrA998DN98MXHyx\nbW/EiPBvhxKfc5N+jRqxjiLhZGRYov78c+CTT4CePYHJk4vOs3o18NhjwIQJpa/n1VeBOXPsytfV\nqwMnnwzk5wPXXw9ccYUl9vx8K3t07w78+9/Av/5l03v3BtasiezrDNabbwJvvGGvw1cD5u2PP4D/\n/hcYMMAeV60KzJ4NLFwIPPdceBP/4cNAx45WQurUCdiyxZL+f/8LbNsWvu1QklBVR/yzUNxOOEH1\nyBGl8uXmqv70k+r336vOn686e7bqL7/4nvcvf1EdOLDw8Zo1qs2aqb74our776teeqlqw4aqzzyj\nWru26saNJdexbZtqrVqqmzeXfO7YMdWPPrL11Kih2r696qZNReeZNEn19NNV9+8P+iUH7ehR1cmT\n7T0r7ptvVOvVs9f88ceqDRqorl1b+roGDFDt1avk9D17VNu1U+3aVXXHjvDEPWqU6o03qubnF53+\n7LOq997re5kFC1QXLgzP9sm53HkzsFwb6AKR+ve/pJ+bq1qxoqrLFea3J/Hs36968cWqZ56p2qGD\namqqapculrCef75okpgxQ7Vly5JtaUaGaseOqp07q06daolbVfXVV1Wvuqrkx3DnnarPPVd+bOnp\nvpOrqmqfPqrXXVcyiXn75htrHJo2VW3d2hqQTp1U160rf9uleeIJ1bp1Vdu0UV2xomisdetao+kx\ncaJq48aqv/9ecj1//GEN35Ytvrdz7Jg1pKecovrOO6HtyseOqTZporpkScnnsrKsofJ+Laqqixer\n1qmjWr++xepLZqY1GNdco3rBBaqnnWbruvxye58mTbJOQaS/hqNHq06ZEtlteMvNVX35ZdVXXgnv\nevPzY9NPTYykv3evdTOpTBkZqueeq/rUUyW/mBkZ1gB07mxf7qws1VNPLZrUypOXZ4l2/PjCad99\nZ4kwOzu02I8dU73iCtX77rOPu7j//MeS8OzZdqTw66+qS5eqDhtmveijR32vd+5c68n78sknqs2b\nW0M5caIlxX/8w5Jiq1a2zeLefNOWWbGi6Hs8aJDqww+X/zpXrFA97zzr9WdllT7fO+/4PmpQVX33\nXfssS/PGG6rXXlv4eM0aS95ffqnat6/qX/9acpn8fGtAe/VSnTPHGpQNG1R37lT9+mvVf/5T9bbb\n7LPu1s32p0jIzFQ9+WRr2AcPVi0oCM96P/tM9cMPVfftK5zmcqlOn26N2w03qNasadv3V3q6fSdK\n8/TTqt27Bx9zsBIj6W/caN+0JJWZab3ZFSvsy/jtt1Yy8LZ5s+28w4aV3hPLy7Pk1KiRlQYeeCDw\nWFassN7qrl32hWzfXvW99wJfjy/796s++qgl3+HDVQ8ftpj79LEjl/T0ksu4XPaF9S5ReSxdautq\n0cJet/f7smGDvY5lywqnbd1qia9qVdXHHy89zjFjrARWt64lwjfeKL285cuxY6o9eqiOGOH7+YIC\nO6KpU8eOtIq/3nPOscRc1vpbtrQGb9s2OyrwNHzZ2fZ+zJxZdJnnnrOjuLKOtFStcR00yBqRTz8t\ne978fJvn6aet3OiPgQNVe/e2ff6SS1RvuSX0DsXWrfb53HCDarVqdiQ8dKg1jK1aFb6XDz+sOmSI\nf+tcv171pJOszObL3r3WeNWq5bskGkmJkfR//lm1bdswvzXO8OefqosW+e4xbNhgpZOaNe1LfO65\ndtjdoYPVx884wxL36NHWA3vjDf+2OXu2HbJ793oCMWiQ6s03W2/04ovDf7ifnq56663WOF12mZUb\nDhwoff7MTCtfederN22yadOmWQPZoYPqXXdZ0jpyxHanMWNKrqugQPWrr8ruwXls3qw6YYKVRIYO\nDew1rlhhYyW+jlBmz1Y9/3wbk6lfv2gDP2uWHSmU955/+qmVrM46y8py3ubPt/3F855+8YU9DqSX\nu3ix7ZN33aW6alXRo5bsbNsnW7Sw933gQOu5X3ihHSWWlsQPHLAk6Rnzyc1VveceO5JbsED1xx9V\nly+39279etWDB/2L9e67baxDVTUnxxrDp56yhO0pXaraeM0pp5RfksnLs9fVu7c1fr7iGDxY9aGH\nbCzsySf9izNcEiPpp6VZkTnBHD1qL6t5c9vZeve2nXvrVtW//c0qWi+9pHroUMll8/Nt5x8zxg7X\no1kDzcmxnnfVqtabjpQff1T997/9S8CzZlli+fNPO2Jo3dqW9Th82HrXV15pieT222M/RNSpk+/S\n03XXWYOqqtqvnw22e1x+ueoHH5S/bpfL9q1+/Xw/36uXJaUtW+yI5dtvA48/O9sSWqtWqpUrW+ek\nTRvbl3v0sNKfR36+NS7dulmi/O23kut76SVL0MVfx8iR1rlISbEGoE0bO6qtXFm1ShVrfG66yXe5\n7OefreH0t4Ho1s13Wc/b8OG2HxUUWKeseIN/8KAdpaWn2/tbq5bv73CkJEbSnz7djs0SzCOP2E5W\nUGCDgyNGWC+ucmU7EyTYnng0LFliO7+TPPKI6h13WL3bV+8qP99KRa1b+58EIumLL0r22j1lJ09v\n88gRS6pTpljvunlz/xpB1bLr4Z4xnRYtVF97LfjX4OFy2VjITz/5Huj2NmGCbdd77CY72xofX41B\nWdvMyrLS50MPWbnGu+fuclnD+uab/q9z4UJrREp771assIS+das93rjROmfeg+OvvGL7occtt/g+\nqoyUxEj6kydbk5pA/vMfSz6+eif+fqmpqMOHLUHedFPZtelwDQ6GqqDA4vUeTO/bV7V//6LzLVli\nCTE11com4fLNN6qPPRabI54BA+yoxXM21+uvW3IMVl6eDY4/9FDh65k9245IvRuC8rhcVkKdNq3k\nc7m5dpQxYULR6b17Fx5R5eTYkcXKlYXPL1xon3O09rvESPpjxlg3Lg5NmWI7iXevfdEi+xL7Gpik\n0GRlxVej+fbbhQexhw6VftrnwIHWwzx8OLrxRUpBgTXO995rybRRI6vXh+LQIRsLGT7c9oGzz/ad\nvMszZYo1SN5cLmuMb7yxZCO5c6d9bjt22FHF9deXXLZtWxsrioZgkr7z7pw1YoRdVOTll2MdUkBy\nc4FGjYDLLgPS0oAOHYAbbrCXM2ECcO21sY6QYi0nB2jWDFi0CFiwAJg7F/jss5Lz5ecDO3fapSsS\nhedXxZUr26+Xv/wy9HXu2mWXpEhJATIz7dfQEtA9pOy9Pv104KOPgFatgPffB8aOte/zwoVAvXol\nl+nf336RPX++zX/JJUWff+cd+1xnzQr+tfkrmDtnxbyH7/kHT09/wAA7gTrG9u2zX6n6Ww+eMsXO\ni1e1XsjHH1utb+zYyMVI8ef5523g/uyzreSSTHbssLPQvv8+fOtctcqOpH39eM1fr79u4yc1a9pA\n+rx5ZZdn/vhDtXp1+62JL0eO2FhNNI7ukRA9/UcfBc46y/6PEVW7ZsymTcCOHXaFyAceAC6/vPSe\nRJcuwD33AHfeGd1YKb5kZgItWgDNm9t1jgLtmcY71fC/ZpcLqBDCVcQOHwbee8++57569r5MnWqf\n4/nn+35+0CDgyBG7/lQkBdPTrxipYIKWlRXzK2y+/bYdXv/8s13l+b33gEcesVv2Ll5sh6jeduwA\nli2zC5kRlaVePaBPH6Bt2+RL+EBkXnMoCR+wCwf26hXYMrfcUvbzjzwCfPBB8DFFkvN6+jfeCDz4\noHW1Y2DtWqs9fvcdcOaZhdNVgdtuA9q0scsIexs+3K52+J//RDdWIkpuwfT0nXdp5RheVvnoUbvc\n8PDhRRM+YD0Uz12Sdu4snK5qA7X33x/dWImIguG8pB/D8s6zz1qd7qGHfD/fvLkdBg4cWDjt+++t\n7JOSEp21w8YhAAANnklEQVQYiYhC4byafozumjVrFjBlit08o6y648CBdhSwZAlw0UWFvfxkrM8S\nUfxxXk8/BuWd6dPt7JxPPgFq1y573mrVgH/8A+jbFzh0yM7Hvfvu6MRJRBQqZw3kulzACSdYNj3h\nhKhs98MPgSefBL74Amjf3r9lXC7gwguBBg2shz9zZmRjJCLyJf5P2Tx61LJolBL+uHHAkCHA118D\n55zj/3IVKtiAbseOzr/RNxGRN2cl/SiWdt56y670kJYGtGwZ+PKXX25HB1dfHfbQiIgixllJP0pn\n7nz5JTBsmJ1507x58Ovp2jV8MRERRYOzkn4UztxZuxa4915g2rTQEj4RUTxy1tk7ES7v7NsHdOsG\nvPpqySvjERElA7+Svoh0EZF1IpIuIv19PF9dRGaIyAoRWS0i9/m7bBERLO/k5QG33grcfLP19ImI\nklG55R0RqQBgNIBOAHYBWCYi01V1nddsjwJYo6o3ikgdAOtF5D0ALj+WLRSm8k52tl2os2JFO62y\nYUO7cFqVKnZ9eyKiZOVPTT8FwAZV3QoAIjIFQHcA3olbAVRz/10NwD5VzReRDn4sWyhM5Z158+zG\nCGPHAhkZdrOFAwfsssfHHRfy6omI4pY/Sb8RgO1ej3fAGgJvowHMEJFdAKoCuD2AZQuFqbwzc6Zd\nG/v00+0fERGZcA3kXgvgF1VtCOB8AGNEpGrAawlDeaegwK6j061bSKshIkpI/vT0dwJo4vW4sXua\nt/sBjAAAVf1dRDYDaOXnsv/zwjff2F0mcnORmpqK1NRUP8IraulSWwVPxySiRJOWloa0tLSQ1lHu\ntXdE5DgA62GDsRkAlgLoqaprveYZA2CPqr4oIvUALAfQFkBWect6rUO1Z0/g+uuBu+4K+gUNGmT/\nDx8e9CqIiOJCRK69o6oFIvIYgLmwctB4VV0rIr3saR0L4CUAE0VklXuxZ1R1vzuoEsuWurEwlHdm\nzLBr6hARUUl+/SJXVecAOLPYtLe9/s6A1fX9WrZUIZ69s3kzsHcvb2hCRFQaZ/0iN8Szd2bOtOpQ\nqDdKJiJKVM5KjyGWd2bMsPuqExGRb866iUqtWkB6evm3r/IhKws49VT7MVaVKhEIkIjIYYIZyHVW\nTz8ry+5HGISvvrJr3DPhExGVzllJv1Il4Pjjg1p0xgz+IIuIqDzOKu/Uqwfs3h3wsvn59oOslSuB\nxo0jEBwRkQPFf3knyEHcOXOA005jwiciKk/cJ31V+/Vtv34RiIeIKME4K+kH8cOshQvtjli33BKB\neIiIEoyzkn4QPf3hw4EBA3idfCIif8R10l+2DFi3LqTrsxERJZW4TvrDhwP/939Bn+VJRJR0/Lrg\nWtQEUNNfswb44Qfg/fcjGA8RUYKJ257+yy8DTzwBVK4cwXiIiBJMXCb9TZuAL78E/v73CMdDRJRg\nnJX0/SzvfPCBDd6GcOl9IqKk5Kyk72dPf948oEuXCMdCRJSA4i7pZ2cDP/8MdOwYhXiIiBKMs5K+\nH/WahQuBCy7gJZSJiILhrKTvR09/3jzg6qujEAsRUQJi0iciSiLOup5+Xh5QsfTfi+3cCbRpA+zZ\nw2vtEBHF//X0y0j4gPXyO3ViwiciCpazkn45WNohIgqNs8o7ZcTicgENGgBLlgDNmkUvLiIip4r/\n8k4ZVq+2cV4mfCKi4MVN0mdph4godEz6RERJJC5q+rm5wCmnANu3AzVrRjkwIiKHStia/vffA+ec\nw4RPRBSquEj6CxYAnTvHOgoiovgXF0l/yxbgjDNiHQURUfyLi6SfkWHn6BMRUWiY9ImIkkhcJP1d\nu5j0iYjCwfFJPyfHTtk8+eRYR0JEFP8cn/Q9pR0J6ExUIiLyJW6SPhERhc7xSZ/1fCKi8HF80mdP\nn4gofJj0iYiSSFwk/YYNYx0FEVFiiIukz54+EVF4OD7pcyCXiCh8HJ/02dMnIgofR99E5dgxoGpV\n+0VuBcc3T0RE0ZVwN1HZvRuoV48Jn4goXBydTlnaISIKL0cl/by8oo85iEtEFF6OSvrbtxd9zJ4+\nEVF4OSrpb9pU9DGTPhFReDkq6W/eXPQxf41LRBRejkr6xXv6rOkTEYWXX0lfRLqIyDoRSReR/j6e\n7yciv4jIzyKyWkTyRaSm+7ktIrLS/fzSsrbD8g4RUWRVLG8GEakAYDSATgB2AVgmItNVdZ1nHlUd\nCWCke/4bAPRV1T/dT7sApKrqgfK2xaRPRBRZ/vT0UwBsUNWtqpoHYAqA7mXM3xPAh16Pxc/tFEn6\n+fnAvn1A3br+LElERP7wJxk3AuB9MuUO97QSROQkAF0ATPWarADmicgyEXm4rA0dPQpkZdnfe/YA\ndeoAFcs9FiEiIn+FeyC3G4DvvEo7AHCpqrYD0BXAoyJyWWkLt2hReAYPB3GJiMLPn370TgBNvB43\ndk/z5Q4ULe1AVTPc/+8Vkc9h5aLvfC187NgLGDECaN0aOP74VDRokOpHeEREySEtLQ1paWkhraPc\nq2yKyHEA1sMGcjMALAXQU1XXFpuvBoBNABqrao57WmUAFVQ1W0SqAJgL4EVVnetjO/rkk4qGDYF+\n/YCxY4GlS4Fx40J6fURECSuYq2yW29NX1QIReQyWsCsAGK+qa0Wklz2tY92z9gDwlSfhu9UD8LmI\nqHtb7/tK+B4tWgC//WZ/88wdIqLw82uYVFXnADiz2LS3iz2eBGBSsWmbAZznbzAtWgCzZtnfGRlA\n27b+LklERP5w1C9yW7QoPG2TA7lEROHnqKTfrBmwbRtQUMDyDhFRJDgq6Z94IlC7tvXymfSJiMLP\nUUkfsBLPxo3246z69WMdDRFRYnFk0l+6FKhRAzj++FhHQ0SUWByZ9L/7jqUdIqJIcGTSX7yYSZ+I\nKBIcl/SbNwf272fSJyKKBMcl/RYt7H8mfSKi8HNc0q9f307dZNInIgo/xyX9ChWsxMMbohMRhZ/j\nkj4APPYYcNFFsY6CiCjxlHtp5WgREXVKLERE8SCYSys7sqdPRESRwaRPRJREmPSJiJIIkz4RURJh\n0iciSiJM+kRESYRJn4goiTDpExElESZ9IqIkwqRPRJREmPSJiJIIkz4RURJh0iciSiJM+kRESYRJ\nn4goiTDpExElESb9AKSlpcU6BL8wzvBinOHFOGOLST8A8bITMM7wYpzhxThji0mfiCiJMOkTESUR\nR90YPdYxEBHFm0BvjO6YpE9ERJHH8g4RURJh0iciSiIxT/oi0kVE1olIuoj0j3U8HiIyXkQyRWSV\n17STRWSuiKwXka9EpEYsY3TH1FhE5ovIGhFZLSJ9nBariJwgIktE5Bd3jEOcFqM3EakgIj+LyAz3\nY8fFKSJbRGSl+z1d6uA4a4jIJyKy1r2PXuS0OEXkDPf7+LP7/ywR6eO0ON2xPikiv4rIKhF5X0SO\nDzTOmCZ9EakAYDSAawGcDaCniLSKZUxeJsDi8jYAwNeqeiaA+QAGRj2qkvIBPKWqZwO4GMCj7vfQ\nMbGq6lEAV6rq+QDOA3CdiKQ4KcZingDwm9djJ8bpApCqqueraop7mhPj/BeA2araGkBbAOvgsDhV\nNd39PrYD0B7AYQCfw2FxikhDAI8DaKeqbQBUBNATgcapqjH7B6ADgC+9Hg8A0D+WMRWLrymAVV6P\n1wGo5/67PoB1sY7RR8zTAHR2aqwAKgNYDuBCJ8YIoDGAeQBSAcxw6ucOYDOA2sWmOSpOANUB/O5j\nuqPiLBbbNQC+dWKcABoC2ArgZHfCnxHMdz3W5Z1GALZ7Pd7hnuZUdVU1EwBUdTeAujGOpwgRaQbr\nSf8I2wkcE6u7ZPILgN0A5qnqMqfF6PY6gP8D4H1amxPjVADzRGSZiDzknua0OJsD+ENEJrhLJ2NF\npDKcF6e32wF84P7bUXGq6i4ArwHYBmAngCxV/RoBxhnrpB/vHHO+q4hUBfApgCdUNRslY4tprKrq\nUivvNAaQIiJn+4gppjGKyPUAMlV1BYCyzn12wud+qVo5oiuspHc5HPZ+wnqj7QCMccd6GHY077Q4\nAQAiUgnAjQA+cU9yVJwiUhNAd1gFoiGAKiJyl4+4yowz1kl/J4AmXo8bu6c5VaaI1AMAEakPYE+M\n4wEAiEhFWMKfrKrT3ZMdGauqHgSQBqALnBfjpQBuFJFNAD4EcJWITAaw22FxQlUz3P/vhZX0UuC8\n93MHgO2qutz9eCqsEXBanB7XAfhJVf9wP3ZanJ0BbFLV/apaABt3uAQBxhnrpL8MwOki0lREjgdw\nB6xO5RSCoj2+GQDuc/99L4DpxReIkXcA/Kaq//Ka5phYRaSO54wCETkJwNUA1sJBMQKAqg5S1Saq\n2gK2L85X1bsBzISD4hSRyu4jO4hIFVgdejWc935mAtguIme4J3UCsAYOi9NLT1hj7+G0OLcB6CAi\nJ4qIwN7P3xBonA4YOOkCYD2ADQAGxDoer7g+ALALwFH3m30/bADla3e8cwHUdECclwIoALACwC8A\nfna/p7WcEiuAc91xrQCwCsCz7umOidFHzFegcCDXUXHCauWez3u153vjtDjdMbWFde5WAPgMQA2H\nxlkZwF4A1bymOTHOIbAO0yoAkwBUCjROXoaBiCiJxLq8Q0REUcSkT0SURJj0iYiSCJM+EVESYdIn\nIkoiTPpEREmESZ+IKIkw6RMRJZH/D0e5dNrL9uwYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a357b90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_accuracies, color='b')\n",
    "plt.plot(test_accuracies, color='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T11:25:06.332640",
     "start_time": "2016-06-10T11:25:06.330136"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T11:25:06.337015",
     "start_time": "2016-06-10T11:25:06.333848"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8926"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best error is still only "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-10T11:25:06.341535",
     "start_time": "2016-06-10T11:25:06.338143"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.740000009536743"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100.0*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}