src/04b-ng-clf-eeap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embed, Encode, Attend and Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras.layers import Input\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import GRU\n",
    "from keras.layers.pooling import GlobalAveragePooling1D\n",
    "from keras.layers.wrappers import TimeDistributed, Bidirectional\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "import custom_attn\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "\n",
    "VOCAB_FILE = os.path.join(DATA_DIR, \"ng-vocab.tsv\")\n",
    "MIN_OCCURS = 5\n",
    "\n",
    "GLOVE_FILE = os.path.join(DATA_DIR, \"glove.840B.300d.txt\")\n",
    "\n",
    "# covers about 95% of input data\n",
    "MAX_SENTS = 40 # maximum number of sentences per document\n",
    "MAX_WORDS = 60 # maximum number of words per sentence\n",
    "\n",
    "WORD_EMBED_SIZE = 300\n",
    "SENT_EMBED_SIZE = 100\n",
    "DOC_EMBED_SIZE = 50\n",
    "NUM_CLASSES = 20\n",
    "\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 10\n",
    "\n",
    "logging.basicConfig()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Load Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 40730\n"
     ]
    }
   ],
   "source": [
    "word2id = {\"PAD\": 0, \"UNK\": 1}\n",
    "fvocab = open(VOCAB_FILE, \"rb\")\n",
    "for i, line in enumerate(fvocab):\n",
    "    word, count = line.strip().split(\"\\t\")\n",
    "    if int(count) <= MIN_OCCURS:\n",
    "        break\n",
    "    word2id[word] = i\n",
    "fvocab.close()\n",
    "id2word = {v:k for k, v in word2id.items()}\n",
    "vocab_size = len(word2id)\n",
    "print(\"vocab_size: {:d}\".format(vocab_size))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load GloVe Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40730, 300)\n"
     ]
    }
   ],
   "source": [
    "E = np.zeros((vocab_size, WORD_EMBED_SIZE))\n",
    "E[1] = np.random.random(WORD_EMBED_SIZE)\n",
    "fglove = open(GLOVE_FILE, \"rb\")\n",
    "for line in fglove:\n",
    "    cols = line.strip().split(\" \")\n",
    "    word = cols[0]\n",
    "    if not word2id.has_key(word):\n",
    "        continue\n",
    "    vec = np.array([float(x) for x in cols[1:]])\n",
    "    idx = word2id[word]\n",
    "    E[idx] = vec\n",
    "fglove.close()\n",
    "print(E.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Document Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846\n"
     ]
    }
   ],
   "source": [
    "ng_data = fetch_20newsgroups(subset='all',\n",
    "                             data_home=DATA_DIR,\n",
    "                             shuffle=True, \n",
    "                             random_state=42)\n",
    "num_docs = len(ng_data.data)\n",
    "print(num_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['like', 'a', 'mouse']\n",
      "['PAD', 'The', 'cat', 'fought', 'like', 'a', 'mouse']\n"
     ]
    }
   ],
   "source": [
    "def pad_or_truncate(xs, maxlen):\n",
    "    if len(xs) > maxlen:\n",
    "        xs = xs[len(xs) - maxlen:]\n",
    "    elif len(xs) < maxlen:\n",
    "        xs = [\"PAD\"] * (maxlen - len(xs)) + xs\n",
    "    return xs\n",
    "\n",
    "xs = [\"The\", \"cat\", \"fought\", \"like\", \"a\", \"mouse\"]\n",
    "print(pad_or_truncate(xs, 3))\n",
    "print(pad_or_truncate(xs, 7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 40, 60)\n"
     ]
    }
   ],
   "source": [
    "X = np.zeros((num_docs, MAX_SENTS, MAX_WORDS))\n",
    "for docid in range(num_docs):\n",
    "    text = ng_data.data[docid]\n",
    "    sents = pad_or_truncate(nltk.sent_tokenize(text), MAX_SENTS)\n",
    "    for sid, sent in enumerate(sents):\n",
    "        words = nltk.word_tokenize(sent)\n",
    "        words = pad_or_truncate(words, MAX_WORDS)\n",
    "        for wid, word in enumerate(words):\n",
    "            try:\n",
    "                word_id = word2id[word]\n",
    "            except KeyError:\n",
    "                word_id = word2id[\"UNK\"]\n",
    "            X[docid, sid, wid] = word_id\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18846, 20)\n"
     ]
    }
   ],
   "source": [
    "y = ng_data.target\n",
    "Y = to_categorical(y, num_classes=NUM_CLASSES)\n",
    "print(Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13192, 40, 60) (13192, 20) (5654, 40, 60) (5654, 20)\n"
     ]
    }
   ],
   "source": [
    "Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, train_size=0.7)\n",
    "print(Xtrain.shape, Ytrain.shape, Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Network\n",
    "\n",
    "### Sentence Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 60)                0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 60, 300)           12219000  \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 60, 200)           240600    \n",
      "_________________________________________________________________\n",
      "attention_mc_1 (AttentionMC) (None, 200)               3860      \n",
      "=================================================================\n",
      "Total params: 12,463,460\n",
      "Trainable params: 12,463,460\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sent_inputs = Input(shape=(MAX_WORDS,), dtype=\"int32\")\n",
    "\n",
    "sent_emb = Embedding(input_dim=vocab_size,\n",
    "                     output_dim=WORD_EMBED_SIZE,\n",
    "                     mask_zero=True,\n",
    "                     weights=[E])(sent_inputs)\n",
    "sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "                             return_sequences=True))(sent_emb)\n",
    "\n",
    "sent_att = custom_attn.AttentionMC()(sent_enc)\n",
    "\n",
    "sent_model = Model(inputs=sent_inputs, outputs=sent_att)\n",
    "\n",
    "# sent_enc = Bidirectional(GRU(SENT_EMBED_SIZE,\n",
    "#                              return_sequences=False))(sent_emb)\n",
    "\n",
    "# sent_model = Model(inputs=sent_inputs, outputs=sent_enc)\n",
    "\n",
    "sent_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 40, 60)            0         \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 40, 200)           12463460  \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 40, 100)           75300     \n",
      "_________________________________________________________________\n",
      "attention_mc_2 (AttentionMC) (None, 100)               1740      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 20)                1020      \n",
      "=================================================================\n",
      "Total params: 12,546,570\n",
      "Trainable params: 12,546,570\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "doc_inputs = Input(shape=(MAX_SENTS, MAX_WORDS), dtype=\"int32\")\n",
    "\n",
    "doc_emb = TimeDistributed(sent_model)(doc_inputs)\n",
    "\n",
    "doc_enc = Bidirectional(GRU(DOC_EMBED_SIZE,\n",
    "                            return_sequences=True))(doc_emb)\n",
    "\n",
    "doc_att = custom_attn.AttentionMC()(doc_enc)\n",
    "\n",
    "fc1_dropout = Dropout(0.2)(doc_att)\n",
    "fc1 = Dense(50, activation=\"relu\")(fc1_dropout)\n",
    "fc2_dropout = Dropout(0.2)(fc1)\n",
    "doc_pred = Dense(NUM_CLASSES, activation=\"softmax\")(fc2_dropout)\n",
    "\n",
    "model = Model(inputs=doc_inputs, outputs=doc_pred)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\",\n",
    "              metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 11872 samples, validate on 1320 samples\n",
      "Epoch 1/10\n",
      "11872/11872 [==============================] - 258s - loss: 2.4765 - acc: 0.1875 - val_loss: 1.6304 - val_acc: 0.4371\n",
      "Epoch 2/10\n",
      "11872/11872 [==============================] - 255s - loss: 1.3235 - acc: 0.5241 - val_loss: 0.9788 - val_acc: 0.6735\n",
      "Epoch 3/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.6980 - acc: 0.7645 - val_loss: 0.7566 - val_acc: 0.7538\n",
      "Epoch 4/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.3642 - acc: 0.8875 - val_loss: 0.7289 - val_acc: 0.7826\n",
      "Epoch 5/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.1867 - acc: 0.9489 - val_loss: 0.7937 - val_acc: 0.7902\n",
      "Epoch 6/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.1108 - acc: 0.9709 - val_loss: 0.8700 - val_acc: 0.7909\n",
      "Epoch 7/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.0626 - acc: 0.9845 - val_loss: 0.9153 - val_acc: 0.8076\n",
      "Epoch 8/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.0396 - acc: 0.9909 - val_loss: 0.9682 - val_acc: 0.8030\n",
      "Epoch 9/10\n",
      "11872/11872 [==============================] - 254s - loss: 0.0313 - acc: 0.9925 - val_loss: 1.0149 - val_acc: 0.8053\n",
      "Epoch 10/10\n",
      "11872/11872 [==============================] - 253s - loss: 0.0258 - acc: 0.9934 - val_loss: 1.1216 - val_acc: 0.7977\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(Xtrain, Ytrain, batch_size=BATCH_SIZE,\n",
    "                   epochs=NUM_EPOCHS, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VNX9//HXJyEQ9t2CLAZblEUtlKBY6tK6IVqg9YtY\nl6r9Kv22bmiXH7bWWleqXdSvWkWlra2VKm7UUhEruHwFa3ADWUQQJKASVgEDJPD5/XFmzCQkZGEm\nd2byfj4e9zF37r0z98wQ8s4599xzzN0RERFJNzlRF0BERKQ6CigREUlLCigREUlLCigREUlLCigR\nEUlLCigREUlLCigREUlLCigREUlLCiiRNGSB/n9Kk6b/ACL7YGYTzWy5mW01s0Vm9q2EfReb2eKE\nfV+Jbe9lZk+YWYmZbTCzu2LbrzOzvya8vsDM3MyaxZ7PMbObzOz/gM+Ag83swoRzrDCz71cp32gz\ne8vMPo2Vc4SZjTWz+VWOu8rMnk7dNyWSfM2iLoBImlsOHAN8DIwF/mpmXwK+BlwHjAGKgC8CZWaW\nCzwDvACcB+wGCutxvvOAU4GlgAGHAqcDK4BjgX+Z2evu/oaZHQk8BPwX8G+gO9AW+AC4z8z6u/vi\nhPe9sSFfgEhUVIMS2Qd3f8zd17r7Hnf/O7AMOBK4CLjV3V/34H13XxXbdyDwE3ff7u473P2Vepzy\nT+7+rruXu3uZu//T3ZfHzvEi8BwhMAH+G5ji7rNi5Vvj7kvcfSfwd+BcADMbCBQQglMkYyigRPbB\nzL4ba0LbbGabgcOALkAvQu2qql7AKncvb+ApV1c5/6lmNs/MNsbOPzJ2/vi5qisDwJ+Bs83MCLWn\nR2PBJZIxFFAiNTCzg4D7gUuBzu7eAVhIaHpbTWjWq2o10Dt+XamK7UCrhOfdqjnm8+kFzKwF8Djw\nG+ALsfPPiJ0/fq7qyoC7zwN2EWpbZwN/qf5TiqQvBZRIzVoTAqMEwMwuJNSgAB4AfmxmQ2I97r4U\nC7T/AB8Bk8ystZnlm9nw2GveAo41s95m1h64upbzNwdaxM5fbmanAicn7H8QuNDMTjCzHDPrYWb9\nEvY/BNwFlNWzmVEkLSigRGrg7ouA3wJzgU+Aw4H/i+17DLgJ+BuwFXgK6OTuu4FvAl8CPgSKgXGx\n18wiXBt6B5hPLdeE3H0rcDnwKLCJUBOanrD/P8CFwO+BLcCLwEEJb/EXQqD+FZEMZJqwUCQ7mVlL\nYB3wFXdfFnV5ROpLNSiR7PUD4HWFk2Qq3QclkoXMbCWhM8WYiIsi0mBq4hMRkbSkJj4REUlLadfE\n16VLFy8oKIi6GCIikiLz589f7+5dazuu1oAysymEscDWufth1ew34A7CHe6fARe4+xuxfecD18QO\nvdHd/1zb+QoKCigqKqrtMBERyVBmtqoux9Wlie9PwIh97D8V6BtbxgN/iBWgE/BL4CjC+GS/NLOO\ndSmUiIhIrTUod3/JzAr2ccho4CEPvS3mmVkHM+sOHA/McveNAGY2ixB0j+xvoUVEpIHcw9LQ9dxc\nyM9vlKIm4xpUDyoPcFkc21bT9r2Y2XhC7YvevXsnoUgikhbKy6G0NCy7doXnZWV7P9Z1WyqO37On\n4pew+97P67IvFa+B/Q+TxPVkGTMGnnwyee+3D2nRScLdJwOTAQoLC9XvXSTVyspCaHz2WcVS9Xky\ntu3a1TifJzcX8vKgWbPwmLi+r20tW0JODpiFJXG96lLTvlS+BtJvvW/f5P7b7UMyAmoNYdj/uJ6x\nbWsIzXyJ2+ck4XwiTc+uXbB5M2zZEh7jS9Xnn35at/Aob8BsIDk50KpV5aVly/DYoQMceGDlbYnH\ntGwJzZvXLTTqu61Zs4pfnpJVkhFQ04FLzWwqoUPEFnf/yMxmAjcndIw4mdpHbxbJPu6wY8e+g6W2\nbaWl+z5Hbi60bw/t2kHr1hXh0KXL3oFRU4jUdkxenoJAGlVdupk/QqgJdTGzYkLPvDwAd7+XMD/N\nSOB9QjfzC2P7NprZDcDrsbe6Pt5hQiRjlZXBe+/BBx/UHCzVBU1Z2b7ft3nzUAuJL+3bQ69elZ8n\n7q+6rXVrhYdknbQb6qiwsNB1H5REbs+eEEILF1Zeli6tPmzizVxVg6S2YIk/z89XwEiTYWbz3b2w\ntuPSopOESGTcYe3avYNo0aJwrSauTx847DD45jfD45e+BJ06hXBp3z7UgEQkqRRQ0nRs3Lh3EC1c\nCJs2VRzTrVsIoPHjw+Nhh8GAAdC2bXTlFmmiFFCSfbZtCzWgqkH00UcVx7RvD4cfDuPGVQTRwIGh\nU4GIpAUFlGSuXbvCNaF4AC1YEB4/+KDimJYtQw3o5JNDCB1+eHg88EBd8xFJcwooSX+7d8OKFXvX\niN57r+J+nmbN4NBD4cgj4Xvfq6gV9ekTumCLSMZRQEn6cYfXXoOpU+Hll0Nz3Y4dFfsPPjiEz5gx\nFbWiQw5RRwWRLKOAkvTgDu+8A488An//O6xcCS1awDHHwA9/WFEj6t8f2rSJurQi0ggUUBKtpUtD\nTWnqVFiyJDTHnXQS/OpXMHp06MwgIk2SAkoa36pVoZY0dSq8+WborHDccTBhApxxhnrSiQiggJLG\n8vHH8NhjoQlv7tyw7aij4Pe/hzPPDL3qREQSKKAkdTZuhMcfDzWlOXPC8EFHHAG33BLuP+rTJ+oS\nikgaU0BJcm3dCk8/HUJp5szQDbxvX7jmmhBKAwZEXUIRyRAKKNl/paUwY0YIpWeeCV3Ce/WCK6+E\ns86CwYN1U6yI1JsCShpm1y6YNSuE0lNPheGFDjgALroIvvMdGDYsTHAnTZI77NwZfiy2bQvj7pqF\n+6lzc8OSuF71eXxdf9c0bQooqbvdu+HFF0MoPf54uMbUsWOoJZ11VuiJ10w/Uplmz54QIPEwSday\ne/f+l82s7mFWn+BLXOLfgXt4TFyvbltj7Y/PYt+8efWPydxXn+PbtoXOnff/37Yu9NtE9s0d5s0L\nofToo6E3XuvWYRSHs84KY9xl6AgO7uGX6K5dYYqn+GPiek2P8RGW4tOpuVde39/Hhr62vBy2b68I\nicT1mpbt2+v+neXkhF9QbdpUXrp1q1hv3Xrv/a1aVXzf5eXhsep6bc8bui/x+Y4de+8zC0tOTlji\n69Vti88uX9P+2l5f1/1moWy1/Vxu3177z2x8PVlGjw6NJo1BASV7c4e3364Y1WHVqjCqw2mnhVA6\n7bTwGyeJp9u2DUpKYP368Bhf37q19rCoS6DUtC+btWixd1C0aRNuM6tue12WFi3U7JaJqv4x1tD/\nT2Vl0LNn45VbASUVliypGNVh6dLw5+JJJ8ENN4Q/m9q1q9Pb7N4dWv8Sg6a68Elc37mz5verrQmi\nalNEq1a1H1uXpo19HRv/SxoqP1a3bX8f63NsTk5FLSYvr07/XNIExK//ZVoLfJ2Ka2YjgDuAXOAB\nd59UZf/vga/HnrYCDnD3DrF9u4EFsX0fuvuoZBRckmT3bpg8Ge67L9SazOD44+Gqq+Db34YuXSgt\njQXJ+7UHTUlJCKd4k1NV7dqFv+C7dg1/iQ0aFNa7dq3Ynrjetq3+YhdpqmoNKDPLBe4GTgKKgdfN\nbLq7L4of4+5XJhx/GTA44S1K3X1Q8oosyeLL3ufDsycyt6gZK3teRMnxX6Wkcz/Wf9aKkvuh5OYQ\nOjVdo8jJqQiSLl3CWK41BU3XruHCaosWjfsZRSRz1aUGdSTwvruvADCzqcBoYFENx38H+GVyiifJ\ntHMnvPEGzH11D6/+dQVz327FWp8WdhZDq42VA6Vfv33Xbjp0UE9yEUmdugRUD2B1wvNi4KjqDjSz\ng4A+wAsJm/PNrAgoBya5+179P8xsPDAeoHfv3nUrudTqo4/CsHevvhoei4riHQNyKCCX47sv5as/\nbMnRIzty6KHhuoWISLpI9iWzs4Bp7p54B8RB7r7GzA4GXjCzBe6+PPFF7j4ZmAxQWFhYw9UL2Zfy\n8jCdUjyQXn01TKkE4cJ+YaFz2XHv8NWXb+Xo3P/Q/Y6JYeZZXeARkTRVl4BaA/RKeN4ztq06ZwGX\nJG5w9zWxxxVmNodwfWr53i+V+tiwIdyeFK8dvfZauNkSoHt3+OpX4bLLwuPgLqtpcclF8NxzcMIJ\n8ODzcNBB0X4AEZFa1CWgXgf6mlkfQjCdBZxd9SAz6wd0BOYmbOsIfObuO82sCzAcuDUZBW9K9uyB\nxYsr146WLg37cnNDT7j//m84+ugQSL17xypG7vDnP8MVV4Qq1t13w//8jy4ciUhGqDWg3L3czC4F\nZhK6mU9x93fN7HqgyN2nxw49C5jqXqmDcX/gPjPbA+QQrkHV1LlCYj79FP7zn4owmjcPtmwJ+zp3\nDiF0/vnhsbCwhmtHa9fC978fBm899lj44x/h4IMb9XOIiOwP85puWIlIYWGhFxUVRV2MRuMOy5dX\nNNW9+iosWBC2m8HAgSGIvvrVUEPq27eWy0bu8Le/hfa9HTvC3EuXXaZak4ikDTOb7+6FtR2XYfcV\nZ77S0tCbLl47mjs33NwK4SbWYcPgW98KgXTUUdC+fT3e/JNP4Ac/gCefDGn2pz/BIYek4mOIiKSc\nAqqRLFwYLgW99FLFQKOHHAIjR1bUkPr3rxhdud4eewx++MMweN1tt4W5mBr8ZiIi0VNApVhZGdx6\nK/zqV6E29OMfhzAaNizc7Lrf1q+HSy8Ng7oOHRpqTZq1VkSygAIqhd55By64AN58E848E+66K0mh\nFPfUU6EjxKZNcNNN8NOfZt5okCIiNdCV8xTYtSvUmIYMgTVrYNq0UMFJWjht2gTnnRcuVh14YLio\n9bOfKZxEJKvoN1qSvfVWqDW9/TacfTbccUcYuy5p/vlPuPji0LPil7+En/9c8yqISFZSDSpJdu2C\na68Nl4E+/jh0pHv44SSG05YtYWii008PN0O99hpcd53CSUSylmpQSTB/Plx4Ybh/6bzz4PbboVOn\nJJ7guefCUBFr14amvGuv1bwVIpL1VIPaDzt3hha2o44KY+P94x/w0ENJDKetW8PQRKecEqZJnTs3\ndIZQOIlIE6AaVAO9/nqoNb37brjm9LvfQceOSTzB7NmhSW/VqtA3/YYbID8/iScQEUlvqkHV044d\nMHFiuI9p82aYMSMMc5e0cNq+HS6/HL7xjdAr75VXwo23CicRaWJUg6qHefNCrWnJknBJ6Le/redQ\nRLV55ZVwgvffDyF1yy3QqlUSTyAikjlUg6qD0lL4yU9g+PBQwXn2WXjggSSGU2kp/OhHYdTx3bth\nzpzQP13hJCJNmGpQtXj11VCpee89GD8+tLa1a5fEE7z2Wpg7Y+nSMNDrrbeGDhEiIk2calA1+Owz\nuOoq+NrXQm+9WbPgvvuSGE47d8LVV4eB+UpLwwnuuUfhJCISoxpUNV5+OXSge//9MED4pEnQtm0S\nTzB/fqg1vfsuXHRRuJiV1GqZiEjmUw0qwfbtYUqM444Ll4JeeCHMkp60cNq1KwxPdNRRYTy9GTPg\n/vsVTiIi1ahTQJnZCDNbambvm9nEavZfYGYlZvZWbLkoYd/5ZrYstpyfzMIn04svwhFHwJ13wiWX\nhJHIv/71JJ7g44/DJILXXw/nnBMmiDr11CSeQEQku9TaxGdmucDdwElAMfC6mU1390VVDv27u19a\n5bWdgF8ChYAD82Ov3ZSU0ifBtm3hvqa774YvfjF0oDvuuCSfZNUqOPHEMFTRk0/CmDFJPoGISPap\nSw3qSOB9d1/h7ruAqcDoOr7/KcAsd98YC6VZwIiGFTX5XngBDj889E2YMCGMQJ70cFq6NPS0WL8e\nnn9e4SQiUkd16STRA1id8LwYOKqa484ws2OB94Ar3X11Da/tUfWFZjYeGA/Qu3fvupV8P2zdGub2\nu/de6Ns3dIoYPjwFJ3r7bTjppLA+ezYMGpSCk4hIpikrK6O4uJgdO3ZEXZSUys/Pp2fPnuQ1cNaF\nZPXi+wfwiLvvNLPvA38GvlHXF7v7ZGAyQGFhoSepTNV6/vkwCsTq1eHe2OuvT9H9sHPnwsiRodv4\n88/DoYem4CQikomKi4tp27YtBQUFmFnUxUkJd2fDhg0UFxfTp0+fBr1HXZr41gC9Ep73jG1LLMgG\nd98Ze/oAMKSur20sW7aEG21POglatoT/+z/4zW9SFE7//nc4UZcuYfgihZOIJNixYwedO3fO2nAC\nMDM6d+68X7XEugTU60BfM+tjZs2Bs4DpVQrSPeHpKGBxbH0mcLKZdTSzjsDJsW2N6tln4bDD4MEH\nQ9Pem2+GDnUpMX16qDn16RPaDg86KEUnEpFMls3hFLe/n7HWJj53LzezSwnBkgtMcfd3zex6oMjd\npwOXm9kooBzYCFwQe+1GM7uBEHIA17v7xv0qcT1s3hya8aZMgf79w7BFR1V39SxZ/vY3+O534Stf\nCamY1FkLRUSaljrdB+XuM9z9EHf/orvfFNt2bSyccPer3X2gu3/Z3b/u7ksSXjvF3b8UW/6Ymo+x\nt3/+M9Sa/vznMKLQG2+kOJzuuw/OPTf02Pv3vxVOIpK2Nm/ezD333FPv140cOZLNmzenoETVy7qR\nJDZtChMInn56mKNp3jy4+eYUT6d0221h5tuRI+Ff/0ryuEgiIslVU0CVl5fv83UzZsygQ4cOqSrW\nXrIyoJ56Cq65BoqKoLAwhSdzDyf66U/hzDPhiSdCDwwRkTQ2ceJEli9fzqBBgxg6dCjHHHMMo0aN\nYsCAAQCMGTOGIUOGMHDgQCZPnvz56woKCli/fj0rV66kf//+XHzxxQwcOJCTTz6Z0tLSpJcz6waL\nPfhgWLkSUh7ye/aEu3v/93/DgK/33gu5uSk+qYhknQkT4K23kvuegwbB7bfXuHvSpEksXLiQt956\nizlz5nDaaaexcOHCz7uDT5kyhU6dOlFaWsrQoUM544wz6Ny5c6X3WLZsGY888gj3338/Z555Jo8/\n/jjnnntuUj9G1gUUNEI4lZfDxRfDn/4EV14ZRiNvAj1yRCQ7HXnkkZXuVbrzzjt58sknAVi9ejXL\nli3bK6D69OnDoNjgA0OGDGHlypVJL1dWBlRK7doVBnudNg2uuw6uvVbhJCINt4+aTmNp3br15+tz\n5szh+eefZ+7cubRq1Yrjjz++2nuZWrRo8fl6bm6umvgi99lncMYZoQv5734Xak8iIhmmbdu2bN26\ntdp9W7ZsoWPHjrRq1YolS5Ywb968Ri5dBQVUXW3ZAt/8ZhgZ4v77w3UnEZEM1LlzZ4YPH85hhx1G\ny5Yt+cIXvvD5vhEjRnDvvffSv39/Dj30UIYNGxZZOc09pUPf1VthYaEXFRVFXYzK1q+HESPC4K9/\n/SuMGxd1iUQkgy1evJj+/ftHXYxGUd1nNbP57l5rH2vVoGqzdm0YV2/FitB//bTToi6RiEiToIDa\nlw8+CBMNrlsXbsA9/vioSyQi0mQooGqyeHEIp9LSMHTRkUdGXSIRkSYl60aSSIo33oBjj4Xdu+HF\nFxVOIiIRUEBV9cor8PWvh4miXn45zAkvIiKNTgGV6Lnn4OSToVu3EFR9+0ZdIhGRJksBFffEE+E+\np0MOgZdegl69an+NiEgT0KZNm0jOq4ACeOghGDsWhgyB2bMh4aY1ERGJhnrx3X03XHopnHBCuM8p\nor8UREQay8SJE+nVqxeXXHIJANdddx3NmjVj9uzZbNq0ibKyMm688UZGjx4daTmbdkDdcgv87Gcw\nahT8/e8pntVQRGRvEcy2wbhx45gwYcLnAfXoo48yc+ZMLr/8ctq1a8f69esZNmwYo0aNwiIcDLtO\nAWVmI4A7gFzgAXefVGX/VcBFQDlQAnzP3VfF9u0GFsQO/dDdRyWp7A3nHuaB//Wv4eyzw7QZeXlR\nl0pEpFEMHjyYdevWsXbtWkpKSujYsSPdunXjyiuv5KWXXiInJ4c1a9bwySef0K1bt8jKWWtAmVku\ncDdwElAMvG5m0919UcJhbwKF7v6Zmf0AuBWID1hX6u6DklzuhtuzJzTp/eEP8P3vwz33QI4uxYlI\nNKKabWPs2LFMmzaNjz/+mHHjxvHwww9TUlLC/PnzycvLo6CgoNppNhpTXX4zHwm87+4r3H0XMBWo\n1DDp7rPd/bPY03lAz+QWM0nKy+GCC0I4/fSn4VHhJCJN0Lhx45g6dSrTpk1j7NixbNmyhQMOOIC8\nvDxmz57NqlWroi5inQKqB7A64XlxbFtN/hv4V8LzfDMrMrN5ZjamuheY2fjYMUUlJSV1KFID7NwZ\neur95S9w000waZImGhSRJmvgwIFs3bqVHj160L17d8455xyKioo4/PDDeeihh+jXr1/URUxuJwkz\nOxcoBI5L2HyQu68xs4OBF8xsgbsvT3ydu08GJkOYbiOZZQJg+3b41rdg1iy480647LKkn0JEJNMs\nWLDg8/UuXbowd+7cao/btm1bYxWpkroE1Bog8a7VnrFtlZjZicDPgePcfWd8u7uviT2uMLM5wGBg\nedXXp8zmzWGKjHnz4I9/DE18IiKS9urSxPc60NfM+phZc+AsYHriAWY2GLgPGOXu6xK2dzSzFrH1\nLsBwILFzRWqtWxfG1Xv9dXj0UYWTiEgGqbUG5e7lZnYpMJPQzXyKu79rZtcDRe4+HbgNaAM8Fusz\nH+9O3h+4z8z2EMJwUpXef6lTXBymy/jwQ5g+PcyIKyKSJtw90nuMGsP+zthep2tQ7j4DmFFl27UJ\n6yfW8LpXgcYfDnz58jAyxMaNMHMmHHNMoxdBRKQm+fn5bNiwgc6dO2dtSLk7GzZsIH8/BkDIvpEk\n3nsvzHy7a1cYV2/IkKhLJCJSSc+ePSkuLiZlvZbTRH5+Pj17Nvyuo+wLqG7dwgSDN98MAwZEXRoR\nkb3k5eXRp0+fqIuR9rIvoNq1C4O+iohIRtMwCiIikpYUUCIikpZsf7sBJpuZlQDJGASqC7A+Ce/T\nVOj7qh99X/Wj76v+svk7O8jdu9Z2UNoFVLKYWZG7F0Zdjkyh76t+9H3Vj76v+tN3piY+ERFJUwoo\nERFJS9kcUJOjLkCG0fdVP/q+6kffV/01+e8sa69BiYhIZsvmGpSIiGQwBZSIiKSlrAsoMxthZkvN\n7H0zmxh1edKZmfUys9lmtsjM3jWzK6IuU6Yws1wze9PMnom6LOnOzDqY2TQzW2Jmi83s6KjLlM7M\n7MrY/8eFZvaImTV8OPAMl1UBZWa5wN3AqcAA4DtmphFja1YO/MjdBwDDgEv0fdXZFcDiqAuRIe4A\nnnX3fsCX0fdWIzPrAVwOFLr7YYQ5+M6KtlTRyaqAAo4E3nf3Fe6+C5gKjI64TGnL3T9y9zdi61sJ\nvzh6RFuq9GdmPYHTgAeiLku6M7P2wLHAgwDuvsvdN0dbqrTXDGhpZs2AVsDaiMsTmWwLqB7A6oTn\nxegXbp2YWQEwGHgt2pJkhNuBnwJ7oi5IBugDlAB/jDWJPmBmraMuVLpy9zXAb4APgY+ALe7+XLSl\nik62BZQ0gJm1AR4HJrj7p1GXJ52Z2enAOnefH3VZMkQz4CvAH9x9MLAd0LXhGphZR0KrTx/gQKC1\nmZ0bbamik20BtQbolfC8Z2yb1MDM8gjh9LC7PxF1eTLAcGCUma0kNCF/w8z+Gm2R0loxUOzu8Zr5\nNEJgSfVOBD5w9xJ3LwOeAL4acZkik20B9TrQ18z6mFlzwsXF6RGXKW2ZmRGuDSx2999FXZ5M4O5X\nu3tPdy8g/Hy94O5N9i/c2rj7x8BqMzs0tukEYFGERUp3HwLDzKxV7P/nCTThTiVZNaOuu5eb2aXA\nTELvlynu/m7ExUpnw4HzgAVm9lZs28/cfUaEZZLscxnwcOyPxhXAhRGXJ225+2tmNg14g9DL9k2a\n8JBHGupIRETSUrY18YmISJZQQImISFpSQImISFpSQImISFpSQImISFpSQImISFpSQImISFpSQImI\nSFpSQImISFpSQImISFpSQImISFpSQImISFpSQImkiJmtNLMToy6HSKZSQImISFpSQImISFpSQImk\nmJm1MLPbzWxtbLndzFrE9nUxs2fMbLOZbTSzl80sJ7bv/5nZGjPbamZLzeyEaD+JSOPKqhl1RdLU\nz4FhwCDAgaeBa4BfAD8CioGusWOHAR6bIv1SYKi7rzWzAsIs0SJNhmpQIql3DnC9u69z9xLgV8B5\nsX1lQHfgIHcvc/eXPUxzvRtoAQwwszx3X+nuyyMpvUhEFFAiqXcgsCrh+arYNoDbgPeB58xshZlN\nBHD394EJwHXAOjObamYHItKEKKBEUm8tcFDC896xbbj7Vnf/kbsfDIwCropfa3L3v7n712KvdeDX\njVtskWgpoERS7xHgGjPramZdgGuBvwKY2elm9iUzM2ALoWlvj5kdambfiHWm2AGUAnsiKr9IJBRQ\nIql3I1AEvAMsAN6IbQPoCzwPbAPmAve4+2zC9adJwHrgY+AA4OrGLbZItCxcjxUREUkvqkGJiEha\nUkCJiEhaUkCJiEhaUkCJiEhaSruhjrp06eIFBQVRF0NERFJk/vz56929a23HpV1AFRQUUFRUFHUx\nREQkRcxsVe1HqYlPRETSVPYF1O7dcMMNsGlT1CUREZH9kH0B9c47cOONMHIkbNsWdWlERKSB0u4a\n1H4bPBimToWxY2H0aPjnPyE/P+pSiYh8rqysjOLiYnbs2BF1UVIqPz+fnj17kpeX16DXZ19AAXzr\nWzBlCpx/PowbB9OmQQO/IBGRZCsuLqZt27YUFBQQxgnOPu7Ohg0bKC4upk+fPg16j+xr4ov77nfh\nrrtg+nS48ELYo4GgRSQ97Nixg86dO2dtOAGYGZ07d96vWmJ21qDiLrkEtmyBn/8c2rWDu++GLP6B\nEJHMkc3hFLe/nzG7Awrg6qtDSN16K7RvD7fcEnWJRESkDrK3iS/ODCZNgv/5n/A4aVLUJRIRidTm\nzZu555576v26kSNHsnnz5hSUqHrZH1AQQuruu+Hss0ONqgH/MCIi2aKmgCovL9/n62bMmEGHDh1S\nVay9ZH8TX1xODvzpT7B1a7g21a4dnHtu1KUSEWl0EydOZPny5QwaNIi8vDzy8/Pp2LEjS5Ys4b33\n3mPMmDGIJVi5AAAT6klEQVSsXr2aHTt2cMUVVzB+/HigYii6bdu2ceqpp/K1r32NV199lR49evD0\n00/TsmXLpJaz6QQUhK7mjz4abuK94AJo0wbGjIm6VCLSlE2YAG+9ldz3HDQIbr+9xt2TJk1i4cKF\nvPXWW8yZM4fTTjuNhQsXft4dfMqUKXTq1InS0lKGDh3KGWecQefOnSu9x7Jly3jkkUe4//77OfPM\nM3n88cc5N8l/9DeNJr5E+fnw9NNQWBjukXr++ahLJCISqSOPPLLSvUp33nknX/7ylxk2bBirV69m\n2bJle72mT58+DBo0CIAhQ4awcuXKpJeradWg4tq2hRkz4PjjQw1q1iw4+uioSyUiTdE+ajqNpXXr\n1p+vz5kzh+eff565c+fSqlUrjj/++GrvZWrRosXn67m5uZSWlia9XE2vBhXXqRM89xx07x6a/N5+\nO+oSiYg0irZt27J169Zq923ZsoWOHTvSqlUrlixZwrx58xq5dBWabkABdOsWmvjatIGTT4b33ou6\nRCIiKde5c2eGDx/OYYcdxk9+8pNK+0aMGEF5eTn9+/dn4sSJDBs2LKJSgrl7ak9g1gt4CPgC4MBk\nd7+jpuMLCwu90ScsXLIEjj02XJ965RXo3btxzy8iTcrixYvp379/1MVoFNV9VjOb7+6Ftb22MWpQ\n5cCP3H0AMAy4xMwGNMJ5665fP5g5Ez79FE48ET75JOoSiYg0eSkPKHf/yN3fiK1vBRYDPVJ93nob\nPDhMzbFmTWju04SHIiKRatRrUGZWAAwGXquyfbyZFZlZUUlJSWMWqbLhw+HJJ0OTnyY8FBGJVKMF\nlJm1AR4HJrj7p4n73H2yuxe6e2HXrl0bq0jVO/lkeOQR+M9/Qhf0LJ9QTEQkXTVKQJlZHiGcHnb3\nJxrjnPvl298OEx7++9/wne9ALeNTiYhI8qU8oCxMCPIgsNjdf5fq8yXN+efDnXfCU0/B976nCQ9F\nRBpZY4wkMRw4D1hgZvEBp37m7jMa4dz757LLwlxSv/hFGH3irrs04aGINDlt2rRhWwTX5FMeUO7+\nCpC5v9V//vMQUr/5TZjw8Oaboy6RiEiT0DTH4qsPszAb76efhtl427eH//f/oi6ViEiDTZw4kV69\nenHJJZcAcN1119GsWTNmz57Npk2bKCsr48Ybb2T06NGRllMBVRdmYZLDTz+FiRPDXFI/+EHUpRKR\nLBDBbBuMGzeOCRMmfB5Qjz76KDNnzuTyyy+nXbt2rF+/nmHDhjFq1CgswssaCqi6ys2Fhx4K90bF\nJzw855yoSyUiUm+DBw9m3bp1rF27lpKSEjp27Ei3bt248soreemll8jJyWHNmjV88skndOvWLbJy\nKqDqI3HCw/PPDx0nRo2KulQiksGimm1j7NixTJs2jY8//phx48bx8MMPU1JSwvz588nLy6OgoKDa\naTYaU9MezbwhWraE6dPhK1+BM88M90qJiGSYcePGMXXqVKZNm8bYsWPZsmULBxxwAHl5ecyePZtV\nq1ZFXUQFVIO0bQv/+hf07QujR0OE86WIiDTEwIED2bp1Kz169KB79+6cc845FBUVcfjhh/PQQw/R\nr1+/qIuYfU1827eHVref/ARGjEjhiTp3DhMeHnMMnHoqvPgiHHFECk8oIpJcCxYs+Hy9S5cuzJ07\nt9rjorgHCrKwBrV6NRQXh8z49rchpbXU7t3DhIetW4cx/JYtS+HJRESalqwLqH794J13wi1LM2dC\n//5w002wc2eKTlhQEEJq9+4wl9SHH6boRCIiTUvWBRRAixbhdqXFi0OHu2uugcMPD4GVEvEJDzdv\nhpNOgnXrUnQiEckWqZ7NPB3s72fMyoCK690bpk0L2WEWrkmdcUaKKjlf+UqY8HD1ajjllBBWIiLV\nyM/PZ8OGDVkdUu7Ohg0byM/Pb/B7WLp9QYWFhV5UVJT09925E373O7jhhvD8mmvgRz8Kta2kmjkT\nvvlNGDo0dKJo3TrJJxCRTFdWVkZxcXHk9xmlWn5+Pj179iQvL6/SdjOb7+6Ftb2+yQRU3IcfwpVX\nwhNPhF7i//u/ocKTVI8/Hu6ROuEE+Mc/UpCCIiKZq64BldVNfNXp3Tvkx7PPhucjRsB//VeSm/3O\nOAMefBBmzdKEhyIiDdTkAirulFNgwYLQw2/GjNDb75Zbktjb74IL4I474MknNeGhiEgDNNmAgtDy\n9rOfhd5+p5wS1o84Ilw6SorLL4frr4e//CWsp1lzqohIOmvSARV30EHhmtS//hUqOqecAmPHhg55\n++2aa+Cqq+Duu8PMvCIiUicKqAQjRsDChXDjjaHHeL9+MGkS7Nq1H29qFmbjveii0J44fjysX5+0\nMouIZCsFVBUtWoRZ3uPNfldfHZr9Zs3ajzc1g3vvDTWpKVNC98E77oCysqSVW0Qk2yigahBv9psx\nI3TCO/nk/Wz2y82F3/42jMN05JFhGs0jjqjoTigiIpUooGpx6qmh2e+GG+CZZ0Kz369/vR/NfgMG\nhFD6xz9C8p16Kpx+Orz3XlLLLSKS6RRQdZCfH/o6LF4chtqbODFUfp5/voFvaBZC6d134bbb4KWX\nYODAMLSFhkgSEQEUUPVSUABPPRU6UJSXh7A688wwvUeDNG8OP/5xmKbjggvg97+HQw6ByZPD6Ogi\nIk2YAqoBRo4MzX7XXx9a6vr1g1tv3Y9mvy98Ae6/H4qK4NBD4fvfhyFDwiSIIiJNVJMbiy/ZPvgg\njO339NMhqO66KwzB12Du8NhjYUrgDz8M4zDddluovomIpJA7bNoEH31UsaxdW/n5UUeFX0n7o65j\n8WXdlO+NrU+fima/yy8PcxaeeWbosNezZwPe0Cy8wTe/Ge6fuuWWUE378Y/Dxa82bZL+GUQku+3Z\nE26/rCl0Epfqhntr0yZMIH7ggdChQ+OVO+U1KDObApwOrHP3w2o7PtNqUIl27AhNfbfcEnqV//KX\ncMUV4VJTg61eHYLpb38LPx2//jWcfTbkqHVWpKkrLw/zo+4reNauhU8+qX7M6g4dwq+V7t33XhK3\nJ/vv4rSZbsPMjgW2AQ9le0DFffBBuM1p+vQwCO1dd8E3vrGfb/rqqyHtiopCHfuOO8KjiGQV91CL\niQdPTaHz0UdQUlL9ONRdu9YeOt26QcuWjf/5II0CKlaYAuCZphJQcc88EzJlxQoYNy40+/XosR9v\nuGdPGHh24kT4+GM477wwFtOBByatzCJSuz17oLQUtm9PzVJd6OTkhP5U+wqd7t3DMfvVatMIMiqg\nzGw8MB6gd+/eQ1atWpXyMjWWqs1+Y8aEyXYLC2HwYGjVqgFvunUr3HxzmCI4Ly8Mw37VVeGGLRH5\n3O7d4Rf+1q2wbVvFY9X1+gbIZ5/Vrxw5OWFy7fosVWtBBxwQfodkg4wKqETZVINKtGIFXHstzJkD\na9aEbTk54f7ceGAVFoYbgOs8Ae+KFaHzxJNPhl5+v/kNfPvboaOFSIaJN21VFyINfaxPkDRrVvfw\naNOmfmHTooX+WyZSQKWxjz4Kl5Jef73iMT7AeV5eCKl4aA0dGkZHarav/pYvvBAuei1YAMcdF65P\nffnLjfJZpOlJbN767LPKtYrq1uPPawuTbdvqPvl0Tg60bRuCYn8eE8Mm3ZvFsokCKoO4h1ueEgNr\n/nzYsiXsb9kSBg2qHFqHHFKlI195ebjZ9xe/CDcyXHxxGECwa9dIPpNEx73y9ZHagqM+69u3h/eu\nr5Yt9z9MEh/z81UjyWRpE1Bm9ghwPNAF+AT4pbs/WNPxTTGgqrNnDyxfXjm03nijosmibdsw2ES8\naXDo0HBPlm3eBL/6Veg62KZN6Ot+ySX68zDDuIcw2LixfsumTfW/PgLhF368OapVq9rX63pcq1Zh\n0V0RkihtAqq+FFA12707DFib2Dz41lsVQyx16pQQWN1WU/j41fR48WHskEPCOH8jR0b7AZqgPXvg\n00/rHzQbN+57urAWLcK/d9WlQ4eKJqu6BkrLltlz8V0ygwKqidi1K4wLmBhaCxZUjDXbreMOCne9\nSuH2Fxk6xCm84zwOGN432kJnmHhtZsuWMNh8/LGuNZrqugzHtW5dfdDUtrRsqSYuyVwKqCastBTe\nfjuhefA/zpKl4B5+o/Vuu5HC49tQeHRzhg4NTYUdO0Zc6BTasaNyuMSXxOc1rceX2gaXb9++/iHT\nsWM9emyKZBEFlFSydSu88e9NFE16nqLXynk9ZxjL9/T5fH/btuEyVYsWtT/W5ZhkHNu8eQiG+gZK\n1X21jTJvFj5/hw4haNq3r7xe9Xl8vXPnima1ffayFJFKFFBSszffhAkT2PjSAub3GUvRCT+lpO0X\n2bUr3Ieycyefr1d9rG1fFD9OrVrVP1wS19u21UV8kcakgJJ9c4fHHw83+q5aBcccE4a5GD0avvjF\nBr/l7t0NC7aa9uXm7jtc2rUL946JSOZQQEndlJbCnXfCww+H3hUQ7gwePTosQ4eqeiEiSaWAkvpb\nsSIMwT59Orz0UqgOde8e5qYaNSrMxKjx/kRkPymgZP9s3AgzZoSpgp99NoxD07o1nHJKqFmddlro\nJSAiUk8KKEmenTth9uwQVtOnh8locnLga1+raAps4HUrEWl6FFCSGnv2hIECp08PgaXrViJSTwoo\naRz7um41enSYSljXrUQkgQJKGp+uW4lIHSigJFq6biUiNVBASfqo6brVwIEhqEaN0nUrkSZEASXp\nS9etRJo0BZRkhtquW514YggvzS0hkjUUUJJ5qrtuBWHgvQEDKi/9+0OvXgoukQykgJLMFr9u9dpr\nsGhRxVJSUnFMmzYhqKqGV0GBrmeJpLG6BpRmsZH0lJMTOk4MHVp5e0lJmPc+HliLF8OsWfDnP1cc\n07Il9OtXubY1YEDoNaiJm0Qyhv63Smbp2jUsxx5befvmzZWDa9EiePnlMEp7XPPmcMghe9e4+vYN\n+0QkrSigJDt06ABHHx2WRNu2wZIllYNr/nx47LGK2RVzc0NIVW0uPPTQUBsTkUgooCS7tWkDhYVh\nSVRaCkuXVm4qXLQodM7YvTscYwYHH7x3jatfv/C+IpJSCihpmlq2hEGDwpJo1y5YtqxyjWvRotAF\nvqys4rhu3cKwTZ06Vb907Lj3tnbt1OtQpB4UUCKJmjcPI1wMHFh5e3l5uME4HlgffBDu4dq4May/\n8UZY37695vfOzd07uKoLsqpLhw7q3CFNkn7qReqiWbPQweKQQ2DMmJqP27kTNm2qCK+qS+K+devC\n9bGNG0Mnj31p1672IIuHWZs24Wbn+GPr1uoEIhlJASWSTC1ahOa/bt3q97rdu0NIVRdk1S2rV1cc\nV15e+/vn5e0dWtWt13d/y5a650xSplECysxGAHcAucAD7j6pMc4rkjFyc8M1rfpOR+IeeiomBtv2\n7WFb4mNN20pKYOXKytt27qxfGeKBta9Qa9UqhHdeXliaN6/+MRn7dJ0va6Q8oMwsF7gbOAkoBl43\ns+nuvijV5xbJembQtm1YDjooOe9ZXl63cKttW0lJ5W1lZaETSqpHr2nWrH7BlpsbaoF1Wczqfuz+\nvt6s4vj6rKf6NQccAIcfntp/w/g/ZSOc40jgfXdfAWBmU4HRgAJKJB01awbt24clFXbvDkEVD6yy\nssrr+9qWqn179tR9ca/f8bW9R6YZMwaefLJRTtUYAdUDWJ3wvBg4KvEAMxsPjAfo3bt3IxRJRCKT\nmxuuXekm6KC6wNu9O2yPL/FQq+t6Kl/TqVOjfTVp0UnC3ScDkyEMFhtxcUREGo9ZCO3c3KhLknYa\no/vNGqBXwvOesW0iIiI1aoyAeh3oa2Z9zKw5cBYwvRHOKyIiGaxR5oMys5HA7YRu5lPc/aZ9HFsC\nrErCabsA65PwPk2Fvq/60fdVP/q+6i+bv7OD3L1rbQel3YSFyWJmRXWZEEsCfV/1o++rfvR91Z++\ns8Zp4hMREak3BZSIiKSlbA6oyVEXIMPo+6offV/1o++r/pr8d5a116BERCSzZXMNSkREMpgCSkRE\n0lLWBZSZjTCzpWb2vplNjLo86czMepnZbDNbZGbvmtkVUZcpU5hZrpm9aWbPRF2WdGdmHcxsmpkt\nMbPFZnZ01GVKZ2Z2Zez/40Ize8TM8qMuU1SyKqASpvY4FRgAfMfMBkRbqrRWDvzI3QcAw4BL9H3V\n2RXA4qgLkSHuAJ51937Al9H3ViMz6wFcDhS6+2GEwQ3OirZU0cmqgCJhag933wXEp/aQarj7R+7+\nRmx9K+EXR49oS5X+zKwncBrwQNRlSXdm1h44FngQwN13uXst89s3ec2AlmbWDGgFrI24PJHJtoCq\nbmoP/cKtAzMrAAYDr0VbkoxwO/BTIAMn82l0fYAS4I+xJtEHzKx11IVKV+6+BvgN8CHwEbDF3Z+L\ntlTRybaAkgYwszbA48AEd/806vKkMzM7HVjn7vOjLkuGaAZ8BfiDuw8GtgO6NlwDM+tIaPXpAxwI\ntDazc6MtVXSyLaA0tUc9mVkeIZwedvcnoi5PBhgOjDKzlYQm5G+Y2V+jLVJaKwaK3T1eM59GCCyp\n3onAB+5e4u5lwBPAVyMuU2SyLaA0tUc9mJkRrg0sdvffRV2eTODuV7t7T3cvIPx8veDuTfYv3Nq4\n+8fAajM7NLbpBGBRhEVKdx8Cw8ysVez/5wk04U4laTGjbrK4e7mZXQrMpGJqj3cjLlY6Gw6cByww\ns7di237m7jMiLJNkn8uAh2N/NK4ALoy4PGnL3V8zs2nAG4Retm/ShIc80lBHIiKSlrKtiU9ERLKE\nAkpERNKSAkpERNKSAkpERNKSAkpERNKSAkpERNKSAkpERNLS/wf9Ym9Mso2MXAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3dc8059990>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"r\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"b\", label=\"val\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy score: 0.807\n",
      "\n",
      "confusion matrix\n",
      "\n",
      "[[193   0   0   0   0   0   0   0   0   0   1   7   0   3   1  18   0   9   3   9]\n",
      " [  0 226  10   6  17  24   6   1   0   1   1   0   1   3   4   1   0   0   2   0]\n",
      " [  0  19 206  27   6  20   0   3   2   1   5   1   0   1   0   1   0   1   1   0]\n",
      " [  0  10  18 195  30   8   7   1   0   0   0   1   7   0   1   0   0   0   0   1]\n",
      " [  0   9   4  31 218   3   7   0   1   0   0   5   7   4   0   0   0   0   0   0]\n",
      " [  1  17  21   3   1 256   2   1   0   2   0   2   2   1   2   3   0   0   0   0]\n",
      " [  1   5   3  24  10   1 201   7  11   7   7   0  10   2   0   1   0   1   1   1]\n",
      " [  0   1   4   1   3   0   4 230  18   3   5   0  12   1   0   1   0   0   5   0]\n",
      " [  1   0   0   2   0   0   3   3 250   2   1   1   2   5   0   0   0   0   5   1]\n",
      " [  0   0   2   0   0   1   3   1   3 250  42   0   0   1   1   1   0   4   7   2]\n",
      " [  1   0   2   0   0   0   4   1   2   6 294   1   0   0   1   1   0   0   2   0]\n",
      " [  4   0   1   1   3   4   0   0   0   1   0 281   2   1   2   0   4   1   2   2]\n",
      " [  2   5   3   9  29   1   6   3   3   0   0   4 227   1   5   0   0   0   1   0]\n",
      " [  2   3   1   0   1   1   4   1   0   1   0   0   1 261   2   0   0   0   2  10]\n",
      " [  0  11   1   0   0   0   0   0   0   0   0   3   0   6 266   1   0   2   7   1]\n",
      " [ 14   6   0   0   1   3   0   0   0   1   3   0   1   0   0 242   1   8   2   8]\n",
      " [  1   0   2   0   0   0   0   6   0   0   0   2   2   0   0   2 203   7  37   6]\n",
      " [  2   0   1   0   0   0   0   0   0   2   2   0   0   0   0   3   1 249  13   0]\n",
      " [  0   0   0   0   0   0   0   0   5   0   0   2   1   4   6   2   5   5 190   5]\n",
      " [ 15   0   0   0   0   1   0   0   1   1   0   3   1   6   2  24   2   6   5 122]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(linewidth=120)\n",
    "Ytest_ = model.predict(Xtest)\n",
    "ytest_ = np.argmax(Ytest_, axis=1)\n",
    "ytest = np.argmax(Ytest, axis=1)\n",
    "print(\"accuracy score: {:.3f}\".format(accuracy_score(ytest, ytest_)))\n",
    "print(\"\\nconfusion matrix\\n\")\n",
    "print(confusion_matrix(ytest, ytest_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}