from_preprocessing_to_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import fashion_mnist\n",
    "from keras.utils import to_categorical\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# dataset\n",
    "(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\n",
    "x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\n",
    "\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "#Learning data\n",
    "x_train_s, x_test_s, x_test_b = [], [], []\n",
    "x_ref, y_ref = [], []\n",
    "\n",
    "x_train_shape = x_train.shape\n",
    "\n",
    "for i in range(len(x_train)):\n",
    "    if y_train[i] == 7: #Sneakers is 7\n",
    "        temp = x_train[i]\n",
    "        x_train_s.append(temp.reshape((x_train_shape[1:])))\n",
    "    else:\n",
    "        temp = x_train[i]\n",
    "        x_ref.append(temp.reshape((x_train_shape[1:])))\n",
    "        y_ref.append(y_train[i])\n",
    "\n",
    "x_ref = np.array(x_ref)\n",
    "\n",
    "#6000 randomly extracted from ref data\n",
    "number = np.random.choice(np.arange(0,x_ref.shape[0]),6000,replace=False)\n",
    "\n",
    "x, y = [], []\n",
    "\n",
    "x_ref_shape = x_ref.shape\n",
    "\n",
    "for i in number:\n",
    "    temp = x_ref[i]\n",
    "    x.append(temp.reshape((x_ref_shape[1:])))\n",
    "    y.append(y_ref[i])\n",
    "\n",
    "x_train_s = np.array(x_train_s)\n",
    "x_ref = np.array(x)\n",
    "y_ref = to_categorical(y)\n",
    "\n",
    "#test data\n",
    "for i in range(len(x_test)):\n",
    "    if y_test[i] == 7: #Sneakers is 7\n",
    "        temp = x_test[i,:,:,:]\n",
    "        x_test_s.append(temp.reshape((x_train_shape[1:])))\n",
    "\n",
    "    if y_test[i] == 9: #Boots is 9\n",
    "        temp = x_test[i,:,:,:]\n",
    "        x_test_b.append(temp.reshape((x_train_shape[1:])))\n",
    "\n",
    "x_test_s = np.array(x_test_s)\n",
    "x_test_b = np.array(x_test_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "from PIL import Image\n",
    "\n",
    "def resize(x):\n",
    "    x_out = []\n",
    "\n",
    "    for i in range(len(x)):\n",
    "        img = cv2.cvtColor(x[i], cv2.COLOR_GRAY2RGB)\n",
    "        img = cv2.resize(img,dsize=(96,96))\n",
    "        x_out.append(img)\n",
    "\n",
    "    return np.array(x_out)\n",
    "\n",
    "X_train_s = resize(x_train_s)\n",
    "X_ref = resize(x_ref)\n",
    "X_test_s = resize(x_test_s)\n",
    "X_test_b = resize(x_test_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model build...\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 96, 96, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_pad (ZeroPadding2D)       (None, 97, 97, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Conv1 (Conv2D)                  (None, 48, 48, 16)   432         Conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bn_Conv1 (BatchNormalization)   (None, 48, 48, 16)   64          Conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_relu (ReLU)               (None, 48, 48, 16)   0           bn_Conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise (Depthw (None, 48, 48, 16)   144         Conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_BN (Bat (None, 48, 48, 16)   64          expanded_conv_depthwise[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_relu (R (None, 48, 48, 16)   0           expanded_conv_depthwise_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project (Conv2D)  (None, 48, 48, 8)    128         expanded_conv_depthwise_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project_BN (Batch (None, 48, 48, 8)    32          expanded_conv_project[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand (Conv2D)         (None, 48, 48, 48)   384         expanded_conv_project_BN[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_BN (BatchNormali (None, 48, 48, 48)   192         block_1_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_relu (ReLU)      (None, 48, 48, 48)   0           block_1_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_pad (ZeroPadding2D)     (None, 49, 49, 48)   0           block_1_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise (DepthwiseCon (None, 24, 24, 48)   432         block_1_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_BN (BatchNorm (None, 24, 24, 48)   192         block_1_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_relu (ReLU)   (None, 24, 24, 48)   0           block_1_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project (Conv2D)        (None, 24, 24, 16)   768         block_1_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project_BN (BatchNormal (None, 24, 24, 16)   64          block_1_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand (Conv2D)         (None, 24, 24, 96)   1536        block_1_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_BN (BatchNormali (None, 24, 24, 96)   384         block_2_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_relu (ReLU)      (None, 24, 24, 96)   0           block_2_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise (DepthwiseCon (None, 24, 24, 96)   864         block_2_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_BN (BatchNorm (None, 24, 24, 96)   384         block_2_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_relu (ReLU)   (None, 24, 24, 96)   0           block_2_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project (Conv2D)        (None, 24, 24, 16)   1536        block_2_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project_BN (BatchNormal (None, 24, 24, 16)   64          block_2_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_add (Add)               (None, 24, 24, 16)   0           block_1_project_BN[0][0]         \n",
      "                                                                 block_2_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand (Conv2D)         (None, 24, 24, 96)   1536        block_2_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_BN (BatchNormali (None, 24, 24, 96)   384         block_3_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_relu (ReLU)      (None, 24, 24, 96)   0           block_3_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_pad (ZeroPadding2D)     (None, 25, 25, 96)   0           block_3_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_3_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_3_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_3_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project (Conv2D)        (None, 12, 12, 16)   1536        block_3_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project_BN (BatchNormal (None, 12, 12, 16)   64          block_3_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand (Conv2D)         (None, 12, 12, 96)   1536        block_3_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_4_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_4_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_4_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_4_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_4_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project (Conv2D)        (None, 12, 12, 16)   1536        block_4_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project_BN (BatchNormal (None, 12, 12, 16)   64          block_4_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_add (Add)               (None, 12, 12, 16)   0           block_3_project_BN[0][0]         \n",
      "                                                                 block_4_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand (Conv2D)         (None, 12, 12, 96)   1536        block_4_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_5_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_5_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_5_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_5_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_5_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project (Conv2D)        (None, 12, 12, 16)   1536        block_5_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project_BN (BatchNormal (None, 12, 12, 16)   64          block_5_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_5_add (Add)               (None, 12, 12, 16)   0           block_4_add[0][0]                \n",
      "                                                                 block_5_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand (Conv2D)         (None, 12, 12, 96)   1536        block_5_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_6_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_6_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_pad (ZeroPadding2D)     (None, 13, 13, 96)   0           block_6_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise (DepthwiseCon (None, 6, 6, 96)     864         block_6_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_BN (BatchNorm (None, 6, 6, 96)     384         block_6_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_relu (ReLU)   (None, 6, 6, 96)     0           block_6_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project (Conv2D)        (None, 6, 6, 32)     3072        block_6_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project_BN (BatchNormal (None, 6, 6, 32)     128         block_6_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand (Conv2D)         (None, 6, 6, 192)    6144        block_6_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_7_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_7_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_7_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_7_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_7_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project (Conv2D)        (None, 6, 6, 32)     6144        block_7_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project_BN (BatchNormal (None, 6, 6, 32)     128         block_7_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_add (Add)               (None, 6, 6, 32)     0           block_6_project_BN[0][0]         \n",
      "                                                                 block_7_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand (Conv2D)         (None, 6, 6, 192)    6144        block_7_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_8_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_8_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_8_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_8_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_8_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project (Conv2D)        (None, 6, 6, 32)     6144        block_8_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project_BN (BatchNormal (None, 6, 6, 32)     128         block_8_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_8_add (Add)               (None, 6, 6, 32)     0           block_7_add[0][0]                \n",
      "                                                                 block_8_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand (Conv2D)         (None, 6, 6, 192)    6144        block_8_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_9_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_9_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_9_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_9_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_9_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project (Conv2D)        (None, 6, 6, 32)     6144        block_9_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project_BN (BatchNormal (None, 6, 6, 32)     128         block_9_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_9_add (Add)               (None, 6, 6, 32)     0           block_8_add[0][0]                \n",
      "                                                                 block_9_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand (Conv2D)        (None, 6, 6, 192)    6144        block_9_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_BN (BatchNormal (None, 6, 6, 192)    768         block_10_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_relu (ReLU)     (None, 6, 6, 192)    0           block_10_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise (DepthwiseCo (None, 6, 6, 192)    1728        block_10_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_BN (BatchNor (None, 6, 6, 192)    768         block_10_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_relu (ReLU)  (None, 6, 6, 192)    0           block_10_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project (Conv2D)       (None, 6, 6, 48)     9216        block_10_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project_BN (BatchNorma (None, 6, 6, 48)     192         block_10_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand (Conv2D)        (None, 6, 6, 288)    13824       block_10_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_11_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_11_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise (DepthwiseCo (None, 6, 6, 288)    2592        block_11_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_BN (BatchNor (None, 6, 6, 288)    1152        block_11_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_relu (ReLU)  (None, 6, 6, 288)    0           block_11_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project (Conv2D)       (None, 6, 6, 48)     13824       block_11_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project_BN (BatchNorma (None, 6, 6, 48)     192         block_11_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_add (Add)              (None, 6, 6, 48)     0           block_10_project_BN[0][0]        \n",
      "                                                                 block_11_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand (Conv2D)        (None, 6, 6, 288)    13824       block_11_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_12_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_12_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise (DepthwiseCo (None, 6, 6, 288)    2592        block_12_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_BN (BatchNor (None, 6, 6, 288)    1152        block_12_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_relu (ReLU)  (None, 6, 6, 288)    0           block_12_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project (Conv2D)       (None, 6, 6, 48)     13824       block_12_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project_BN (BatchNorma (None, 6, 6, 48)     192         block_12_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_12_add (Add)              (None, 6, 6, 48)     0           block_11_add[0][0]               \n",
      "                                                                 block_12_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand (Conv2D)        (None, 6, 6, 288)    13824       block_12_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_13_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_13_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_pad (ZeroPadding2D)    (None, 7, 7, 288)    0           block_13_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise (DepthwiseCo (None, 3, 3, 288)    2592        block_13_pad[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_BN (BatchNor (None, 3, 3, 288)    1152        block_13_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_relu (ReLU)  (None, 3, 3, 288)    0           block_13_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project (Conv2D)       (None, 3, 3, 80)     23040       block_13_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project_BN (BatchNorma (None, 3, 3, 80)     320         block_13_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand (Conv2D)        (None, 3, 3, 480)    38400       block_13_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_14_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_14_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_14_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_14_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_14_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project (Conv2D)       (None, 3, 3, 80)     38400       block_14_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project_BN (BatchNorma (None, 3, 3, 80)     320         block_14_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_add (Add)              (None, 3, 3, 80)     0           block_13_project_BN[0][0]        \n",
      "                                                                 block_14_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand (Conv2D)        (None, 3, 3, 480)    38400       block_14_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_15_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_15_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_15_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_15_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_15_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project (Conv2D)       (None, 3, 3, 80)     38400       block_15_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project_BN (BatchNorma (None, 3, 3, 80)     320         block_15_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_15_add (Add)              (None, 3, 3, 80)     0           block_14_add[0][0]               \n",
      "                                                                 block_15_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand (Conv2D)        (None, 3, 3, 480)    38400       block_15_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_16_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_16_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_16_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_16_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_16_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project (Conv2D)       (None, 3, 3, 160)    76800       block_16_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project_BN (BatchNorma (None, 3, 3, 160)    640         block_16_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1 (Conv2D)                 (None, 3, 3, 1280)   204800      block_16_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1_bn (BatchNormalization)  (None, 3, 3, 1280)   5120        Conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "out_relu (ReLU)                 (None, 3, 3, 1280)   0           Conv_1_bn[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 706,224\n",
      "Trainable params: 536,288\n",
      "Non-trainable params: 169,936\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 96, 96, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_pad (ZeroPadding2D)       (None, 97, 97, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "Conv1 (Conv2D)                  (None, 48, 48, 16)   432         Conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bn_Conv1 (BatchNormalization)   (None, 48, 48, 16)   64          Conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Conv1_relu (ReLU)               (None, 48, 48, 16)   0           bn_Conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise (Depthw (None, 48, 48, 16)   144         Conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_BN (Bat (None, 48, 48, 16)   64          expanded_conv_depthwise[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_depthwise_relu (R (None, 48, 48, 16)   0           expanded_conv_depthwise_BN[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project (Conv2D)  (None, 48, 48, 8)    128         expanded_conv_depthwise_relu[0][0\n",
      "__________________________________________________________________________________________________\n",
      "expanded_conv_project_BN (Batch (None, 48, 48, 8)    32          expanded_conv_project[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand (Conv2D)         (None, 48, 48, 48)   384         expanded_conv_project_BN[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_BN (BatchNormali (None, 48, 48, 48)   192         block_1_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_1_expand_relu (ReLU)      (None, 48, 48, 48)   0           block_1_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_pad (ZeroPadding2D)     (None, 49, 49, 48)   0           block_1_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise (DepthwiseCon (None, 24, 24, 48)   432         block_1_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_BN (BatchNorm (None, 24, 24, 48)   192         block_1_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_1_depthwise_relu (ReLU)   (None, 24, 24, 48)   0           block_1_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project (Conv2D)        (None, 24, 24, 16)   768         block_1_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_1_project_BN (BatchNormal (None, 24, 24, 16)   64          block_1_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand (Conv2D)         (None, 24, 24, 96)   1536        block_1_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_BN (BatchNormali (None, 24, 24, 96)   384         block_2_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_2_expand_relu (ReLU)      (None, 24, 24, 96)   0           block_2_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise (DepthwiseCon (None, 24, 24, 96)   864         block_2_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_BN (BatchNorm (None, 24, 24, 96)   384         block_2_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_2_depthwise_relu (ReLU)   (None, 24, 24, 96)   0           block_2_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project (Conv2D)        (None, 24, 24, 16)   1536        block_2_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_2_project_BN (BatchNormal (None, 24, 24, 16)   64          block_2_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_2_add (Add)               (None, 24, 24, 16)   0           block_1_project_BN[0][0]         \n",
      "                                                                 block_2_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand (Conv2D)         (None, 24, 24, 96)   1536        block_2_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_BN (BatchNormali (None, 24, 24, 96)   384         block_3_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_3_expand_relu (ReLU)      (None, 24, 24, 96)   0           block_3_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_pad (ZeroPadding2D)     (None, 25, 25, 96)   0           block_3_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_3_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_3_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_3_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_3_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project (Conv2D)        (None, 12, 12, 16)   1536        block_3_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_3_project_BN (BatchNormal (None, 12, 12, 16)   64          block_3_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand (Conv2D)         (None, 12, 12, 96)   1536        block_3_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_4_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_4_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_4_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_4_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_4_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_4_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_4_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project (Conv2D)        (None, 12, 12, 16)   1536        block_4_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_4_project_BN (BatchNormal (None, 12, 12, 16)   64          block_4_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_4_add (Add)               (None, 12, 12, 16)   0           block_3_project_BN[0][0]         \n",
      "                                                                 block_4_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand (Conv2D)         (None, 12, 12, 96)   1536        block_4_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_5_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_5_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_5_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise (DepthwiseCon (None, 12, 12, 96)   864         block_5_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_BN (BatchNorm (None, 12, 12, 96)   384         block_5_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_5_depthwise_relu (ReLU)   (None, 12, 12, 96)   0           block_5_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project (Conv2D)        (None, 12, 12, 16)   1536        block_5_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_5_project_BN (BatchNormal (None, 12, 12, 16)   64          block_5_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_5_add (Add)               (None, 12, 12, 16)   0           block_4_add[0][0]                \n",
      "                                                                 block_5_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand (Conv2D)         (None, 12, 12, 96)   1536        block_5_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_BN (BatchNormali (None, 12, 12, 96)   384         block_6_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_6_expand_relu (ReLU)      (None, 12, 12, 96)   0           block_6_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_pad (ZeroPadding2D)     (None, 13, 13, 96)   0           block_6_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise (DepthwiseCon (None, 6, 6, 96)     864         block_6_pad[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_BN (BatchNorm (None, 6, 6, 96)     384         block_6_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_6_depthwise_relu (ReLU)   (None, 6, 6, 96)     0           block_6_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project (Conv2D)        (None, 6, 6, 32)     3072        block_6_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_6_project_BN (BatchNormal (None, 6, 6, 32)     128         block_6_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand (Conv2D)         (None, 6, 6, 192)    6144        block_6_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_7_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_7_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_7_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_7_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_7_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_7_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_7_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project (Conv2D)        (None, 6, 6, 32)     6144        block_7_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_7_project_BN (BatchNormal (None, 6, 6, 32)     128         block_7_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_7_add (Add)               (None, 6, 6, 32)     0           block_6_project_BN[0][0]         \n",
      "                                                                 block_7_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand (Conv2D)         (None, 6, 6, 192)    6144        block_7_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_8_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_8_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_8_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_8_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_8_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_8_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_8_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project (Conv2D)        (None, 6, 6, 32)     6144        block_8_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_8_project_BN (BatchNormal (None, 6, 6, 32)     128         block_8_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_8_add (Add)               (None, 6, 6, 32)     0           block_7_add[0][0]                \n",
      "                                                                 block_8_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand (Conv2D)         (None, 6, 6, 192)    6144        block_8_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_BN (BatchNormali (None, 6, 6, 192)    768         block_9_expand[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "block_9_expand_relu (ReLU)      (None, 6, 6, 192)    0           block_9_expand_BN[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise (DepthwiseCon (None, 6, 6, 192)    1728        block_9_expand_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_BN (BatchNorm (None, 6, 6, 192)    768         block_9_depthwise[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "block_9_depthwise_relu (ReLU)   (None, 6, 6, 192)    0           block_9_depthwise_BN[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project (Conv2D)        (None, 6, 6, 32)     6144        block_9_depthwise_relu[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "block_9_project_BN (BatchNormal (None, 6, 6, 32)     128         block_9_project[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_9_add (Add)               (None, 6, 6, 32)     0           block_8_add[0][0]                \n",
      "                                                                 block_9_project_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand (Conv2D)        (None, 6, 6, 192)    6144        block_9_add[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_BN (BatchNormal (None, 6, 6, 192)    768         block_10_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_10_expand_relu (ReLU)     (None, 6, 6, 192)    0           block_10_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise (DepthwiseCo (None, 6, 6, 192)    1728        block_10_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_BN (BatchNor (None, 6, 6, 192)    768         block_10_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_10_depthwise_relu (ReLU)  (None, 6, 6, 192)    0           block_10_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project (Conv2D)       (None, 6, 6, 48)     9216        block_10_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_10_project_BN (BatchNorma (None, 6, 6, 48)     192         block_10_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand (Conv2D)        (None, 6, 6, 288)    13824       block_10_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_11_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_11_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_11_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise (DepthwiseCo (None, 6, 6, 288)    2592        block_11_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_BN (BatchNor (None, 6, 6, 288)    1152        block_11_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_11_depthwise_relu (ReLU)  (None, 6, 6, 288)    0           block_11_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project (Conv2D)       (None, 6, 6, 48)     13824       block_11_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_11_project_BN (BatchNorma (None, 6, 6, 48)     192         block_11_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_11_add (Add)              (None, 6, 6, 48)     0           block_10_project_BN[0][0]        \n",
      "                                                                 block_11_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand (Conv2D)        (None, 6, 6, 288)    13824       block_11_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_12_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_12_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_12_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise (DepthwiseCo (None, 6, 6, 288)    2592        block_12_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_BN (BatchNor (None, 6, 6, 288)    1152        block_12_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_12_depthwise_relu (ReLU)  (None, 6, 6, 288)    0           block_12_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project (Conv2D)       (None, 6, 6, 48)     13824       block_12_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_12_project_BN (BatchNorma (None, 6, 6, 48)     192         block_12_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_12_add (Add)              (None, 6, 6, 48)     0           block_11_add[0][0]               \n",
      "                                                                 block_12_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand (Conv2D)        (None, 6, 6, 288)    13824       block_12_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_BN (BatchNormal (None, 6, 6, 288)    1152        block_13_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_13_expand_relu (ReLU)     (None, 6, 6, 288)    0           block_13_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_pad (ZeroPadding2D)    (None, 7, 7, 288)    0           block_13_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise (DepthwiseCo (None, 3, 3, 288)    2592        block_13_pad[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_BN (BatchNor (None, 3, 3, 288)    1152        block_13_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_13_depthwise_relu (ReLU)  (None, 3, 3, 288)    0           block_13_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project (Conv2D)       (None, 3, 3, 80)     23040       block_13_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_13_project_BN (BatchNorma (None, 3, 3, 80)     320         block_13_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand (Conv2D)        (None, 3, 3, 480)    38400       block_13_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_14_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_14_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_14_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_14_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_14_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_14_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_14_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project (Conv2D)       (None, 3, 3, 80)     38400       block_14_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_14_project_BN (BatchNorma (None, 3, 3, 80)     320         block_14_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_14_add (Add)              (None, 3, 3, 80)     0           block_13_project_BN[0][0]        \n",
      "                                                                 block_14_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand (Conv2D)        (None, 3, 3, 480)    38400       block_14_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_15_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_15_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_15_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_15_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_15_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_15_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_15_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project (Conv2D)       (None, 3, 3, 80)     38400       block_15_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_15_project_BN (BatchNorma (None, 3, 3, 80)     320         block_15_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "block_15_add (Add)              (None, 3, 3, 80)     0           block_14_add[0][0]               \n",
      "                                                                 block_15_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand (Conv2D)        (None, 3, 3, 480)    38400       block_15_add[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_BN (BatchNormal (None, 3, 3, 480)    1920        block_16_expand[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "block_16_expand_relu (ReLU)     (None, 3, 3, 480)    0           block_16_expand_BN[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise (DepthwiseCo (None, 3, 3, 480)    4320        block_16_expand_relu[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_BN (BatchNor (None, 3, 3, 480)    1920        block_16_depthwise[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "block_16_depthwise_relu (ReLU)  (None, 3, 3, 480)    0           block_16_depthwise_BN[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project (Conv2D)       (None, 3, 3, 160)    76800       block_16_depthwise_relu[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "block_16_project_BN (BatchNorma (None, 3, 3, 160)    640         block_16_project[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1 (Conv2D)                 (None, 3, 3, 1280)   204800      block_16_project_BN[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "Conv_1_bn (BatchNormalization)  (None, 3, 3, 1280)   5120        Conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "out_relu (ReLU)                 (None, 3, 3, 1280)   0           Conv_1_bn[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 1280)         0           out_relu[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           12810       global_average_pooling2d_1[0][0] \n",
      "==================================================================================================\n",
      "Total params: 719,034\n",
      "Trainable params: 549,098\n",
      "Non-trainable params: 169,936\n",
      "__________________________________________________________________________________________________\n",
      "x_target is 6000 samples\n",
      "x_ref is 6000 samples\n",
      "training...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5\n",
      "Descriptive loss: 2.2780318\n",
      "Compact loss 0.0025045697\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl4VOX5xvHvkxBIIOwEREACAsq+JQIGgqIQBBVELCqiiKhUlM26tbai9tdqi6wqiytWFK0IIjsoEHYJGEAWZVMBWSIo+5bw/v7ISGMMZJAkJ5O5P9eVy8nMOzPPHJx7Tp45533NOYeIiASHEK8LEBGRvKPQFxEJIgp9EZEgotAXEQkiCn0RkSCi0BcRCSIKfRGRIKLQFxEJIgp9EZEgUsjrAjIrV66ci46O9roMEZGAsmrVqh+dc1HZjct3oR8dHU1SUpLXZYiIBBQz+86fcWrviIgEEYW+iEgQUeiLiASRfNfTF5Hccfr0aXbu3MmJEye8LkUuQnh4OJUrVyYsLOx33V+hLxIkdu7cSfHixYmOjsbMvC5HfgfnHPv372fnzp1Uq1btdz2G2jsiQeLEiROULVtWgR/AzIyyZcte1F9rCn2RIKLAD3wX+29YYELfOcc/Zmxk+49HvS5FRCTfKjChv/3Ho0z84nvaD09k7MKtpKad8bokEckkNDSURo0aUbduXRo2bMhLL73EmTO5915NSkqiX79+5x3z7bff8t57713QffyxYMECbrzxxot+nJxWYEK/elQk8wa1pnWtKP45cxNdRi9l055DXpclIhlERESQnJzM+vXrmTt3LjNnzuTZZ5/NledKTU0lJiaGkSNHnndc5tD35z6BrMCEPkD5EuGM7dGUl+9szK6fjnPTqMUMm/sNp1K11y+S35QvX55x48bx8ssv45wjLS2Nxx57jNjYWBo0aMDYsWMB2L17N/Hx8TRq1Ih69eqxaNEiAGbNmkWTJk1o2LAh1113HQCDBw+mR48exMXF0aNHj1/tbf9yW4sWLahZsyavvfYaAE8++SSLFi2iUaNGDBs27Ox9zpw5Q3R0ND///PPZmmvWrMnevXtJSUnh1ltvJTY2ltjYWJYsWXLe13rgwAE6d+5MgwYNaN68OWvXrgVg4cKFNGrUiEaNGtG4cWMOHz58ztebUwrcIZtmxo0NLuXqy8vx3KfrGfHZZmav38OLtzagYZVSXpcnki88++l6NvyQs38J17m0BM/cVPeC7lO9enXS0tLYt28fn3zyCSVLlmTlypWcPHmSuLg42rVrx8cff0xCQgJ/+ctfSEtL49ixY6SkpHD//feTmJhItWrVOHDgwNnH3LBhA4sXLyYiIoIFCxb86vnWrl3L8uXLOXr0KI0bN6Zjx4688MILDBkyhGnTpgGcvU9ISAidOnVi8uTJ3HvvvaxYsYKqVatSoUIF7rzzTgYOHEjLli35/vvvSUhIYOPGjed8nc888wyNGzdmypQpfP7559x9990kJyczZMgQXnnlFeLi4jhy5Ajh4eGMGzfuN683JxW40P9FmWKFGX57Y25udCl//vgrbnl1Cfe3qs7AtrUIDwv1ujwRyWTOnDmsXbuWjz76CICDBw+yefNmYmNj6dWrF6dPn6Zz5840atSIBQsWEB8ff/ZY9TJlypx9nJtvvpmIiIgsn6NTp05EREQQERHBtddeyxdffEGpUufeGezWrRvPPfcc9957LxMnTqRbt24AzJs3jw0bNpwdd+jQIY4cOUJkZGSWj7N48WImTZoEQJs2bdi/fz+HDh0iLi6OQYMG0b17d7p06ULlypWzfL05qcCG/i/aXFmBOYPK8M8ZmxibuO3sXn+z6mW9Lk3EMxe6R55btm3bRmhoKOXLl8c5x6hRo0hISPjNuMTERKZPn07Pnj0ZNGgQpUuXPudjFitW7Jy3ZT7cMbvDH1u0aMGWLVtISUlhypQpPP300wCcOXOG5cuXEx4eft77Z+fJJ5+kY8eOzJgxg7i4OGbPnk18fPxvXu/dd999Uc+TUYHq6Z9LifAw/tmlPu/1bkaac3Qbt5y/TvmKIydTvS5NJGilpKTQp08fHn74YcyMhIQERo8ezenTpwH45ptvOHr0KN999x0VKlTg/vvvp3fv3qxevZrmzZuTmJjI9u3bAX7V3jmfTz75hBMnTrB//34WLFhAbGwsxYsX5/Dhw1mONzNuueUWBg0aRO3atSlbNn1nsV27dowaNersuOTk5PM+b6tWrZgwYQKQ3j4qV64cJUqUYOvWrdSvX58nnniC2NhYNm3alOXrzUkFfk8/o6trlGP2gHiGzP6Gt5Zu5/NN+/hHl/q0rpXtugMikgOOHz9Oo0aNOH36NIUKFaJHjx4MGjQIgN69e/Ptt9/SpEkTnHNERUUxZcoUFixYwL///W/CwsKIjIzknXfeISoqinHjxtGlSxfOnDlD+fLlmTt3brbP36BBA6699lp+/PFH/vrXv3LppZcSFRVFaGgoDRs2pGfPnjRu3PhX9+nWrRuxsbG8/fbbZ68bOXIkffv2pUGDBqSmphIfH8+YMWPO+byDBw+mV69eNGjQgKJFizJ+/HgAhg8fzvz58wkJCaFu3brccMMNTJw48TevNyeZcy5HH/BixcTEuLxYRGXVdz/xxKS1bNl3hFubVOavN9amVNHCuf68Il7ZuHEjtWvX9roMzwwePJjIyEj+9Kc/eV3KRcvq39LMVjnnYrK7b1C0d7LStGpppvdrycPX1mBK8i6uH5rIrK92e12WiEiuCqr2TmZFCoXyp4QruKH+JTz+0Vr6vLuaDvUv4dmb6xFVvIjX5YlIDho8eLDXJeQLQbunn1HdS0sypW8cjyVcwbyN+2g7bCGTv9xJfmt9iVws/T8d+C7231Ch7xMWGkLfa2swo18rLo+KZOAHa+j19kp++Pm416WJ5Ijw8HD279+v4A9gv8ynfzGHigbtF7nnk3bG8c6yb/nXrK8JDTGe6nAld8ReRkiIpqWVwKWVswqGc62c5e8XuQr98/h+/zGemryWJVv207x6GV68tQFVy577xA8REa/o6J0ccFnZorx7XzNe6FKf9bsOkTA8kdcXbSPtTP76oBQR8ZdCPxtmxu1XXcbcQa1pWaMcf5++kVtHL+WbvVmfwScikp8p9P10SclwXrs7hhG3N+L7A8e4ceRiRn62mdNarEVEAohC/wKYGZ0aVWLuwHgS6l3C0LnfcNOoxazbedDr0kRE/KLQ/x3KRhZh1B2Nee3uGA4cPUXnV5fw4qxNnDid5nVpIiLnpdC/CG3rVGDuoNZ0bVKZ0Qu20mHkIpK+9W+2PxERLyj0L1LJiDBe7NqAd+9rxqnUM9w2dhmDp67nqKZtFpF8SKGfQ1rWTJ+2+Z4W0Yxf9i0JwxNZtDnF67JERH4l29A3sypmNt/MNpjZejPrn8WYa8zsoJkl+37+luG29mb2tZltMbMnc/oF5CfFihRi8M11+e+DLShcKIQeb3zB4x+t4eDx016XJiIC+Lennwo86pyrAzQH+ppZnSzGLXLONfL9PAdgZqHAK8ANQB3gjnPct0CJiS7DjH6t+OM1lzNp9S7aDl3InPV7vC5LRCT70HfO7XbOrfZdPgxsBCr5+fhXAVucc9ucc6eAiUCn31tsIAkPC+WJ9lfySd84ykYW4YH/rOKR979k/5GTXpcmIkHsgnr6ZhYNNAZWZHFzCzNbY2YzzeyXVZcrATsyjNlJFh8YZvaAmSWZWVJKSsHqg9erVJKpD8fxaNtazP5qD22HJfJJ8i7NdCginvA79M0sEpgEDHDOHcp082qgqnOuITAKmHIhRTjnxjnnYpxzMVFRBW+92rDQEB65ribT+7XksjJF6T8xmd7jk9hzULMdikje8iv0zSyM9MCf4Jz7OPPtzrlDzrkjvsszgDAzKwfsAqpkGFrZd11QqlmhOJP+eDVPd6zNkq0/0nboQiZ+8b32+kUkz/hz9I4BbwAbnXNDzzHmEt84zOwq3+PuB1YCNc2smpkVBm4HpuZU8YEoNMTo3ao6swfEU69SSZ78eB13vbGCHQeOeV2aiAQBf/b044AeQJsMh2R2MLM+ZtbHN6Yr8JWZrQFGAre7dKnAw8Bs0r8A/tA5tz4XXkfAqVq2GBN6N+Mft9RnzY6DtBuWyJuLt2vaZhHJVVpEJR/44efj/GXyOuZ/nUKTy0rxr64NqFG+uNdliUgA0SIqAeTSUhG82TOWYd0asu3Ho3QYsZhX5m/RtM0ikuMU+vmEmXFL48rMHdiatnUq8O/ZX9P5lSWs/0HTNotIzlHo5zNRxYvwSvcmjLmrKfsOn6TTy0sYMvtrTqZq2mYRuXgK/Xyqfb1LmDewNZ0bV+Ll+VvoOHIxq777yeuyRCTAKfTzsZJFwxhyW0PG97qK46fS6DpmKc99uoFjpzRts4j8Pgr9ANC6VhSzB8bTo3lV3lyynfbDF7F0y49elyUiAUihHyAiixTiuU71+OCB5oSGGHe+voKnPl7LoROatllE/KfQDzDNqpdlZv9WPBhfnQ9W7qDd0EQ+27jX67JEJEAo9ANQeFgoT3WozeSH4ihVNIz7xifRf+KXHDh6yuvSRCSfU+gHsIZVSjH14ZYMuL4mM9btpu3QhUxb+4MmcBORc1LoB7jChUIYcH0tpj3SisqlI3j4vS958D+r2HdI0zaLyG8p9AuIKy5Jn7b5zx2uZOE3KVw/dCEfJu3QXr+I/IpCvwApFBrCA/GXM2tAPFdeUoLHP1rL3W9+oWmbReQshX4BVK1cMSY+0JznO9dj9Xc/kTA8kfFLv+WMpm0WCXoK/QIqJMTo0bwqcwa1Jja6DM9MXU+3ccvYlnLE69JExEMK/QKuUqkI3r43liG3NeSbvUdoP2IRYxZuJVXTNosEJYV+EDAzujatzNxB8bS5ojwvzNzELa8uZePuzOvbi0hBp9APIuWLhzOmR1Ne7d6E3QePc9OoxYyYt1l7/SJBRKEfhDrUr8jcga25sUFFhs37hlvHLGOrev0iQUGhH6RKFyvM8Nsb82r3Jny3/ygdRy7SET4iQUChH+Q61K/InAHxNK9elmemrueet75gz0GdzStSUCn0hfIlwnmrZyz/d0s9kr79iXbDFvJJ8i6vyxKRXKDQFyD9CJ/uzaoys38rapSPpP/EZB5+bzU/H9PMnSIFiUJffiW6XDE+fLAFjyVcwez1e2g3LJEFX+/zuiwRySEKffmNQqEh9L22BlP6ps/X3/OtlTw9ZZ3W5hUpABT6ck51Ly3J1Idbcn+rakxY8T0dRy5m9fc/eV2WiFwEhb6cV3hYKH/pWIf372/OqdQzdB29lJfmfM2pVJ3QJRKIFPril+bVyzJrQCu6NKnMqM+30GX0EjbvPex1WSJygRT64rfi4WEMua0hY3s05YefT9Bx1GJeX7RNJ3SJBJBsQ9/MqpjZfDPbYGbrzaz/ecbGmlmqmXXNcF2amSX7fqbmVOHinYS6lzB7QDzxNcvx9+kb6f76Cnb9fNzrskTED/7s6acCjzrn6gDNgb5mVifzIDMLBV4E5mS66bhzrpHv5+aLrljyhajiRXjt7hj+dWsD1u78mfbDEpm0aqeWZxTJ57INfefcbufcat/lw8BGoFIWQx8BJgE6qDtImBl/iK3CrAHx1K5Ygkf/u4Y/vrua/UdOel2aiJzDBfX0zSwaaAysyHR9JeAWYHQWdws3syQzW25mnX9nnZKPVSlTlPcfaM6fO1zJ55v2kTB8EZ9t3Ot1WSKSBb9D38wiSd+TH+Ccy7z6xnDgCedcVsfxVXXOxQB3AsPN7PIsHvsB3wdDUkpKygWUL/lFaIjxQPzlTH0kjqjiRbhvfBJPTlrLkZM6oUskPzF/erBmFgZMA2Y754Zmcft2wHy/lgOOAQ8456ZkGvc2MM0599G5nismJsYlJSX5/QIk/zmZmsbweZsZu3ArlUpHMPQPjYiNLuN1WSIFmpmt8u1gn5c/R+8Y8AawMavAB3DOVXPORTvnooGPgIecc1PMrLSZFfE9TjkgDthwAa9DAlCRQqE80f5KPnywBYbxh7HL+OfMjZxMTfO6NJGg5097Jw7oAbTJcOhlBzPrY2Z9srlvbSDJzNYA84EXnHMK/SARE12Gmf1bcXtsFcYu3Eanl5doXV4Rj/nV3slLau8UTJ9v2svjH63j0PHTDGpXi/tbVSc0xLK/o4j4JcfaOyI5oc2VFZgzMJ7rapfnhZmbuH3cMnYcOOZ1WSJBR6EveaZMscK82r0JQ//QkE27D9N+eCIfrPxeJ3SJ5CGFvuQpM6NLk8rMGhhPg8qleGLSOu5/J4mUwzqhSyQvKPTFE5VKRTChdzP+emMdEjf/SMLwRGZ9tcfrskQKPIW+eCYkxLivZTWmP9KSSqUi6PPuKh79cA2HTpz2ujSRAkuhL56rWaE4Hz90Nf3a1GBK8i5uGL6IZVv3e12WSIGk0Jd8ISw0hEHtruCjPi0oXCiEO15bzvPTNnDitE7oEslJCn3JVxpfVprp/VrSo3lV3li8nZtGLearXQe9LkukwFDoS75TtHAhnu9cj/G9ruLQidN0fmUJL3++mdQ0rcsrcrEU+pJvta4VxewB8dxQvyJD5nzDbWOXsf3Ho16XJRLQFPqSr5UqWphRdzRm5B2N2brvCB1GLOI/y7/TCV0iv5NCXwLCzQ0vZc7A1sREl+avU76i51sr2XvohNdliQQchb4EjEtKhvNOr6t4vlNdVmzfT8LwRKat/cHrskQCikJfAoqZ0aNFNDP6taJq2WI8/N6X9J/4JQeP6YQuEX8o9CUgVY+KZFKfFgxqW4vpa3eTMDyRRZu11KZIdhT6ErAKhYbQ77qafPzQ1RQrEkqPN77gmU++4vgpndAlci4KfQl4DSqXYnq/VvSKq8b4Zd/RcdQi1uz42euyRPIlhb4UCOFhofztpjq817sZJ06l0WX0UobN/YbTOqFL5FcU+lKgXF2jHDMHxNOp4aWM+Gwzt45eypZ9R7wuSyTfUOhLgVMyIoyh3RoxunsTdhw4RseRi3hryXbOnNEJXSIKfSmwbqhfkdkD4rn68rI8++kGery5gh9+Pu51WSKeUuhLgVa+RDhv9ozlH7fU58vvfyZheCJTvtylaRwkaCn0pcAzM+5sdhkz+7eiVoXiDPggmYff+5Kfjp7yujSRPKfQl6BRtWwxPnywBY+3v4I5G/bQbngi87/e53VZInlKoS9BJTTEeOiaGkzpG0eZooW5962V/HnyOo6eTPW6NJE8odCXoFT30pJ88nAcD8RX5/0vvqfDyEWs+u4nr8sSyXUKfQla4WGh/LlDbSbe35zUNMdtY5by79mbOJWqE7qk4FLoS9BrVr0sswa0omvTyrwyfyudX1nC13sOe12WSK5Q6IsAxcPD+FfXhozr0ZS9h05w08uLeS1xm07okgJHoS+SQbu6lzB7YDyta0XxfzM2csdry9lx4JjXZYnkmGxD38yqmNl8M9tgZuvNrP95xsaaWaqZdc1w3T1mttn3c09OFS6SW8pFFmFcj6b8u2sD1v9wiBtGLOK/STt0QpcUCP7s6acCjzrn6gDNgb5mVifzIDMLBV4E5mS4rgzwDNAMuAp4xsxK50ThIrnJzLgtpgoz+7eizqUleOyjtTz4n1X8eOSk16WJXJRsQ985t9s5t9p3+TCwEaiUxdBHgElAxrNdEoC5zrkDzrmfgLlA+4uuWiSPVClTlIn3N+cvHWqz4OsU2gxZwDvLviVNvX4JUBfU0zezaKAxsCLT9ZWAW4DRme5SCdiR4fedZP2BIZJvhYQY98dXZ0b/VtSvXJK/fbKem19erOP6JSD5HfpmFkn6nvwA59yhTDcPB55wzv2uA5zN7AEzSzKzpJQUrXMq+VON8pG8e18zXrmzCfuPnOLW0Ut57L9r1PKRgGL+fDllZmHANGC2c25oFrdvB8z3azngGPAAEAFc45x70DduLLDAOff+uZ4rJibGJSUlXejrEMlTR0+mMurzLby+aBtFC4fyp4Qr6N6sKqEhlv2dRXKBma1yzsVkOy670DczA8YDB5xzA/x44reBac65j3xf5K4CmvhuXg00dc4dONf9FfoSSLbsO8LgqetZvOVH6lQswfOd69K0ahmvy5Ig5G/o+9PeiQN6AG3MLNn308HM+phZn/Pd0RfuzwMrfT/PnS/wRQJNjfKR/Oe+q3jlzib8dOwUt45exp/U8pF8zK/2Tl7Snr4EqqMnU3l5fnrLJzwslEfb1uKu5lUpFKpzICX35eSevoj4oViRQjzR/kpmDYinUZVSDP50Aze9vISkb/XHreQfCn2RHHZ5VCTv9LqKV7s34edjp+g6ZhmPfriGlMNq+Yj3FPoiucDM6FC/Ip892po/XnM5U9fsos1LC3h7yXZS0zR1s3hHoS+Si4oW/m3L58ZRi1mplo94RKEvkgd+afmM7t6EQ8dPc9uYZQz6MFktH8lzCn2RPGJm3FC/IvMebc1D11zOp2t+oM2QBbyllo/kIYW+SB4rWrgQj//S8rmsFM+q5SN5SKEv4pFfWj5j7srQ8vkgmX2HT3hdmhRgCn0RD5kZ7eult3z6Xns5n679geuGLOTNxWr5SO5Q6IvkA0ULF+KxhCuZPSCexlVL89y09JbPF9vV8pGcpdAXyUeqR0Uy/t5YxtzVlMMnUvnD2GUMVMtHcpBCXySfSW/5XMK8Qa15+NoaTF+7m+uGLOQNtXwkByj0RfKpCN88/bMHprd8nve1fFZs2+91aRLAFPoi+Vy1csV+1fLpNm55esvnkFo+cuEU+iIBIKuWT5uX1PKRC6fQFwkgGVs+TX0tn44j1fIR/yn0RQJQtXLFePveWMb2aMqRk+ktnwETv1TLR7Kl0BcJUGZGQt30ls8jbWowY90e2ry0kNcXbeO0Wj5yDgp9kQAXUTiUR9ult3xiokvz9+kb6ThyEcvV8pEsKPRFCohq5YrxVs9YxvVoytGTadw+bjn9J37JXrV8JAOFvkgBYma087V8+rWpwcyv9tBmyAK1fOQshb5IARRROJRB7a5gzoB4rqpW5mzLZ9lWtXyCnUJfpACLLleMN3vG8trdMRw7lcYdry2n3/tq+QQzhb5IAWdmtK1TIb3lc11NZq1Pb/m8lqiWTzBS6IsEifCwUAa1rcXcgektn/+bsZEOI9TyCTYKfZEgU7Xs/1o+x0+nt3weef9L9hxUyycYKPRFglDmls/s9Xu47qUFjEvcqpZPAafQFwliGVs+zaqX5R8zNtFhxCKWbv3R69Iklyj0ReRsy+d1X8vnztdWqOVTQCn0ReSs630tn/5q+RRY2Ya+mVUxs/lmtsHM1ptZ/yzGdDKztWaWbGZJZtYyw21pvuuTzWxqTr8AEclZ4WGhDPS1fJr7Wj43jFjE0i1q+RQE5pw7/wCzikBF59xqMysOrAI6O+c2ZBgTCRx1zjkzawB86Jy70nfbEedcpL8FxcTEuKSkpN/zWkQkF8zbsJdnp61nx4Hj3NigIk93rMMlJcO9LksyMbNVzrmY7MZlu6fvnNvtnFvtu3wY2AhUyjTmiPvfp0cx4PyfJCISMK6vU4G5A1sz4PqazN2wlzYvLWDswq2cSlXLJxBdUE/fzKKBxsCKLG67xcw2AdOBXhluCve1fJabWedzPO4DvjFJKSkpF1KSiOSB8LBQBlxfi7kDW3P15WX558xN3DAikSVq+QScbNs7Zwemt3AWAv/nnPv4POPigb855673/V7JObfLzKoDnwPXOee2nuv+au+I5H+fbdzLs59u4PsDx+jYoCJPd6xNxZIRXpcV1HKsveN7sDBgEjDhfIEP4JxLBKqbWTnf77t8/90GLCD9LwURCWDX1a7AnIHxDLy+FvM27OW6lxYyRi2fgODP0TsGvAFsdM4NPceYGr5xmFkToAiw38xKm1kR3/XlgDhgQ1aPISKBJTwslP7X12TeoNZcfXk5XvC1fBZvVssnP/NnTz8O6AG0yXDoZQcz62NmfXxjbgW+MrNk4BWgm++L3dpAkpmtAeYDL2Q86kdEAl+VMkV5/Z4Y3uwZw+k0x11vrKDvhNXsPnjc69IkC3739POKevoigevE6TTGLtzGqwu2EGLGwLY16RVXjUKhOg80t+VoT19ExB8ZWz5xNdJP7Lp19FK+3nPY69LER6EvIjmuSpmivHZ3DKPuaMyOn45z46hFjJi3WV/05gMKfRHJFWbGTQ0vZe7AeNrXq8iwed9w88uLWbfzoNelBTWFvojkqrKRRRh1R2PG9WjKgaOn6PzqEv41axMnTqd5XVpQUuiLSJ5oV/cS5g5sTZfGlXh1wVY6jlzEqu8OeF1W0FHoi0ieKVk0jH/f1pDxva7ixOkzdB2zjOc+3cCxU6lelxY0FPoikuda14pi9sB47mpWlTeXbKf9cE3dnFcU+iLiicgihXi+cz0mPtCcEIM7X1/BUx+v49CJ016XVqAp9EXEU82rl2Vm/3jub1WND1Z+T8KwROZ/vc/rsgoshb6IeC6icCh/6ViHSX+8msgihbj3rZUM+jCZn4+d8rq0AkehLyL5RuPLSjOtX0seaVODqck/cP3QRGZ9tdvrsgoUhb6I5CtFCoXyaLsr+OThOCqUKEKfd1fTd8JqUg6f9Lq0AkGhLyL5Ut1LSzKlbxyPJVzB3A17aTtsIVO+3EV+myQy0Cj0RSTfCgsNoe+1NZjeryXVyhVjwAfJ9B6fxJ6DJ7wuLWAp9EUk36tZoTgf9bmapzvWZsnWH2k7dCETv/hee/2/g0JfRAJCaIjRu1V1ZvWPp26lEjz58Tp6vPEFOw4c87q0gKLQF5GAEl2uGO/1bs7fO9cjecfPJAxP5O0l2zlzRnv9/lDoi0jACQkx7mpeldkD44mNLsPgTzfQbdwytqUc8bq0fE+hLyIBq1KpCN6+N5YhtzXk6z2HaT9iEWMWbiU1TYu1nItCX0QCmpnRtWll5g1qzbVXRPHCzE10Gb2UTXsOeV1avqTQF5ECoXyJcMbc1ZRX7mzCrp+Oc9OoxQyf942WaMxEoS8iBYaZ0bFBReYOak2H+hUZPm+zlmjMRKEvIgVOmWKFGXF7Y167O+bsEo0vzNQSjaDQF5ECrG2dCswd1JquTSozZuFWOoxcRNJlCcZ/AAAK1UlEQVS3wb1Eo0JfRAq0khFhvNi1Af+57ypOnj7DbWOXMXjq+qBdolGhLyJBoVXNKOYMjOfu5lV5e+m3JAxPZEkQLtGo0BeRoFGsSCGe7VSPDx9sQaGQELq/voKnPl4bVEs0KvRFJOhcVa0MM/u34sH46nywcgfthiby+aa9XpeVJxT6IhKUwsNCeapDbSY/FEeJiEL0ejuJQR8k89PRgr1EY7ahb2ZVzGy+mW0ws/Vm1j+LMZ3MbK2ZJZtZkpm1zHDbPWa22fdzT06/ABGRi9GwSik+faQl/a6rydQ1P9B22EJmriu4SzRadvNRm1lFoKJzbrWZFQdWAZ2dcxsyjIkEjjrnnJk1AD50zl1pZmWAJCAGcL77NnXO/XSu54uJiXFJSUkX/cJERC7Uhh8O8fikNXy16xA31LuE5zrVI6p4Ea/L8ouZrXLOxWQ3Lts9fefcbufcat/lw8BGoFKmMUfc/z49ipEe8AAJwFzn3AFf0M8F2vv/MkRE8k6dS0sw5aE4Hm9/BZ9t2kfbYQuZ/OXOArVYywX19M0sGmgMrMjitlvMbBMwHejlu7oSsCPDsJ1k+sAQEclPCoWG8NA1NZjRrxXVyxVj4Adr6PX2SnYfPO51aTnC79D3tXAmAQOcc7+Zvs45N9k5dyXQGXj+Qoowswd83wUkpaSkXMhdRURyRY3ykfy3z9X87cY6LN92gHZDE3lvReAv0ehX6JtZGOmBP8E59/H5xjrnEoHqZlYO2AVUyXBzZd91me8zzjkX45yLiYqK8rt4EZHcFBpi9GpZjdkD4qlXqSR/nryO7q+v4Pv9gbtEoz9H7xjwBrDROTf0HGNq+MZhZk2AIsB+YDbQzsxKm1lpoJ3vOhGRgHFZ2aK8d38z/nFLfdbuPEjC8ETeCtAlGgv5MSYO6AGsM7Nk33V/Bi4DcM6NAW4F7jaz08BxoJvvi90DZvY8sNJ3v+ecc8E925GIBCQz485ml3HNFVH8ZfI6nv10A9PX7ubFrg24PCrS6/L8lu0hm3lNh2yKSH7nnGPyl7t49tMNHD+dxsDra3F/q2oUCvXufNccO2RTRER+zczo0qQycwfF0+aK8rw4axO3vLqUjbvz/xKNCn0Rkd+pfPFwxvRoyqvdm7D7YPoSjcPm5u8lGhX6IiIXqUP9iswd2JqbGl7KiM82c9Ooxazd+bPXZWVJoS8ikgNKFyvMsG6NeOOeGA4eP03nV5bwz5kb890SjQp9EZEcdF3tCswZFE+32CqMXbiNDiMWsTIfLdGo0BcRyWElwsP4Z5cGTOjdjFNpZ/iDb4nGoye9X6JRoS8ikkviapRj9oB47mkRzfhl6Us0Lt7s7RKNCn0RkVxUrEghBt9clw8fbEHh0BDuemMFT07ybolGhb6ISB6IjS7DjP6teLB1dT5M2kHboQv5bGPeL9Go0BcRySPhYaE8dUP6Eo2lIgpz3/gkBkz8Mk+XaFToi4jksV+WaBxwfU2mrd1N22ELmZFHSzQq9EVEPFC4UAgDrq/Fp4+0pGLJCB6asJq+E1bn+syd/syyKSIiuaR2xRJMfuhqXl+8nSMnUgkJsVx9PoW+iIjHCoWG0Kf15XnyXGrviIgEEYW+iEgQUeiLiAQRhb6ISBBR6IuIBBGFvohIEFHoi4gEEYW+iEgQMedy95TfC2VmKcB3F/EQ5QBvJ6zOmuq6MKrrwqiuC1MQ66rqnIvKblC+C/2LZWZJzrkYr+vITHVdGNV1YVTXhQnmutTeEREJIgp9EZEgUhBDf5zXBZyD6rowquvCqK4LE7R1FbievoiInFtB3NMXEZFzCMjQN7P2Zva1mW0xsyezuL2ImX3gu32FmUXnk7p6mlmKmSX7fnrnUV1vmtk+M/vqHLebmY301b3WzJrkk7quMbODGbbX3/KoripmNt/MNpjZejPrn8WYPN9mftaV59vMzMLN7AszW+Or69ksxuT5e9LPujx5T/qeO9TMvjSzaVnclnvbyzkXUD9AKLAVqA4UBtYAdTKNeQgY47t8O/BBPqmrJ/CyB9ssHmgCfHWO2zsAMwEDmgMr8kld1wDTPNheFYEmvsvFgW+y+LfM823mZ115vs182yDSdzkMWAE0zzTGi/ekP3V58p70Pfcg4L2s/r1yc3sF4p7+VcAW59w259wpYCLQKdOYTsB43+WPgOvMLHfXIPOvLk845xKBA+cZ0gl4x6VbDpQys4r5oC5POOd2O+dW+y4fBjYClTINy/Nt5mddec63DY74fg3z/WT+sjDP35N+1uUJM6sMdAReP8eQXNtegRj6lYAdGX7fyW//xz87xjmXChwEyuaDugBu9bUDPjKzKrlck7/8rd0LLXx/ns80s7p5/eS+P6sbk76XmJGn2+w8dYEH28zXqkgG9gFznXPn3F55+J70py7w5j05HHgcOHOO23NtewVi6AeyT4Fo51wDYC7/+ySXrK0m/dTyhsAoYEpePrmZRQKTgAHOuUN5+dznk01dnmwz51yac64RUBm4yszq5cXzZsePuvL8PWlmNwL7nHOrcvu5shKIob8LyPhpXNl3XZZjzKwQUBLY73Vdzrn9zrmTvl9fB5rmck3+8meb5jnn3KFf/jx3zs0AwsysXF48t5mFkR6sE5xzH2cxxJNtll1dXm4z33P+DMwH2me6yYv3ZLZ1efSejANuNrNvSW8DtzGzdzONybXtFYihvxKoaWbVzKww6V9yTM00Zipwj+9yV+Bz5/tGxMu6MvV8bya9J5sfTAXu9h2R0hw46Jzb7XVRZnbJL31MM7uK9P9fcz0ofM/5BrDROTf0HMPyfJv5U5cX28zMosyslO9yBNAW2JRpWJ6/J/2py4v3pHPuKedcZedcNOk58blz7q5Mw3JtexXKiQfJS865VDN7GJhN+hEzbzrn1pvZc0CSc24q6W+M/5jZFtK/KLw9n9TVz8xuBlJ9dfXM7boAzOx90o/qKGdmO4FnSP9SC+fcGGAG6UejbAGOAffmk7q6An80s1TgOHB7Hnx4Q/qeWA9gna8fDPBn4LIMtXmxzfypy4ttVhEYb2ahpH/IfOicm+b1e9LPujx5T2Ylr7aXzsgVEQkigdjeERGR30mhLyISRBT6IiJBRKEvIhJEFPoiIkFEoS+Sgyx9lsvfzJookl8o9EVEgohCX4KSmd3lm2s92czG+ibmOmJmw3xzr39mZlG+sY3MbLlvUq7JZlbad30NM5vnm9xstZld7nv4SN/kXZvMbEIezPAq4jeFvgQdM6sNdAPifJNxpQHdgWKknxFZF1hI+hnCAO8AT/gm5VqX4foJwCu+yc2uBn6ZhqExMACoQ/r6CnG5/qJE/BRw0zCI5IDrSJ9Ya6VvJzyC9Kl3zwAf+Ma8C3xsZiWBUs65hb7rxwP/NbPiQCXn3GQA59wJAN/jfeGc2+n7PRmIBhbn/ssSyZ5CX4KRAeOdc0/96kqzv2Ya93vnKDmZ4XIaep9JPqL2jgSjz4CuZlYewMzKmFlV0t8PXX1j7gQWO+cOAj+ZWSvf9T2Ahb6Vq3aaWWffYxQxs6J5+ipEfgftgUjQcc5tMLOngTlmFgKcBvoCR0lfaONp0ts93Xx3uQcY4wv1bfxvRs0ewFjf7Iingdvy8GWI/C6aZVPEx8yOOOciva5DJDepvSMiEkS0py8iEkS0py8iEkQU+iIiQUShLyISRBT6IiJBRKEvIhJEFPoiIkHk/wHvuIDksDbmdAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ0AAAEKCAYAAADJvIhZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzs3Xd8FVX6+PHPk0ICoUPooQcwtAAh0uuqiK4sigqrAoIgCnbdRb+7+/PLfr/fXV0VsQBLU0AUEEGxYaGISgkB6TVAkNB7T0KS5/fHnbjXGJILJHdSnvfrdV/ce+bMmWcG7n04M2fOiKpijDHG+EOA2wEYY4wpPizpGGOM8RtLOsYYY/zGko4xxhi/saRjjDHGbyzpGGOM8Rufko6I9BKRHSKSICKjs1keIiJznOWrRaSu17LnnfIdInKLUxYhIktFZKuIbBGRJ7zqvygiB0RkvfPqnVNbTnl5EZknIttFZJuItHfKK4rINyKyy/mzwrUcJGOMMXkj16QjIoHA28CtQBQwQESislQbCpxS1YbAWOAlZ90ooD/QFOgFjHfaSwOeUdUooB0wMkubY1U12nl9kUtbAOOARaraBGgJbHPKRwOLVTUSWOx8NsYY4xJfejqxQIKq7lHVVGA20CdLnT7AdOf9PKCniIhTPltVU1R1L5AAxKrqIVVdB6Cq5/AkiZq5xJFtWyJSDugCTHXaS1XV09nENR34gw/7a4wxJp8E+VCnJrDf63MScOOV6qhqmoicASo55auyrPur5OKcimsFrPYqHiUiA4F4PD2iUzm0dQk4BrwjIi2BtcATqnoBqKqqh5z6h4Gq2e2giAwHhgOEhYW1adKkSXbVjDHGXMHatWuPq2p4bvV8STr5RkRKAx8BT6rqWad4AvB3QJ0/XwWG5NBMENAaeExVV4vIODyn0f7qXUlVVUSynfNHVScBkwBiYmI0Pj7+2nfKGGOKIRHZ50s9X06vHQAivD7XcsqyrSMiQUA54ERO64pIMJ6EM0tV52dWUNUjqpquqhnAZDyn93KKIwlIUtXMntI8PEkI4IiIVHe2Vx046sP+GmOMySe+JJ01QKSI1BOREngu5i/MUmchMMh53w9Yop6ZRBcC/Z3RbfWASCDOud4zFdimqq95N5SZJBx9gc1e2/hNW6p6GNgvIo2dej2BrdnENQj4xIf9NcYYk09yPb3mXKMZBXwFBALTVHWLiIwB4lV1IZ4EMlNEEoCTeBITTr25eJJAGjBSVdNFpBPwALBJRNY7m3rBGan2sohE4zm9lgg8nFNbzrqPAbOcpLgHeNAp/ycwV0SGAvuAe67tMBljjMkLYo82+DW7pmNMwXD58mWSkpJITk52OxTjJTQ0lFq1ahEcHPyrchFZq6oxua3v6kACY4y5kqSkJMqUKUPdunXxnJE3blNVTpw4QVJSEvXq1bumNmwaHGNMgZScnEylSpUs4RQgIkKlSpWuq/dpSccYU2BZwil4rvfvxJKOMYXAoTOXmL8uiYwMuwZrCjdLOsYUcNsPn6Xv2yt4eu4Gvtl2xO1wipXDhw/Tv39/GjRoQJs2bejduzc7d+50O6xfvPvuuxw8eDDbZYMHD2bevHl+jih3lnSMKcBW7j7B3RNWAlCzfEneWLwLG3HqH6pK37596datG7t372bt2rX84x//4MiRgpP4c0o6BZUlHWMKqE83HGTQtDiqlQtl/qMdePJ3kWw5eJbF22xiDX9YunQpwcHBjBgx4peyli1b0rlzZ1SV5557jmbNmtG8eXPmzJkDwLJly+jatSt9+vShfv36jB49mlmzZhEbG0vz5s3ZvXs34OmFjBgxgpiYGBo1asRnn30GQGJiIp07d6Z169a0bt2aFStW/LLtl156iebNm9OyZUtGjx7NvHnziI+P57777iM6OppLly5dcV8WL15Mq1ataN68OUOGDCElJQWA0aNHExUVRYsWLXj22WcB+PDDD2nWrBktW7akS5cueXtQsSHTxhRIU3/Yy98/20ps3YpMHhhDuVLB/KFVTd5cksC4xbvoeUOVYnWR/b8/3cLWg2dzr3gVomqU5f/9vukVl2/evJk2bdpku2z+/PmsX7+eDRs2cPz4cdq2bfvLD/SGDRvYtm0bFStWpH79+jz00EPExcUxbtw43nzzTV5//XXAk2Di4uLYvXs33bt3JyEhgSpVqvDNN98QGhrKrl27GDBgAPHx8Xz55Zd88sknrF69mlKlSnHy5EkqVqzIW2+9xSuvvEJMzJVvj0lOTmbw4MEsXryYRo0aMXDgQCZMmMADDzzAggUL2L59OyLC6dOeyfnHjBnDV199Rc2aNX8py0vW0zGmAMnIUP738638/bOt3NqsGjOGxlKulOcmvODAAEZ2b8CmA2dYusN6O2764YcfGDBgAIGBgVStWpWuXbuyZs0aANq2bUv16tUJCQmhQYMG3HzzzQA0b96cxMTEX9q45557CAgIIDIykvr167N9+3YuX77MsGHDaN68OXfffTdbt3pm9Pr222958MEHKVWqFAAVK1b0OdYdO3ZQr149GjVqBMCgQYNYvnw55cqVIzQ0lKFDhzJ//vxf2u7YsSODBw9m8uTJpKen59T0NbGejjEFREpaOs9+uNFzWq19Hf72+6YEBvy6N3Nn61q8sTiBcYsT6N64+PR2cuqR5JemTZte04X4kJCQX94HBAT88jkgIIC0tLRflmX9uxMRxo4dS9WqVdmwYQMZGRmEhoZeY/S5CwoKIi4ujsWLFzNv3jzeeustlixZwsSJE1m9ejWff/45bdq0Ye3atVSqVCnPtms9HWMKgLPJlxk8bQ2fbjjI6Fub8OIdv004kNnbaciG/af5bucxFyItPnr06EFKSgqTJk36pWzjxo18//33dO7cmTlz5pCens6xY8dYvnw5sbGxObT2Wx9++CEZGRns3r2bPXv20LhxY86cOUP16tUJCAhg5syZv/Q0brrpJt555x0uXrwIwMmTJwEoU6YM586dy3E7jRs3JjExkYSEBABmzpxJ165dOX/+PGfOnKF3796MHTuWDRs2ALB7925uvPFGxowZQ3h4OPv378+p+atmSccYlx05m8w9E1eyJvEkY+9tyYiuDXLswfRrU4ua5Usyzkay5SsRYcGCBXz77bc0aNCApk2b8vzzz1OtWjX69u1LixYtaNmyJT169ODll1+mWrVqV9V+7dq1iY2N5dZbb2XixImEhoby6KOPMn36dFq2bMn27dsJCwsDoFevXtxxxx3ExMQQHR3NK6+8AvxnQEJOAwlCQ0N55513uPvuu2nevDkBAQGMGDGCc+fOcfvtt9OiRQs6derEa695Jvx/7rnnaN68Oc2aNaNDhw60bNnyOo7ib9mEn1nYhJ/GnxKOnmPQtDWcvpjKxAfa0Dky1wcvAvDeqn385ePNzBwa6/M6hc22bdu44YYb3A4jXwwePJjbb7+dfv36uR3KNcnu78bXCT+tp2OMS9YknuSuCStJTc9gzsPtryp53B1Ti+rlQhn3rfV2TOFiSccYFyzafJj7p6ymUlgJ5j/SgWY1y13V+iFBgTzSrQHx+06xcveJfIrS5Jd333230PZyrpclHWP8bObKRB6ZtZaoGmWZ90gHIiqWuqZ27omJoGrZEMYt3pW3ARYg1osreK7378SSjjF+oqq8vGg7f/1kCz2bVOX9h9pRMazENbcXGhzIiK4NWL33JKv2FL3eTmhoKCdOnLDEU4BkPk/neoZy2306xvjB5fQM/vzRRuavO8CA2Nr8vU9TggKv//98A2JrM37Zbt5YvIt29fPuXoqCoFatWiQlJXHsmA0NL0gynxx6rSzpGJPPzqek8eisdSzfeYynb2rEYz0a5tlNnaHBgTzcpT7/8/k21iSepG1d3+9UL+iCg4Ov+emUpuCy02vG5KNj51IYMGkVPyYc5+W7WvB4z8g8n0XgvhvrULl0CcZ9W3Sv7Ziiw6ekIyK9RGSHiCSIyOhsloeIyBxn+WoRqeu17HmnfIeI3OKURYjIUhHZKiJbROQJr/ovisgBEVnvvHrn1JZTnigim5z68V7l0SKyKrNcRK7ulmFjrsPe4xe4c8KPJBw9z5SBMdzTNiJftlOyRCDDu9Tnh4TjrN13Ml+2YUxeyTXpiEgg8DZwKxAFDBCRqCzVhgKnVLUhMBZ4yVk3CugPNAV6AeOd9tKAZ1Q1CmgHjMzS5lhVjXZeX+TSVqbuTn3vm5NeBv5bVaOBvzmfjcl36/ef5q4JK7iQks4Hw9vRvUmVfN3e/e3qUDGsBOMWJ+Trdoy5Xr70dGKBBFXdo6qpwGygT5Y6fYDpzvt5QE/xnEPoA8xW1RRV3QskALGqekhV1wGo6jlgG1AzlziybSuXdRQo67wvBxSupx2ZQmnJ9iMMmLSK0iFBzH+kA9ER5fN9m6VKBDGsc32W7zzGTz+fyvftGXOtfEk6NQHvGd+S+G2C+KWOqqYBZ4BKvqzrnIprBaz2Kh4lIhtFZJqIVPAhDgW+FpG1IjLcq86TwL9EZD/wCvB8djsoIsOd02/xNlLGXI/ZcT8zbMZaIquW5qNHOlC3cpjftj2wfR0qlArmjSJ8344p/FwdSCAipYGPgCdVNfMJTROABkA0cAh41YemOqlqazynAEeKSObj7h4BnlLVCOApYGp2K6vqJFWNUdWY8PCiOY+VyV+qyrhvdzF6/iY6NazMB8PaEV4mJPcV81BYSBAPda7P0h3H2JiU9w/fMiYv+JJ0DgDeV0BrOWXZ1hGRIDynsk7ktK6IBONJOLNUdX5mBVU9oqrpqpoBTOY/p9Cu2JaqZv55FFjgtc4gILPtD8n9dJwxVy0tPYMXFmxi7Lc76demFlMGxRAW4s7dCAPb16FcSevtmILLl6SzBogUkXoiUgLPxfyFWeosxPMDD9APWKKe24gXAv2d0W31gEggzrneMxXYpqqveTckItW9PvYFNnttI7u2wkSkjLNuGHCz1zoHga7O+x6AfRNNnrqUms6I99byQdx+RnVvyL/6tSA4D276vFZlQoMZ2qke3247yuYDZ1yLw5gryfW/Y6qaJiKjgK+AQGCaqm4RkTFAvKouxJNAZopIAnAST2LCqTcX2IpnxNpIVU0XkU7AA8AmEVnvbOoFZ6TayyISjec6TSLwcC5tVQUWOPc+BAHvq+oip81hwDin95UMeF/vMea6nLyQypB317Ax6TR//0MzHmhXx+2QABjcsS6Tv9/DG4t3MWlgrjPNG+NX9jydLOx5OsYX+09eZOC0OA6evsQbA1pxS9Ore4BXfhv7zU7GLd7FF493JqpG2dxXMOY62fN0jMknmw+coe/4FZy8kMqsh24scAkHYEjHepQJCeLNJXZG2RQslnSMuQrLdx7j3n+vJCQogI8eaU9MAZ3rrFypYAZ3rMuXmw+z/fDZ3Fcwxk8s6Rjjo/nrkhjy7hpqVwpj/qMdaFiljNsh5Whop3qElQjkzSU2S4EpOCzpGJMLVWXCst08PXcDsfUqMufhdlQte+3PE/GX8qVKMKhDXb7YdIhdR865HY4xgCUdY3KUnqG8uHALLy3azh0ta/Dug7GUDQ12OyyfPdS5PiWDrbdjCg5LOsZcQfLldEa9v47pK/cxvEt9Xr83mhJBhesrUzGsBAPb1+XTjQdJOHre7XCMsaRjTHbOXLzMwKlxLNpymL/eHsULvW8gICBvn4PjL8M61yM0KJC3l1pvx7jPko4xWRw4fYl+E1ewfv9p3hzQiqGdCvfTKyuVDuGB9nX4ZP0B9hyz3o5xlyUdY7xsP3yWu8av4PDZZKYPieX2FjXcDilPDOtcnxJBAby9dLfboZhizpKOMY6Vu09w94SVAMwb0YH2DSq5HFHeCS8Twn031uHj9QfYd+KC2+GYYsySjjHApxsOMmhaHNXLhzL/0Q40rlaw78G5Fg93qU9QgNi1HeMqSzqm2Jvy/R4e++AnomuX58OHO1CjfEm3Q8oXVcqGMiC2NvPXHWD/yYtuh2OKKUs6ptjKyFD+57Ot/M/n2+jdvBozhsRSrlThuQfnWozo2oAAEcYvs96OcYclHVMspaSl88Sc9Uz5YS+DO9TlzQGtCQ0OdDusfFetXCj9YyP4MD6JpFPW2zH+Z0nHFDtnky8zeNoaPt1wkOdvbcL/+30UgYX0Hpxr8Ui3zN6OjWQz/mdJxxQrR84mc8/ElaxJPMnYe1vycNcGOA8ALDaqlyvJ3TG1+DB+PwdPX3I7HFPMWNIxxUbC0XPcOX4F+09e5J0H29K3VS23Q3LNo90bAjDBejvGzyzpmGJhTeJJ7pqwktT0DOY83J7OkeFuh+SqmuVL0q9NLeas2c/hM8luh2OKEUs6pshbtPkw909ZTaWwEsx/pAPNapZzO6QC4dFuDclQZeJ31tsx/mNJxxRpM1cm8sistUTVKMu8RzoQUbGU2yEVGBEVS3Fn65q8H/czR89ab8f4h09JR0R6icgOEUkQkdHZLA8RkTnO8tUiUtdr2fNO+Q4RucUpixCRpSKyVUS2iMgTXvVfFJEDIrLeefXOqS2nPFFENjn147PE9piIbHe28/LVHBxTeKkqLy/azl8/2ULPJlV5/6F2VAwr4XZYBc7I7g1Jz1AmfrfH7VBMMRGUWwURCQTeBm4CkoA1IrJQVbd6VRsKnFLVhiLSH3gJuFdEooD+QFOgBvCtiDQC0oBnVHWdiJQB1orIN15tjlXVV7LEkW1bqpruVOmuqsezrNMd6AO0VNUUEani85Exhdbl9Az+/NFG5q87wIDY2vy9T1OCAq1Tn506lcL4Q3RNZq3ex4hu9alSpuA/EdUUbr58E2OBBFXdo6qpwGw8P+Te+gDTnffzgJ7iGYfaB5itqimquhdIAGJV9ZCqrgNQ1XPANqBmLnFk21Yu6zwC/FNVU5xtHfVhf00hdj4ljaHT45m/7gBP39SI/+vbzBJOLkb1aMjl9AwmL7fejsl/vnwbawL7vT4n8dsE8UsdVU0DzgCVfFnXORXXCljtVTxKRDaKyDQRqeBDHAp8LSJrRWS4V51GQGfnlN93ItI2ux0UkeEiEi8i8ceOHcuuiikEjp1LYcCkVfyYcJyX72rB4z0ji909ONeiXuUw+kTX5L1VP3P8fIrb4ZgiztX/AopIaeAj4ElVPesUTwAaANHAIeBVH5rqpKqtgVuBkSLSxSkPAioC7YDngLmSza+Qqk5S1RhVjQkPL95DaQurvccvcOeEH0k4ep4pA2O4p22E2yEVKqN6NCQlLZ3J31tvx+QvX5LOAcD7G1zLKcu2jogEAeWAEzmtKyLBeBLOLFWdn1lBVY+oarqqZgCT+c8ptCu2paqZfx4FFnitkwTMV484IAOo7MM+m0Jk/f7T3DVhBRdS0vlgeDu6N7FLd1erQXhpft+yBjNX7uPkhVS3wzFFmC9JZw0QKSL1RKQEnov5C7PUWQgMct73A5aoqjrl/Z3RbfWASCDO6W1MBbap6mveDYlIda+PfYHNXtvIrq0wZzACIhIG3Oy1zsdAd2dZI6AE8KvBBqZwW7L9CAMmraJ0SBDzH+lAdER5t0MqtB7r0ZBLl9OZYr0dk49yHb2mqmkiMgr4CggEpqnqFhEZA8Sr6kI8CWSmiCQAJ/EkJpx6c4GteEasjVTVdBHpBDwAbBKR9c6mXlDVL4CXRSQaz3WaRODhXNqqCixwzpoFAe+r6iKnzWnANBHZDKQCg5xkaIqA2XE/818fb6ZpjbJMHdSW8DIhbodUqDWsUobbmldn+opEhnWuTwUbYm7ygdhv8K/FxMRofHx87hWNa1SVNxYnMPbbnXRtFM74+1oTFpLr/5+MD3YcPsctry/nsR4Neebmxm6HYwoREVmrqjG51bOxpKZQSUvP4IUFmxj77U76tanFlEExlnDyUONqZejdvBrv/pjImYuX3Q7HFEGWdEyhcSk1nRHvreWDuP2M6t6Qf/VrQbDdg5PnHusRybmUNKb9uNftUEwRZN9YUyicvJDKgMmrWLL9KH//QzOevaWx3YOTT26oXpZbmlZl2o97OXPJejsmb1nSMQXe/pMXuWvCCrYdOsuE+9vwQLs6bodU5D3WI5JzyWlMX5HodiimiLGkYwq0zQfO0Hf8Ck5eSGXWQzdyS9NqbodULDSrWY7f3VCVqT/s5Vyy9XZM3rGkYwqs5TuPce+/VxISFMBHj7Qnpm5Ft0MqVp7oGcmZS5eZsXKf26GYIsSSjimQ5q9LYsi7a6hdKYz5j3agYZUybodU7DSvVY4eTaow+fs9nE9JczscU0RY0jEFiqoyYdlunp67gRvrV2TOw+2oWtam23fL4z0jOX3xMjNWJrodiikiLOmYAiM9Q3lx4RZeWrSdPtE1eGdwLGVDg90Oq1iLjihP10bhTPl+Lxest2PygCUdUyAkX05n1PvrmL5yHw93qc/Ye6IpEWT/PAuCx3tGcvJCKu+tsms75vrZt9q47szFywycGseiLYf52+1RPN/7BgIC7B6cgqJNnQp0jqzMpOV7uJSanvsKxuTAko5x1YHTl+g3cQXr95/mzQGtGNKpntshmWw80TOSExdSmbXaejvm+ljSMa7Zdugsd47/kcNnk5kxNJbbW9RwOyRzBTF1K9KhQSUmfreH5MvW2zHXzpKOcUV84knumbgSQZg3ogPt6ldyOySTiyd6RnL8fArvr/7Z7VBMIWZJx/jd9sNnefDdNYSXDWH+ox1oXM3uwSkMbqxfiRvrVWTid7utt2OumSUd41dJpy4yaFocpUoEMnPojdQoX9LtkMxVeOJ3kRw9l8KcNfvdDsUUUpZ0jN+cupDKwGlxXExNZ/qQWGpawil02tevRGzdikxYtpuUNOvtmKtnScf4xaXUdIZMX0PSqUtMGRhDk2pl3Q7JXAMR4fGekRw+m8zc+CS3wzGFkCUdk+/S0jMY9f46Nuw/zRv9W3GjDRoo1Do2rESbOhWYsDSB1LQMt8MxhYwlHZOvVJUXFmxi8fajjOnTjF7N7NEEhV1mb+fgmWTmrbXejrk6PiUdEeklIjtEJEFERmezPERE5jjLV4tIXa9lzzvlO0TkFqcsQkSWishWEdkiIk941X9RRA6IyHrn1TuntpzyRBHZ5NSPzya+Z0RERaSyrwfG5I1Xvt7B3PgkHu8Zyf328LUio0tkZaIjyvO29XbMVco16YhIIPA2cCsQBQwQkags1YYCp1S1ITAWeMlZNwroDzQFegHjnfbSgGdUNQpoB4zM0uZYVY12Xl/k0lam7k79mCzxRwA3A3ZzgZ9NX5HI20t3MyC2Nk/9LtLtcEweEhGe6BnJgdOXWPCT9XaM73zp6cQCCaq6R1VTgdlAnyx1+gDTnffzgJ7ieYB9H2C2qqao6l4gAYhV1UOqug5AVc8B24CaucSRbVs+xD8W+BOgPtQ1eeTzjYd48dMt3BRVlb/3aYrnn4MpSro1DqdFrXK8tTSBy+nW2zG+8SXp1AS8B+Un8dsE8UsdVU0DzgCVfFnXORXXCljtVTxKRDaKyDQRqeBDHAp8LSJrRWS4V9t9gAOquiGnHRSR4SISLyLxx44dy6mq8cGK3cd5as562tSuwJsDWhEUaJcOiyIR4fEekew/eYmPfzrgdjimkHD110BESgMfAU+q6lmneALQAIgGDgGv+tBUJ1VtjecU4EgR6SIipYAXgL/ltrKqTlLVGFWNCQ8Pv5ZdMY6tB8/y8Iy11KlUiimDYggNDsx9JVNo9byhCs1qluWtpQmkWW/H+MCXpHMAiPD6XMspy7aOiAQB5YATOa0rIsF4Es4sVZ2fWUFVj6hquqpmAJP5zym0K7alqpl/HgUWOOs0AOoBG0Qk0am/TkRs+FQ+2X/yIoPeiaN0aBDTh8RSvlQJt0My+Syzt7PvxEUWbjjodjimEPAl6awBIkWknoiUwHMxf2GWOguBQc77fsASVVWnvL8zuq0eEAnEOdd7pgLbVPU174ZEpLrXx77AZq9tZNdWmIiUcdYNwzNoYLOqblLVKqpaV1Xr4jkd11pVD/uwz+YqnTifwsBpcaSmZTBjSKxNb1OM3BRVlRuql+WtJQmkZ9ilU5OzXJOOc41mFPAVngv+c1V1i4iMEZE7nGpTgUoikgA8DYx21t0CzAW2AouAkaqaDnQEHgB6ZDM0+mVn+PNGoDvwVC5tVQV+EJENQBzwuaouur7DYq7GxdQ0hkyP5+DpS0wdFENkVZvAszjxjGRryJ7jF/hso/V2TM7E0yExmWJiYjQ+/je3+pgruJyewbAZ8SzfeYx/PxDDTVFV3Q7JuCAjQ7l13PekZWTw9VNdCbQnvxY7IrI26y0r2bFhReaaqSp//mgjy3Yc4//6NreEU4wFBHhmKdh97AJfbDrkdjimALOkY67ZS4t2MH/dAZ6+qRH9Y2u7HY5x2a3NqhFZpTRvLtlFhl3bMVdgScdck2k/7GXid7u578baPNajodvhmAIgIEB4rGckO4+c58vNNl7HZM+SjrlqCzccZMxnW+nVtBpj+jSz2QbML25rXp0G4WHW2zFXZEnHXJUfdh3nmbnria1Xkdf7R9sFY/MrgQHCYz0i2X74HF9vtd6O+S1LOsZnmw+c4eGZ8TQIL83kgTbbgMne7S2qU69yGOMWJ2CjY01WlnSMT34+cZHB76yhfKkSvPtgLOVKBrsdkimgggIDGNW9IdsOneWbrUfcDscUMJZ0TK6On09h4LTVpGVkMH1ILNXKhbodking+kTXoE6lUryxZJf1dsyvWNIxOTqfksaD76zh8Nlkpg1uS8Mqpd0OyRQCQYEBjOzekM0HzrJk+1G3wzEFiCUdc0WpaRk88t5ath46y/j7WtO6doXcVzLG0bdVTSIqluSNxdbbMf9hScdkKyND+dO8DXy/6zj/uLM5PZrYbAPm6gQHBjCyW0M2JJ1h2U57TpXxsKRjsvWPL7fx8fqDPHdLY+6Jich9BWOycWfrWtQsX5Jx31pvx3hY0jG/MXn5HiZ/v5dB7evwaLcGbodjCrESQQE82r0B6/ef5vtdx90OxxQAlnTMr3z80wH+94tt3Na8On/7fVObbcBct35talGjXCjj7NqOwZKO8bJ85zGe/XAD7epX5LV7W9psAyZPhAQF8ki3Bqzdd4oVu0+4HY5xmSUXhj/UAAAfV0lEQVQdA8DGpNOMeG8tkVXLMGlgDCFBNtuAyTv3tI2gWtlQu7ZjLOkY2Hv8Ag++s4aKYSWY/mBbyobabAMmb2X2duIST7Jqz0m3wzEusqRTzB09l8zAaatRYMaQWKqUtdkGTP64t20EVcqEMG7xTrdDMS6ypFOMnUu+zIPvrOH4uVSmDW5L/XCbbcDkn9DgQEZ0bcCqPSdZvceu7RRXlnSKqZS0dEa8t5Ydh88x/v7WREeUdzskUwz88cbaVC4dwhtLdrkdinGJT0lHRHqJyA4RSRCR0dksDxGROc7y1SJS12vZ8075DhG5xSmLEJGlIrJVRLaIyBNe9V8UkQMist559c6pLac8UUQ2OfXjvcr/JSLbRWSjiCwQEftlxTPbwDNzN/BjwgleuqsF3RtXcTskU0x4ejv1+THhBPGJdm2nOMo16YhIIPA2cCsQBQwQkags1YYCp1S1ITAWeMlZNwroDzQFegHjnfbSgGdUNQpoB4zM0uZYVY12Xl/k0lam7k79GK+yb4BmqtoC2Ak8n/shKdpUlb9/vpXPNh5i9K1NuKtNLbdDMsXMH2+sTaWwEoxbbL2d4siXnk4skKCqe1Q1FZgN9MlSpw8w3Xk/D+gpnrsK+wCzVTVFVfcCCUCsqh5S1XUAqnoO2AbUzCWObNvKaQVV/VpV05yPq4Bi/wv77+V7eOfHRIZ0rMfDXeq7HY4phkqVCGJ4l/p8v+s4634+5XY4xs98STo1gf1en5P4bYL4pY7zI38GqOTLus6puFbAaq/iUc4psWkikjm1cU5tKfC1iKwVkeFX2I8hwJfZLRCR4SISLyLxx44V3YkJP1qbxD+/3M7vW9bgL7fdYLMNGNfc364OFcNK8Ib1doodVwcSiEhp4CPgSVU96xRPABoA0cAh4FUfmuqkqq3xnAIcKSJdsmznv/Cc0puV3cqqOklVY1Q1Jjw8/Np2poBbuuMof/poIx0bVuKVu1sQYLMNGBeFhQTxUOd6LNtxjA37T7sdjvEjX5LOAcB7muFaTlm2dUQkCCgHnMhpXREJxpNwZqnq/MwKqnpEVdNVNQOYzH9OoV2xLVXN/PMosMBrHURkMHA7cJ8W01uh1+8/zaPvraNJtTJMvL+NzTZgCoSB7etSvlSw9XaKGV+SzhogUkTqiUgJPBfzF2apsxAY5LzvByxxfuAXAv2d0W31gEggzrneMxXYpqqveTckItW9PvYFNnttI7u2wkSkjLNuGHBz5joi0gv4E3CHql70YV+LnD3HzjPk3TWElwnh3QdjKWOzDZgConRIEA91qsfi7UfZlHTG7XCMn+SadJxrNKOAr/Bc8J+rqltEZIyI3OFUmwpUEpEE4GlgtLPuFmAusBVYBIxU1XSgI/AA0CObodEvO8OfNwLdgadyaasq8IOIbADigM9VdZHT1ltAGeAbZxsTr/E4FUpHzyYzcFocgme2gfAyIW6HZMyvDOpQl7KhQXbfTjEixfSM0xXFxMRofHx87hULuLPJl7n336vYd+ICs4e3o0Utu0XJFEzjvt3F2G938vnjnWhao5zb4ZhrJCJrs9yyki2bkaAISklLZ/iMeHYdOcfE+9tYwjEF2uCOdSkTGsSbixPcDsX4gSWdIiY9Q3l6zgZW7TnJK3e3pEujojkazxQd5UoG82DHeizacphth87mvoIp1CzpFCGqyphPt/D5pkP8V+8b+EOr3O63NaZgGNqxHqVDgnhrifV2ijpLOkXI+GW7mb5yH8M612OYzTZgCpFypYIZ3KEuX2w+xM4j59wOx+QjSzpFxNw1+/nXVzv4Q3QNnr/1BrfDMeaqDe1Uj1LBgbxpvZ0izZJOEbB42xGeX7CJzpGVeblfS5ttwBRKFcJKMLBDXT7beJCEo9bbKaos6RRya/edYuT762haoywT729DiSD7KzWF17DO9SkZHGjXdoow+4UqxBKOnmPo9DVUKxvKtMFtCQsJcjskY65LxbASPNCuDgs3HGTPsfNuh2PygSWdQurwmWQGTo0jKCCAGUNupHJpm23AFA3DutQnJMh6O0WVJZ1C6MylywyaFsfZ5DTefbAttSuVcjskY/JM5dIh3N+uNh+vP0Di8Qtuh2PymCWdQib5cjrDZsSz5/h5/v1AG5rVtGlDTNEzrEt9ggMDeGup9XaKGks6hUh6hvLk7PXE7T3Jq/dE07FhZbdDMiZfVCkTyn031mHBTwf4+USxnCC+yLKkU0ioKn/7ZDOLthzmb7dHcUfLGm6HZEy+erhrfQIDhLett1OkWNIpJN5cksCs1T8zomsDhnSq53Y4xuS7qmVD+WNsbT5al8T+k9bbKSos6RQCH8T9zGvf7OTO1jX5c6/GbodjjN883LU+ASKMX7bb7VBMHrGkU8B9veUw/7VgE90ah/PSXS3wPHTVmOKhermS3Ns2gnlr93Pg9CW3wzF5wJJOARafeJLHPviJ5rXKM/6+1gQH2l+XKX5GdGsAwIRldm2nKLBfsQJq55FzDHl3DTXLl+SdwW0pVcJmGzDFU83yJbk7JoK5a5I4dMZ6O4WdJZ0C6ODpSwyaFkdIcCDTh8RSMayE2yEZ46pHuzUgQ5WJdm2n0LOkU8CcvpjKoGlxnE9OY/qDsURUtNkGjKlVoRT92tTigzX7OXI22e1wzHXwKemISC8R2SEiCSIyOpvlISIyx1m+WkTqei173infISK3OGURIrJURLaKyBYRecKr/osickBE1juv3jm15ZQnisgmp368V3lFEflGRHY5f1a42gPkT8mX03loejz7Tlxk0sAYomqUdTskYwqMkd0bkp6hTPzOejuFWa5JR0QCgbeBW4EoYICIRGWpNhQ4paoNgbHAS866UUB/oCnQCxjvtJcGPKOqUUA7YGSWNseqarTz+iKXtjJ1d+rHeJWNBharaiSw2PlcIKWlZzDq/Z9Y+/MpXu8fTfsGldwOyZgCJaJiKe5sVZP3V//MUevtFFq+9HRigQRV3aOqqcBsoE+WOn2A6c77eUBP8Yzt7QPMVtUUVd0LJACxqnpIVdcBqOo5YBtQM5c4sm3Lh3Uy45oO/CGX+q5QVf76yWa+3XaEF3/flN7Nq7sdkjEF0qgeDUnLUCYt3+N2KOYa+ZJ0agL7vT4n8dsE8UsdVU0DzgCVfFnXORXXCljtVTxKRDaKyDSvU2I5taXA1yKyVkSGe9WpqqqHnPeHgarZ7aCIDBeReBGJP3bsWHZV8tXYb3fxQdx+RnZvwKAOdf2+fWMKizqVwugTXYP3Vu/j2LkUt8Mx18DVgQQiUhr4CHhSVc86xROABkA0cAh41YemOqlqazynAEeKSJesFVRV8SSn31DVSaoao6ox4eHh17An1+69Vft4Y/Eu7ompxbM322wDxuRmVPeGpKZlMOV76+0URr4knQNAhNfnWk5ZtnVEJAgoB5zIaV0RCcaTcGap6vzMCqp6RFXTVTUDmMx/TqFdsS1VzfzzKLDAa50jIlLd2V514KgP++s3izYf4q+fbKZnkyr8X9/mNtuAMT6oH16aO1rWYMbKfZw4b72dwsaXpLMGiBSReiJSAs/F/IVZ6iwEBjnv+wFLnJ7FQqC/M7qtHhAJxDnXe6YC21T1Ne+GMpOEoy+w2Wsb2bUVJiJlnHXDgJuzrJMZ1yDgEx/21y9W7znB47PXEx1Rnrf+2Jogm23AGJ+N6hFJclo6U37Y63Yo5irlepu7qqaJyCjgKyAQmKaqW0RkDBCvqgvxJJCZIpIAnMSTmHDqzQW24hmxNlJV00WkE/AAsElE1jubesEZqfayiETjORWWCDycS1tVgQVOLyEIeF9VFzlt/hOYKyJDgX3APdd+qPLO9sNneWhGPBEVSjJtUFtKlgjMfSVjzC8aVinN7S1qMGNFIsM716eC3UBdaIinQ2IyxcTEaHx8fO4Vr1HSqYvcNWEFAPMf7UjN8iXzbVvGFGW7jpzj5teXM7JbQ569xa6Huk1E1ma5ZSVbdk7Hj05d8Mw2cDE1nelDYi3hGHMdIquWoXfz6ry7IpHTF1PdDsf4yJKOn1xKTWfI9DXsP3WJKQNjaFLNZhsw5no91qMh51PSmPZjotuhGB9Z0vEDz2wD69iw/zRv9G/FjfVttgFj8kKTamW5tVk13vlxL2cuXXY7HOMDSzr5TFV5YcEmFm8/ypg+zejVrJrbIRlTpDzWI5JzyWm886ONZCsMLOnks1e/3snc+CQe7xnJ/e3quB2OMUVOVI2y3BxVlWk/7OVssvV2CjpLOvlo+opE3lqawIDY2jz1u0i3wzGmyHq8ZyRnk9OYbtd2CjxLOvnk842HePHTLdwUVZW/92lqsw0Yk4+a1SzH726owpQf9nI+Jc3tcEwOLOnkgxW7j/PUnPW0qV2BNwe0stkGjPGDx3tGcubSZaavSHQ7FJMD+zXMY1sPnuXhGWupU6kUUwbFEBpssw0Y4w8tapWne+Nwpny/hwvW2ymwLOnkof0nLzLonThKhwYxfUgs5UvZ1BzG+NPjPSM5dfEyM1ftczsUcwWWdPLIifMpDJoWR2paBjOGxFLDZhswxu9a1a5Al0bhTF6+h4up1tspiCzp5JExn23lwOlLTB0UQ2TVMm6HY0yx9UTPSE5cSGXWqp/dDsVkw5JOHvnb7VG8M7gtMXUruh2KMcVamzoV6NSwMv9evpukUxfdDsdkYUknj1QqHUKHhpXdDsMYAzx7S2POp6TR89XvePXrHXaqrQCxpGOMKXKiI8qz5Jlu3NK0Gm8uSaD7K8tY8FMSGRn2KBe3WdIxxhRJNcqX5I0BrZg3oj1VyoTy1JwN3DlhBT/9fMrt0Io1SzrGmCItpm5FPhnZkX/1a8GB05foO34FT81Zz+EzyW6HVixZ0jHGFHkBAcLdMREsfbYbj3ZrwOebDtH9lWW8uXgXyZfT3Q6vWLGkY4wpNkqHBPGnXk1Y/HRXujUO59VvdtLz1e/4bONBVO16jz9Y0jHGFDsRFUsx4f42fDCsHWVLBjPq/Z+499+r2HzgjNuhFXk+JR0R6SUiO0QkQURGZ7M8RETmOMtXi0hdr2XPO+U7ROQWpyxCRJaKyFYR2SIiT3jVf1FEDojIeufVO6e2vJYFishPIvKZV1lPEVnntPODiDS8moNjjCna2jeoxGePdeL/+jZn97Hz/P6tH/jTvA0cPWfXe/JLrklHRAKBt4FbgShggIhEZak2FDilqg2BscBLzrpRQH+gKdALGO+0lwY8o6pRQDtgZJY2x6pqtPP6Ipe2Mj0BbMsS1wTgPlWNBt4H/pLb/hpjipfAAOGPN9Zm6XPdeKhTPeavO0CPV75j4ne7SUmz6z15zZeeTiyQoKp7VDUVmA30yVKnDzDdeT8P6CmeB8j0AWaraoqq7gUSgFhVPaSq6wBU9RyeZFEzlziybQtARGoBtwFTsqyjQFnnfTngoA/7a4wphsqGBvNft0Xx9VNdaFe/Iv/8cjs3j13OV1sO2/WePORL0qkJ7Pf6nMRvE8QvdVQ1DTgDVPJlXedUXCtgtVfxKBHZKCLTRKSCD3G8DvwJyMgS10PAFyKSBDwA/DOH/TTGGOqHl2bKoLbMGBJLicAAHp65lvunrmb74bNuh1YkuDqQQERKAx8BT6pq5t/oBKABEA0cAl7NpY3bgaOqujabxU8BvVW1FvAO8NoV2hguIvEiEn/s2LFr2xljTJHSpVE4Xz7Rmf++oymbD5yl97jv+cvHmzh5IdXt0Ao1X5LOASDC63MtpyzbOiIShOdU1omc1hWRYDwJZ5aqzs+soKpHVDVdVTOAyTin0HJoqyNwh4gk4jn110NE3hORcKClqmb2oOYAHbLbQVWdpKoxqhoTHh6ey+EwxhQXQYEBDOpQl++e68bA9nX5IG4/3f61lKk/7OVyetYTK8YXviSdNUCkiNQTkRJ4LuYvzFJnITDIed8PWKKek6ALgf7O6LZ6QCQQ51zvmQpsU9Vf9T5EpLrXx77AZq9t/KYtVX1eVWupal0ntiWqej9wCignIo2c9W/itwMNjDEmV+VLleDFO5qy6InOtIwoz98/28otry9n6Y6jbodW6ATlVkFV00RkFPAVEAhMU9UtIjIGiFfVhXgSyEwRSQBO4vnxx6k3F9iKZ8TaSFVNF5FOeK6xbBKR9c6mXnBGqr0sItF4BgEkAg/n1FYucQ8DPhKRDDxJaMhVHR1jjPESWbUMM4bEsmT7Uf7n8208+M4aujUO5y+3RdGwSmm3wysUxEZl/FpMTIzGx8e7HYYxpoBLTctgxspExn27i0uX03mgfR2e7NmIcqWC3Q7NFSKyVlVjcqtnMxIYY8w1KBEUwEOd67P0uW7cHRPB9BWJdHtlKTNXJpJm13uuyJKOMcZch8qlQ/jHnc357LHONK5Whr9+soXb3viBH3Yddzu0AsmSjjHG5IGoGmX5YFg7Jt7fmouX07h/6mqGzYgn8fgFt0MrUCzpGGNMHhERejWrzjdPdeW5WxqzIuE4N439jn98sY1zyZfdDq9AsKRjjDF5LDQ4kJHdG7L02W70ia7Jv5fvofsry5gd9zPpxfyR2ZZ0jDEmn1QpG8ord7dk4aiO1KkUxuj5m7jjrR9YveeE26G5xpKOMcbksxa1yjNvRHveGNCKUxdSuXfSKkbOWsf+kxfdDs3vLOkYY4wfiAh3tKzB4me68dTvGrF4+xF6vvYdr3y1gwspaW6H5zeWdIwxxo9Klgjkid9FsuSZbtzarBpvLU2gx6vLmL8uiYxicL3Hko4xxrigRvmSjOvfio8e6UC1sqE8PXcDfSesYN3Pp9wOLV9Z0jHGGBe1qVOBBY925NW7W3Lo9CXuHL+CJ2f/xKEzl9wOLV9Y0jHGGJcFBAh3tanF0me7MbJ7A77YfJger3znmdcttWg9MtuSjjHGFBBhIUE8d0sTFj/dle5Nwhn77U5+99p3fLrhYJF5ZLYlHWOMKWAiKpZi/H1tmD28HeVKBvPYBz9xz79XsinpjNuhXTdLOsYYU0C1q1+JTx/rxD/ubM6eYxe44+0feO7DDRw9l+x2aNfMko4xxhRggQHCgNjaLH2uG8M61+fj9Qfo/q9lTFi2m5S0wne9x5KOMcYUAmVDg3mh9w18/VRX2jeozEuLtnPTa8tZtPlwobreY0nHGGMKkXqVw5gyKIaZQ2MJDQ5gxHtr+ePk1Ww7dNbt0HxiSccYYwqhzpHhfPF4Z8b0acq2w2e57Y3veWHBJk6cT3E7tBxZ0jHGmEIqKDCAge3rsuzZbgxsX5c5a/bT7ZVlTPl+D6lpBfOR2T4lHRHpJSI7RCRBREZnszxEROY4y1eLSF2vZc875TtE5BanLEJElorIVhHZIiJPeNV/UUQOiMh659U7p7a8lgWKyE8i8plXmYjI/4rIThHZJiKPX83BMcaYwqB8qRK8eEdTvnqyM61qV+B/Pt9Gr9eXs3T7UbdD+41ck46IBAJvA7cCUcAAEYnKUm0ocEpVGwJjgZecdaOA/kBToBcw3mkvDXhGVaOAdsDILG2OVdVo5/VFLm1legLYliWuwUAE0ERVbwBm57a/xhhTWDWsUobpD7Zl2uAYAB58dw2DpsWRcPScy5H9hy89nVggQVX3qGoqnh/uPlnq9AGmO+/nAT1FRJzy2aqaoqp7gQQgVlUPqeo6AFU9hydZ1MwljmzbAhCRWsBtwJQs6zwCjFHVDGdbBS/tG2NMHhIRejSpyqInu/CX225g3c+nuOX173lx4RZOX0x1Ozyfkk5NYL/X5yR+myB+qaOqacAZoJIv6zqn4loBq72KR4nIRhGZJiIVfIjjdeBPQNaTmA2Ae0UkXkS+FJHIHPfUGGOKiBJBATzUuT7Lnu1G/7YRzFiZSLdXljFjZSJp6e5d73F1IIGIlAY+Ap5U1czxfhPwJIto4BDwai5t3A4cVdW12SwOAZJVNQaYDEy7QhvDncQUf+zYsWvbGWOMKYAqlQ7hf/s25/PHO3NDtbL87ZMt9H7je77f5c5vnS9J5wCe6yKZajll2dYRkSCgHHAip3VFJBhPwpmlqvMzK6jqEVVNd06JTcY5hZZDWx2BO0QkEc+pvx4i8p5TJwnIbHsB0CK7HVTVSaoao6ox4eHhVz4SxhhTSN1QvSzvD7uRife3IflyBg9MjeOh6fHsPX7Br3H4knTWAJEiUk9ESuC5mL8wS52FwCDnfT9giXpukV0I9HdGt9UDIoE453rPVGCbqr7m3ZCIVPf62BfY7LWN37Slqs+rai1VrevEtkRV73fW+Rjo7rzvCuz0YX+NMaZIEhF6NavGN0934c+9mrBy93FuHvsd//fFNs4mX/ZLDEG5VVDVNBEZBXwFBALTVHWLiIwB4lV1IZ4EMlNEEoCTeH78cerNBbbiGbE2UlXTRaQT8ACwSUTWO5t6wRmp9rKIRAMKJAIP59RWLuH/E5glIk8B54GHfDssxhhTdIUEBfJItwbc1aYmr3y1g8nf72H+uiTeGNCKDg0q5+u2pTDN2eMPMTExGh8f73YYxhjjN5uSzvDyV9v5V7+WVCsXek1tiMha5/p5jnLt6RhjjCnamtcqx8yhN/plWzYNjjHGGL+xpGOMMcZvLOkYY4zxG0s6xhhj/MaSjjHGGL+xpGOMMcZvLOkYY4zxG0s6xhhj/MZmJMhCRI4B+65x9crA8TwMJ69YXFfH4ro6FtfVKahxwfXFVkdVc50x2ZJOHhKReF+mgfA3i+vqWFxXx+K6OgU1LvBPbHZ6zRhjjN9Y0jHGGOM3lnTy1iS3A7gCi+vqWFxXx+K6OgU1LvBDbHZNxxhjjN9YT8cYY4zfWNIxxhjjN5Z0roGI9BKRHSKSICKjs1keIiJznOWrRaRuAYlrsIgcE5H1zivfH98tItNE5KiIbL7CchGRN5yYN4pI6/yOyce4uonIGa9j9Tc/xRUhIktFZKuIbBGRJ7Kp4/dj5mNcfj9mIhIqInEissGJ67+zqeP376OPcfn9++i17UAR+UlEPstmWf4eL1W111W8gEBgN1AfKAFsAKKy1HkUmOi87w/MKSBxDQbe8vPx6gK0BjZfYXlv4EtAgHbA6gISVzfgMxf+fVUHWjvvywA7s/l79Psx8zEuvx8z5xiUdt4HA6uBdlnquPF99CUuv38fvbb9NPB+dn9f+X28rKdz9WKBBFXdo6qpwGygT5Y6fYDpzvt5QE8RkQIQl9+p6nLgZA5V+gAz1GMVUF5EqheAuFyhqodUdZ3z/hywDaiZpZrfj5mPcfmdcwzOOx+DnVfW0VF+/z76GJcrRKQWcBsw5QpV8vV4WdK5ejWB/V6fk/jtl++XOqqaBpwBKhWAuADuck7JzBORiHyOyRe+xu2G9s7pkS9FpKm/N+6c1miF53/J3lw9ZjnEBS4cM+dU0XrgKPCNql7xePnx++hLXODO9/F14E9AxhWW5+vxsqRTvHwK1FXVFsA3/Od/M+a31uGZS6ol8CbwsT83LiKlgY+AJ1X1rD+3nZNc4nLlmKlquqpGA7WAWBFp5o/t5saHuPz+fRSR24Gjqro2v7d1JZZ0rt4BwPt/JLWcsmzriEgQUA444XZcqnpCVVOcj1OANvkcky98OZ5+p6pnM0+PqOoXQLCIVPbHtkUkGM8P+yxVnZ9NFVeOWW5xuXnMnG2eBpYCvbIscuP7mGtcLn0fOwJ3iEginlPwPUTkvSx18vV4WdK5emuASBGpJyIl8FxoW5ilzkJgkPO+H7BEnatybsaV5bz/HXjOy7ttITDQGZHVDjijqofcDkpEqmWexxaRWDzflXz/oXK2ORXYpqqvXaGa34+ZL3G5ccxEJFxEyjvvSwI3AduzVPP799GXuNz4Pqrq86paS1Xr4vmNWKKq92eplq/HKyivGiouVDVNREYBX+EZMTZNVbeIyBggXlUX4vlyzhSRBDwXq/sXkLgeF5E7gDQnrsH5HZeIfIBnVFNlEUkC/h+ei6qo6kTgCzyjsRKAi8CD+R2Tj3H1Ax4RkTTgEtDfD/9xAM//RB8ANjnXAwBeAGp7xebGMfMlLjeOWXVguogE4klyc1X1M7e/jz7G5ffv45X483jZNDjGGGP8xk6vGWOM8RtLOsYYY/zGko4xxhi/saRjjDHGbyzpGGOM8RtLOsYUIeKZ6fk3MwcbU1BY0jHGGOM3lnSMcYGI3O88b2W9iPzbmRzyvIiMdZ6/slhEwp260SKyypkYcoGIVHDKG4rIt84Em+tEpIHTfGlnAsntIjLLDzOcG+MzSzrG+JmI3ADcC3R0JoRMB+4DwvDcFd4U+A7PLAkAM4A/OxNDbvIqnwW87Uyw2QHInAqnFfAkEIXn+Uod832njPGRTYNjjP/1xDO54xqnE1ISz/T3GcAcp857/7+9+1WJKIjiOP79WQRRFIPFoC/iOxgUQRAfwCcQ1vfQuFnQLhgEk8lkNNlFUFBEjmEH/FOEhR03fD/p3mHuMCcM586EM8BZkkVgqaquWvsQOE2yAKxW1TlAVb0CtPFuquqhvd8C68D15MOS/mbSkfoLMKyqwx+NydGvfuPWqHr79vyB61xTxOM1qb9LYCvJCkCS5SRrjNbjVuuzC1xX1RPwmGSjte8BV+32zockm22M2SRzXaOQxuAfkNRZVd0lGQAXSWaAd+AAeGF02deA0XHbTvtkHzhuSeWer6rSe8BJqxD8Dmx3DEMai1WmpSmR5Lmq5v97HtIkebwmSerGnY4kqRt3OpKkbkw6kqRuTDqSpG5MOpKkbkw6kqRuPgGqIYLCb7J7SQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from keras.applications.mobilenetv2 import MobileNetV2#, VGG16\n",
    "from keras.optimizers import SGD\n",
    "from keras.models import Model\n",
    "from keras.layers import GlobalAveragePooling2D, Dense\n",
    "from keras import backend as K\n",
    "from keras.engine.network import Network\n",
    "\n",
    "input_shape = (96, 96, 3)\n",
    "classes = 10\n",
    "batchsize = 128\n",
    "#feature_out = 512 #secondary network out for VGG16\n",
    "feature_out = 1280 #secondary network out for MobileNet\n",
    "alpha = 0.5 #for MobileNetV2\n",
    "lambda_ = 0.1 #for compact loss\n",
    "\n",
    "#Loss function\n",
    "def original_loss(y_true, y_pred):\n",
    "    lc = 1/(classes*batchsize) * batchsize**2 * K.sum((y_pred -K.mean(y_pred,axis=0))**2,axis=[1]) / ((batchsize-1)**2)\n",
    "    return lc\n",
    "\n",
    "#Learning\n",
    "def train(x_target, x_ref, y_ref, epoch_num):\n",
    "\n",
    "    #Read VGG16, S network\n",
    "    print(\"Model build...\")\n",
    "    #mobile = VGG16(include_top=False, input_shape=input_shape, weights='imagenet')\n",
    "\n",
    "    #Read mobile net, S network\n",
    "    mobile = MobileNetV2(include_top=True, input_shape=input_shape, alpha=alpha, weights='imagenet')\n",
    "\n",
    "    #Delete last layer\n",
    "    mobile.layers.pop()\n",
    "\n",
    "    #Fixed weight\n",
    "    for layer in mobile.layers:\n",
    "        if layer.name == \"block_13_expand\": # \"block5_conv1\": for VGG16\n",
    "            break\n",
    "        else:\n",
    "            layer.trainable = False\n",
    "\n",
    "    model_t = Model(inputs=mobile.input,outputs=mobile.layers[-1].output)\n",
    "\n",
    "    #R network　S and Weight sharing\n",
    "    model_r = Network(inputs=model_t.input,\n",
    "                      outputs=model_t.output,\n",
    "                      name=\"shared_layer\")\n",
    "\n",
    "    #Apply a Fully Connected Layer to R\n",
    "    prediction = Dense(classes, activation='softmax')(model_t.output)\n",
    "    model_r = Model(inputs=model_r.input,outputs=prediction)\n",
    "\n",
    "    #Compile\n",
    "    optimizer = SGD(lr=5e-5, decay=0.00005)\n",
    "    model_r.compile(optimizer=optimizer, loss=\"categorical_crossentropy\")\n",
    "    model_t.compile(optimizer=optimizer, loss=original_loss)\n",
    "\n",
    "    model_t.summary()\n",
    "    model_r.summary()\n",
    "\n",
    "    print(\"x_target is\",x_target.shape[0],'samples')\n",
    "    print(\"x_ref is\",x_ref.shape[0],'samples')\n",
    "\n",
    "    ref_samples = np.arange(x_ref.shape[0])\n",
    "    loss, loss_c = [], []\n",
    "\n",
    "    print(\"training...\")\n",
    "\n",
    "    #Learning\n",
    "    for epochnumber in range(epoch_num):\n",
    "        x_r, y_r, lc, ld = [], [], [], []\n",
    "\n",
    "        #Shuffle target data\n",
    "        np.random.shuffle(x_target)\n",
    "\n",
    "        #Shuffle reference data\n",
    "        np.random.shuffle(ref_samples)\n",
    "        for i in range(len(x_target)):\n",
    "            x_r.append(x_ref[ref_samples[i]])\n",
    "            y_r.append(y_ref[ref_samples[i]])\n",
    "        x_r = np.array(x_r)\n",
    "        y_r = np.array(y_r)\n",
    "\n",
    "        for i in range(int(len(x_target) / batchsize)):\n",
    "\n",
    "            #Load data for batch size\n",
    "            batch_target = x_target[i*batchsize:i*batchsize+batchsize]\n",
    "            batch_ref = x_r[i*batchsize:i*batchsize+batchsize]\n",
    "            batch_y = y_r[i*batchsize:i*batchsize+batchsize]\n",
    "\n",
    "            #target data\n",
    "            #Get loss while learning\n",
    "            lc.append(model_t.train_on_batch(batch_target, np.zeros((batchsize, feature_out))))\n",
    "\n",
    "            #reference data\n",
    "            #Get loss while learning\n",
    "            ld.append(model_r.train_on_batch(batch_ref, batch_y))\n",
    "\n",
    "        loss.append(np.mean(ld))\n",
    "        loss_c.append(np.mean(lc))\n",
    "\n",
    "        if (epochnumber+1) % 5 == 0:\n",
    "            print(\"epoch:\",epochnumber+1)\n",
    "            print(\"Descriptive loss:\", loss[-1])\n",
    "            print(\"Compact loss\", loss_c[-1])\n",
    "\n",
    "    #Result graph\n",
    "    plt.plot(loss,label=\"Descriptive loss\")\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "    plt.plot(loss_c,label=\"Compact loss\")\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.legend()\n",
    "    plt.show()    \n",
    "\n",
    "    return model_t\n",
    "\n",
    "model = train(X_train_s, X_ref, y_ref, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/neighbors/lof.py:236: FutureWarning: default contamination parameter 0.1 will change in version 0.22 to \"auto\". This will change the predict method behavior.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XecVNX9//HXZxdpUgUhSAkroAYUVlnE+nPtgAULEbvYMCpRQ2LUFEW/Gk3E2Bs2ggEWS1RUElDjxhJFQAEpIm2FBRRDk8UFKZ/fH/fuOCxbZsvsMDPv5+OxD+beOffez5ld7mfOOfeea+6OiIgIQEaiAxARkd2HkoKIiEQoKYiISISSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoKkHDMrMLNiMysys6/NbLSZNSlV5ggz+7eZbTSzDWb2upl1L1WmmZk9YGbLwn0tDpdb122NROqOkoKkqtPcvQmQDRwM3FLyhpkdDkwBXgP2AbKAWcCHZrZvWKY+8A7QA+gHNAMOB9YAh8YraDOrF699i8RCSUFSmrt/DUwmSA4l/gKMcfcH3X2ju6919z8AHwMjwjIXA52AM919nrvvcPfV7v5/7j6prGOZWQ8ze8vM1prZN2b2u3D9aDO7M6pcrpkVRi0XmNlNZjYb2BS+fqnUvh80s4fC183N7BkzW2VmK8zsTjPLrOFHJQIoKUiKM7MOQH9gUbjcGDgCeLGM4i8AJ4avTwD+5e5FMR6nKfA28C+C1kdXgpZGrM4DTgFaAHnAgHCfhCf8c4BxYdnRwLbwGAcDJwFXVOFYIuVSUpBU9aqZbQSWA6uB28L1exH83a8qY5tVQMl4QatyypTnVOBrd7/P3TeHLZCpVdj+IXdf7u7F7v4V8ClwZvjeccD37v6xmbUFBgA3uPsmd18N3A+cW4VjiZRLSUFS1Rnu3hTIBQ7gx5P9OmAH0K6MbdoB/wtfrymnTHk6AourFWlgeanlcQStB4Dz+bGV8FNgD2CVma03s/XAk0CbGhxbJEJJQVKau/+HoLtlZLi8CfgI+HkZxc/hxy6ft4GTzWzPGA+1HNi3nPc2AY2jln9SVqilll8EcsPurzP5MSksB7YArd29RfjTzN17xBinSIWUFCQdPACcaGa9wuWbgUvM7Doza2pmLcOB4MOB28MyzxOcgF82swPMLMPMWpnZ78xsQBnHeANoZ2Y3mFmDcL99w/dmEowR7GVmPwFuqCxgd/8WyAeeA5a6+/xw/SqCK6fuCy+ZzTCzLmZ2TDU+F5FdKClIygtPsGOAW8PlD4CTgbMIxg2+IhiwPcrdF4ZlthAMNn8BvAV8B3xC0A21y1iBu28kGKQ+DfgaWAgcG779PMElrwUEJ/QJMYY+LoxhXKn1FwP1gXkE3WEvUbWuLpFymR6yIyIiJdRSEBGRCCUFERGJUFIQEZEIJQUREYlIusm3Wrdu7Z07d67Wtps2bWLPPWO97Dw1qM7pQXVODzWp84wZM/7n7ntXVi7pkkLnzp2ZPn16tbbNz88nNze3dgPazanO6UF1Tg81qbOZfRVLOXUfiYhIhJKCiIhEKCmIiEiEkoKIiEQoKYiISETckoKZPWtmq81sTjnvm5k9ZGaLzGy2mR0Sr1hERCQ28WwpjCZ44Hl5+gPdwp+hwONxjEVERGIQt/sU3P09M+tcQZGBBA9Pd+BjM2thZu3C+eJFRFLWuKnLeG3miipv12zHFuJ9a0Yib15rz86PICwM1+2SFMxsKEFrgrZt25Kfn1+tAxYVFVV722SlOqcH1bn25C/fykcrt9X6fqMtWLcDgP1bVq2zplGj7XH/PSfFHc3uPgoYBZCTk+PVvaNPd0CmB9U5PVS3zpV9S5+6dBMAfbP2qm5olerbAgZmt+f8vp2qtF1d/J4TmRRWEDzsvESHcJ2IpIHqdqGUWL++mMcXfFTl7aYuXQuUf9Lvm7VXtU7YqSKRSWEiMMzM8oC+wAaNJ4gkp+qc4Cs7OcdLup/0KxO3pGBm44FcoLWZFQK3AXsAuPsTwCRgALAI+B64NF6xiEj5avqNHap3gq/pyTnoSjm8WttK+eJ59dF5lbzvwLXxOr5IOqvKib42vrHr23fqSIqBZhEpW8nJv3T/elVO9DqhSzQlBZHdQHW7cEpO/qUvbdSJXqpLSUGkjpWVAKrbhVNy8t+neIn616VWKCmIxFnpJFBWAqj5oOuSmgUpElJSEKlFsbQC1LUjuzMlBZEKVLWvPx6tAJG6pKQgQvkn/6r29SsBSLJTUpC0VpIMyjv56yQv6UZJQVJSrN0+0clAJ38RJQVJEdFJYP36Yhas+xyovNtHyUBkZ0oKknR0hY9I/CgpyG6rKoO/0UlAE6WJVJ+SguyWxk1dxu9eKbsLSK0AkfhRUpDdRnTLoKQ18KczD9LJX6QOKSlInYulW0itAZHEUFKQuKnqDWFKBCKJp6QgtSaWid9KlnXyF9k9KSlIrShrYFgnf5Hko6QgVVJZl5AGhkWSm5KCVKqsq4LUJSSSmpQUZBcVjQ3o5C+S2pQUJKK8GUOVCETSh5JCmqps/iAlAZH0pKSQRiobG1AyEBElhRSXv3wrjz/5EaCxARGpnJJCChs3dRmj5/4ArFUiEJGYKCmkqOibyXTvgIjESkkhxZS+gmhIj/pKCCISMyWFFFHW5aQDs9uzT/GSBEcmIslESSEFlJ53KHrcID9fSUFEYqekkOQ0diAitSmuScHM+gEPApnA0+5+T6n3OwF/A1qEZW5290nxjCkV6AllIhIvcUsKZpYJPAqcCBQC08xsorvPiyr2B+AFd3/czLoDk4DO8YopFZTuKtJlpiJSm+LZUjgUWOTuSwDMLA8YCEQnBQeaha+bAyvjGE9KKGkhqGUgIvFg7h6fHZsNAvq5+xXh8kVAX3cfFlWmHTAFaAnsCZzg7jPK2NdQYChA27Zte+fl5VUrpqKiIpo0aVKtbXcH+cu3MnruD+zfMoNb+jaKaZtkr3N1qM7pQXWummOPPXaGu+dUVi7RA83nAaPd/T4zOxx43swOdPcd0YXcfRQwCiAnJ8dzc3OrdbD8/Hyqu22i7Dx+sAmAS3J7kBtjKyEZ61xTqnN6UJ3jI55JYQXQMWq5Q7gu2uVAPwB3/8jMGgKtgdVxjCtpaPxAROpaPJPCNKCbmWURJINzgfNLlVkGHA+MNrOfAQ2Bb+MYU9LQpaYikggZ8dqxu28DhgGTgfkEVxnNNbM7zOz0sNivgSvNbBYwHhji8RrkSCJKCCKSKHEdUwjvOZhUat2tUa/nAUfGM4ZkUnqqCiUEEalriR5oFsqft0gJQUTqmpJCglU0b5GISF1TUqhjpZ+NrK4iEdmdKCnUodKtgpJ/1ToQkd2FkkId0ACyiCQLJYU405iBiCQTJYU40v0GIpJs4nbzWrpTQhCRZKSkEAdKCCKSrGLqPjKz+kAnd18U53iSmgaURSTZVZoUzOwU4K9AfSDLzLKB29z9zHgHl0w0oCwiqSCWlsIdQF/gXQB3n2lmXeMaVZJRd5GIpIpYxhS2uvv6UuvSfibTaHpEpoikilhaCvPN7BwgI3w2wnXAx/ENK/n0zdpLCUFEkl4sLYVhQG9gB/APYAtwfTyDSibjpi6LDCyLiCS7WFoKJ7v7TcBNJSvM7CyCBJHWoscSBma3T3A0IiI1F0tL4Q9lrPt9bQeSbDS4LCKpqNyWgpmdDPQD2pvZX6PeakbQlZS2lBBEJFVV1H20GpgDbAbmRq3fCNwcz6B2d7raSERSVblJwd0/Az4zs7HuvrkOY9qtlQws62ojEUlFsQw0tzezu4DuQMOSle6+X9yi2o2VtBI0sCwiqSiWpDAauBMYCfQHLiUNb14rmddo3qrv1EoQkZQVy9VHjd19MoC7L3b3PxAkh7RSkhC6t2umVoKIpKxYWgpbzCwDWGxmvwBWAE3jG9buJXocYcJVhyc6HBGRuIklKfwK2JNgeou7gObAZfEManejcQQRSReVJgV3nxq+3AhcBGBmaXN21NVGIpJOKhxTMLM+ZnaGmbUOl3uY2RhgakXbpQpNYyEi6abcpGBmdwNjgQuAf5nZCIJnKswC0uJyVN2kJiLppqLuo4FAL3cvNrO9gOXAQe6+pG5CSyx1G4lIOqqo+2izuxcDuPta4Mt0SgjqNhKRdFRRS2FfMyuZHtsIns8cmS7b3c+qbOdm1g94EMgEnnb3e8oocw4wguCGuFnufn7s4deukhvUSp6PoG4jEUk3FSWFs0stP1KVHZtZJvAocCJQCEwzs4nuPi+qTDfgFuBId19nZm2qcozaFn3H8sDs9koIIpJ2KpoQ750a7vtQYFFJl5OZ5RGMU8yLKnMl8Ki7rwuPubqGx6yx7u2a6QY1EUlbsdy8Vl3tCQanSxQCfUuV2Q/AzD4k6GIa4e7/Kr0jMxsKDAVo27Yt+fn51QqoqKiowm3Xry8GqPb+d0eV1TkVqc7pQXWOj3gmhViP3w3IBToA75nZQe6+PrqQu48CRgHk5OR4bm5utQ6Wn59PeduOm7qMBes+p2/WXuTmpk5LoaI6pyrVOT2ozvERy4R4AJhZgyruewXQMWq5Q7guWiEw0d23uvtS4EuCJFHnNJWFiEgMScHMDjWzz4GF4XIvM3s4hn1PA7qZWZaZ1QfOBSaWKvMqQSuB8K7p/YA6v+xV9ySIiARiaSk8BJwKrAFw91nAsZVt5O7bgGHAZGA+8IK7zzWzO8zs9LDYZGCNmc0juFv6RndfU/Vq1IxaCSIigVjGFDLc/Sszi163PZadu/skYFKpdbdGvXZgePiTUGoliIjElhSWm9mhgIf3HvySoO9fRERSTCzdR1cTfJPvBHwDHBauExGRFBNLS2Gbu58b90hERCThYmkpTDOzSWZ2iZml1WM4RUTSTaVJwd27AHcCvYHPzexVM1PLQUQkBcV085q7/9fdrwMOAb4jePhOSii5R0FERGK7ea2JmV1gZq8DnwDfAkfEPbI6onsURER+FMtA8xzgdeAv7v5+nONJCN2jICISiCUp7OvuO+IeiYiIJFy5ScHM7nP3XwMvm5mXfj+WJ6+JiEhyqailMCH8t0pPXBMRkeRV0ZPXPglf/szdd0oMZjYMqOmT2UREZDcTyyWpl5Wx7vLaDkRERBKvojGFwQTPQMgys39EvdUUWF/2ViIikswqGlP4hOAZCh2AR6PWbwQ+i2dQIiKSGBWNKSwFlgJv1104IiKSSBV1H/3H3Y8xs3VA9CWpRvB8nL3iHp2IiNSpigaaSx652RrYO+qnZDnpad4jEZGdlZsUou5i7ghkuvt24HDgKmDPOogt7jTvkYjIzmK5JPVVgkdxdgGeA7oB4+IaVR3SvEciIj+KJSnscPetwFnAw+7+K0BfrUVEUlAsSWGbmf0cuAh4I1y3R/xCEhGRRIn1juZjCabOXmJmWcD4+IYlIiKJUOnU2e4+x8yuA7qa2QHAIne/K/6hiYhIXas0KZjZ0cDzwAqCexR+YmYXufuH8Q5ORETqViwP2bkfGODu8wDM7GcESSInnoGJiEjdi2VMoX5JQgBw9/lA/fiFJCIiiRJLS+FTM3sC+Hu4fAGaEE9EJCXFkhR+AVwH/DZcfh94OG4RiYhIwlSYFMzsIKAL8Iq7/6VuQhIRkUQpd0zBzH5HMMXFBcBbZlbWE9hERCSFVDTQfAHQ091/DvQBrq7qzs2sn5ktMLNFZnZzBeXONjM3M13RJCKSQBUlhS3uvgnA3b+tpOwuzCyT4Ilt/YHuwHlm1r2Mck2B64GpVdl/TWnabBGRXVU0prBv1LOZDegS/axmdz+rkn0fSnD38xIAM8sDBgLzSpX7P+DPwI1VCbymNG22iMiuKkoKZ5dafqSK+24PLI9aLgT6Rhcws0OAju7+ppmVmxTMbCgwFKBt27bk5+dXMZRAUVFRZNv164vZv2UG+xQvIT9/SbX2lwyi65wuVOf0oDrHR0XPaH4nngc2swzgr8CQysq6+yhgFEBOTo7n5uZW65j5+fmUbPv4go8AyM09vFr7ShbRdU4XqnN6UJ3jo0rjBFW0guCpbSU6hOtKNAUOBPLNrAA4DJiowWYRkcSJZ1KYBnQzsywzqw+cC0wsedPdN7h7a3fv7O6dgY+B0919ehxjEhGRCsScFMysQVV27O7bgGHAZGA+8IK7zzWzO8zs9KqFKSIidSGWqbMPBZ4BmgOdzKwXcIW7/7Kybd19EjCp1LpbyymbG0vAtaHkctS+WXvV1SFFRJJCLC2Fh4BTgTUA7j6L4ElsSUuXo4qIlC2WpJDh7l+VWrc9HsHUhehWwvl9OyU6HBGR3Uoss6QuD7uQPLxL+ZfAl/ENK37UShARKV8sLYWrgeFAJ+AbgktHqzwP0u5ErQQRkbJV2lJw99UEl5OKiEiKi+Xqo6cAL73e3YfGJSIREUmYWMYU3o563RA4k53nNBIRkRQRS/fRhOhlM3se+CBuEYmISMJUZ5qLLKBtbQciIiKJF8uYwjp+HFPIANYC5T5FTUREkleFScHMDOjFj7Ob7nD3XQadRUQkNVTYfRQmgEnuvj38UUIQEUlhsYwpzDSzg+MeiYiIJFy53UdmVi+c/vpgYJqZLQY2ETyv2d39kDqKUURE6khFYwqfAIcAevaBiEiaqCgpGIC7L66jWEREJMEqSgp7m9nw8t5097/GIR4REUmgipJCJtCEsMUgIiKpr6KksMrd76izSEREJOEquiRVLQQRkTRTUVI4vs6iqCP5y7cydenaRIchIrLbKjcpuHvKnT0/WrkN0KM4RUTKU51ZUpOaHsUpIlK+tEsKIiJSPiUFERGJUFIQEZEIJQUREYlQUhARkQglBRERiVBSEBGRiLgmBTPrZ2YLzGyRmd1cxvvDzWyemc02s3fM7KfxjEdERCoWt6RgZpnAo0B/oDtwnpl1L1XsMyDH3XsCLwF/iVc8IiJSuXi2FA4FFrn7Enf/AcgDBkYXcPd33f37cPFjoEMc4xERkUpUNHV2TbUHlkctFwJ9Kyh/OfDPst4ws6HAUIC2bduSn59frYC2b9/O+vXrq719MioqKkqr+oLqnC5U5/iIZ1KImZldCOQAx5T1vruPAkYB5OTkeG5ubrWOc/fUf9KiRQtycw+vZqTJJz8/n+p+XslKdU4PqnN8xDMprAA6Ri13CNftxMxOAH4PHOPuW+IYj4iIVCKeYwrTgG5mlmVm9YFzgYnRBczsYOBJ4HR3Xx3HWEREJAZxSwruvg0YBkwG5gMvuPtcM7vDzE4Pi91L8BzoF81spplNLGd3IiJSB+I6puDuk4BJpdbdGvX6hHgeX0REqkZ3NIuISISSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoKIiEQoKYiISISSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoKIiEQoKYiISISSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoKIiEQoKYiISISSgoiIRCgpiIhIRL1EByDpa+vWrRQWFrJ58+Za3W/z5s2ZP39+re5zd6c6p4dY6tywYUM6dOjAHnvsUa1jKClIwhQWFtK0aVM6d+6MmdXafjdu3EjTpk1rbX/JQHVOD5XV2d1Zs2YNhYWFZGVlVesY6j6ShNm8eTOtWrWq1YQgks7MjFatWtWo9a2kIAmlhCBSu2r6f0pJQUREIpQUJK1lZmaSnZ1Njx496NWrF/fddx87duyo9eP88MMP3HDDDXTt2pVu3boxcOBACgsLa7TPMWPGcOCBB3LQQQdx1FFHMXLkSACGDBnCSy+9VBths3LlSgYNGhRZPu+88+jZsyf3338/t956K2+//XaV91lQUMC4ceMiy9OnT+e6666rdozFxcUcc8wxbN++PbLugQceoGHDhmzYsCGybvTo0QwbNmynbXNzc5k+fToARUVFXHXVVXTp0oXevXuTm5vL1KlTqx0XBH381113HV27dqVnz558+umnZZabMGECPXv2pEePHtx0002R9Vu2bGHw4MF07dqVvn378tVXXwHw+eefM2TIkBrFVh4NNEtaa9SoETNnzgRg9erVnH/++Xz33XfcfvvttXqc3/3ud2zcuJEFCxaQmZnJc889x1lnncXUqVOr1dz/5z//yQMPPMCUKVPYZ599+N///scrr7xSqzED7LPPPpEE8/XXXzNt2jQWLVpUo32WJIXzzz8fgJycHHJycqq9v2effZazzjqLzMzMyLrx48fTp08f/vGPf3DppZfGtJ8rrriCrKwsFi5cSEZGBkuXLmXevHnVjguC39PChQtZuHAhU6dO5eqrr94l0axZs4Ybb7yRGTNmsPfee3PJJZfwzjvvcPzxx/PMM8/QsmVLFi1aRF5eHrfddhsvv/wyBx10EIWFhSxbtoxOnTrVKMbSlBRkt3D763OZt/K7WtnX9u3byczMpPs+zbjttB4xb9emTRtGjRpFnz59GDFiBDt27ODmm28mPz+fLVu2cO2113LVVVcBcO+99/LCCy+wZcsWzjzzTG6//XYKCgro168fvXv35tNPP6VHjx6MGTMGgOeee46lS5dGTlyXXnopzz77LP/+97/p0qUL/fv356ijjuK///0v7du357XXXqNRo0YsXryYa6+9lm+//ZbGjRvz1FNPccABB3D33XczcuRI9tlnHwAaNGjAlVdeuUud7rjjDl5//XWKi4s54ogjePLJJzEzHnroIZ544gnq1atH9+7dycvL4z//+Q/XX389EPRLv/fee6xZs4ZTTz2VOXPmcNJJJ7FixQqys7N5+OGHeeaZZzj11FMZNGgQ06ZN4/rrr2fTpk00aNCAd955hzVr1nDRRRexadMmAB555BGOOOIIbr75ZubPn092djaXXHIJBx98MCNHjuSNN95g7dq1XHbZZSxZsoTGjRszatQoevbsyYgRI1i2bBlLlixh2bJl3HDDDZGT/dixY3dqeSxevJiioiIee+wx7rrrrpiSwuLFi5k6dSpjx44lIyPoQMnKyqr2FTwlXnvtNS6++GLMjMMOO4z169ezatUq2rVrFymzZMkSunXrxt577w3ACSecwMsvv8zxxx/Pa6+9xogRIwAYNGgQ1157Le6OmXHaaaeRl5fHb3/72xrFWFradB+Nm7qMBetqv1tAUsu+++7L9u3bWb16Nc888wzNmzdn2rRpTJs2jaeeeoqlS5cyZcoUFi5cyCeffMLMmTOZMWMG7733HgALFizgmmuuYf78+TRr1ozHHnuMRYsW0alTJ5o1a7bTsXJycpg7dy4ACxcu5Nprr2Xu3Lm0aNGCl19+GYChQ4fy8MMPM2PGDEaOHMk111wDwJw5c+jdu3el9Rk2bBjTpk1jzpw5FBcX88YbbwBwzz338NlnnzF79myeeOIJAEaOHMmjjz7KzJkzef/992nUqNFO+5o4cSJdunRh5syZHH300ZH1P/zwA4MHD+bBBx9k1qxZvP322zRq1Ig2bdrw1ltv8emnnzJhwoRIF9E999zD0UcfzcyZM/nVr3610zFuu+02Dj74YGbPns2f/vQnLr744sh7X3zxBZMnT+aTTz7h9ttvZ+vWrfzwww8sWbKEzp07R8rl5eVx7rnncvTRR7NgwQK++eabSj+nuXPnkp2dvVNrozyDBw8mOzt7l5+SLwDRVqxYQceOHSPLHTp0YMWKFTuV6dq1KwsWLKCgoIBt27bx6quvsnz58l22r1evHs2aNWPNmjVA8Pfz/vvvVxpvVaVNS+G1mcEvYmB2+wRHImWpyjf6ytTW9etTpkxh9uzZke6TDRs2sHDhQqZMmcKUKVM4+OCDgaAveuHChXTq1ImOHTty5JFHAnDhhRfy0EMPcdJJJ1V6rKysLLKzswHo3bs3BQUFFBUV8d///pef//znkXJbtmypUh3effdd/vKXv/D999+zdu1aevTowWmnnUbPnj254IILOOOMMzjjjDMAOPLIIxk+fDgXXHABZ511Fh06dIjpGAsWLKBdu3b06dMHIJL8Nm3axLBhw5g5cyaZmZl8+eWXle7rgw8+iCTE4447jjVr1vDdd0EL8pRTTqFBgwY0aNCANm3asHr1ajZv3kyLFi122sf48eN55ZVXyMjI4Oyzz+bFF19k2LBh5XbTVbX7bsKECVUqX5mWLVvy+OOPM3jwYDIyMjjiiCNYvHhxpdu1adOGlStX1mosEOekYGb9gAeBTOBpd7+n1PsNgDFAb2ANMNjdC+IVz/4tMzi/b+32v0lqWbJkCZmZmbRp0wZ35+GHH+bkk0/eqczkyZO55ZZbIl1JJQoKCnY5wZgZXbp0YdmyZbskqxkzZnDqqacCQfdPiczMTIqLi9mxYwctWrSIjHlE69GjBzNmzOC4444rty6bN2/mmmuuYfr06XTs2JERI0ZErl9/8803ee+993j99de56667+Pzzz7n55ps55ZRTmDRpEkceeSSTJ0+mYcOGMX5yu7r//vtp27Yts2bNYseOHTXaF+z6GW3bto1GjRrtdE3+559/zsKFCznxxBOBoBWTlZXFsGHDaNWqFevWrdtpn2vXrqV169a0aNGCWbNmRboeKzJ48GAWLFiwy/rhw4fv1LIBaN++feRbPwQ3bLZvv+sX09NOO43TTjsNgFGjRkViKNm+Q4cObNu2je+++45WrVoBwe+3dGuuNsSt+8jMMoFHgf5Ad+A8M+teqtjlwDp37wrcD/w5XvGIVObbb7/lF7/4ReRb5cknn8zjjz/O1q1bAfjyyy/ZtGkTJ598Ms8++yxFRUVA0MRfvXo1AMuWLeOjjz4CYNy4cRx11FHsueeeXHLJJQwfPjxyhcyYMWP4/vvvKzypN2vWjKysLF588UUguJJl1qxZANxyyy3ceOONfP3110Bw8nv66ad32r7kZNm6dWuKiooiLZ4dO3awfPlyjj32WP785z+zYcMGioqKWLx4MQcddBA33XQTffr04Ysvvojpc9t///1ZtWoV06ZNA4KW2rZt29iwYQPt2rUjIyOD559/PlL3pk2bsnHjxjL3dfTRRzN27FgA8vPzad269S7dbtFatmzJ9u3bI3UdP348I0aMoKCggIKCAlauXMnKlSv56quv6NOnDx9++GHkM5s+fTpbtmyhY8eOdOnShZycHG677TbcHQiS/JtvvrnnzWPmAAALqklEQVTLMSdMmMDMmTN3+SmdEABOP/10xowZg7vz8ccf07x5853GE0qU/P2sW7eOxx57jCuuuCKy/d/+9jcAXnrpJY455pjIF48vv/ySAw88sNzPprri2VI4FFjk7ksAzCwPGAhED+cPBEaEr18CHjEz85LfikicFRcXk52dzdatW6lXrx4XXXQRw4cPB4KrUQoKCjjkkENwd/bee29effVVTjrpJObPn8/hhx8OQJMmTfj73/9OZmYm+++/P48++iiXXXYZ3bt35+qrrwbg7rvv5je/+Q377bcfGRkZHHDAAbzyyiuVdl2MHTuWq6++mjvvvJOtW7dy7rnn0qtXLwYMGMA333zDCSecgLvj7pETSYkWLVpw5ZVXcuCBB/KTn/wk0r2zfft2LrzwQjZs2BC5ZLJFixb88Y9/5N133yUjI4MePXrQv39/Vq1aVelnWL9+fSZMmMAvf/lLiouLadSoEW+//TbXXHMNZ599NmPGjKFfv37sueeeAPTs2ZPMzEx69erFkCFDIt1wACNGjOCyyy6jZ8+eNG7cOHJCrMhJJ53EBx98wAknnEBeXh6TJk3a6f0zzzyTvLw8brrpJh588EEGDBjAjh07aNKkCePHj48MLD/99NP8+te/pmvXrjRq1IjWrVtz7733Vnr8igwYMIBJkybRtWtXGjduzHPPPRd5Lzs7O9IKvP766yMJ/9Zbb2W//fYD4PLLL+eiiy6ia9eu7LXXXjsl/nfffZdTTjmlRvGVxeJ1/jWzQUA/d78iXL4I6Ovuw6LKzAnLFIbLi8My/yu1r6HAUIC2bdv2zsvLq3I8Y+dvYevWrQzp2aS6VUpKRUVFNGmye9a5efPmdO3atdb3G0sXQDx89dVXnHPOOTW+tr06ElXnRCqp88yZM3n00Ud56qmnEh1S3JXUecuWLfTv358pU6ZQr96u3+0XLVq00z0aAMcee+wMd6/02t+kGGh291HAKICcnBzPzc2t8j5yc4PmaHW2TWa7c53nz58flwnNEjVRWpMmTcjIyEjIsdN5crijjz6ahQsX0rhx45RPjCV1/vrrr7n33ntp2bJlmeUaNmy4UwusKuKZFFYAHaOWO4TryipTaGb1gOYEA84iSadz587MmTMn0WGkpcsuuyzRIdSpbt260a1bt7jsO573KUwDuplZlpnVB84FJpYqMxG4JHw9CPi3xhPSi37dIrWrpv+n4pYU3H0bMAyYDMwHXnD3uWZ2h5mdHhZ7BmhlZouA4cDN8YpHdj8NGzZkzZo1SgwitaTkeQo1ufw3rmMK7j4JmFRq3a1RrzcDPy+9naSHDh06UFhYyLffflur+928eXONr4lPNqpzeoilziVPXquupBholtS0xx571HhumbLk5+dXe5AtWanO6aEu6pw2cx+JiEjllBRERCRCSUFERCLidkdzvJjZt8BX1dy8NfC/SkulFtU5PajO6aEmdf6pu+9dWaGkSwo1YWbTY7nNO5WozulBdU4PdVFndR+JiEiEkoKIiESkW1IYlegAEkB1Tg+qc3qIe53TakxBREQqlm4tBRERqYCSgoiIRKRkUjCzfma2wMwWmdkuM6+aWQMzmxC+P9XMOtd9lLUrhjoPN7N5ZjbbzN4xs58mIs7aVFmdo8qdbWZuZkl/+WIsdTazc8Lf9VwzG1fXMda2GP62O5nZu2b2Wfj3PSARcdYWM3vWzFaHT6Ys630zs4fCz2O2mR1SqwGUPN81VX6ATGAxsC9QH5gFdC9V5hrgifD1ucCERMddB3U+Fmgcvr46HeoclmsKvAd8DOQkOu46+D13Az4DWobLbRIddx3UeRRwdfi6O1CQ6LhrWOf/BxwCzCnn/QHAPwEDDgOm1ubxU7GlcCiwyN2XuPsPQB4wsFSZgUDJE8FfAo63yp6gvnurtM7u/q67fx8ufkzwJLxkFsvvGeD/gD8Dm+syuDiJpc5XAo+6+zoAd19dxzHWtljq7ECz8HVzYGUdxlfr3P09YG0FRQYCYzzwMdDCzNrV1vFTMSm0B5ZHLReG68os48HDgDYAreokuviIpc7RLif4ppHMKq1z2Kzu6O5v1mVgcRTL73k/YD8z+9DMPjazfnUWXXzEUucRwIVmVkjw/JZf1k1oCVPV/+9VoucppBkzuxDIAY5JdCzxZGYZwF+BIQkOpa7VI+hCyiVoDb5nZge5+/qERhVf5wGj3f0+MzsceN7MDnT3HYkOLBmlYkthBdAxarlDuK7MMmZWj6DJuaZOoouPWOqMmZ0A/B443d231FFs8VJZnZsCBwL5ZlZA0Pc6MckHm2P5PRcCE919q7svBb4kSBLJKpY6Xw68AODuHwENCSaOS1Ux/X+vrlRMCtOAbmaWZWb1CQaSJ5YqMxG4JHw9CPi3hyM4SarSOpvZwcCTBAkh2fuZoZI6u/sGd2/t7p3dvTPBOMrp7j49MeHWilj+tl8laCVgZq0JupOW1GWQtSyWOi8Djgcws58RJIXafcbr7mUicHF4FdJhwAZ3X1VbO0+57iN332Zmw4DJBFcuPOvuc83sDmC6u08EniFoYi4iGNA5N3ER11yMdb4XaAK8GI6pL3P30xMWdA3FWOeUEmOdJwMnmdk8YDtwo7snbSs4xjr/GnjKzH5FMOg8JJm/5JnZeILE3jocJ7kN2APA3Z8gGDcZACwCvgcurdXjJ/FnJyIitSwVu49ERKSalBRERCRCSUFERCKUFEREJEJJQUREIpQUZLdjZtvNbGbUT+cKynYubzbJKh4zP5yJc1Y4RcT+1djHL8zs4vD1EDPbJ+q9p82sey3HOc3MsmPY5gYza1zTY0t6UFKQ3VGxu2dH/RTU0XEvcPdeBJMl3lvVjd39CXcfEy4OAfaJeu8Kd59XK1H+GOdjxBbnDYCSgsRESUGSQtgieN/MPg1/jiijTA8z+yRsXcw2s27h+guj1j9pZpmVHO49oGu47fHhPP2fh/PcNwjX32M/Pp9iZLhuhJn9xswGEcwvNTY8ZqPwG35O2JqInMjDFsUj1YzzI6ImQjOzx81sugXPUbg9XHcdQXJ618zeDdedZGYfhZ/ji2bWpJLjSBpRUpDdUaOorqNXwnWrgRPd/RBgMPBQGdv9AnjQ3bMJTsqF4bQHg4Ejw/XbgQsqOf5pwOdm1hAYDQx294MIZgC42sxaAWcCPdy9J3Bn9Mbu/hIwneAbfba7F0e9/XK4bYnBQF414+xHMK1Fid+7ew7QEzjGzHq6+0MEU0kf6+7HhlNf/AE4IfwspwPDKzmOpJGUm+ZCUkJxeGKMtgfwSNiHvp1gTp/SPgJ+b2YdgH+4+0IzOx7oDUwLp/doRJBgyjLWzIqBAoLpl/cHlrr7l+H7fwOuBR4heD7DM2b2BvBGrBVz92/NbEk4Z81C4ADgw3C/VYmzPsG0JdGf0zlmNpTg/3U7ggfOzC617WHh+g/D49Qn+NxEACUFSR6/Ar4BehG0cHd5aI67jzOzqcApwCQzu4rg6VR/c/dbYjjGBdET5pnZXmUVCufjOZRgErZBwDDguCrUJQ84B/gCeMXd3YIzdMxxAjMIxhMeBs4ysyzgN0Afd19nZqMJJoYrzYC33P28KsQraUTdR5IsmgOrwjnyLyKYHG0nZrYvsCTsMnmNoBvlHWCQmbUJy+xlsT+fegHQ2cy6hssXAf8J++Cbu/skgmTVq4xtNxJM312WVwiennUeQYKgqnGGE779ETjMzA4gePLYJmCDmbUF+pcTy8fAkSV1MrM9zaysVpekKSUFSRaPAZeY2SyCLpdNZZQ5B5hjZjMJnqUwJrzi5w/AFDObDbxF0LVSKXffTDAD5Ytm9jmwA3iC4AT7Rri/Dyi7T3408ETJQHOp/a4D5gM/dfdPwnVVjjMcq7iPYCbUWQTPZv4CGEfQJVViFPAvM3vX3b8luDJqfHicjwg+TxFAs6SKiEgUtRRERCRCSUFERCKUFEREJEJJQUREIpQUREQkQklBREQilBRERCTi/wN+Ytuy/ZjtrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "train = model.predict(X_train_s)\n",
    "test_s = model.predict(X_test_s)\n",
    "test_b = model.predict(X_test_b)\n",
    "\n",
    "train = train.reshape((len(X_train_s),-1))\n",
    "test_s = test_s.reshape((len(X_test_s),-1))\n",
    "test_b = test_b.reshape((len(X_test_b),-1))\n",
    "\n",
    "#Convert to 0-1\n",
    "ms = MinMaxScaler()\n",
    "train = ms.fit_transform(train)\n",
    "test_s = ms.transform(test_s)\n",
    "test_b = ms.transform(test_b)\n",
    "\n",
    "# fit the model\n",
    "clf = LocalOutlierFactor(n_neighbors=5)\n",
    "y_pred = clf.fit(train)\n",
    "\n",
    "#Abnormal score\n",
    "Z1 = -clf._decision_function(test_s)\n",
    "Z2 = -clf._decision_function(test_b)\n",
    "\n",
    "#Drawing of ROC curve\n",
    "y_true = np.zeros(len(test_s)+len(test_b))\n",
    "y_true[len(test_s):] = 1 #0:Normal, 1：Abnormal\n",
    "\n",
    "#Calculate FPR, TPR(, Threshould)\n",
    "fpr, tpr, _ = metrics.roc_curve(y_true, np.hstack((Z1, Z2)))\n",
    "\n",
    "#AUC\n",
    "auc = metrics.auc(fpr, tpr)\n",
    "\n",
    "#Plot the ROC curve\n",
    "plt.plot(fpr, tpr, label='DeepOneClassification(AUC = %.2f)'%auc)\n",
    "plt.legend()\n",
    "plt.title('ROC curve')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}