TransferLearning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import dataset_ops\n",
    "import numpy as np  # noqa\n",
    "import matplotlib.pyplot as plt  # noqa\n",
    "from model_helper import make_model\n",
    "from metrics import F1, Precision, Recall, soft_dice_loss, remove_clutter_one_sample, ClassPrecision, ClassRecall\n",
    "import datetime\n",
    "import cuda\n",
    "import pandas_format  # noqa\n",
    "from pathlib import Path\n",
    "try:\n",
    "    from tqdm import notebook as tqdm\n",
    "except ImportError:\n",
    "    tqdm = None\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext tensorboard\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "cuda.initialize()\n",
    "\n",
    "inputs = ('SpeedFts', 'Pitch', 'Roll', 'Yaw', 'current_altitude', )\n",
    "outputs= ('elev', 'ai', 'rdr', 'throttle', 'Flaps')\n",
    "\n",
    "mp_dataset_manager = dataset_ops.MicroPilotTestsManager(dataset_dir=Path('h5'), runs_filename='runs.hdf')\n",
    "mp_all_runs = mp_dataset_manager.get_all_available_tests()\n",
    "\n",
    "mp_selected_runs = mp_all_runs.loc[(mp_all_runs['Test Length'] > 200) & (mp_all_runs['Test Length'] < 20000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data..."
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f897aacd199744439a361625537897d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=888.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "done\n",
      "({'signals': TensorSpec(shape=(None, 18000, 10), dtype=tf.float32, name=None), 'mask': TensorSpec(shape=(None, 18000, 1), dtype=tf.float32, name=None)}, TensorSpec(shape=(None, 18000, 25), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "import transfer_learning\n",
    "def create_prec_recall_f1(tolerance):\n",
    "    prec = Precision(name=f'prec_{tolerance}', tolerance=tolerance)\n",
    "    recl = Recall(name=f'recall_{tolerance}', tolerance=tolerance)\n",
    "\n",
    "    return [\n",
    "        # ClassPrecision(),\n",
    "        # ClassRecall(),\n",
    "        prec,\n",
    "        recl,\n",
    "        F1(prec, recl),\n",
    "    ]\n",
    "\n",
    "evaluation_metrics = create_prec_recall_f1(5) # + create_prec_recall_f1(15) + create_prec_recall_f1(5)\n",
    "\n",
    "_, mp_train_dataset, mp_test_dataset, mp_validation_dataset = dataset_ops.load_and_split(\n",
    "    mp_dataset_manager, mp_selected_runs, inputs+outputs, batch_size=25,\n",
    "    split_ratio=(18, 1, 1), max_length=18000\n",
    ")\n",
    "print(mp_train_dataset.element_spec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## First, train mp model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "signals (InputLayer)            [(None, 18000, 10)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "mask (InputLayer)               [(None, 18000, 1)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "mask_stealing_layer (MaskSteali (None, 18000, 10)    0           signals[0][0]                    \n",
      "                                                                 mask[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "conv_3 (Conv1D)                 (None, 18000, 64)    1984        mask_stealing_layer[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv_5 (Conv1D)                 (None, 18000, 64)    20544       conv_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv_10 (Conv1D)                (None, 18000, 64)    41024       conv_5[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv_15 (Conv1D)                (None, 18000, 64)    61504       conv_10[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv_20 (Conv1D)                (None, 18000, 64)    81984       conv_15[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       (None, 18000, 128)   74496       conv_20[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "gru_1 (GRU)                     (None, 18000, 128)   99072       gru[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 18000, 128)   16512       gru_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)         (None, 18000, 128)   0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 18000, 25)    3225        leaky_re_lu[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 400,345\n",
      "Trainable params: 400,345\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(lr=3e-5)\n",
    "\n",
    "# MP:\n",
    "mp_model_params = dict(\n",
    "    inputs=inputs, outputs=outputs,\n",
    "    input_length=18000,\n",
    "    n_states=mp_dataset_manager.count_states(),\n",
    "    convs=[(64, 3), (64, 5), (64, 10), (64, 15), (64, 20)],\n",
    "    grus=[128, 128],\n",
    "    skip_denses=0\n",
    ")\n",
    "\n",
    "model = make_model(**mp_model_params)\n",
    "model.compile(loss=soft_dice_loss, optimizer=optimizer, metrics=evaluation_metrics)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# load_model = 'models/paparazzi-20200810-163206-500.h5'\n",
    "# load_model = 'models/paparazzi-paparazzi-20200812-184311-500.h5'\n",
    "# load_model = 'models/paparazzi-paparazzi-20200812-184311-500.h5'\n",
    "load_model = 'models/TL-20200820-211542-500.h5'\n",
    "# load_model = None\n",
    "if load_model:\n",
    "    model.load_weights(load_model)\n",
    "else:\n",
    "    epochs = 500\n",
    "    # epochs = 5\n",
    "\n",
    "    training_start_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "    log_dir=\"logs/fit/\" + training_start_time\n",
    "    tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "\n",
    "    history = model.fit(mp_train_dataset,\n",
    "                        epochs=epochs,\n",
    "                        validation_data=mp_validation_dataset,\n",
    "                        callbacks=[\n",
    "                            tensorboard_callback,\n",
    "                            # tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5),\n",
    "                            tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10),\n",
    "                        ])\n",
    "    model.save(f'models/TL-{training_start_time}-{epochs}.h5')\n",
    "    # tf.keras.utils.plot_model(model, show_shapes=True, to_file=f'models/paparazzi-{training_start_time}-{epochs}.png')\n",
    "    # history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "prec_5            0.59\n",
       "recall_5          0.76\n",
       "prec_15           0.72\n",
       "recall_15         0.89\n",
       "prec_25           0.81\n",
       "recall_25         0.93\n",
       "class_precision   0.88\n",
       "class_recall      0.95\n",
       "dtype: float16"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mp_evaluation = transfer_learning.evaluate_model(model, mp_validation_dataset)\n",
    "mp_evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Now do some Paparazzi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data..."
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba3d7817ce944e26a82e09f96bb53819",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=379.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "done\n",
      "({'signals': TensorSpec(shape=(None, 18000, 10), dtype=tf.float32, name=None), 'mask': TensorSpec(shape=(None, 18000, 1), dtype=tf.float32, name=None)}, TensorSpec(shape=(None, 18000, 20), dtype=tf.float32, name=None))\n"
     ]
    }
   ],
   "source": [
    "pp_optimizer = tf.keras.optimizers.Adam(lr=3e-4)\n",
    "pp_dataset_manager = dataset_ops.PaparazziTestManager(dataset_dir=Path('pprz_h5'), runs_filename='pprz_runs.hdf')\n",
    "pp_selected_runs = pp_dataset_manager.get_all_available_tests().sample(frac=1, axis=1)#, random_state=55)\n",
    "_, pp_train_dataset, pp_test_dataset, pp_validation_dataset = dataset_ops.load_and_split(\n",
    "    pp_dataset_manager, pp_selected_runs, inputs+outputs, batch_size=25,\n",
    "    split_ratio=(10, 1, 2), max_length=18000\n",
    ")\n",
    "pp_train_dataset: tf.data.Dataset\n",
    "pp_test_dataset: tf.data.Dataset\n",
    "pp_validation_dataset: tf.data.Dataset\n",
    "print(pp_train_dataset.element_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results = dict()\n",
    "training_histories = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# model_path = Path('models/transfer_intermediates/freeze_after_16')\n",
    "model_path = Path('models/transfer_intermediates/')\n",
    "model_path.mkdir(parents=True, exist_ok=True)\n",
    "unfreeze_schedule = {\n",
    "    0: 3,\n",
    "    # 0: None,\n",
    "    16: 1\n",
    "}\n",
    "if unfreeze_schedule and unfreeze_schedule.get(0, None) is not None:\n",
    "    layers_set = (unfreeze_schedule[0], )\n",
    "else:\n",
    "    layers_set = (1, 3)\n",
    "    unfreeze_schedule = None\n",
    "\n",
    "# for X in (5, 10, 15, 20, 25):\n",
    "# for i, X in enumerate((20, 15, )*10, start=20):\n",
    "for i, X in enumerate([*range(5, 80,5)]*10, start=380):\n",
    "    for layers in layers_set:\n",
    "        if unfreeze_schedule:\n",
    "            col_name = f'layer_dynamic__x_{X}__trial_{i}'\n",
    "        else:\n",
    "            col_name = f'layer_{layers}__x_{X}__trial_{i}'\n",
    "        if (model_path / (col_name+'.h5')).exists():\n",
    "            new_model, evaluation_result = transfer_learning.load_model(\n",
    "                file_name=model_path / (col_name+'.h5'),\n",
    "                validation_dataset=pp_validation_dataset,\n",
    "            )\n",
    "        else:\n",
    "            # shuffled_training = tf.data.Dataset.from_tensors(pp_train_dataset_concrete)\n",
    "            new_model, evaluation_result, training_history = transfer_learning.train_and_test_transfer_model(\n",
    "                X=X,\n",
    "                optimizer=pp_optimizer,\n",
    "                training_dataset=pp_train_dataset.shuffle(20*X),\n",
    "                validation_dataset=pp_validation_dataset,\n",
    "                loss=soft_dice_loss,\n",
    "                metrics_=evaluation_metrics,\n",
    "                plot=False,\n",
    "                summary=False,\n",
    "                epochs=30,\n",
    "                unfreeze_schedule=unfreeze_schedule,\n",
    "                # model args:\n",
    "                original_model=model,\n",
    "                original_model_params=mp_model_params,\n",
    "                layers_to_drop=layers,\n",
    "                n_states=pp_dataset_manager.count_states(),\n",
    "            )\n",
    "            training_history['layers'] = layers\n",
    "            training_history['X'] = X\n",
    "            training_histories[col_name] = training_history\n",
    "\n",
    "            training_history.to_csv(model_path / (col_name + '_history.csv'))\n",
    "            new_model.save(model_path / (col_name+'.h5'))\n",
    "\n",
    "        evaluation_result.rename(col_name, inplace=True)\n",
    "        evaluation_result['layers'] = layers\n",
    "        evaluation_result['X'] = X\n",
    "        results[col_name] = evaluation_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame(results).T\n",
    "results_df.loc[results_df.index.str.contains('dynamic'), 'layers'] = 'dynamic'\n",
    "# results_df.to_hdf('dynamic_results.h5', key='dynamic_results')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "with pandas_format.PandasFloatFormatter('{:,.2f}'):\n",
    "    # display(pd.concat(results, axis=1).T)\n",
    "    display(results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# results_df.groupby(['layers', 'X']).mean()\n",
    "results_df.groupby(['layers', 'X']).median()\n",
    "results_df.groupby(['layers', 'X']).median().to_csv('tl_median.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df.sort_values(['layers', 'X']).set_index(['layers', 'X'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df.to_csv(model_path/'summary.csv')\n",
    "results_df.to_hdf(model_path/'results.h5', 'results')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# pd.concat(training_histories, keys=[x.name for x in results])\n",
    "concatenated_histories = pd.concat(training_histories)\n",
    "for idx in concatenated_histories.index.get_level_values(0).unique():\n",
    "    concatenated_histories.loc[idx].drop(['X', 'layers'], axis=1).plot()\n",
    "    plt.title(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results = dict()\n",
    "files = [file for file in Path('models/transfer_intermediates/').iterdir()\n",
    "    if file.suffix == '.h5']\n",
    "\n",
    "new_model = transfer_learning.prepare_transfer_model(\n",
    "    original_model=model,\n",
    "    original_model_params=mp_model_params,\n",
    "    layers_to_drop=1,\n",
    "    n_states=pp_dataset_manager.count_states(),\n",
    ")\n",
    "new_model.compile(loss=soft_dice_loss, optimizer=pp_optimizer, metrics=evaluation_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    colname = file.name[:-3]\n",
    "    parts = colname.split('_')\n",
    "    new_model.load_weights(str(file))\n",
    "    evaluation_results = transfer_learning.evaluate_model(new_model, pp_validation_dataset)\n",
    "    # new_model, evaluation_results = transfer_learning.load_model(file, pp_validation_dataset)\n",
    "    results[colname] = evaluation_results\n",
    "    results[colname]['layers'] = 300 if parts[1] == 'dynamic' else int(parts[1])\n",
    "    results[colname]['X'] = int(parts[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    colname = file.name[:-3]\n",
    "    parts = colname.split('_')\n",
    "    layers = parts[1] if parts[1] == 'dynamic' else int(parts[1])\n",
    "    X = int(parts[4])\n",
    "    results[colname] = results[colname].astype('float64')\n",
    "    results[colname]['layers'] = 300 if parts[1] == 'dynamic' else int(parts[1])\n",
    "    results[colname]['X'] = int(parts[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Transfer Learning Baseline 1\n",
    "Training the same architecture and hyper params from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "training_histories, results = dict(), dict()\n",
    "\n",
    "model_path = Path('models/transfer_baselines/1/')\n",
    "model_path.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "for i, X in enumerate([*range(5, 80, 5)]*10):\n",
    "    col_name = f'baseline_1__x_{X}__trial_{i}'\n",
    "    model_file_path = model_path / (col_name+'.h5')\n",
    "    if model_file_path.exists():\n",
    "        new_model, evaluation_result = transfer_learning.load_model(\n",
    "            file_name=model_file_path,\n",
    "            validation_dataset=pp_validation_dataset,\n",
    "        )\n",
    "    else:\n",
    "        # shuffled_training = tf.data.Dataset.from_tensors(pp_train_dataset_concrete)\n",
    "        new_model, evaluation_result, training_history = transfer_learning.train_and_test_transfer_model(\n",
    "            X=X,\n",
    "            optimizer=pp_optimizer,\n",
    "            training_dataset=pp_train_dataset.shuffle(20*X),\n",
    "            validation_dataset=pp_validation_dataset,\n",
    "            loss=soft_dice_loss,\n",
    "            metrics_=evaluation_metrics,\n",
    "            plot=False,\n",
    "            summary=False,\n",
    "            epochs=30,\n",
    "            unfreeze_schedule=None,\n",
    "            # model args:\n",
    "            original_model=model,\n",
    "            original_model_params=mp_model_params,\n",
    "            layers_to_drop=-1,\n",
    "            n_states=pp_dataset_manager.count_states(),\n",
    "        )\n",
    "        training_history['X'] = X\n",
    "        training_histories[col_name] = training_history\n",
    "\n",
    "        training_history.to_csv(model_path / (col_name + '_history.csv'))\n",
    "        new_model.save(model_file_path)\n",
    "\n",
    "    evaluation_result.rename(col_name, inplace=True)\n",
    "    evaluation_result['X'] = X\n",
    "    results[col_name] = evaluation_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df = pd.DataFrame(results).T\n",
    "results_df.to_hdf(model_path / 'baseline1_results.h5', 'baseline1')\n",
    "results_df.to_csv(model_path / 'baseline1_results.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "results_df.groupby('X').median().to_csv(model_path / 'baseline1_results_summary.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Second baseline\n",
    "Using the same mp model with the same weights without fine-tuning anything on Paparazzi's data\n",
    "\n",
    "1. Generate the mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training dataset size: tf.Tensor(292, shape=(), dtype=int32)\n",
      "expected output (292, 18000) input_length (292,)\n",
      "pred 14 [ 2  3  4  5  6  7  8  9 10 11 12 13 14 24]\n",
      "xout 19 [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 17 18 19]\n",
      "cont (19, 14) [[    28     56   3038  33630   2784    384     40    708   1141      0\n",
      "     174    928   1324  24158]\n",
      " [   124    182   3653   5636    944      4    126   3536    160      6\n",
      "      23     60    322     43]\n",
      " [    14     18   9476  25893   1557     67     26   1076     85      6\n",
      "       6     34   1132   4811]\n",
      " [     0      0  32307 123181   6683    992     99   1157   1079      0\n",
      "     277    210  12469  25201]\n",
      " [     0      0   1472   5328    273    115     21     73     70      0\n",
      "      17      0    818   1314]\n",
      " [     0      0  21203  85277   3799    606     34    506    445      0\n",
      "      99    150   6960  12311]\n",
      " [     0      0   2345  11210    649     73      0    174     41      0\n",
      "       6     70    786   4158]\n",
      " [     0      0   2148   9155    378     46      0    103    248      0\n",
      "      29      0    993   1433]\n",
      " [     0      0   1711   9033   1127     87     74     28     65      0\n",
      "      52      0   1645   4590]\n",
      " [     0      0   2663  11581    427     56     12     19    142      0\n",
      "      25      4   1442   5112]\n",
      " [     0      0     25   1920      0      0      0      0      0      0\n",
      "       0      0    319   1057]\n",
      " [     0      0      0    279     71      0      0      0      0      0\n",
      "       0      0      0      0]\n",
      " [     0      0      0    231     52      0      0     77     74      0\n",
      "       0     38      0    280]\n",
      " [     0      0     88    245     36     28      0     22     38      0\n",
      "       5     53     44    368]\n",
      " [     0      0     32    253     39     20      0     30     36      0\n",
      "       0     39      7    646]\n",
      " [     0      0      0    944      0      0      0      0      0      0\n",
      "       0      0      0    249]\n",
      " [     0      0      0      8      8      0      0      0      0      0\n",
      "       0      0      0     19]\n",
      " [     0      0     58    216     44      0      0      0      0      0\n",
      "       0      0      0    263]\n",
      " [     0      0      0      0      0      0      0      0      0      0\n",
      "       0      0      0     10]]\n",
      "argm (19,) [ 3  3  3  3  3  3  3  3  3  3  3  3 13 13 13  3 13 13 13]\n",
      "{0: 5, 1: 5, 2: 5, 3: 5, 4: 5, 5: 5, 6: 5, 7: 5, 8: 5, 9: 5, 10: 5, 11: 5, 12: 24, 13: 24, 14: 24, 15: 5, 17: 24, 18: 24, 19: 24}\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAN2UlEQVR4nO3dfaye9V3H8fdn7Rh2D+Wh2tSWeTB2U4IxkBNkmZlznYbBQkkkhMVlHWlsMuecY9FV/QOj/0DUPSULWMdcZyYDcZFGpgvpIKiRZqdDeSgqlfHQWiiTcTQ2bqN8/eO+tMeu3bl7P56e3/uVnNzX8/XNL+d87uv+Xff1O6kqJEltecW0C5AkTZ7hL0kNMvwlqUGGvyQ1yPCXpAatnHYBAGvWrKmZmZlplyFJp5W9e/d+o6q+f5B9l0T4z8zMMDc3N+0yJOm0kuSpQfe120eSGmT4S1KDDH9JapDhL0kNMvwlqUGGvyQ1yPCXpAYZ/pLUIMNfkhq0JJ7wffjgPDPb7x5o3ydvvGLE1UjS8ueVvyQ1yPCXpAYZ/pLUIMNfkhq0aPgn+UySw0keWbDsnCT3JHm8ez27W54kn0yyP8lDSS4eZ/GSpMH0c+X/WeCy45ZtB3ZX1UZgdzcP8A5gY/ezDbh5NGVKkkZp0fCvqvuBF45bvBnY2U3vBK5asPxz1fMAcFaSdaMqVpI0GoP2+a+tqkPd9LPA2m56PfDMgu0OdMu+S5JtSeaSzB09Mj9gGZKkQQx9w7eqCqgB9ttRVbNVNbti1ephy5AknYJBw/+5/+3O6V4Pd8sPAuct2G5Dt0yStIQMGv67gC3d9BbgrgXL39N96+dSYH5B95AkaYlYdGyfJLcBbwXWJDkA3ADcCNyRZCvwFHBNt/mXgMuB/cAR4Lox1Pz/DDomUD8cN0jScrVo+FfVu06yatMJti3g/cMWJUkaL5/wlaQGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQYa/JDXI8JekBhn+ktQgw1+SGmT4S1KDDH9JapDhL0kNMvwlqUGGvyQ1yPCXpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0aKvyTfCjJo0keSXJbkjOTnJ9kT5L9SW5PcsaoipUkjcbA4Z9kPfArwGxVXQisAK4FbgI+VlU/AnwT2DqKQiVJozNst89K4PuSrARWAYeAtwF3dut3AlcNeQ5J0ogNHP5VdRD4feBpeqE/D+wFXqyql7rNDgDrhy1SkjRaw3T7nA1sBs4HfhB4NXDZKey/LclckrmjR+YHLUOSNIBhun3eDny9qp6vqu8AXwTeDJzVdQMBbAAOnmjnqtpRVbNVNbti1eohypAknaphwv9p4NIkq5IE2ATsA+4Fru622QLcNVyJkqRRG6bPfw+9G7tfAx7ujrUD+AhwfZL9wLnArSOoU5I0QisX3+TkquoG4IbjFj8BXDLMcSVJ4+UTvpLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQYa/JDXI8JekBhn+ktQgw1+SGmT4S1KDDH9JapDhL0kNMvwlqUGGvyQ1yPCXpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNWio8E9yVpI7k/xTkseSvCnJOUnuSfJ493r2qIqVJI3GsFf+nwD+uqp+FPgJ4DFgO7C7qjYCu7t5SdISMnD4J1kNvAW4FaCqvl1VLwKbgZ3dZjuBq4YtUpI0WsNc+Z8PPA/8cZIHk3w6yauBtVV1qNvmWWDtiXZOsi3JXJK5o0fmhyhDknSqhgn/lcDFwM1VdRHwXxzXxVNVBdSJdq6qHVU1W1WzK1atHqIMSdKpGib8DwAHqmpPN38nvTeD55KsA+heDw9XoiRp1AYO/6p6FngmyRu7RZuAfcAuYEu3bAtw11AVSpJGbuWQ+38A+HySM4AngOvovaHckWQr8BRwzZDnkCSN2FDhX1X/AMyeYNWmYY4rSRovn/CVpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQYa/JDXI8JekBhn+ktQgw1+SGmT4S1KDDH9JapDhL0kNMvwlqUGGvyQ1yPCXpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBQ4d/khVJHkzyl938+Un2JNmf5PYkZwxfpiRplFaO4BgfBB4DXtfN3wR8rKq+kOQWYCtw8wjOM3Ez2+8e27GfvPGKsR1bkhYz1JV/kg3AFcCnu/kAbwPu7DbZCVw1zDkkSaM3bLfPx4FfB17u5s8FXqyql7r5A8D6E+2YZFuSuSRzR4/MD1mGJOlUDBz+Sd4JHK6qvYPsX1U7qmq2qmZXrFo9aBmSpAEM0+f/ZuDKJJcDZ9Lr8/8EcFaSld3V/wbg4PBlSpJGaeAr/6r6jaraUFUzwLXAV6rqF4B7gau7zbYAdw1dpSRppMbxPf+PANcn2U/vHsCtYziHJGkIo/iqJ1V1H3BfN/0EcMkojitJGg+f8JWkBhn+ktQgw1+SGmT4S1KDDH9JapDhL0kNMvwlqUGGvyQ1yPCXpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQYa/JDXI8JekBhn+ktQgw1+SGmT4S1KDDH9JapDhL0kNMvwlqUEDh3+S85Lcm2RfkkeTfLBbfk6Se5I83r2ePbpyJUmjMMyV/0vAh6vqAuBS4P1JLgC2A7uraiOwu5uXJC0hA4d/VR2qqq910/8JPAasBzYDO7vNdgJXDVukJGm0RtLnn2QGuAjYA6ytqkPdqmeBtSfZZ1uSuSRzR4/Mj6IMSVKfhg7/JK8B/hz41ar6j4XrqqqAOtF+VbWjqmaranbFqtXDliFJOgVDhX+SV9IL/s9X1Re7xc8lWdetXwccHq5ESdKoDfNtnwC3Ao9V1UcXrNoFbOmmtwB3DV6eJGkc0uuZGWDH5KeAvwEeBl7uFv8mvX7/O4DXA08B11TVC9/rWK9at7HWbfn4QHXouz154xXTLkHSBCTZW1Wzg+y7ctCTVtXfAjnJ6k2DHleSNH4+4StJDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQYa/JDXI8JekBg08sJuWrpntd4/1+I4aKp3+vPKXpAYZ/pLUIMNfkhpk+EtSgwx/SWqQ4S9JDTL8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lqkOEvSQ0y/CWpQY7nr1M2zv8X4P8KkCbDK39JapBX/lpSxv1fyMbFTyw63Yzlyj/JZUn+Ocn+JNvHcQ5J0uBGfuWfZAXwKeBngQPAV5Psqqp9oz6XtFR4H0Snm3Fc+V8C7K+qJ6rq28AXgM1jOI8kaUDj6PNfDzyzYP4A8JPHb5RkG7Ctm/3WUze985Ex1HI6WgN8Y9pFLBG2BZCbANtiIdvimDcOuuPUbvhW1Q5gB0CSuaqanVYtS4ltcYxtcYxtcYxtcUySuUH3HUe3z0HgvAXzG7plkqQlYhzh/1VgY5Lzk5wBXAvsGsN5JEkDGnm3T1W9lOSXgS8DK4DPVNWji+y2Y9R1nMZsi2Nsi2Nsi2Nsi2MGbotU1SgLkSSdBhzeQZIaZPhLUoMmGv6LDfuQ5FVJbu/W70kyM8n6JqmPtrg+yb4kDyXZneSHplHnJPQ7HEiSn09SSZbt1/z6aYsk13S/G48m+dNJ1zgpffyNvD7JvUke7P5OLp9GneOW5DNJDic54bNQ6flk104PJbm4rwNX1UR+6N38/Vfgh4EzgH8ELjhum18CbummrwVun1R9k/zpsy1+BljVTb+v5bbotnstcD/wADA77bqn+HuxEXgQOLub/4Fp1z3FttgBvK+bvgB4ctp1j6kt3gJcDDxykvWXA38FBLgU2NPPcSd55d/PsA+bgZ3d9J3ApiSZYI2TsmhbVNW9VXWkm32A3vMSy1G/w4H8LnAT8N+TLG7C+mmLXwQ+VVXfBKiqwxOucVL6aYsCXtdNrwb+bYL1TUxV3Q+88D022Qx8rnoeAM5Ksm6x404y/E807MP6k21TVS8B88C5E6lusvppi4W20ntnX44WbYvuY+x5VXV6jvfcv35+L94AvCHJ3yV5IMllE6tusvppi98G3p3kAPAl4AOTKW3JOdU8ARzPf8lL8m5gFvjpadcyDUleAXwUeO+US1kqVtLr+nkrvU+D9yf58ap6capVTce7gM9W1R8keRPwJ0kurKqXp13Y6WCSV/79DPvwf9skWUnvo9y/T6S6yeprCIwkbwd+C7iyqr41odombbG2eC1wIXBfkifp9WnuWqY3ffv5vTgA7Kqq71TV14F/ofdmsNz00xZbgTsAqurvgTPpDfrWmoGG1Jlk+Pcz7MMuYEs3fTXwleruaCwzi7ZFkouAP6QX/Mu1XxcWaYuqmq+qNVU1U1Uz9O5/XFlVAw9otYT18zfyF/Su+kmyhl430BOTLHJC+mmLp4FNAEl+jF74Pz/RKpeGXcB7um/9XArMV9WhxXaaWLdPnWTYhyS/A8xV1S7gVnof3fbTu8Fx7aTqm6Q+2+L3gNcAf9bd8366qq6cWtFj0mdbNKHPtvgy8HNJ9gFHgV+rqmX36bjPtvgw8EdJPkTv5u97l+PFYpLb6L3hr+nub9wAvBKgqm6hd7/jcmA/cAS4rq/jLsO2kiQtwid8JalBhr8kNcjwl6QGGf6S1CDDX5IaZPhLUoMMf0lq0P8AqZlmGr7+h7MAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ict520c/.virtualenvs/mp/lib/python3.6/site-packages/ipykernel_launcher.py:48: RuntimeWarning: divide by zero encountered in log\n"
     ]
    },
    {
     "data": {
      "text/plain": "<Figure size 288x390.857 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAFRCAYAAAB0YhgKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAPs0lEQVR4nO3df5BV9XnH8c/D/nBhkd8rIKAyCUFJimiuRlNLtEYHqA3pTJLW6pRM7dDO6CS2tgmWmSR1Jp2m/qCdSSeWRKNNDE6baHVSJShNaztWk7VxCKAVVCwLCItEYRfI/nr6x71MkeEuzH3unrvO837NMLt7zz77Pfvrzbl395w1dxeAvMY0egcANBYRAJIjAkByRABIjggAyREBILlREQEzW2Jm/2Nm281sVcFrzzGzH5vZVjPbYmafL3L94/ajycx+ZmY/bMDak8zs+2b2spm9ZGaXF7z+H1c+9pvNbJ2ZtY3weveb2T4z23zcbVPM7Ckz21Z5Orng9e+sfPw3mdmjZjZppNY/UcMjYGZNkv5O0lJJCyRdb2YLCtyFAUm3ufsCSZdJurng9Y/5vKSXGrCuJP2tpPXufr6kC4vcDzObJelzkkru/iFJTZJ+Z4SXfUDSkhNuWyVpo7vPk7Sx8nKR6z8l6UPuvlDSK5JuH8H136XhEZB0qaTt7v6au/dJeljS8qIWd/c97v7flecPqfwNMKuo9SXJzGZL+g1J3ypy3craEyUtlnSfJLl7n7u/XfBuNEsaa2bNksZJ2j2Si7n7M5IOnHDzckkPVp5/UNIni1zf3Te4+0DlxeckzR6p9U80GiIwS9LO417uUsHfhMeY2XmSLpL0fMFL/42kL0gaKnhdSZorqVvStyt3R75lZu1FLe7uuyTdJel/Je2R9I67byhq/eNMd/c9lefflDS9AftwzO9LerKoxUZDBEYFMxsv6QeSbnX3gwWue52kfe7+QlFrnqBZ0sWSvuHuF0nq1cgeCr9L5b73cpVjdLakdjO7saj1T8bLv0vfkN+nN7PVKt9FfaioNUdDBHZJmnPcy7MrtxXGzFpUDsBD7v5IkWtL+lVJnzCzHSrfFfp1M/tuget3Sepy92NHP99XOQpF+bik19292937JT0i6aMFrn/MXjObKUmVp/uK3gEz+6yk6yTd4AWe1DMaIvBTSfPMbK6Ztar8oNDjRS1uZqby/eGX3P2eotY9xt1vd/fZ7n6eyu/7v7p7Yf8Tuvubknaa2fzKTVdL2lrU+irfDbjMzMZVPhdXqzEPkD4uaUXl+RWSHitycTNbovJdwk+4++Ei15a7N/yfpGUqPyL6qqTVBa99hcqHfpskvVj5t6xBH4crJf2wAesuktRZ+Rj8s6TJBa//F5JelrRZ0ncknTHC661T+fGHfpWPhG6SNFXlnwpsk/S0pCkFr79d5cfGjn0N3lvUx98qOwUgqdFwdwBAAxEBIDkiACRHBIDkRlUEzGwl6+dcP/P73uj1R1UEJDX0E8H6DV0/8/ve0PVHWwQAFKzQ3xNobWn3trbqp0n39feqtaX6uStDzbFmDU4dHH77wcNqmjCu6vazx74TWn/3kYmh9f1IU2j9lp7hP9f9fb1qaa3+8e+baKH1Nab6+oOHetV05vDnLf3KtBmh5bfu3Ft128CRXjWPHX79wehVDmyY97+nV03jh19/xvjav/7e2nVUPb/oP+knsLnmt1qDtrZJuqR0c83zR6e2htY/+Hux84K+tOBfQvNf2XJdaL5/U+w6EzOe6w/N77w2FqGh9uEjfCqdf/CF0PzCW9eE5g/Oj+2/t8Xmv3h57ScWfu1T1c9P4+4AkBwRAJIjAkByoQg08gKhAOqj5giMgguEAqiDyJFAQy8QCqA+IhE4rQuEmtlKM+s0s86+/t7AcgBGwog/MOjua9295O6l4X4RCEBjRCLQ8AuEAoiLRKChFwgFUB81/9qwuw+Y2S2SfqTyn46639231G3PABQidO6Auz8h6Yk67QuABuA3BoHkiACQXKGnEg+0jdGB+bWflD0wNnY+e2tz7FTOtTsXh+Z79owPzX/gy8+G5vevvDw0P2XeW6H5gfXTQvNLJt0Umj/rg3ND8+N3xy4ocOD82KnwD/9gWe1rd22vuo0jASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiu0OsJtPQMaPp/1n5O+tAZLaH1Z16/JzR/xcRtofmvdtV+Prgk9XzmstD8lK1HQvPbL479afTpbw+F5qOat7wemm9tnxeaPzIn+P4v7q599pWBqps4EgCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkCr2ewLwPztL6zq8WueS7PLy9FJr/zPh3QvOPnbc7NP/zq84JzXc8PzY0rzHVz0k/HXuviJ1P/5N/uC80f8HqNaH5pr7QuJaXfhKaXzCu9q+fN1p7q27jSABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiOCADJEQEgOSIAJEcEgOSIAJAcEQCSK/R6Ao226tlPhea/OWdfaH7303NC81O7PTQ/eWtPaL75aHtoXrHdl1bGxs954heh+R/97I7Q/Hlfvys0/1j7oppn3+x5teo2jgSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiOCADJEQEguVTXE/jcJRtD8787YUto/rqjnw3NH9g6LTQ/YUdraH7/hRaat8HQeNhbF00KzV94y5rQ/MLf3hGaP6f9QM2z/zjuSNVtHAkAyREBIDkiACRHBIDkQg8MmtkOSYckDUoacPdSPXYKQHHq8dOBq9x9fx3eDoAG4O4AkFw0Ai5pg5m9YGYnvSq8ma00s04z6+zu7g4uB6DeohG4wt0vlrRU0s1mtvjEV3D3te5ecvdSR0dHcDkA9RaKgLvvqjzdJ+lRSZfWY6cAFKfmCJhZu5mdeex5SddK2lyvHQNQjMhPB6ZLetTMjr2d77n7+rrsFYDC1BwBd39N0oV13BcADcCPCIHkiACQXKrrCdy75ddC86/MnRGaP/Dy1ND8WS94aP7fNqwKzS+8NXY+/aRt/aH5qGmPxK4H8fbSBaH5TS/ODc13z2+vefbwQEvVbRwJAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHKpricwNGSh+Z6B1tB8y6FYc1t6BkLzUS2HYtcz2L+w+jntRei7+P2h+efW3RaaP/fv7wzN794zuebZ/v7q3+ocCQDJEQEgOSIAJEcEgOSIAJAcEQCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByqa4nMDjQ1ND1j87sj81Pjn26PnLD3aH5Sbt/GZpv7T0jNB/VN6mxX+5jxsc+/7POervm2bdaql+LgiMBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiOCADJEQEgOSIAJEcEgOSIAJAcEQCSIwJAcqlOJR47ri80P6E5eCrtvtiHu7V3KDR/5s+7Q/NDE8aG5o8saAvNR7V1xz7/14z5dGh+6NsfDs23NA3WPGtWfRtHAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiOCADJEQEgOSIAJEcEgOSIAJBcqusJnH13S2h+/c0XhOZfW/0noXnEPP0fqxu6/vlfWhOaf+2XM2qe7Tta/VudIwEgOSIAJEcEgOSIAJAcEQCSO2UEzOx+M9tnZpuPu22KmT1lZtsqTyeP7G4CGCmncyTwgKQlJ9y2StJGd58naWPlZQDvQaeMgLs/I+nACTcvl/Rg5fkHJX2yzvsFoCC1PiYw3d33VJ5/U9L0aq9oZivNrNPMOru7Y3/8AkD9hR8YdHeX5MNsX+vuJXcvdXR0RJcDUGe1RmCvmc2UpMrTffXbJQBFqjUCj0taUXl+haTH6rM7AIp2Oj8iXCfpvyTNN7MuM7tJ0l9JusbMtkn6eOVlAO9BpzyL0N2vr7Lp6jrvC4AG4DcGgeSs/OB+MUqlknd2dha2Xr1dsDp2PvjE14dC889977bQPPIysxfcvXSybRwJAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHKnvLIQ/t+Ygdh889HY9QQa7SM33h17A8FLVzz/UGOvp/CxpV8Lzf/7k18MzUeuZ9E2Y/aHq23jSABIjggAyREBIDkiACRHBIDkiACQHBEAkiMCQHJEAEiOCADJEQEgOSIAJEcEgOSIAJAcEQCSS3U9gUtW3BN7AzMtNN58OHY9gauv+svQ/MYf/3lofsJrh0PzfRNbQ/NR1469MTQ/sGxRaH7xb/51aH7okpaaZ32Y/+45EgCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkUl1PYPyuvtB879lnhOZ9TOx6BE29/aH5qMG22JeLeZ12pEYbjnw3NP+xZbHrAYz96auh+f6rPlDzLNcTAFAVEQCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRn7sWd5F0qlbyzs7Ow9ert3HvvDM2/8Ud/Vqc9QUYLbl9T8+zrD9yjI3t2nvSCFhwJAMkRASA5IgAkRwSA5E4ZATO738z2mdnm4277ipntMrMXK/+WjexuAhgpp3Mk8ICkJSe5fY27L6r8e6K+uwWgKKeMgLs/I+lAAfsCoAEijwncYmabKncXJtdtjwAUqtYIfEPS+yQtkrRH0t3VXtHMVppZp5l1dnd317gcgJFSUwTcfa+7D7r7kKRvSrp0mNdd6+4ldy91dHTUup8ARkhNETCzmce9+FuSNld7XQCj2yn/uJyZrZN0paRpZtYl6cuSrjSzRZJc0g5JfziC+whgBJ0yAu5+/Uluvm8E9gVAA/Abg0ByRABILvYH55NZcMcbofmld9wSmn9y99dD83hvm/30OzXPdh0crLqNIwEgOSIAJEcEgOSIAJAcEQCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABILtX1BD766btC8wdveF9ofsrL/aF55NZ1zcSaZ/u7mqpu40gASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIjAkByRABIjggAyREBIDkiACRHBIDkiACQHBEAkkt1PYFn/+lPG70LQM3OXbez5tldB/qqbuNIAEiOCADJEQEgOSIAJEcEgOSIAJAcEQCSIwJAckQASI4IAMkRASA5IgAkRwSA5IgAkBwRAJIzdy9uMbNuSW8M8yrTJO0vaHdYf3Stn/l9L2L9c92942QbCo3AqZhZp7uXWD/f+pnf90avz90BIDkiACQ32iKwlvXTrp/5fW/o+qPqMQEAxRttRwIACkYEgOSIAJAcEQCSIwJAcv8HKPb8knGCrIIAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sklearn.metrics\n",
    "\n",
    "training_dataset = pp_train_dataset\n",
    "\n",
    "expected_output = np.concatenate([*training_dataset.map(lambda X, y: tf.argmax(y, axis=-1)).as_numpy_iterator()])\n",
    "input_length = np.concatenate([*training_dataset.map(lambda X, y: tf.argmin(tf.squeeze(X['mask']), axis=-1)).as_numpy_iterator()])\n",
    "\n",
    "evaluation_result = transfer_learning.evaluate_model(model, training_dataset)\n",
    "\n",
    "model_prediction = model.predict(training_dataset)\n",
    "model_prediction = tf.math.argmax(model_prediction, axis=-1)\n",
    "\n",
    "print('training dataset size:', training_dataset.unbatch().reduce(0, lambda a, b: a+1))\n",
    "print('expected output', expected_output.shape, 'input_length', input_length.shape)\n",
    "\n",
    "# input_length[] = length\n",
    "# expected_output[batch#, t] = state_id\n",
    "ari_scores = []\n",
    "x_outputs = []\n",
    "predictions = []\n",
    "for _x_output, _prediction, l in zip(expected_output, model_prediction, input_length):\n",
    "    x_output = _x_output[:l]\n",
    "    prediction = _prediction[:l]\n",
    "\n",
    "    x_outputs.append(x_output)\n",
    "    predictions.append(prediction)\n",
    "\n",
    "    ari = sklearn.metrics.adjusted_rand_score(x_output, prediction)\n",
    "    ari_scores.append(ari)\n",
    "\n",
    "predictions = tf.concat(predictions, axis=0).numpy()\n",
    "x_outputs = tf.concat(x_outputs, axis=0).numpy()\n",
    "ari_scores = np.array(ari_scores)\n",
    "plt.hist(ari_scores)\n",
    "plt.xlim([0, 1])\n",
    "plt.show()\n",
    "\n",
    "def create_mappings(predictions, expected_outputs):\n",
    "    from sklearn.metrics.cluster import contingency_matrix\n",
    "\n",
    "    expected_outputs_u, expected_outputs_idx = np.unique(expected_outputs, return_inverse=True)\n",
    "    predictions_u, predictions_idx = np.unique(predictions, return_inverse=True)\n",
    "    contingency = contingency_matrix(expected_outputs, predictions, sparse=False)\n",
    "    # print('pred', len(predictions_u), predictions_u)\n",
    "    # print('xout', len(expected_outputs_u), expected_outputs_u)\n",
    "    # print('cont', contingency.shape, contingency)\n",
    "    # print('argm', np.argmax(contingency, axis=1).shape, np.argmax(contingency, axis=1))\n",
    "    plt.matshow(np.log(contingency))\n",
    "    plt.show()\n",
    "    mapping = {expected_outputs_u[i]: predictions_u[j] for i, j in enumerate(np.argmax(contingency, axis=1))}\n",
    "    # print(mapping)\n",
    "    return mapping\n",
    "\n",
    "mapping = create_mappings(predictions, x_outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "2. make predictions and evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "HBox(children=(FloatProgress(value=0.0, max=10.0), HTML(value='')))",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "894d212a9cd94035a72a4841d416d0e0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "results = dict()\n",
    "\n",
    "model_path = Path('models/transfer_baselines/2/')\n",
    "model_path.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "def _map(inp_tensor):\n",
    "    return tf.map_fn(lambda old: mapping.get(int(old), int(old)), inp_tensor)\n",
    "\n",
    "def _clip(_):\n",
    "    t, l = _\n",
    "    # return tf.slice(t, 0, l)\n",
    "    return t[:l]\n",
    "\n",
    "# for i, X in enumerate([*range(5, 80, 5)]*10):\n",
    "for i in tqdm.tqdm(range(10)):\n",
    "    col_name = f'baseline_2__trial_{i}'\n",
    "    training_dataset = pp_validation_dataset.shuffle(1600)\n",
    "\n",
    "    expected_output = np.concatenate([*training_dataset.map(lambda X, y: tf.argmax(y, axis=-1)).as_numpy_iterator()])\n",
    "    input_length = np.concatenate([*training_dataset.map(lambda X, y: tf.argmin(tf.squeeze(X['mask']), axis=-1)).as_numpy_iterator()])\n",
    "\n",
    "    evaluation_result = transfer_learning.evaluate_model(\n",
    "        model,\n",
    "        training_dataset,\n",
    "        transfer_learning.metrics_reporting[:-2]\n",
    "    )\n",
    "\n",
    "    # Classification prediction happens after applying the mapping\n",
    "    model_prediction = tf.math.argmax(model.predict(training_dataset), axis=-1)\n",
    "    for metric in transfer_learning.metrics_reporting[-2:]:\n",
    "        evaluation_result[metric.name] = 0\n",
    "    evaluation_results_dict = defaultdict(list)\n",
    "\n",
    "    for x_output, prediction in zip(\n",
    "                  map(_clip, zip(expected_output, input_length)),\n",
    "        map(_map, map(_clip, zip(model_prediction, input_length))),\n",
    "    ):\n",
    "        x_output = tf.one_hot(tf.reshape(x_output, [1, -1]), depth=25)\n",
    "        prediction = tf.one_hot(tf.reshape(prediction, [1, -1]), depth=25)\n",
    "        for metric in transfer_learning.metrics_reporting[-2:]:\n",
    "            metric.reset_states()\n",
    "            metric.update_state(x_output, prediction)\n",
    "            evaluation_results_dict[metric.name].append(metric.result())\n",
    "\n",
    "    for metric in transfer_learning.metrics_reporting[-2:]:\n",
    "        evaluation_result[metric.name] = np.mean(evaluation_results_dict[metric.name])\n",
    "    evaluation_results_dict.clear()\n",
    "\n",
    "    evaluation_result.rename(col_name, inplace=True)\n",
    "    results[col_name] = evaluation_result\n",
    "\n",
    "results_df = pd.DataFrame(results).T\n",
    "results_df.to_csv(model_path / 'results.csv')\n",
    "results_df.median().to_csv(model_path / 'results-median.csv')\n",
    "results_df.mean().to_csv(model_path / 'results-mean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "ins, ground_truth = training_dataset\n",
    "prediction = model.predict_on_batch(ins)\n",
    "\n",
    "ground_truth = tf.squeeze(ground_truth)\n",
    "\n",
    "for metric in metrics_reporting:\n",
    "    metric.reset_states()\n",
    "    metric.update_state(ground_truth, prediction)\n",
    "\n",
    "prediction = tf.math.argmax(prediction, axis=-1)\n",
    "prediction = tf.map_fn(lambda old: mapping[old], prediction)\n",
    "no_clutter = tf.map_fn(remove_clutter_one_sample, prediction)\n",
    "ground_truth = tf.math.argmax(ground_truth, axis=-1)\n",
    "\n",
    "mask = tf.squeeze(ins['mask'], axis=-1)\n",
    "run_length = tf.math.minimum(tf.argmin(mask, axis=-1), N)\n",
    "\n",
    "max_run_length_in_batch = int(tf.math.reduce_max(run_length))\n",
    "\n",
    "# results.append(\n",
    "#     [set_name] + [float(metric.result()) for metric in metrics_reporting]\n",
    "# )\n",
    "\n",
    "for prednc, truth, idx in zip(no_clutter, ground_truth, run_length):\n",
    "    truth, prednc = truth[:idx], prednc[:idx]\n",
    "\n",
    "    concat = tf.stack((prednc, truth), axis=0)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "ins, ground_truth = training_dataset\n",
    "prediction = model.predict_on_batch(ins)\n",
    "\n",
    "ground_truth = tf.squeeze(ground_truth)\n",
    "\n",
    "for metric in metrics_reporting:\n",
    "    metric.reset_states()\n",
    "    metric.update_state(ground_truth, prediction)\n",
    "\n",
    "prediction = tf.math.argmax(prediction, axis=-1)\n",
    "prediction = tf.map_fn(lambda old: mapping[old], prediction)\n",
    "no_clutter = tf.map_fn(remove_clutter_one_sample, prediction)\n",
    "ground_truth = tf.math.argmax(ground_truth, axis=-1)\n",
    "\n",
    "mask = tf.squeeze(ins['mask'], axis=-1)\n",
    "run_length = tf.math.minimum(tf.argmin(mask, axis=-1), N)\n",
    "\n",
    "max_run_length_in_batch = int(tf.math.reduce_max(run_length))\n",
    "\n",
    "# results.append(\n",
    "#     [set_name] + [float(metric.result()) for metric in metrics_reporting]\n",
    "# )\n",
    "\n",
    "for prednc, truth, idx in zip(no_clutter, ground_truth, run_length):\n",
    "    truth, prednc = truth[:idx], prednc[:idx]\n",
    "\n",
    "    concat = tf.stack((prednc, truth), axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "ins, ground_truth = training_dataset\n",
    "prediction = model.predict_on_batch(ins)\n",
    "\n",
    "ground_truth = tf.squeeze(ground_truth)\n",
    "\n",
    "for metric in metrics_reporting:\n",
    "    metric.reset_states()\n",
    "    metric.update_state(ground_truth, prediction)\n",
    "\n",
    "prediction = tf.math.argmax(prediction, axis=-1)\n",
    "prediction = tf.map_fn(lambda old: mapping[old], prediction)\n",
    "no_clutter = tf.map_fn(remove_clutter_one_sample, prediction)\n",
    "ground_truth = tf.math.argmax(ground_truth, axis=-1)\n",
    "\n",
    "mask = tf.squeeze(ins['mask'], axis=-1)\n",
    "run_length = tf.math.minimum(tf.argmin(mask, axis=-1), N)\n",
    "\n",
    "max_run_length_in_batch = int(tf.math.reduce_max(run_length))\n",
    "\n",
    "# results.append(\n",
    "#     [set_name] + [float(metric.result()) for metric in metrics_reporting]\n",
    "# )\n",
    "\n",
    "for prednc, truth, idx in zip(no_clutter, ground_truth, run_length):\n",
    "    truth, prednc = truth[:idx], prednc[:idx]\n",
    "\n",
    "    concat = tf.stack((prednc, truth), axis=0)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit ('mp': venv)",
   "language": "python",
   "name": "python36964bitmpvenvc0be145795914919ad5cb1b44742ad1c"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}