cloud-rn34-fpn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T14:37:24.238089Z",
     "iopub.status.busy": "2020-11-03T14:37:24.237063Z",
     "iopub.status.idle": "2020-11-03T14:41:43.267949Z",
     "shell.execute_reply": "2020-11-03T14:41:43.268868Z"
    },
    "papermill": {
     "duration": 259.06189,
     "end_time": "2020-11-03T14:41:43.269112",
     "exception": false,
     "start_time": "2020-11-03T14:37:24.207222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pip/_internal/commands/install.py:235: UserWarning: Disabling all use of wheels due to the use of --build-option / --global-option / --install-option.\r\n",
      "  cmdoptions.check_install_build_global(options)\r\n",
      "Processing /kaggle/input/apex-325f5a0/apex-master\r\n",
      "Skipping wheel build for apex, due to binaries being disabled for it.\r\n",
      "Installing collected packages: apex\r\n",
      "    Running setup.py install for apex ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\r\n",
      "\u001b[?25hSuccessfully installed apex-0.1\r\n"
     ]
    }
   ],
   "source": [
    "!pip install -q git+https://github.com/qubvel/segmentation_models.pytorch\n",
    "!pip install -q -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/apex-325f5a0/apex-master/\n",
    "from apex import amp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.046483,
     "end_time": "2020-11-03T14:41:43.361249",
     "exception": false,
     "start_time": "2020-11-03T14:41:43.314766",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Importing and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:43.481338Z",
     "iopub.status.busy": "2020-11-03T14:41:43.479926Z",
     "iopub.status.idle": "2020-11-03T14:41:47.333222Z",
     "shell.execute_reply": "2020-11-03T14:41:47.332664Z"
    },
    "papermill": {
     "duration": 3.925853,
     "end_time": "2020-11-03T14:41:47.333376",
     "exception": false,
     "start_time": "2020-11-03T14:41:43.407523",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pytorch version: 1.6.0\n",
      "albumentations version: 0.4.6\n",
      "segmentation_models_pytorch version: 0.1.2\n"
     ]
    }
   ],
   "source": [
    "import os, gc, time, json, copy\n",
    "import pickle, random, itertools, collections\n",
    "import tqdm\n",
    "from functools import partial\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import cv2\n",
    "\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n",
    "\n",
    "import albumentations as albu\n",
    "from albumentations import pytorch as AT\n",
    "\n",
    "import segmentation_models_pytorch as smp\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "print(\"pytorch version:\", torch.__version__)\n",
    "print(\"albumentations version:\", albu.__version__)\n",
    "print(\"segmentation_models_pytorch version:\", smp.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:47.738011Z",
     "iopub.status.busy": "2020-11-03T14:41:47.737221Z",
     "iopub.status.idle": "2020-11-03T14:41:47.742015Z",
     "shell.execute_reply": "2020-11-03T14:41:47.741492Z"
    },
    "papermill": {
     "duration": 0.375684,
     "end_time": "2020-11-03T14:41:47.742126",
     "exception": false,
     "start_time": "2020-11-03T14:41:47.366442",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "SEED = 2019\n",
    "\n",
    "# Data parameters\n",
    "path = '../input/understanding_cloud_organization'\n",
    "img_paths = '../input/understanding-clouds-resized'\n",
    "output_model_file = \"best_model_optim.pt\"\n",
    "n_splits = 5\n",
    "which_fold = 2  # should be int in [0, n_splits-1]\n",
    "\n",
    "# Learning parameters\n",
    "batch_size = 32\n",
    "lr = 1.3e-2\n",
    "epochs_for_sched = 10\n",
    "checkpoint_iter = 200\n",
    "warmup_proportion = 0.25\n",
    "grad_accumulation_steps = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.031156,
     "end_time": "2020-11-03T14:41:47.806120",
     "exception": false,
     "start_time": "2020-11-03T14:41:47.774964",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:48.028045Z",
     "iopub.status.busy": "2020-11-03T14:41:47.990791Z",
     "iopub.status.idle": "2020-11-03T14:41:48.031475Z",
     "shell.execute_reply": "2020-11-03T14:41:48.030904Z"
    },
    "papermill": {
     "duration": 0.19462,
     "end_time": "2020-11-03T14:41:48.031609",
     "exception": false,
     "start_time": "2020-11-03T14:41:47.836989",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_torch(seed=SEED):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "def get_img(x, folder: str='train_images_525/train_images_525'):\n",
    "    \"\"\"\n",
    "    Return image based on image name and folder.\n",
    "    \"\"\"\n",
    "    data_folder = f\"{img_paths}/{folder}\"\n",
    "    image_path = os.path.join(data_folder, x)\n",
    "    img = cv2.imread(image_path)\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    return img\n",
    "\n",
    "def rle_decode(mask_rle: str = '', shape: tuple = (1400, 2100)):\n",
    "    \"\"\"\n",
    "    Decode run-length encoded mask.\n",
    "    \n",
    "    :param mask_rle: run-length as string formatted (start length)\n",
    "    :param shape: (height, width) of array to return \n",
    "    Returns numpy array, 1 - mask, 0 - background\n",
    "    \"\"\"\n",
    "    s = mask_rle.split()\n",
    "    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n",
    "    starts -= 1\n",
    "    ends = starts + lengths\n",
    "    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n",
    "    for lo, hi in zip(starts, ends):\n",
    "        img[lo:hi] = 1\n",
    "    return img.reshape(shape, order='F')\n",
    "\n",
    "def make_mask(image_name: str='img.jpg', shape: tuple=(350, 525)):\n",
    "    \"Create mask (ndarray, values between 0-1) based on image name and shape.\"\n",
    "    masks = np.zeros((shape[0], shape[1], 4), dtype=np.float32)\n",
    "    for classidx, classid in enumerate([\"Fish\", \"Flower\", \"Gravel\", \"Sugar\"]):\n",
    "        mask = cv2.imread(\"../input/understanding-clouds-resized/train_masks_525/train_masks_525/\" + classid + image_name)\n",
    "        if mask is None:\n",
    "            continue\n",
    "        if mask[:,:,0].shape != (350,525):\n",
    "            mask = cv2.resize(mask, (525,350))\n",
    "        masks[:, :, classidx] = mask[:,:,0]\n",
    "    masks = masks / 255\n",
    "    return masks\n",
    "\n",
    "def mask2rle(img):\n",
    "    \"\"\"\n",
    "    Convert mask to run-length encoding.\n",
    "    img: numpy array, 1 - mask, 0 - background\n",
    "    Returns run length as string formated\n",
    "    \"\"\"\n",
    "    pixels= img.T.flatten()\n",
    "    pixels = np.concatenate([[0], pixels, [0]])\n",
    "    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n",
    "    runs[1::2] -= runs[::2]\n",
    "    return ' '.join(str(x) for x in runs)\n",
    "\n",
    "\n",
    "def visualize(image, mask, original_image=None, original_mask=None):\n",
    "    \"\"\"\n",
    "    Plot images and masks.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - original ones: If given, show them all\n",
    "    \"\"\"\n",
    "    fontsize = 14\n",
    "    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n",
    "    \n",
    "    if original_image is None and original_mask is None:\n",
    "        f, ax = plt.subplots(1, 5, figsize=(24, 24))\n",
    "\n",
    "        ax[0].imshow(image)\n",
    "        for i in range(4):\n",
    "            ax[i + 1].imshow(mask[:, :, i])\n",
    "            ax[i + 1].set_title(f'Mask {class_dict[i]}', fontsize=fontsize)\n",
    "    else:\n",
    "        f, ax = plt.subplots(2, 5, figsize=(24, 12))\n",
    "\n",
    "        ax[0, 0].imshow(original_image)\n",
    "        ax[0, 0].set_title('Original image', fontsize=fontsize)\n",
    "                \n",
    "        for i in range(4):\n",
    "            ax[0, i + 1].imshow(original_mask[:, :, i])\n",
    "            ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n",
    "        \n",
    "        ax[1, 0].imshow(image)\n",
    "        ax[1, 0].set_title('Transformed image', fontsize=fontsize)        \n",
    "        \n",
    "        for i in range(4):\n",
    "            ax[1, i + 1].imshow(mask[:, :, i])\n",
    "            ax[1, i + 1].set_title(f'Transformed mask {class_dict[i]}', fontsize=fontsize)\n",
    "            \n",
    "            \n",
    "def visualize_with_raw(image, mask, original_image=None, original_mask=None, raw_image=None, raw_mak=None):\n",
    "    \"\"\"\n",
    "    Plot images and masks.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - original ones / raw ones: If given, show them all\n",
    "    \"\"\"\n",
    "    fontsize = 14\n",
    "    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n",
    "\n",
    "    f, ax = plt.subplots(3, 5, figsize=(24, 12))\n",
    "\n",
    "    ax[0, 0].imshow(original_image)\n",
    "    ax[0, 0].set_title('Original image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[0, i + 1].imshow(original_mask[:, :, i])\n",
    "        ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n",
    "\n",
    "    ax[1, 0].imshow(raw_image)\n",
    "    ax[1, 0].set_title('Original image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[1, i + 1].imshow(raw_mak[:, :, i])\n",
    "        ax[1, i + 1].set_title(f'Raw predicted mask {class_dict[i]}', fontsize=fontsize)\n",
    "        \n",
    "    ax[2, 0].imshow(image)\n",
    "    ax[2, 0].set_title('Transformed image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[2, i + 1].imshow(mask[:, :, i])\n",
    "        ax[2, i + 1].set_title(f'Predicted mask with processing {class_dict[i]}', fontsize=fontsize)\n",
    "            \n",
    "            \n",
    "def plot_with_augmentation(image, mask, augment):\n",
    "    \"\"\"\n",
    "    Plot images, masks and augmentation results.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - augment: transformation from albumentations\n",
    "    \"\"\"\n",
    "    augmented = augment(image=image, mask=mask)\n",
    "    image_flipped = augmented['image']\n",
    "    mask_flipped = augmented['mask']\n",
    "    visualize(image_flipped, mask_flipped, original_image=image, original_mask=mask)\n",
    "\n",
    "\n",
    "def post_process(probability, threshold, min_size):\n",
    "    \"\"\"\n",
    "    Post processing of each predicted mask, components with lesser number of pixels\n",
    "    than `min_size` are ignored\n",
    "    \n",
    "    Input\n",
    "    - probability: predicted probability mask, ndarray (350, 525)\n",
    "    - threshold: value to binarize probability mask\n",
    "    - min_size: ??\n",
    "    \"\"\"\n",
    "    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n",
    "    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n",
    "    predictions = np.zeros((350, 525), np.float32)\n",
    "    num = 0\n",
    "    for c in range(1, num_component):\n",
    "        p = (component == c)\n",
    "        if p.sum() > min_size:\n",
    "            predictions[p] = 1\n",
    "            num += 1\n",
    "    return predictions, num\n",
    "\n",
    "def get_training_augmentation():\n",
    "    train_transform = [\n",
    "        albu.Resize(256, 384),\n",
    "        albu.HorizontalFlip(p=0.25),\n",
    "        albu.VerticalFlip(p=0.25),\n",
    "        albu.ShiftScaleRotate(scale_limit=0.35, rotate_limit=10, shift_limit=0.1, border_mode=0),\n",
    "    ]\n",
    "    return albu.Compose(train_transform)\n",
    "\n",
    "def get_validation_augmentation():\n",
    "    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n",
    "    test_transform = [\n",
    "        albu.Resize(256, 384)\n",
    "    ]\n",
    "    return albu.Compose(test_transform)\n",
    "\n",
    "def to_tensor(x, **kwargs):\n",
    "    \"\"\"\n",
    "    Convert image or mask.\n",
    "    \"\"\"\n",
    "    return x.transpose(2, 0, 1).astype('float32')\n",
    "\n",
    "def get_preprocessing(preprocessing_fn):\n",
    "    \"\"\"Construct preprocessing transform\n",
    "    \n",
    "    Args:\n",
    "        preprocessing_fn (callbale): data normalization function \n",
    "            (can be specific for each pretrained neural network)\n",
    "    Return:\n",
    "        transform: albumentations.Compose\n",
    "    \n",
    "    \"\"\"\n",
    "    _transform = [\n",
    "        albu.Lambda(image=preprocessing_fn),\n",
    "        albu.Lambda(image=to_tensor, mask=to_tensor),\n",
    "    ]\n",
    "    return albu.Compose(_transform)\n",
    "\n",
    "\n",
    "def compute_dice(img1, img2):\n",
    "    img1 = np.asarray(img1).astype(np.bool)\n",
    "    img2 = np.asarray(img2).astype(np.bool)\n",
    "\n",
    "    intersection = np.logical_and(img1, img2)\n",
    "\n",
    "    return 2. * intersection.sum() / (img1.sum() + img2.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.060942,
     "end_time": "2020-11-03T14:41:48.128892",
     "exception": false,
     "start_time": "2020-11-03T14:41:48.067950",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data overview\n",
    "\n",
    "Let's have a look at the data first.\n",
    "We have folders with train and test images, file with train image ids and masks and sample submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:48.209123Z",
     "iopub.status.busy": "2020-11-03T14:41:48.208172Z",
     "iopub.status.idle": "2020-11-03T14:41:50.693606Z",
     "shell.execute_reply": "2020-11-03T14:41:50.692609Z"
    },
    "papermill": {
     "duration": 2.529071,
     "end_time": "2020-11-03T14:41:50.693772",
     "exception": false,
     "start_time": "2020-11-03T14:41:48.164701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(f'{path}/train.csv')\n",
    "sub = pd.read_csv(f'{path}/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:50.761299Z",
     "iopub.status.busy": "2020-11-03T14:41:50.760605Z",
     "iopub.status.idle": "2020-11-03T14:41:50.902328Z",
     "shell.execute_reply": "2020-11-03T14:41:50.903573Z"
    },
    "papermill": {
     "duration": 0.178827,
     "end_time": "2020-11-03T14:41:50.904073",
     "exception": false,
     "start_time": "2020-11-03T14:41:50.725246",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 5546 images in train dataset\n",
      "There are 3698 images in test dataset\n"
     ]
    }
   ],
   "source": [
    "n_train = len(os.listdir(f'{img_paths}/train_images_525/train_images_525'))\n",
    "n_test = len(os.listdir(f'{img_paths}/test_images_525/test_images_525'))\n",
    "print(f'There are {n_train} images in train dataset')\n",
    "print(f'There are {n_test} images in test dataset')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.03271,
     "end_time": "2020-11-03T14:41:50.970684",
     "exception": false,
     "start_time": "2020-11-03T14:41:50.937974",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Preparing data for modelling\n",
    "\n",
    "At first, let's create a list of unique image ids and the count of masks for each image. This will allow us to make a stratified split based on this count."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.044271Z",
     "iopub.status.busy": "2020-11-03T14:41:51.043485Z",
     "iopub.status.idle": "2020-11-03T14:41:51.047615Z",
     "shell.execute_reply": "2020-11-03T14:41:51.047025Z"
    },
    "papermill": {
     "duration": 0.044183,
     "end_time": "2020-11-03T14:41:51.047713",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.003530",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_val_split(id_mask_count, n_splits=None, which_fold=None):\n",
    "    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n",
    "    cv_indices = [(tr_idx, val_idx) for tr_idx, val_idx in kf.split(id_mask_count['img_id'], id_mask_count['count'])]\n",
    "    if which_fold is not None:\n",
    "        return cv_indices[which_fold]\n",
    "    return cv_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.130880Z",
     "iopub.status.busy": "2020-11-03T14:41:51.130070Z",
     "iopub.status.idle": "2020-11-03T14:41:51.170317Z",
     "shell.execute_reply": "2020-11-03T14:41:51.169748Z"
    },
    "papermill": {
     "duration": 0.089947,
     "end_time": "2020-11-03T14:41:51.170419",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.080472",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "id_mask_count = (train.loc[~train['EncodedPixels'].isnull(), 'Image_Label']\n",
    "                    .apply(lambda x: x.split('_')[0])\n",
    "                    .value_counts()\n",
    "                    .reset_index()\n",
    "                    .rename(columns={'index': 'img_id', 'Image_Label': 'count'}))\n",
    "\n",
    "trn_idx, val_idx = train_val_split(id_mask_count, n_splits, which_fold)\n",
    "train_ids, valid_ids = id_mask_count['img_id'].values[trn_idx], id_mask_count['img_id'].values[val_idx]\n",
    "test_ids = sub['Image_Label'].apply(lambda x: x.split('_')[0]).drop_duplicates().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.245258Z",
     "iopub.status.busy": "2020-11-03T14:41:51.244452Z",
     "iopub.status.idle": "2020-11-03T14:41:51.248967Z",
     "shell.execute_reply": "2020-11-03T14:41:51.249449Z"
    },
    "papermill": {
     "duration": 0.045337,
     "end_time": "2020-11-03T14:41:51.249564",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.204227",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4437, 1109)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ids), len(valid_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.327337Z",
     "iopub.status.busy": "2020-11-03T14:41:51.326625Z",
     "iopub.status.idle": "2020-11-03T14:41:51.337344Z",
     "shell.execute_reply": "2020-11-03T14:41:51.336792Z"
    },
    "papermill": {
     "duration": 0.053731,
     "end_time": "2020-11-03T14:41:51.337441",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.283710",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>db06c8a.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e8ff27c.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>87dc46a.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8078d91.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e8b4c68.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5541</th>\n",
       "      <td>b3cc504.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5542</th>\n",
       "      <td>fc15eb7.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5543</th>\n",
       "      <td>d8a17b7.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5544</th>\n",
       "      <td>62c2891.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5545</th>\n",
       "      <td>9934d3b.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5546 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           img_id  count\n",
       "0     db06c8a.jpg      4\n",
       "1     e8ff27c.jpg      4\n",
       "2     87dc46a.jpg      4\n",
       "3     8078d91.jpg      4\n",
       "4     e8b4c68.jpg      4\n",
       "...           ...    ...\n",
       "5541  b3cc504.jpg      1\n",
       "5542  fc15eb7.jpg      1\n",
       "5543  d8a17b7.jpg      1\n",
       "5544  62c2891.jpg      1\n",
       "5545  9934d3b.jpg      1\n",
       "\n",
       "[5546 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_mask_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.413887Z",
     "iopub.status.busy": "2020-11-03T14:41:51.412783Z",
     "iopub.status.idle": "2020-11-03T14:41:51.417461Z",
     "shell.execute_reply": "2020-11-03T14:41:51.416893Z"
    },
    "papermill": {
     "duration": 0.045514,
     "end_time": "2020-11-03T14:41:51.417572",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.372058",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    2372\n",
       "3    1560\n",
       "1    1348\n",
       "4     266\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_mask_count['count'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.036457,
     "end_time": "2020-11-03T14:41:51.491815",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.455358",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Only 266 images have all four masks."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.035473,
     "end_time": "2020-11-03T14:41:51.562937",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.527464",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Setting up data loader, model, solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.649768Z",
     "iopub.status.busy": "2020-11-03T14:41:51.648404Z",
     "iopub.status.idle": "2020-11-03T14:41:51.651371Z",
     "shell.execute_reply": "2020-11-03T14:41:51.652103Z"
    },
    "papermill": {
     "duration": 0.054393,
     "end_time": "2020-11-03T14:41:51.652242",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.597849",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CloudDataset(Dataset):\n",
    "    def __init__(self, dataset_type: str='train', img_ids: np.array=None,\n",
    "                 transforms=albu.Compose([albu.HorizontalFlip(), AT.ToTensor()]),\n",
    "                 preprocessing=None, label_smooth=0):\n",
    "        if dataset_type != 'test':\n",
    "            self.data_folder = f\"{img_paths}/train_images_525/train_images_525\"\n",
    "        else:\n",
    "            self.data_folder = f\"{img_paths}/test_images_525/test_images_525\"\n",
    "        self.img_ids = img_ids\n",
    "        self.transforms = transforms\n",
    "        self.preprocessing = preprocessing\n",
    "        self.label_smooth = label_smooth\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        image_name = self.img_ids[idx]\n",
    "        mask = make_mask(image_name)\n",
    "        img = cv2.imread(os.path.join(self.data_folder, image_name))\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        augmented = self.transforms(image=img, mask=mask)\n",
    "        img = augmented['image']\n",
    "        mask = augmented['mask']\n",
    "        if self.label_smooth > 0:\n",
    "            mask = np.where(mask==1, 1-label_smooth, label_smooth).astype('float32')\n",
    "        if self.preprocessing:\n",
    "            preprocessed = self.preprocessing(image=img, mask=mask)\n",
    "            img = preprocessed['image']\n",
    "            mask = preprocessed['mask']\n",
    "        return img, mask\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.img_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.740307Z",
     "iopub.status.busy": "2020-11-03T14:41:51.739537Z",
     "iopub.status.idle": "2020-11-03T14:41:51.743384Z",
     "shell.execute_reply": "2020-11-03T14:41:51.742827Z"
    },
    "papermill": {
     "duration": 0.052889,
     "end_time": "2020-11-03T14:41:51.743502",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.690613",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ENCODER = 'resnet34'\n",
    "ENCODER_WEIGHTS = 'imagenet'\n",
    "ACTIVATION = None\n",
    "\n",
    "preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)\n",
    "\n",
    "def model_optimizer_init(ft_lrs, num_labels=4):\n",
    "    print(\"Building model and optimizer...\")\n",
    "    model = smp.FPN(\n",
    "        encoder_name=ENCODER, \n",
    "        encoder_weights=ENCODER_WEIGHTS, \n",
    "        classes=4, \n",
    "        activation=ACTIVATION\n",
    "    )\n",
    "    param_g1 = [p for c in list(model.encoder.children())[:5] for p in c.parameters()]\n",
    "    param_g2 = [p for c in list(model.encoder.children())[5:] for p in c.parameters()]\n",
    "    optimizer = torch.optim.Adam([\n",
    "        {'params': param_g1, 'lr': ft_lrs[0]},\n",
    "        {'params': param_g2, 'lr': ft_lrs[1]}, \n",
    "        {'params': model.decoder.parameters(), 'lr': ft_lrs[2]},\n",
    "    ])\n",
    "    return model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.825009Z",
     "iopub.status.busy": "2020-11-03T14:41:51.824044Z",
     "iopub.status.idle": "2020-11-03T14:41:51.828352Z",
     "shell.execute_reply": "2020-11-03T14:41:51.827782Z"
    },
    "papermill": {
     "duration": 0.046966,
     "end_time": "2020-11-03T14:41:51.828463",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.781497",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def dice_loss(input, target):\n",
    "#     input = torch.sigmoid(input)\n",
    "#     smooth = 1.0\n",
    "#     iflat = input.view(-1)\n",
    "#     tflat = target.view(-1)\n",
    "#     intersection = (iflat * tflat).sum()\n",
    "#     return ((2.0 * intersection + smooth) / (iflat.sum() + tflat.sum() + smooth))\n",
    "\n",
    "\n",
    "# class FocalLoss(nn.Module):\n",
    "#     def __init__(self, gamma):\n",
    "#         super().__init__()\n",
    "#         self.gamma = gamma\n",
    "\n",
    "#     def forward(self, input, target):\n",
    "#         if not (target.size() == input.size()):\n",
    "#             raise ValueError(\"Target size ({}) must be the same as input size ({})\"\n",
    "#                              .format(target.size(), input.size()))\n",
    "#         max_val = (-input).clamp(min=0)\n",
    "#         loss = input - input * target + max_val + \\\n",
    "#             ((-max_val).exp() + (-input - max_val).exp()).log()\n",
    "#         invprobs = F.logsigmoid(-input * (target * 2.0 - 1.0))\n",
    "#         loss = (invprobs * self.gamma).exp() * loss\n",
    "#         return loss.mean()\n",
    "\n",
    "\n",
    "# class MixedLoss(nn.Module):\n",
    "#     def __init__(self, alpha, gamma):\n",
    "#         super().__init__()\n",
    "#         self.alpha = alpha\n",
    "#         self.focal = FocalLoss(gamma)\n",
    "\n",
    "#     def forward(self, input, target):\n",
    "#         loss = self.alpha * self.focal(input, target) - torch.log(dice_loss(input, target))\n",
    "#         return loss.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:51.919413Z",
     "iopub.status.busy": "2020-11-03T14:41:51.914058Z",
     "iopub.status.idle": "2020-11-03T14:41:51.949042Z",
     "shell.execute_reply": "2020-11-03T14:41:51.948498Z"
    },
    "papermill": {
     "duration": 0.082279,
     "end_time": "2020-11-03T14:41:51.949182",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.866903",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Functions for the training process\n",
    "\n",
    "class NetSolver(object):\n",
    "\n",
    "    def __init__(self, model, criterion, optimizer, scheduler, print_freq, filepath):\n",
    "        self.model = model\n",
    "        self.criterion = criterion\n",
    "        self.optimizer = optimizer\n",
    "        self.scheduler = scheduler\n",
    "        self.print_freq = print_freq\n",
    "        self.filepath = filepath\n",
    "\n",
    "        self.model = self.model.to(device)\n",
    "        self.criterion = self.criterion.to(device)\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self):\n",
    "        \"\"\"Set up some book-keeping variables for optimization.\n",
    "        \"\"\"\n",
    "        self.best_val_loss = 1e4\n",
    "        self.best_val_dice = 0.\n",
    "        self.loss_history = []\n",
    "        self.val_loss_history = []\n",
    "        self.dice_history = []\n",
    "        self.val_dice_history = []\n",
    "        self.models = {}\n",
    "\n",
    "    def save_checkpoint(self, iteration):\n",
    "        \"\"\"Save model checkpoint.\n",
    "        \"\"\"\n",
    "#         self.models[f'ckpt_{iteration}'] = self.model.state_dict()\n",
    "        self.model_state = self.model.state_dict()\n",
    "    \n",
    "    def save_final_state(self):\n",
    "        \"\"\"Save final states.\n",
    "        \"\"\"\n",
    "        state = {'model': self.model_state,\n",
    "                 'optimizer': self.optimizer.state_dict()\n",
    "                 }\n",
    "        torch.save(state, self.filepath)\n",
    "\n",
    "    def forward_pass(self, im, gt):\n",
    "        \"\"\"Forward pass through the network.\n",
    "        \"\"\"\n",
    "        im, gt = im.to(device=device, dtype=torch.float), gt.to(device=device, dtype=torch.float)\n",
    "        scores = self.model(im)\n",
    "        loss = self.criterion(scores, gt)\n",
    "        return loss, torch.sigmoid(scores)\n",
    "\n",
    "    def train(self, loaders, iterations, start_time):\n",
    "        \"\"\"Weight of network updated by apex, grad accumulation, model checkpoint.\n",
    "        \"\"\"\n",
    "        train_loader, val_loader = loaders\n",
    "        loader = iter(train_loader)\n",
    "        running_loss = 0.\n",
    "        self.optimizer.zero_grad()\n",
    "\n",
    "        # start training for iterations\n",
    "        for i in range(iterations):\n",
    "            self.model.train()\n",
    "\n",
    "            try:\n",
    "                im, gt = next(loader)\n",
    "            except:\n",
    "                loader = iter(train_loader)\n",
    "                im, gt = next(loader)\n",
    "            loss, _ = self.forward_pass(im, gt)\n",
    "            \n",
    "#             loss.backward()\n",
    "            with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n",
    "                scaled_loss.backward()\n",
    "\n",
    "            # gradient accumulation for larger batch size effect with smaller memory usage\n",
    "            if (i+1) % grad_accumulation_steps == 0:   # Wait for several backward steps\n",
    "                self.optimizer.step()                  # Now we can do an optimizer step\n",
    "                self.optimizer.zero_grad()\n",
    "                self.scheduler.step()\n",
    "\n",
    "            running_loss += loss.item()\n",
    "            \n",
    "            # verbose and checkpoint\n",
    "            if (i+1) % self.print_freq == 0 or (i+1) == iterations:\n",
    "                print(f'Iteration {i+1}:')\n",
    "                train_dice, _ = self.check_metric(train_loader, num_batches=50)\n",
    "                print('{\"metric\": \"Loss\", \"value\": %.4f}' % (running_loss/(i+1),))\n",
    "                print('{\"metric\": \"DICE\", \"value\": %.4f}' % (train_dice,))\n",
    "                \n",
    "                is_print = False if (i+1) < iterations else True\n",
    "                val_dice, val_loss = self.check_metric(val_loader)\n",
    "                print('{\"metric\": \"Val. Loss\", \"value\": %.4f}' % (val_loss,))\n",
    "                print('{\"metric\": \"Val. DICE\", \"value\": %.4f}' % (val_dice,))\n",
    "                \n",
    "                self.loss_history.append(running_loss/(i+1))\n",
    "                self.val_loss_history.append(val_loss)\n",
    "                self.dice_history.append(train_dice)\n",
    "                self.val_dice_history.append(val_dice)\n",
    "                \n",
    "                if val_loss < self.best_val_loss:\n",
    "                    print('updating best val loss...')\n",
    "                    self.best_val_loss = val_loss\n",
    "                    self.save_checkpoint(i+1)\n",
    "                if val_dice > self.best_val_dice:\n",
    "                    print('updating best val auc...')\n",
    "                    self.best_val_dice = val_dice\n",
    "                \n",
    "                torch.cuda.empty_cache()\n",
    "                print()\n",
    "                \n",
    "            if (time.time() - start_time) > 29000:\n",
    "                break\n",
    "        \n",
    "        self.save_final_state()\n",
    "\n",
    "    def check_metric(self, loader, num_batches=None):\n",
    "        \"\"\"Calculate metrics for validation\n",
    "        \"\"\"\n",
    "        self.model.eval()\n",
    "        losses, dices = [], []\n",
    "        with torch.no_grad():\n",
    "            for t, (im, gt) in enumerate(loader):\n",
    "                l, probs = self.forward_pass(im, gt)\n",
    "                losses.append(l.item())\n",
    "                dices.append(compute_dice((probs.cpu().numpy()>=0.5),\n",
    "                                          gt.cpu().numpy()))\n",
    "                if num_batches is not None and (t+1) == num_batches:\n",
    "                    break\n",
    "\n",
    "        loss = np.mean(losses)\n",
    "        dice = np.mean(dices)\n",
    "\n",
    "        return dice, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:52.066682Z",
     "iopub.status.busy": "2020-11-03T14:41:52.051036Z",
     "iopub.status.idle": "2020-11-03T14:41:52.076075Z",
     "shell.execute_reply": "2020-11-03T14:41:52.075528Z"
    },
    "papermill": {
     "duration": 0.090689,
     "end_time": "2020-11-03T14:41:52.076186",
     "exception": false,
     "start_time": "2020-11-03T14:41:51.985497",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class OneCycleScheduler(object):\n",
    "    # one-cycle scheduler\n",
    "    SCHEDULES = set(['cosine', 'linear', 'linear_cosine'])\n",
    "\n",
    "    def __init__(self, optimizer, iterations, sched_profile='cosine', max_lr=3e-3,\n",
    "                 moms=(.95, .85), div_factor=25, warmup=0.3, final_div=None):\n",
    "\n",
    "        self.optimizer = optimizer\n",
    "        assert sched_profile in self.SCHEDULES\n",
    "        self.sched_profile = sched_profile\n",
    "\n",
    "        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n",
    "            if len(max_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} max_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(max_lr)))\n",
    "            self.max_lrs = list(max_lr)\n",
    "            self.init_lrs = [lr/div_factor for lr in self.max_lrs]\n",
    "        else:\n",
    "            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n",
    "            self.init_lrs = [max_lr/div_factor] * len(optimizer.param_groups)\n",
    "\n",
    "        self.final_div = final_div\n",
    "        if self.final_div is None: self.final_div = div_factor*1e4\n",
    "        self.final_lrs = [lr/self.final_div for lr in self.max_lrs]\n",
    "        self.moms = moms\n",
    "\n",
    "        self.total_iteration = iterations\n",
    "        self.up_iteration = int(self.total_iteration * warmup)\n",
    "        self.down_iteration = self.total_iteration - self.up_iteration\n",
    "\n",
    "        self.curr_iter = 0\n",
    "        self._assign_lr_mom(self.init_lrs, [moms[0]]*len(optimizer.param_groups))\n",
    "\n",
    "    def _assign_lr_mom(self, lrs, moms):\n",
    "        for param_group, lr, mom in zip(self.optimizer.param_groups, lrs, moms):\n",
    "            param_group['lr'] = lr\n",
    "            param_group['betas'] = (mom, 0.999)\n",
    "\n",
    "    def _annealing_cos(self, start, end, pct):\n",
    "        cos_out = np.cos(np.pi * pct) + 1\n",
    "        return end + (start-end)/2 * cos_out\n",
    "\n",
    "    def _annealing_linear(self, start, end, pct):\n",
    "        return start + pct * (end-start)\n",
    "    \n",
    "    def _annealing_function(self, curr_iter):\n",
    "        if self.sched_profile == 'cosine':\n",
    "            return self._annealing_cos\n",
    "        if self.sched_profile == 'linear':\n",
    "            return self._annealing_linear\n",
    "        if self.sched_profile == 'linear_cosine':\n",
    "            if curr_iter <= self.up_iteration:\n",
    "                return self._annealing_linear\n",
    "            else:\n",
    "                return self._annealing_cos\n",
    "    \n",
    "    def step(self):\n",
    "        self.curr_iter += 1\n",
    "        anneal = self._annealing_function(self.curr_iter)\n",
    "\n",
    "        if self.curr_iter <= self.up_iteration:\n",
    "            pct = self.curr_iter / self.up_iteration\n",
    "            curr_lrs = [anneal(min_lr, max_lr, pct) \\\n",
    "                            for min_lr, max_lr in zip(self.init_lrs, self.max_lrs)]\n",
    "            curr_moms = [anneal(self.moms[0], self.moms[1], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "        else:\n",
    "            pct = (self.curr_iter-self.up_iteration) / self.down_iteration\n",
    "            curr_lrs = [anneal(max_lr, final_lr, pct) \\\n",
    "                            for max_lr, final_lr in zip(self.max_lrs, self.final_lrs)]\n",
    "            curr_moms = [anneal(self.moms[1], self.moms[0], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "\n",
    "        self._assign_lr_mom(curr_lrs, curr_moms)\n",
    "\n",
    "def lr_range_test(train_loader, model, optimizer, criterion, start_lr=1e-7,\n",
    "                  end_lr=10, num_it=100, stop_div=True):\n",
    "    epochs = int(np.ceil(num_it/len(train_loader)))\n",
    "    n_groups = len(optimizer.param_groups)\n",
    "\n",
    "    if isinstance(start_lr, list) or isinstance(start_lr, tuple):\n",
    "        if len(start_lr) != n_groups:\n",
    "            raise ValueError(\"expected {} max_lr, got {}\".format(n_groups, len(start_lr)))\n",
    "        start_lrs = list(start_lr)\n",
    "    else:\n",
    "        start_lrs = [start_lr] * n_groups\n",
    "\n",
    "    if isinstance(end_lr, list) or isinstance(end_lr, tuple):\n",
    "        if len(end_lr) != n_groups:\n",
    "            raise ValueError(\"expected {} max_lr, got {}\".format(n_groups, len(end_lr)))\n",
    "        end_lrs = list(end_lr)\n",
    "    else:\n",
    "        end_lrs = [end_lr] * n_groups\n",
    "\n",
    "    curr_lrs = start_lrs*1\n",
    "    for param_group, lr in zip(optimizer.param_groups, curr_lrs):\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    n, lrs_logs, loss_log = 0, [], []\n",
    "\n",
    "    for e in range(epochs):\n",
    "        model.train()\n",
    "        for x, y in train_loader:\n",
    "            x, y = x.to(device=device, dtype=torch.float), y.to(device=device, dtype=torch.float)\n",
    "            scores = model(x)\n",
    "            loss = criterion(scores, y)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "#             loss.backward()\n",
    "            with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                scaled_loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            lrs_logs.append(curr_lrs)\n",
    "            loss_log.append(loss.item())\n",
    "\n",
    "            # update best loss\n",
    "            if n == 0:\n",
    "                best_loss, n_best = loss.item(), n\n",
    "            else:\n",
    "                if loss.item() < best_loss:\n",
    "                    best_loss, n_best = loss.item(), n\n",
    "\n",
    "            # update lr per iter with exponential schedule\n",
    "            n += 1\n",
    "            curr_lrs = [lr * (end_lr/lr) ** (n/num_it) for lr, end_lr in zip(start_lrs, end_lrs)]\n",
    "            for param_group, lr in zip(optimizer.param_groups, curr_lrs):\n",
    "                param_group['lr'] = lr\n",
    "\n",
    "            # stopping condition\n",
    "            if n == num_it or (stop_div and (loss.item() > 4*best_loss or torch.isnan(loss))):\n",
    "                break\n",
    "\n",
    "    print('minimum loss {}, at lr {}'.format(best_loss, lrs_logs[n_best]))\n",
    "    return lrs_logs, loss_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.036272,
     "end_time": "2020-11-03T14:41:52.149731",
     "exception": false,
     "start_time": "2020-11-03T14:41:52.113459",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:52.230438Z",
     "iopub.status.busy": "2020-11-03T14:41:52.229798Z",
     "iopub.status.idle": "2020-11-03T14:41:52.234369Z",
     "shell.execute_reply": "2020-11-03T14:41:52.233816Z"
    },
    "papermill": {
     "duration": 0.048296,
     "end_time": "2020-11-03T14:41:52.234465",
     "exception": false,
     "start_time": "2020-11-03T14:41:52.186169",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_dataset = CloudDataset(dataset_type='train', img_ids=train_ids,\n",
    "                             transforms=get_training_augmentation(),\n",
    "                             preprocessing=get_preprocessing(preprocessing_fn))\n",
    "\n",
    "valid_dataset = CloudDataset(dataset_type='valid', img_ids=valid_ids,\n",
    "                             transforms=get_validation_augmentation(),\n",
    "                             preprocessing=get_preprocessing(preprocessing_fn))\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True,\n",
    "                          num_workers=2, pin_memory=True)\n",
    "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False,\n",
    "                          num_workers=2, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T14:41:52.317436Z",
     "iopub.status.busy": "2020-11-03T14:41:52.316668Z",
     "iopub.status.idle": "2020-11-03T14:42:03.717257Z",
     "shell.execute_reply": "2020-11-03T14:42:03.716677Z"
    },
    "papermill": {
     "duration": 11.447417,
     "end_time": "2020-11-03T14:42:03.717410",
     "exception": false,
     "start_time": "2020-11-03T14:41:52.269993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model and optimizer...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet34-333f7ec4.pth\" to /root/.cache/torch/hub/checkpoints/resnet34-333f7ec4.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abb65ff0b90a41fcbeab9422e72df8fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=87306240.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Each allocated and cached memory in cuda (will clean cache):\n",
      "95.471616M\n",
      "115.34336M\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/cuda/memory.py:346: FutureWarning: torch.cuda.memory_cached has been renamed to torch.cuda.memory_reserved\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# model setup\n",
    "seed_torch(SEED)\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "num_train_steps = int(epochs_for_sched * len(train_loader) / grad_accumulation_steps)\n",
    "ft_lrs = [0.0025*lr, 0.05*lr, lr]\n",
    "\n",
    "model, optimizer = model_optimizer_init(ft_lrs)\n",
    "scheduler = OneCycleScheduler(optimizer, num_train_steps, sched_profile='cosine', max_lr=ft_lrs,\n",
    "                              warmup=warmup_proportion)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = smp.utils.losses.DiceLoss(eps=1., activation='sigmoid').to(device)\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0)\n",
    "solver = NetSolver(model, criterion, optimizer, scheduler, checkpoint_iter, output_model_file)\n",
    "\n",
    "print()\n",
    "print('Each allocated and cached memory in cuda (will clean cache):')\n",
    "print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n",
    "print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T14:42:03.813438Z",
     "iopub.status.busy": "2020-11-03T14:42:03.805892Z",
     "iopub.status.idle": "2020-11-03T15:02:47.782832Z",
     "shell.execute_reply": "2020-11-03T15:02:47.781867Z"
    },
    "papermill": {
     "duration": 1244.02424,
     "end_time": "2020-11-03T15:02:47.782965",
     "exception": false,
     "start_time": "2020-11-03T14:42:03.758725",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training.\n",
      "Iteration 200:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4802}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5718}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4259}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5768}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 400:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4593}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5764}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4336}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5678}\n",
      "\n",
      "Iteration 600:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4473}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5741}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4281}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5730}\n",
      "\n",
      "Iteration 800:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4376}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6070}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4058}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5952}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1000:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4298}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6242}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3948}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6057}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1200:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4220}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6382}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3851}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6151}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1390:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4151}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6435}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3809}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6192}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Training complete in 1244 seconds.\n",
      "\n",
      "Each allocated and cached memory in cuda (will clean cache):\n",
      "430.5024M\n",
      "1566.572544M\n"
     ]
    }
   ],
   "source": [
    "n_iter = num_train_steps * grad_accumulation_steps\n",
    "print('Start training.')\n",
    "t0 = time.time()\n",
    "solver.train((train_loader, valid_loader), n_iter, t0)\n",
    "print('Training complete in {:.0f} seconds.'.format(time.time()-t0))\n",
    "\n",
    "print()\n",
    "print('Each allocated and cached memory in cuda (will clean cache):')\n",
    "print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n",
    "print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T15:02:47.887519Z",
     "iopub.status.busy": "2020-11-03T15:02:47.886502Z",
     "iopub.status.idle": "2020-11-03T15:02:48.173317Z",
     "shell.execute_reply": "2020-11-03T15:02:48.174119Z"
    },
    "papermill": {
     "duration": 0.342711,
     "end_time": "2020-11-03T15:02:48.174356",
     "exception": false,
     "start_time": "2020-11-03T15:02:47.831645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFlCAYAAACukk+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXhV5bn+8e/KzgyZw5QwJMxDJsIsk0gVRak4FKkD2h616lE7nHKU/lq1ntNqj9YqbW2r1rEOpQgoimJREFAGGcMgMwlkADKQEEhChv3+/lghCZAwZVh7J/fnunJp1l5750FtuXnf9T6PZYxBRERERDyDj9MFiIiIiEgthTMRERERD6JwJiIiIuJBFM5EREREPIjCmYiIiIgHUTgTERER8SC+ThfQlKKjo01cXJzTZYiIiIic1/r16/OMMR3OvN6qwllcXBzr1q1zugwRERGR87IsK6O+69rWFBEREfEgCmciIiIiHkThTERERMSDtKpnzkRERNqKiooKMjMzKSsrc7oUOY/AwEC6du2Kn5/fBd2vcCYiIuKFMjMzCQkJIS4uDsuynC5HGmCMIT8/n8zMTOLj4y/oPdrWFBER8UJlZWVERUUpmHk4y7KIioq6qBVOhTMREREvpWDmHS7235PCmYiIiFy0wsJCXnzxxUt67+TJkyksLDznPY899hhLliy5pM8/U1xcHHl5eU3yWS1Bz5yJiIi0AQs2ZvHM4p1kF5YSEx7EzEn9mDo49pI/71Q4e+CBB856raqqCpfL1eB7Fy1adN7Pf/LJJy+5Nm+nlbMLtGBjFqOf/oL4Rz9m9NNfsGBjltMliYiIXJAFG7OYNW8LWYWlGCCrsJRZ87Y06veyRx99lL1795KSksLMmTNZtmwZEyZM4NZbbyUxMRGAqVOnMmTIEAYNGsRLL71U895TK1np6ekMGDCAe+65h0GDBnHVVVdRWloKwF133cXcuXNr7n/88cdJTU0lMTGRHTt2AJCbm8uVV15JamoqP/rRj+jRo8d5V8iee+45EhISSEhI4PnnnwfgxIkTXHvttSQnJ5OQkMA///nPml/jwIEDSUpK4uc///kl/7O6WM26cmZZ1tXAC4ALeMUY83QD9w0DVgO3GGPmVl/7KXA3YIAtwA+MMY6cFz71H3VpRRVQ+x810Kg/dYiIiDSFXy/cxvbsYw2+vvFAIeVV7tOulVZU8d9z03h37YF63zMwJpTHpwxq8DOffvpptm7dyqZNmwBYtmwZa9euZevWrTWnEl999VUiIyMpLS1l2LBh3HTTTURFRZ32Obt37+bdd9/l5ZdfZtq0abz//vvcfvvtZ/286OhoNmzYwIsvvsizzz7LK6+8wq9//WuuuOIKZs2axaeffnpaAKzP+vXree2111izZg3GGEaMGMH48ePZt28fMTExfPzxxwAUFRVRUFDA/Pnz2bFjB5ZlnXcbtik128qZZVku4M/ANcBA4PuWZQ1s4L7fAYvrXIsFHgaGGmMSsMPd9Oaq9XyeWbyzJpidUlpRxTOLdzpUkYiIyIU7M5id7/qlGj58+GntImbPnk1ycjIjR47k4MGD7N69+6z3xMfHk5KSAsCQIUNIT0+v97NvvPHGs+5ZuXIl06fb8eDqq68mIiLinPWtXLmSG264gXbt2tG+fXtuvPFGVqxYQWJiIkuWLOGRRx5hxYoVhIWFERoaSmBgIHfffTfz5s0jODj4Yv9xXLLmXDkbDuwxxuwDsCzrPeB6YPsZ9z0EvA8Mq6e2IMuyKoBgILsZaz2n7MLSi7ouIiLSks61wgUw+ukvyKrn96zY8CD++aNRTVZHu3btav5+2bJlLFmyhFWrVhEcHMzll19ebzuJgICAmr93uVw125oN3edyuaisrATsHmIXo6H7+/bty/r161m0aBGzZs3iqquu4rHHHmPt2rV8/vnnvPfee/zpT3/iiy++uKifd6ma85mzWOBgne8zq6/VqF4huwH4a93rxpgs4FngAJADFBljPqvvh1iWda9lWessy1qXm5vbhOXXigkPqvd6l7DAZvl5IiIiTWnmpH4E+Z3+gH6Qn4uZk/pd8meGhIRQXFzc4OtFRUVEREQQHBzMjh07WL169SX/rIaMGTOGOXPmAPDZZ59x9OjRc94/btw4FixYQElJCSdOnGD+/PmMHTuW7OxsgoODuf322/n5z3/Ohg0bOH78OEVFRUyePJnnn3++Zvu2JTRnOKuvqceZkfV54BFjzGl7hpZlRWCvssUDMUA7y7LO3oAGjDEvGWOGGmOGdujQoQnKPlt9/1ED+FiwP+9Es/xMERGRpjJ1cCxP3ZhIbHgQFvaK2VM3JjbquemoqChGjx5NQkICM2fOPOv1q6++msrKSpKSkvjVr37FyJEjG/ErqN/jjz/OZ599RmpqKp988gldunQhJCSkwftTU1O56667GD58OCNGjODuu+9m8ODBbNmyheHDh5OSksJvfvMbfvnLX1JcXMx1111HUlIS48eP5w9/+EOT198Q62KXBC/4gy1rFPCEMWZS9fezAIwxT9W5Zz+1IS4aKAHuBfyAq40x/1F93wxgpDHm7PO6dQwdOtSsW7euqX8pwNlHkK9J6MzcDZlUVLr57Y2JXJ+igwEiItJyvv32WwYMGOB0GY46efIkLpcLX19fVq1axf3339+iK1wXo75/X5ZlrTfGDD3z3uZ85uwboI9lWfFAFvYD/bfWvcEYU/PUoGVZrwMfGWMWWJY1AhhpWVYwUApMBJondV2gqYNjz/oTxg/HxPPj9zby4/c28fWefJ747iCC/Bvu6yIiIiJN58CBA0ybNg23242/vz8vv/yy0yU1iWYLZ8aYSsuyHsQ+hekCXjXGbLMs677q1/96jveusSxrLrABqAQ2Auc+H+uAmPAg3r1nJC98vps/Ld3DhgNH+fNtqfTt1PCSqoiIiDSNPn36sHHjRqfLaHLNtq3phObc1jyflbvz+Mk/N3H8ZAVPTBnELcO6aeaZiIg0G21repeL2dbUhIAmMqZPNIt+PIahPSJ5dN4WfvzeJorLKpwuS0RERLyMwlkT6hgSyJs/HM7MSf34eEsOU/64kq1ZRU6XJSIiIl5E4ayJ+fhY/OeE3rx370hOVrq58cWvee2r/RfdKE9ERETaJoWzZjIsLpJFD49lXN9ofr1wOz96az2FJeVOlyUiIuKY9u3bA5Cdnc3NN99c7z2XX34553t+/Pnnn6ekpKTm+8mTJzfJ7MsnnniCZ599ttGf01gKZ80oop0/L88Yyq+uG8jSnUe4dvZK1mcUOF2WiIi0RWlz4A8J8ES4/de0OY6VEhMTw9y5cy/5/WeGs0WLFhEeHt4UpXkEhbNmZlkW/zEmnvfvvwyXj8W0v63mL8v24nZrm1NERFpI2hxY+DAUHQSM/deFDzcqoD3yyCO8+OKLNd8/8cQT/P73v+f48eNMnDiR1NRUEhMT+eCDD856b3p6OgkJCQCUlpYyffp0kpKSuOWWW06brXn//fczdOhQBg0axOOPPw7Yw9Szs7OZMGECEyZMACAuLo68vDwAnnvuORISEkhISOD555+v+XkDBgzgnnvuYdCgQVx11VUNzvA8ZdOmTYwcOZKkpCRuuOGGmtFQs2fPZuDAgSQlJdUMXf/yyy9JSUkhJSWFwYMHn3Os1YVozia0UkdS13A+engMs+Zt4Xef7mDVvnyem5ZMdPuA879ZRETkXD55FA5tafj1zG+g6uTp1ypK4YMHYf0b9b+ncyJc83SDHzl9+nR+8pOf8MAD9vCeOXPm8OmnnxIYGMj8+fMJDQ0lLy+PkSNH8t3vfrfB9lJ/+ctfCA4OJi0tjbS0NFJTU2te+81vfkNkZCRVVVVMnDiRtLQ0Hn74YZ577jmWLl1KdHT0aZ+1fv16XnvtNdasWYMxhhEjRjB+/HgiIiLYvXs37777Li+//DLTpk3j/fff5/bb650MCcCMGTP44x//yPjx43nsscf49a9/zfPPP8/TTz/N/v37CQgIqNlKffbZZ/nzn//M6NGjOX78OIGBjZu9rZWzFhQa6Mefvj+Y396QyJp9+Vzzwgq+3pPndFkiItLanRnMznf9AgwePJgjR46QnZ3N5s2biYiIoHv37hhj+MUvfkFSUhLf+c53yMrK4vDhww1+zvLly2tCUlJSEklJSTWvzZkzh9TUVAYPHsy2bdvYvn37OWtauXIlN9xwA+3ataN9+/bceOONrFixAoD4+HhSUlIAGDJkCOnp6Q1+TlFREYWFhYwfPx6AO++8k+XLl9fUeNttt/GPf/wDX197jWv06NH87Gc/Y/bs2RQWFtZcv1RaOWthlmVx64jupPYI5z/f3sBtf1/DQxN68/DEPvi6lJVFROQSnGOFC7CfMSs6ePb1sG7wg48v+cfefPPNzJ07l0OHDtVs8b399tvk5uayfv16/Pz8iIuLo6ys7JyfU9+q2v79+3n22Wf55ptviIiI4K677jrv55yrM0JAQO1OlcvlOu+2ZkM+/vhjli9fzocffsj//M//sG3bNh599FGuvfZaFi1axMiRI1myZAn9+/e/pM8HrZw5pn/nUBY+NIabU7sy+4s93PrKGg4Vnfs/OhERkUsy8THwCzr9ml+Qfb0Rpk+fznvvvcfcuXNrTl8WFRXRsWNH/Pz8WLp0KRkZGef8jHHjxvH2228DsHXrVtLS0gA4duwY7dq1IywsjMOHD/PJJ5/UvCckJKTe57rGjRvHggULKCkp4cSJE8yfP5+xY8de9K8rLCyMiIiImlW3t956i/Hjx+N2uzl48CATJkzg//7v/ygsLOT48ePs3buXxMREHnnkEYYOHcqOHTsu+mfWpZUzBwX7+/LM95IZ1SuKXy7YyjUvLOf305K5on8np0sTEZHWJGma/dfPn4SiTAjragezU9cv0aBBgyguLiY2NpYuXboAcNtttzFlyhSGDh1KSkrKeVeQ7r//fn7wgx+QlJRESkoKw4cPByA5OZnBgwczaNAgevbsyejRo2vec++993LNNdfQpUsXli5dWnM9NTWVu+66q+Yz7r77bgYPHnzOLcyGvPHGG9x3332UlJTQs2dPXnvtNaqqqrj99tspKirCGMNPf/pTwsPD+dWvfsXSpUtxuVwMHDiQa6655qJ/Xl2arekh9uYe58F3NvJtzjHuHdeTn1/VD39fLWyKiEj9NFvTu2i2phfq1aE98x+4jDtG9uCl5fv43t9WcbCg5PxvFBERkVZF4cyDBPq5+J+pCfzltlT25R5n8uwVfLIlx+myREREpAUpnHmgaxK7sOjhsfTs0J77397ALxdsoayiyumyREREpAUonHmobpHB/OtHo7h3XE/+sfoAN7z4NXtzjztdloiIeJDW9Nx4a3ax/54UzjyYv68Pv5g8gNfuGsaholKm/HEl8zZkOl2WiIh4gMDAQPLz8xXQPJwxhvz8/IuaGqDTml7iUFEZD7+3kbX7C7h5SFeevH4Qwf7qhCIi0lZVVFSQmZl53sas4rzAwEC6du2Kn5/fadcbOq2pcOZFKqvczP5iD3/8Yjc9o9vxp1tTGdAl1OmyRERE5BKolUYr4Ovy4WdX9uXtu0dQXFbJ1D9/xdtrMrSkLSIi0ooonHmhy3pFs+jHYxnRM4r/N38rD767kWNlFU6XJSIiIk1A4cxLRbcP4PW7hvHI1f35dOshrpu9ks0HC50uS0RERBpJ4cyL+fhY3H95L+b8aCRVbsPNf/2aV1bs0zaniIiIF1M4awWG9Ijk44fHMKFfR/7342+55811HD1R7nRZIiIicgkUzlqJ8GB//nbHEJ6YMpDlu/KYPHsF36QXOF2WiIiIXCSFs1bEsizuGh3PvAcuI8DXh+kvreZPX+ymyq1tThEREW+hcNYKJcSGsfChMVyb2IVnP9vFna+u5UixmhSKiIh4A4WzViok0I8Xpqfwu5sSWZdRwOQXVrBid67TZYmIiMh5KJy1YpZlccuw7nz44Bgigv2Z8epanlm8g8oqt9OliYiISAMUztqAvp1C+PDBMdwytBt/XrqX77+8muzCUqfLEhERkXoonLURQf4unr4piRemp7A9+xiTZ69gyfbDTpclIiIiZ1A4a2OuT4nlo4fHEhsexN1vruPJhdspr9Q2p4iIiKdQOGuD4qPbMe+By7jrsjhe/Wo/N//1azLyTzhdloiIiKBw1mYF+Lp44ruD+NsdQ0jPO8G1s1fyUVq202WJiIi0eQpnbdykQZ1Z9OOx9O3Ungff2cgv5m+hrKLK6bJERETaLIUzoWtEMP/80Sjuv7wX76w5wNQ/f8WeI8VOlyUiItImKZwJAH4uHx65uj+v/2AYucUnmfLHr/jXuoMYo9FPIiIiLUnhTE5zeb+OLPrxWFK6hTNzbho/m7OZ4ycrnS5LRESkzVA4k7N0Cg3kH3eP4Kff6csHm7L47h9Xsi27yOmyRERE2gSFM6mXy8fix9/pwzv3jOREeSU3vPg1b61K1zaniIhIM1M4k3Ma2TOKRQ+P5bJeUfzqg2088PYGikornC5LRESk1VI4k/OKah/Aq3cO4xeT+/Pv7Ye5dvYKNh446nRZIiIirZLCmVwQHx+Le8f14l/3jQLge39dxUvL9+J2a5tTRESkKSmcyUUZ3D2Cjx8ey5UDO/HbRTv44RvfkH/8pNNliYiItBoKZ3LRwoL8ePG2VP7n+kF8vTefybNXsHpfvtNliYiItAoKZ3JJLMvijlFxzH/gMtr5+3Lry6t5YcluqrTNKSIi0igKZ9Iog2LC+PChMVyfEssfluzi9lfWcORYmdNliYiIeC2FM2m09gG+PDctmWduTmLTwUKueWEFX+7KdbosERERr6RwJk3Csiy+N7QbCx8aTXT7AO58dS1Pf7KDiiq306WJiIh4FYUzaVK9O4bwwYOjuXVEd/765V5u+dsqMo+WOF2WiIiI12jWcGZZ1tWWZe20LGuPZVmPnuO+YZZlVVmWdXOda+GWZc21LGuHZVnfWpY1qjlrlaYT6Ofitzck8qdbB7P78HEmv7CCxdsOOV2WiIiIV2i2cGZZlgv4M3ANMBD4vmVZAxu473fA4jNeegH41BjTH0gGvm2uWqV5XJcUw0cPj6FHVDt+9NZ6nvhwGycrq5wuS0RExKM158rZcGCPMWafMaYceA+4vp77HgLeB46cumBZVigwDvg7gDGm3BhT2Iy1SjPpEdWOufeP4oej43n963Ru+svX7M874XRZIiIiHsu3GT87FjhY5/tMYETdGyzLigVuAK4AhtV5qSeQC7xmWVYysB74sTFGv6t7oQBfF49NGchlvaL4+dzNXDd7BTemxvLFjlyyC0uJCQ9i5qR+TB0c63SpIiIijmvOlTOrnmtndih9HnjEGHPmXpcvkAr8xRgzGDgB1PvMmmVZ91qWtc6yrHW5uWrf4Mm+M7ATix4eS8eQAN5afYCswlIMkFVYyqx5W1iwMcvpEkVERBzXnOEsE+hW5/uuQPYZ9wwF3rMsKx24GXjRsqyp1e/NNMasqb5vLnZYO4sx5iVjzFBjzNAOHTo0Zf3SDGLCgzhZT3uN0ooqnlm804GKREREPEtzhrNvgD6WZcVbluUPTAc+rHuDMSbeGBNnjInDDmAPGGMWGGMOAQcty+pXfetEYHsz1iotKKew/gkCWYWlHCnWdAEREWnbmi2cGWMqgQexT2F+C8wxxmyzLOs+y7Luu4CPeAh427KsNCAF+G1z1SotKyY8qMHXRj/9BQ+/u5F16QUYozmdIiLS9lit6TfAoUOHmnXr1jldhpzHgo1ZzJq3hdKK2kcNg/xc/PTKPhwqOsm/1h+kuKySgV1CufOyHnw3OZYgf5eDFYuIiDQ9y7LWG2OGnnVd4UycsGBjFs8s3lnvac2S8koWbMzmzVXp7DhUTFiQH9OGduX2kT3oEdXO2cJFRESaiMKZeB1jDGv3F/Dmqgw+3XYItzFM6NeRO0b1YHyfDvj41HcgWERExDsonIlXO1RUxjtrD/DOmgPkHT9JXFQwt4/swfeGdCMs2M/p8kRERC6awpm0CuWVbj7ddog3v05nXcZRgvxcTB0cwx0j4xgYE+p0eSIiIhdM4Uxana1ZRby1KoMFm7I4WelmeFwkMy7rwaRBnfFzNWeXGBERkcZTOJNWq7CknH+ty+St1RkcKCihY0gAt47ozq3Du9MxNNDp8kREROqlcCatXpXb8OWuI7y5KoNlO3Px9bG4JrELM0b1YGiPCCxLBwhERMRzNBTOmnPwuUiLcvlYXNG/E1f078T+vBP8Y3UGc9YdZOHmbAZ0CeXOUT24PkU900RExLNp5UxatTN7poUG+jJtaDfuGKWeaSIi4ixta0qbZozhm/SjvLEqncVbD1FlDJf37cCMy+LUM01ERByhbU1p0yzLYnh8JMPjIzl8rIx31hzgnbUH+MFr39AjKpg71DNNREQ8hFbOpM061TPtrVXpfJN+lEA/H6amxDJjlHqmiYhI89O2psg5bMuu7ZlWVuFmWFwEM0bFMWlQZ/x91TNNRESansKZyAUoKqngX+sP8uYqu2dah5AAbh3enVtHdKeTeqaJiEgTUjgTuQhut+HLXbm8sSq9pmfapITO3DkqjmFx6pkmIiKNpwMBIhfBx8diQv+OTOjfkfQ6PdM+Tsuhf+cQ7rwsjutTYgj21/+ERESkaWnlTOQClZZX8cGmLN5YlcG3OccIDfTle0O7ccfIHsRFq2eaiIhcHG1rijQRYwzrMo7yxtfpfLr1EJVuw+X9OjBjVA8u79tRPdNEROSCaFtTpIlYlsWwuEiGxUVy5FgZ76w9wDtrDvDD19fRPbK6Z9rQroQH+ztdqoiIeCGtnIk0gYoqN4u3HeLNrzNYm15AoJ8P1yfHMuOyHgyKCXO6PBER8UDa1hRpIduzj/HW6nTmb7R7pg3tEcEdo3pwTUIX9UwTEZEaCmciLexUz7S3VmeQkV9CdPsAbh3RndvUM01ERFA4E3GM221YvjuXN1dlsHTnEVyW3TNtxsgeDI+PVM80EZE2SgcCRBzi42Nxeb+OXN6vIxn5ds+0f35T2zNtxqg4pg5WzzQREbFp5UzEAaXlVXy4OYs3vs5ge84xQgJ9+d6Qbtwxqgfx6pkmItImaFtTxAMZY1ifcZQ3V2WwaEsOlW7D+L7VPdP6dcSlnmkiIq2WwpmIhztyrIx31x7k7TUZHCk+SbfIIO4Y2YNpQ7upZ5qISCukcCbiJSqq3Hy27TBvrEpn7f4CAnx9uD4lhhmj4kiIVc80EZHWQuFMxAvtOHSMN1dlMH9DFqUVVQzpEcEM9UwTEWkVFM5EvFhRaQVz12fy1qp00k/1TBvejVtH9KBzmHqmiYh4I4UzkVbgVM+0t1Zl8MXOI/hYFpMGdWLGqDhyCkt59rNdZBeWEhMexMxJ/Zg6ONbpkkVEpAHqcybSCtTtmXYgv4R/rLF7pi3acggLOPVHrazCUmbN2wKggCYi4mX00IqIl+oeFcwvJg9g9ayJhAf5ceYaeGlFFf/36Q5HahMRkUuncCbi5YL8XRSVVtT7WnZRGf81ZzPLdh6hosrdwpWJiMil0LamSCsQEx5EVmHpWdeD/V18tv0Q72/IJCLYj2sSuzAlKYbh8ZFqcCsi4qEUzkRagZmT+jFr3hZKK6pqrgX5ufjtDYlck9iZ5bvyWLg5m/kbsnhnzQE6hgRwbVIXrkuKIbV7uIavi4h4EJ3WFGklFmzM4pnFO895WrOkvJIvdhxh4eZslu7MpbzSTWx4ENcl2ytqg2JCFdRERFqIWmmIyGmOlVXw722HWZiWzcrdeVS6DT2j23FdcgzfTe5C744hTpcoItKqKZyJSIOOnijn022HWLg5m1X78jEG+ncOYUpyDFOSYugeFex0iSIirY7CmYhckCPHyli0JYeFaTmszzgKQHK3cKZUP6OmiQQiIk1D4UxELlrm0RI+TsthYVo2W7OOYVkwLC6SKUlduCaxC9HtA5wuUUTEaymciUij7Ms9zkdpOXy4OZs9R47jY8Ho3tFMSYph0qDOhAX7OV2iiIhXUTgTkSZhjGHn4WI+2mwHtQMFJfi5LMb37cCU5Bi+M6AT7QLUpUdE5HwUzkSkyRlj2JJVxMLN2XyUlkNOURmBfj5M7N+JKclduLxfRwL9XE6XKSLikRTORKRZud2G9QeOsnBzNou25JB3vJz2Ab5cOdAOamN6d8DfVxPjREROUTgTkRZTWeVm9b4CFm7O5pOtORwrqyQsyI9rEjozJTmGkT2jND5KRNo8hTMRcUR5pZuVe3JZuDmHz7Yd4kR5FdHtA7g20Q5qqd0j8FFQE5E2SOFMRBxXVlHF0h1HWJiWzeffHuFkpZuYsECuq252mxCr8VEi0nYonImIRzl+spIl2w+zcHM2y3fnUlFliIsKtqcSJMfQt5PGR4lI66ZwJiIeq7CknMXbDrFwcw5f783DbaBvp/ZMSYrhuuQY4qPbOV2iiEiTUzgTEa+QW3yST7bm8NHmHNamFwCQGBvGlOQuXJsUQ2x4kMMViog0DUfCmWVZVwMvAC7gFWPM0w3cNwxYDdxijJlb57oLWAdkGWOuO9/PUzgTaV2yC0vtOZ+bs9mcWQTA0B4RTEmO4ZrEznQM0ZxPEfFeLR7OqoPVLuBKIBP4Bvi+MWZ7Pff9GygDXj0jnP0MGAqEKpyJtG0Z+Sf4KM0OajsOFeNjwaheUVyXFMPVgzoT0c7f6RJFRC6KE+FsFPCEMWZS9fezAIwxT51x30+ACmAY8NGpcGZZVlfgDeA3wM8UzkTklF2Hi/loczYL03LYn3cCXx+LsX2imZIcw5UDOxESqDmfIuL5GgpnzTkALxY4WOf7TGDEGUXFAjcAV2CHs7qeB/4b0JEtETlN304h/Oyqfvz0yr5syz5WMz7qZ3M24+/rwxX9OjIlOYYr+nckyF/jo0TEuzRnOKuvWdGZy3TPA48YY6rq9jayLOs64IgxZr1lWZef84dY1r3AvQDdu3dvVMEi4l0syyIhNoyE2DAeubo/Gw8eZeHmHD7eksOn2w4R7O+yx0clxTC2bzQBvgpqIuL5HN3WtCxrP7UhLhoowQ5aI4A7gEogEAgF5hljbj/Xz9S2pogAVLkNa/bns3BzDp9sze6OmHYAACAASURBVKGwpILQQF+uTujMdUkxXNYrCl+X5nyKiLOceObMF/tAwEQgC/tAwK3GmG0N3P86dZ45q3P9cuDneuZMRC5FRZWblXvyWLg5m8+2Heb4yUqi2vlzTWJnpiTFMCwuUuOjRMQRLf7MmTGm0rKsB4HF2K00XjXGbLMs677q1//aXD9bROQUP5cPE/p1ZEK/jpRVVLFsZy4L07KZuz6Tf6w+QOfQQK5N6sKU5BiSu4ZpfJSIOE5NaEWkTTpxspIl3x7mo7QcvtyZS3mVm26RQUxJssdH9e8cwgebsnlm8U6yC0uJCQ9i5qR+TB0c63TpItJKaEKAiEgDikor+GzbIRam5fDVnjyq3IaOIQEUnCin0l37/5FBfi6eujFRAU1EmoTCmYjIBcg/fpJPth7iyYXbKa9yn/V6l7BAVs2a6EBlItLaNBTOdFxJRKSOqPYB3D6yBxX1BDOAnKIypv1tFX9euoctmUW43a3nD7gi4hmas8+ZiIjXigkPIquw9Kzr7QN8OXGykmcW7+SZxTuJbOfPmN7RjO0Tzbi+HegUqnmfItI4CmciIvWYOakfs+ZtobSiquZakJ+L/52awNTBseQWn2TlnlxW7Mpj+e48PtycDUD/ziE1QW1YXCSBfmp8KyIXR8+ciYg0YMHGrAs6rel2G749dIwVu/NYviuXdelHKa9yE+Drw4ieUYyrDmt9OrZXqw4RqaEDASIiLaSkvJI1+wr4clcuK3bnsjf3BGAfJhjbJ5qxfTowpnc0Ee38Ha5URJykcCYi4pDMoyWs2J3Hit25rNydx7GySiwLkmLDGNe3A+P6diClWzh+Gikl0qYonImIeIDKKjdpWUUs35XL8l25bDpYiNtASIAvo3pFMbZvB8b36UD3qGCnSxWRZqZwJiLigYpKK/h6j32oYPmu3JoTonFRwYztY6+qjeoVRfsAnd8SaW0UzkREPJwxhn15J1ixK5flu/NYtTef0ooqfH0sUntEML5vB8b2iSYhJkzD2kVaAYUz8Sxpc+DzJ6EoE8K6wsTHIGma01WJeJSTlVWszzjK8l3282rbso8BEBHsx5g+HWpOgaq3moh3UjgTz5E2BxY+DBV1Gnz6BcGU2QpoIudwZm+1vOMnAejXKYRxfdVbTcTbKJyJZygvgRcS4UTe2a8FhMFNr0CXJAjp3PK1iXgR9VYT8X4KZ+IMtxsOb4W9X9hfB1ZD1cnzv699J+iSfPpXWDfQbzIi9Wqot1rn0MCaiQXqrSbiWRTOpOUcy4a9S2HfUvuvJdWrZB0HQa8JkPZe/StnobH2yllOGuRstr9yd4CpHp8TFGGHtM5J1YEtBSJ7go96Q4mcKfNoCSt357G8gd5qY/t0YHB39VYTcVKjwpllWT8GXgOKgVeAwcCjxpjPmrrQxlA4c0j5Ccj4unp1bCnkfmtfb9fRDmO9roCel9duVV7MM2cVpXB4O+Rsqg1sR7ZDVbn9un/7OmGt+iu6L7jUdkDklCq3YXNmIct35bJidx4bDxzFbewh7qN6RTFOvdVEHNHYcLbZGJNsWdYk4D+BXwGvGWNSm77US6dw1kLcbji0uTaMHVxjhyXfQOhxGfSsDmSdBjW8DdmY05qV5ZC3szas5WyGQ1ugosR+3TcQOiXYz66dCmwdB4JvQNP8+kW8XEO91XpEBTOuj92u47Le0eqtJtLMGhvO0owxSZZlvQAsM8bMtyxrozFmcHMUe6kUzppRUaYdxPZ+AfuWQWmBfb1TYvXq2AToPspeAXOCuwry95we2HLS4GSR/bqPL3QcULsd2iXZDo/+7ZypV8RDGGPYn3fCnlhQT2+1UwcL1FtNpOk1Npy9BsQC8UAy4MIOaUOautDGUDhrQiePQ/rK6ufGvoC8Xfb19p1P36ps39HJKs/NGDiafkZg21z7DJzlA1F9Tt8S7ZwIQeGOli3iJPVWE2k5jQ1nPkAKsM8YU2hZViTQ1RiT1vSlXjqFs0ZwV9nPddVsVa4FdwX4BkHc6OowNsFeffLmE5PG2AcWcjbDoToHD45l1d4TEV8d1uocPGgX7VzNIg7KLT7JV3vyalbWzuytNrZPB4bHq7eayKVobDgbDWwyxpywLOt2IBV4wRiT0fSlXjqFs4tUeKA2jO1bBmWF9vUuybXPjXUbAX5t4E/Ix3Pt5+jqrrAdTa99PTT27NYeIV28O6iKXCS327DjUDHLd+ee1VtteHwk4/t2UG81kYvQ6GfOsLczk4C3gL8DNxpjxjd1oY2hcHYeZcfsrcq9X9jblfl77OshMXYQ6zXB3qrUKpGttNA+aFA3sOXtAqr/NxMcfXZgi4hTYJM241RvtVNh7czeamP7dmBsnd5qCzZm8czinWQXlhITHsTMSf2YOjjWyV+CiKMaG842GGNSLct6DMgyxvz91LXmKPZSKZydoaoSsjfWPjeW+Q24K8EvGOLG1j47Ft1XgeJCnTwOh7fVOSW6GY58a/9zBXvKQZczWntE9QYfbflI65dVWFo9tP3s3modQwL4cnce5ZXumvuD/Fw8dWOiApq0WY0NZ18CnwI/BMYCudjbnIlNXWhjKJwBBftrw9j+5VBWBFgQk1L73Fi34Wor0ZQqT9q9105r7bG1dhKCX7B90KBuYOvQH1x+ztYt0ozO7K22PuNovffFhgfx1aNXtHB1Ip6hseGsM3Ar8I0xZoVlWd2By40xbzZ9qZeuTYazsiI7hJ1qc3F0v309rFv1NmX1VmVwpJNVtj1VFfYWaN22HofSoPy4/brL327lUTPxIAU6DXSuFYlIM4t/9GMa+t3mthHdGdunA5f1jiI0UH9okbaj0eObLMvqBAyr/natMeZIE9bXJNpEOKuqhKz1tc+NZa6zxxv5t6/eqqx+diyqt7YqPY3bDQX7Tp92kLO59iCG5bJX1E5r7ZEAASHO1i3SBEY//UVNs9u6An19cPlYnCivwuVjkdIt3G6E2zeapNgwfDVeSlqxxq6cTQOeAZYBFvbW5kxjzNwmrrNRWmU4M8b+Df1U89f9y+HkMbtHV0xq7XNjXYdpm8wbGWOfmq3b1iN7E5w49WcfC6J6nRHYkupfCW3M1AWRZrZgYxaz5m2htKKq5tqpZ84mJ3Zh44GjLN9tb4FuySrCGAgN9GV07+jqWaDRdI3QeClpXRo9vgm48tRqmWVZHYAlxpjkJq+0EVpNOCs9Wr1VWd3morC6Y0l49+qVsSsgfpw9CFxap+JDZzfPLTpY+3p49zphLdn+b+Tfv7qweaUiDrnQ05oFJ8r5ao/dBHf5rjwOHSsDoGd0O/sUaJ8OjOwVpfFS4vUaG8621H34v7op7WYdCGgiVRX2ScpTz41lbwDjhoBQO4T1vNwOZJE9tVXZlpUUnB3YCvae+z1h3eCnW1umPpFmYIxhz5HjLN9th7XV+/Ipq3Dj57JI7R5Rs6qm8VLijRobzp7B7nH2bvWlW4A0Y8wjTVplI3lNODMG8vdWr4x9YfceKy+2nzmKHVK7OhY7BFz6k6GcQ9kxuxfb65MbuMGCJwpbtCSR5lRWUcWGjKN8uTuXFbvy2J5TO16q7hZolzAdrhHP1xQHAm4CRmM/c7bcGDO/aUtsPI8OZyUF9jNj+5baK2Sntqgi4msf4o8bq7mOcmn+kHD6tucpIV3gv3a0fD0iLaRmvFT182q5xXYLmz4d2zO2+mDByPgogvzVa1A8T6PDmTfwqHBWWQ6Za2tXx7I3AcZuUtpzXG3Psch4pyuV1iBtDix8+PRnzsCejTrtTeh7lTN1ibQgY+zxUiuqg9qa/QWUV7rxd/kwLD7CDmt9ohnQOVRboOIRLimcWZZVDPW2prEAY4wJbboSG8/RcGaM3dfq1EP86Suh4oS9VdlteO2sypjB2qqU5nHmac2R98Gm9+DwVrh8FoybCT5qSyBtR1lFFWv2F7CiuhHuzsPFAES3D6g+WBDNmD7RdAxpA/ODxSNp5ayx6mtT0OsKe6ty71J7u/JYln1vVO/aMBY3BgI9KsNKW1JeAh/9BNL+CX2vhhv+pq1zabMOHytjRfXBghW78yg4UQ5A/84hjOvbgXF9OjA0LoJAP22BSstQOGuMereMLGoWFQPDq09UVnfkj+jR9DWIXCpjYO3LsHiWfXrzln/YzW1F2jC327A955j9rNquPNZlFFBRZQjw9WFEzyjGVbfs6NupPZZOyUszUThrjIYetg4Igxnz7dE7Gmwtnu7Aaphzpz3y67t/hKTvOV2RiMc4cbKSNfvzWb7LXlnbm3sCgE6hATXPqo3pHU1Ue80llqajcNYYT4TT4KN3alMg3qT4MPzrLjjwNYy4D676X02WEKlHVmEpK6ub4K7ck0dRaQUACbGhjO1jb4EO6RGBv6+e45RLp3DWGA2tnKnBp3ijqgr47Few5i/QfRR873UI6ex0VSIeq8pt2JJVVHOwYMOBo1S6DcH+Lkb2jKqZWtCrQzttgcpFUThrjPqeOdNoHPF2W+bCBw9CYJjdbqP7CKcrEvEKxWUVrNqbX3O4ID2/BIDY8KCaoDa6dxThwf4OVyqeTuGssTRUWlqjQ1vhn7fbK8OTnoLh92hEmMhFOpBfwoo9uSzflcvXe/IpPlmJjwVJXcPtgwV9O5DSLRw/l7ZA5XQKZyJSv9JCmP8j2PUpJN0C1z0P/sFOVyXilSqr3GzOLKw5WLDpYCFuA+0DfBnVyz4FOq5vB3pEtXO6VPEACmci0jC3G5Y/A8uegk4JcMtbml4h0gSKSir4em8ey3fnsXxXLlmF9uMx3SODa7ZAL+sdRWigDua0RQpnInJ+u/8N7/+H/fc3vqKxTyJNyBhDen4Jy3flsmJ3Lqv25nOivAqXj8XgbuE1s0CTYsPw1RZom6BwJiIXpmA//PMOjX0SaWbllW42Hjhac7AgLasIYyA00JfRve3tz7F9oukaoccMWiuFMxG5cOUl8NFPIe09jX0SaSEFJ8r5ao8d1JbvyuPQsTIAeka3q9kCHdUrinYBtfOZF2zM4pnFO8kuLCUmPIiZk/oxdXCsU78EuUgKZyJycYyBb16BTx/V2CeRFmaMYc+R4yyvXlVbvS+fsgo3fi6L1O4RjOvbgSq3m78s20tphbvmfUF+Lp66MVEBzUsonInIpdHYJxHHnaysYn360ZqDBdtzjjV4b2x4EF89ekULVieXSuFMRC6dxj6JeJTc4pMM+82SBl9/8vpBjIiPok/H9vj4qHehp2oonPnWd7OIyGlCOsGdH9aOfcrZrLFPIg7qEBJAbHhQTWuOunwseOyDbQBEBPsxPD6S4fFRjIiPZECXUFwKax5PK2cicnG2zIUPH4KAUI19EnHQgo1ZzJq3hdKKqpprQX4ufntDAkN6RLJ6fz5r9xewZn8+BwvsEBca6MuwuEhG9IxkRHwUg2JC1bbDQY5sa1qWdTXwAuACXjHGPN3AfcOA1cAtxpi5lmV1A94EOgNu4CVjzAvn+3kKZyIt5PA2eO82jX0ScdiFntbMLixlzf581uwrYM3+AvbnnQCgnb+LIXGRjIiPZGTPSBJjw/H3VVhrKS0ezizLcgG7gCuBTOAb4PvGmO313PdvoAx4tTqcdQG6GGM2WJYVAqwHpp753jMpnIm0II19EvFaR46VsaZ6VW3NvgJ2HzkOQKCfD0N6RDCiehs0uVs4gX4uh6ttvZx45mw4sMcYs6+6gPeA64EzA9ZDwPvAsFMXjDE5QE713xdblvUtEFvPe0XEKUHhMP1dWPEsLP0tHN6usU8iXqJjaCBTkmOYkhwDQP7xk9VboPbXH5bswhjw9/UhpVs4I+MjGdEzitTuEQT5K6w1t+ZcObsZuNoYc3f193cAI4wxD9a5JxZ4B7gC+DvwkTFm7hmfEwcsBxKMMWedHbYs617gXoDu3bsPycjIaJZfj4icg8Y+ibQqhSXlfJN+lDX78lmzv4Bt2UW4Dfi5LJK6hjMiPpLh8ZEMjYukfYDOFl4qJ7Y1vwdMOiOcDTfGPFTnnn8BvzfGrLYs63XOCGeWZbUHvgR+Y4yZd76fqW1NEQdp7JNIq1VcVsG6jKPVz6zlsyWziEq3weVjkRATyoie9jbo0LhIwoLUZudCORHORgFPGGMmVX8/C8AY81Sde/YDp54ijgZKgHuNMQssy/IDPgIWG2Oeu5CfqXAm4jCNfRJpE0rKK1mfcdTeCt1XwKaDhZRXubEsGNgllOHxkTXPrUW083e6XI/lRDjzxT4QMBHIwj4QcKsxZlsD979O9cqZZVkW8AZQYIz5yYX+TIUzEQ+gsU8ibU5ZRRUbDxTWHDDYcOAoJyvtsVL9OoXUtO4YHh9Jh5AAh6v1HE610pgMPI/dSuNVY8xvLMu6D8AY89cz7n2d2nA2BlgBbMFupQHwC2PMonP9PIUzEQ9yYA3MmaGxTyJt0MnKKrZkFrFmfwGr9+WzPuMoJeV2P7aeHdoxIj6KkdWBrXNYoMPVOkfjm0Sk5Wnsk4gAFVVutmYV1ZwI/WZ/AcUnKwHoERXMiPjalbVukW2nJY/CmYg4o6qiduxT91Ea+yQiVLkN3+YcY3X1adBv0gsoLKkA7MHtI+Jrpxj0iArGaqVNrhXORMRZGvskIg1wuw27jhTXnAZds6+A/BPlAHQKDaiZDTqyZyS9OrRvNWFN4UxEnKexTyJyAYwx7M09bjfFrQ5sh4+dBCC6vX/tadCekfTtGIKPlw5zVzgTEc+gsU8icpGMMWTkl5w2HzSr0B7mHh7sZw9zj49kZM8oBnQJxeUlYU3hTEQ8h9tdO/apU4LGPonIRcs8WlK7Dbq/gIz8EgBCAn1rwtrw+EgSYsPwc3lmQ2yFMxHxPLv/De/fDRiNfRKRRjlUVFYT1Nbsy2dv7gkAgv1dDOkRwcjqKQZJXcPx9/WMsKZwJiKeqWA/zLkDDmnsk4g0ndziU8Pc7a3QnYeLAQj08yG1e0TNc2uDu4cT6GcPc1+wMYtnFu8ku7CUmPAgZk7qx9TBsc1Wo8KZiHgujX0SkWZ29EQ5a9NrDxhszzmGMeDv8iGlWzjhwb4s25lHeZW75j1Bfi6eujGx2QKawpmIeDaNfRKRFlRUWsG69ALW7i9g9f4CNh8srPe+2PAgvnr0imapoaFwpr0DEfEMlmW31rhrEVSUwivfgbR/OV2ViLRSYUF+TBzQiVmTB/DBf46mofOd2dWnQluSwpmIeJbuI+BHyyFmMMy7Gz55xJ4yICLSjGLCgy7qenNSOBMRzxPSCe78EEY+AGv+Cm9MgeJDTlclIq3YzEn9CKo+GHBKkJ+LmZP6tXgtCmci4plcfnD1U3DT3yFnM/xtPBxY43RVItJKTR0cy1M3JhIbHoSF/axZcx4GOBcdCBARz3d4G/zzdig8oLFPItJq6ECAiHivToPgnqXQ+zvwyUx7/FN5idNViYg0C4UzEfEOQeEw/V2Y8P8gbQ78/Sq7ga2ISCujcCYi3sPHB8b/N9z2Lyg6CC+Nh12fOV2ViEiTUjgTEe/T50q4dxmEd4d3psGy39nD1EVEWgGFMxHxTpHx8MPPIHk6LPstvPd9KK2/w7eIiDdROBMR7+UfDFP/ApOfhT1L4KXL7QHqIiJeTOFMRLybxj6JSCujcCYirYPGPolIK6FwJiKth8Y+iUgroHAmIq2Lxj6JiJdTOBOR1inxZrh7iX1o4PXJsOYlaEXj6kSk9VI4E5HWq2bs05Ua+yQiXkPhTERat6BwmP4OTPilxj6JiFdQOBOR1s/HB8bP1NgnEfEKCmci0nZo7JOIeAGFMxFpWzT2SUQ8nMKZiLQ9p419+lxjn0TEoyiciUjbdGrs0w8WQWWZxj6JiMdQOBORtq3bcLj3S4hN1dgnEfEICmciIiGdYMYHMPI/NfZJRBzn63QBIiIeweUHV//WXkH78CF77NOQO2HTO1CUCWFdYeJjkDTN6UpFpJXTypmISF2nxj6ZKvjyd3ZfNIz914UP241sRUSakcKZiMiZOg0Cl//Z1ytK4fMnW74eEWlTFM5EROpzLLv+60UHYecnUFXZsvWISJuhcCYiUp+wrvVft3zg3enwh0H2KlrBvpatS0RaPYUzEZH6THwM/IJOv+YXBNe/aA9Sj0mBlX+A2YPh9evsZ9EqSp2pVURaFZ3WFBGpz6lTmZ8/Wf9pzf7X2lufm96BjW/BvHsgMAwSp0HqDOiS5FztIuLVLGOM0zU0maFDh5p169Y5XYaItDVuN2SshA1vwfYPoOokdEm2Q1rCzRAU7nSFIuKBLMtab4wZetZ1hTMRkSZUehS2zIX1b8DhLeAbBAOvt4Naj8vssVEiIiiciYi0LGMgZ5O9mrblX3DyGET2gtQ7IPlWeyqBiLRpCmciIk4pL4FvP4QNb0LGV2C5oO/VdlDrfSW49PivSFvUUDjT/yOIiDQ3/2BInm5/5e2xDxBsegd2fgztO8Pg22Dw7RDZ0+lKRcQDaOVMRMQJVRWw+zN7NW33Z2DcEDfWfjZtwJSz23iISKujlTMREU/i8rPbcZzWkuMfaskhIlo5ExHxGGrJIdKm6ECAiIg3abAlxx3QY7Racoi0AgpnIiLeSC05RFqthsJZs87WtCzrasuydlqWtceyrEfPcd8wy7KqLMu6+WLfKyLSqlkWxAyG656D/9oJN/wNQjrDkifguQHw7q2w8xOoqnS6UhFpIs12IMCyLBfwZ+BKIBP4xrKsD40x2+u573fA4ot9r4hIm3K+lhwpt9otOaJ6OV2piDRCc66cDQf2GGP2GWPKgfeA6+u57yHgfeDIJbxXRKRtiu4NV/4afrYdpr8DMSnw1fPwx1R4/TpImwMVpU5XKSKXoDlbacQCB+t8nwmMqHuDZVmxwA3AFcCwi3lvnc+4F7gXoHv37o0uWkTEq6glh0ir05wrZ/UdJTrz9MHzwCPGmKpLeK990ZiXjDFDjTFDO3TocAllioi0EqExMO7n8NAGuHMh9JlkN7n921j42zj45hUoLXS6ShE5j+ZcOcsEutX5viuQfcY9Q4H3LPtIeDQw2bKsygt8r4iI1MfHB+LH2V+T/6+2JcfH/wWL/x8MnKqWHCIerNlaaViW5QvsAiYCWcA3wK3GmG0N3P868JExZu7FvvcUtdIQEWmAWnKIeJwWb6VhjKkEHsQ+hfktMMcYs82yrPssy7rvUt7bXLWKiLR6askh4jXUhFZEpC2r25LjxBG15BBpQZoQICIiDauqgN2f2dueuxeDcUPcWPuk54Ap4BfkdIUirU5D4aw5DwSIiIi3OK0lRw5sfscOaqe15LjDHsQuIs1KK2ciIlI/txsyVtohbfsHUHXSDmepMyDhZggKd7pCEa+mbU0REbl0pUftlhwb3oBDW8A3UC05RBpJ4UxERJpG9ia7ue2WuXCyyG7JMfh2+yBBSGenqxPxGgpnIiLStMpL4NsP7aCW8RVYLug7yd727H0luPRYs8i56ECAiIg0Lf9gSJ5uf9VtybFzUf0tOdLmwOdPQlEmhHWFiY9B0jRnfw0iHkgrZyIi0nQaaskR3dcObpWltff6BcGU2Qpo0mZp5UxERJpfQy050lecfW9Fqb2SpnAmcppmG98kIiJtXGgXGPtf8NAGoIHTnEWZLVqSiDdQOBMRkebl42M/Y1YvA38ZDSt+D0fTW7IqEY+lcCYiIs1v4mNnj4DyDYSk6eDfzt7efCEZXp4Iq160t0RF2ig9cyYiIs3v1HNlDZ3WLDwAW+fB1vdh8SxY/AuIGwMJN8KA66FdlHO1i7QwndYUERHPkre7OqjNhbxd4OMLPSdAwk32QYPAUKcrFGkSakIrIiLexRg4vNVeTdv6vr265gqAPldC4s3QZ5Lda03ES6mVhoiIeBfLgs6J9tfExyFznR3Sts2HHR+BXzvoP9leUes1EXz9na5YpElo5UxERLyLu8oeF7X1fdj+gT2UPTAMBnzXDmpxYzU6SryCtjVFRKT1qaqAfcvsIew7PobyYmjXAQbdYAe1rsPtVh4iHkjbmiIi0vq4/Oxn0PpcaU8c2P1ve0Vtw5uw9iUI61Yb1Lok21ulIh5OK2ciItL6nCyGnZ/YQW3PEnBXQmQvO6Ql3gwd+jldoYi2NUVEpI0qKYBvF9pBLX2FPYy9U4LdQ23QjRAZ73SF0kYpnImIiBQfhu0L7KB2cI19LXaovaI26AZ7HqhIC1E4ExERqavwgN2WY8tcOJQGWJpKIC1K4UxERKQhZ04lsFzQq+5UgjCnK5RWSOFMRETkfM41lSDhJuh7taYSSJNRKw0REZHz0VQC8QBaORMRETkfdxVkfG1ve2oqgTQRbWuKiIg0hVNTCba+D99+VDuVYOBUu4eaphLIBVI4ExERaWp1pxLs+hQqyyC0KyScmkqQoqkE0iCFMxERkeZ02lSCz8FdUTuVIOEm6Njf6QrFwyiciYiItBRNJZALoHAmIiLihOLD9iGCrXPrTCUYAgk3w6CpEBrjbH3iGIUzERERp9U3laDHaHtFbeBUTSVoYxTOREREPImmErR5CmciIiKeqN6pBP7Q5ypNJWjlNCFARETEE505lSBrvb3tWXcqQb9r7B5qva4A3wCnK5ZmppUzERERT9TgVIIp1VMJxsG2efD5k1CUCWFdYeJjkDTN6crlAmlbU0RExFvVN5XAPwQqS8FdWXufXxBMma2A5iW0rSkiIuKtXH7Q50r767rqqQTz7j09mIE9seCjn8KJPIjqDVG9ILy7/X7xGgpnIiIi3sQvCAZ+F+bMqP/18uOweFbt9z6+EN7DDmpRvSGyZ21wC+2qOaAeSOFMRETEG4V1haKD9VzvBvcug/y9kL8HCvZW//1eSF8JFSW19/oGQkR8dXDrZY+bOhXc2nfSXFCHKJyJiIh4o4mPwcKH7a3MU/yC7Ovtou2v7iNOf48xUJxjB7WC6vCWv8/uubb7M6gqr73Xv331KtupFbc6wS04smV+jW2UwpmIiIg3OvXQ/8Wc1rQse1xUaAzECG9OQgAAB8VJREFUjz39NXeVvRKXvxcK9lUHt72QvQm2fwimqvbewPDaoFazVVq98hYY2vS/1jZGpzVFRETk3CrLoTDjjBW36q3SY5mn39uuY3Vw63n6altkT3tlT2rotKaIiIhcGl9/iO5jf52pohQK9td5vq16q3TXZ3DiyOn3hv7/9u4/1qu6juP488XlIlzkR4kZcklIXcw1puzGKpbLyGZLw9Zmtlmbrbm2TK0/WvVHrfVPW62VW6scUDYpxhA2ypY2rcw/JC5IAUINifCGxi0iuwXy69Uf59j9aSJ8v99zvl9ej+3sfu85XzgvPmP3vr+fX6e3KNrGDpPOvqy4RwApziIiIuJcdE+DS64qjrGOvTA8RDpyqHTnBjh2ZPh96iq2/Bg1v608Zs2HSV2t+/fUQIqziIiIaI6pM+HSq4tjrP8cnmBF6V448GSxHchLuqZMsKK0LOJmzO3IFaUpziIiIqL1el5bHPPfMvq8DUN/nXh+295H4dSLw+/t7imLtTHz2y66AnouatvCLcVZRERE1IcEM15fHAuWjb52+nSxAOF/hVt5PL8T9jw0+okJF8yaYP+2soibNnvie/9+XS2eVZriLCIiItrDpEnF3LTZb4DLrxt97dQJOHJgfI/bgc2wYz0wYneKnjnjn5hw+Bn49deK55VCsa3IT+4qXre4QGtqcSbpBuBbQBew0vZXx1xfAXwFOA2cBO6x/UR57dPAxylacwdwu+1jzcwbERERbaqre7inbKwTx+Af+8evKH3mMdi+5uX/zhNHi560TinOJHUB3wauBwaALZI22X56xNseBTbZtqTFwDpgkaR5wF3AVbaPSloH3Ar8oFl5IyIiokN1T4XXLSqOsV4cKlaSfu8d469BMcTZYs182ulSYK/tfbaPA2uBFSPfYHvIw7vgTmdUnyOTgWmSJgM9wMEmZo2IiIjz0QUXwtzFxZYdE5nV29o8NLc4mweMfCLrQHluFEkfkLQHeAj4GIDtvwBfBw4AzwH/tP1IE7NGRETE+Wz5F8c/weClZ5W2WDOLs4nWr457VpTtjbYXATdTzD9D0msoetkWApcC0yXdNuFNpDsk9UvqHxwcbFj4iIiIOI8svgVuurfsQVPx9aZ7O2615gAwso+wl/8zNGn7cUmXS5oDXAf8yfYggKQNwNuBByb4c/cB90HxbM3GxY+IiIjzyuJbKinGxmpmz9kW4EpJCyVNoZjQv2nkGyRdIRU7xElaAkwB/k4xnPlWST3l9eXA7iZmjYiIiKiFpvWc2T4p6U7gYYqtNFbb3iXpE+X17wIfBD4q6QRwFPhQuUBgs6T1wDaKLTaeouwdi4iIiOhkGl4s2f76+vrc399fdYyIiIiIVyRpq+2+seebOawZEREREa9SirOIiIiIGklxFhEREVEjKc4iIiIiaiTFWURERESNpDiLiIiIqJEUZxERERE10lH7nEkaBP7c5NvMAf7W5HucT9KejZc2bby0aWOlPRsvbdpYrWrPy2xfPPZkRxVnrSCpf6IN4+LspD0bL23aeGnTxkp7Nl7atLGqbs8Ma0ZERETUSIqziIiIiBpJcfbq5QHsjZX2bLy0aeOlTRsr7dl4adPGqrQ9M+csIiIiokbScxYRERFRIynOzpCkGyT9QdJeSZ+rOk+7k7Ra0iFJO6vO0ikkzZf0S0m7Je2SdHfVmdqZpKmSfivpd2V7frnqTJ1AUpekpyT9tOosnUDSfkk7JG2X1F91nk4gabak9ZL2lD9P39byDBnWfGWSuoA/AtcDA8AW4MO2n640WBuTdC0wBPzQ9purztMJJM0F5treJmkGsBW4Of9Pz44kAdNtD0nqBp4A7rb9ZMXR2pqkzwB9wEzbN1adp91J2g/02c4eZw0i6X7gN7ZXSpoC9Ng+0soM6Tk7M0uBvbb32T4OrAVWVJyprdl+HDhcdY5OYvs529vK1/8CdgPzqk3VvlwYKr/tLo98mj0HknqB9wErq84SMRFJM4FrgVUAto+3ujCDFGdnah7w7IjvB8gvvagxSQuAa4DN1SZpb+UQ3HbgEPAL22nPc/NN4LPA6aqDdBADj0jaKumOqsN0gDcCg8D3y+H3lZKmtzpEirMzownO5RN01JKkC4EHgXtsv1B1nnZm+5Ttq4FeYKmkDMGfJUk3Aodsb606S4dZZnsJ8F7gk+WUkTh7k4ElwHdsXwP8G2j5PPMUZ2dmAJg/4vte4GBFWSJeVjk36kFgje0NVefpFOWwxq+AGyqO0s6WAe8v50itBd4l6YFqI7U/2wfLr4eAjRTTcOLsDQADI3rJ11MUay2V4uzMbAGulLSwnBx4K7Cp4kwRo5QT2FcBu21/o+o87U7SxZJml6+nAe8G9lSbqn3Z/rztXtsLKH6GPmb7topjtTVJ08vFP5RDb+8BsgL+HNh+HnhW0pvKU8uBli+qmtzqG7Yj2ycl3Qk8DHQBq23vqjhWW5P0Y+CdwBxJA8CXbK+qNlXbWwZ8BNhRzpMC+ILtn1WYqZ3NBe4vV2tPAtbZzvYPUSeXABuLz2VMBn5k++fVRuoInwLWlJ0x+4DbWx0gW2lERERE1EiGNSMiIiJqJMVZRERERI2kOIuIiIiokRRnERERETWS4iwiIiKiRlKcRURERNRIirOIiIiIGklxFhEREVEj/wX5MknBVEU0fgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(solver.loss_history, marker='o', label=\"training loss\")\n",
    "plt.plot(solver.val_loss_history, marker='o', label=\"validation loss\")\n",
    "plt.ylabel('loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T15:02:48.380797Z",
     "iopub.status.busy": "2020-11-03T15:02:48.377659Z",
     "iopub.status.idle": "2020-11-03T15:02:48.887106Z",
     "shell.execute_reply": "2020-11-03T15:02:48.888508Z"
    },
    "papermill": {
     "duration": 0.610866,
     "end_time": "2020-11-03T15:02:48.888746",
     "exception": false,
     "start_time": "2020-11-03T15:02:48.277880",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFlCAYAAACukk+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVjWVd7H8fdhE0QFwR1UMPcFBRE1yyxbbFMzKzNraiqtpml6ZiqzZrKamnGyqewp87HGyjLL1NRWG0uzxRVNQHFfARcQQVT2+zx//MhQ0VC5uVk+r+vy0t/+RRQ+nHN+5xhrLSIiIiJSNXh5ugARERER+ZXCmYiIiEgVonAmIiIiUoUonImIiIhUIQpnIiIiIlWIwpmIiIhIFeLj6QIqUqNGjWxERISnyxARERH5TfHx8RnW2sYn769R4SwiIoLVq1d7ugwRERGR32SM2VXWfnVrioiIiFQhCmciIiIiVYjCmYiIiEgVUqPGnJWlsLCQlJQU8vLyPF2KlIO/vz/h4eH4+vp6uhQRERGPqPHhLCUlhfr16xMREYExxtPlyBlYazl48CApKSlERkZ6uhwRERGPqPHdmnl5eYSGhiqYVQPGGEJDQ9XKKSIitVqND2eAglk1os+ViIjUdrUinHlSVlYWkydPPqdrr7nmGrKyss54zlNPPcWiRYvO6f5n8s477/Dggw8CMGXKFKZPn17hzxAREZFT1fgxZ2dr3tpUJi7cRFpWLi2CA3j0qg4MjQ475/v9Es4eeOCBU44VFxfj7e192mu/+OKL37z/s88+e861ldd9993n9meIiIiIQy1npcxbm8q4uYmkZuVigdSsXMbNTWTe2tRzvufjjz/Otm3b6NGjB48++ihLlizh0ksvZeTIkXTr1g2AoUOH0rNnT7p06cLUqVOPXxsREUFGRgY7d+6kU6dO3HvvvXTp0oUrr7yS3NxcAO68805mz559/Pzx48cTExNDt27d2LhxIwDp6elcccUVxMTEMGbMGFq3bk1GRsYptb799tu0b9+eSy65hB9//PH4/qeffpoXX3wRgK1bt3L55ZfTvXt3YmJi2LZtGwATJ06kV69eREVFMX78+HP++xIREfGUeWtT6TfhWyIf/5x+E749r+//56NWtZw98+l6NqQdPu3xtbuzKCh2nbAvt7CYx2YnMHPl7jKv6dyiAeOv73Lae06YMIGkpCR+/vlnAJYsWcLKlStJSko6/kbitGnTCAkJITc3l169enHjjTcSGhp6wn22bNnCzJkzefPNN7n55puZM2cOo0aNOuV5jRo1Ys2aNUyePJkXX3yRt956i2eeeYbLLruMcePG8dVXX50QAH+xd+9exo8fT3x8PEFBQVx66aVER0efct5tt93G448/zg033EBeXh4ul4uvv/6aLVu2sHLlSqy1DB48mKVLl9K/f//T/r2IiIhUJb800OQWFgO/NtAA59WDdi7UclbKycHst/afq7i4uBOminj11Vfp3r07ffr0Yc+ePWzZsuWUayIjI+nRowcAPXv2ZOfOnWXee9iwYaec88MPPzBixAgABg0aRMOGDU+5bsWKFQwYMIDGjRvj5+fHLbfccso5OTk5pKamcsMNNwDOnGR169bl66+/5uuvvyY6OpqYmBg2btxY5scgIiJSVU1cuOl4MPtFbmExExduqvRaalXL2ZlauAD6TfiW1KzcU/aHBQfw0Zi+FVZHYGDg8T8vWbKERYsWsWzZMurWrcuAAQPKnEqiTp06x//s7e19vFvzdOd5e3tTVFQEOPOHlcdvvSl5uvtYaxk3bhxjxowp13NERESqAmstWw8cYcWOzDK//wOknWa/O6nlrJRHr+pAgO+JA/QDfL159KoO53zP+vXrk5OTc9rj2dnZNGzYkLp167Jx40aWL19+zs86nYsuuohZs2YB8PXXX3Po0KFTzunduzdLlizh4MGDFBYW8vHHH59yToMGDQgPD2fevHkA5Ofnc+zYMa666iqmTZvGkSNHAEhNTeXAgQMV/nGIiIicj2KXJSk1m2k/7GDMe6vp+dwirnh5KX+dl4TXadonWgQHVG6R1LKWs9/yS59yRb6tGRoaSr9+/ejatStXX30111577QnHBw0axJQpU4iKiqJDhw706dPnvD6GsowfP55bb72Vjz76iEsuuYTmzZtTv379E85p3rw5Tz/9NH379qV58+bExMRQXFx8yr3ee+89xowZw1NPPYWvry8ff/wxV155JcnJyfTt67Qu1qtXj/fff58mTZpU+MciIiJSXoXFLhJTs1mxPZOVOw6yeuchcvKdXqWWIQFc1rEJcZEh9I4MYc2uQzzxSdIJXZvn20Bzrkx5u7yqg9jYWLt69eoT9iUnJ9OpUycPVVQ15Ofn4+3tjY+PD8uWLeP+++8//oJCVaTPmYiInIu8wmJ+3pPlhLGdB1mzK+t42LqgcSC924TSOzKEXhEhZbaIVfR0Wr/FGBNvrY09eb9azmqB3bt3c/PNN+NyufDz8+PNN9/0dEkiIiLn7Uh+EWt2HWLFjoOs3JHJuj3ZFBS7MAY6NWvALb1aOmEsMoRG9er85v2GRodV+puZZVE4qwXatWvH2rVrPV2GiIjIeck+VsiqnZms3JnJiu0HSUo7TLHL4u1l6BYWxF39IoiLDCG2dQhBdX09Xe45UzgTERGRKik9J98JYzsyWb79IJv252At+Pl40aNlMA8MuIC4yBBiWjUksE7NiTQ15yMRERGRai0tK5eVOzJZseMgK3Zksj39KOAMzI+NaMi13ZoTFxlC95bB+PuefvnD6k7hTERERCqdtZZdB485rWIlY8ZSDjlzitX39yEuIoRbYlsSFxlC17AgfL1rz+xfCmciIiLidi6XZWu6M+Hriu1OGDuQkw9AaKAfcZEh3H1RJHGRIXRs1gDv0008VgsonFVB9erV48iRI6SlpfHQQw8dX9i8tAEDBvDiiy8SG3vKG7jHvfLKK4wePZq6desCcM011/DBBx8QHBxc6fWKiEjtUuyyJO89fDyMrdqZyaFjhQA0a+BP3wtCj88xdkHjer+5Sk1t4tZwZowZBEwCvIG3rLUTyjhnAPAK4AtkWGsvKXXMG1gNpFprr3NnrcclzIJvnoXsFAgKh4FPQdTNlfLok7Vo0eK8gs4rr7zCqFGjjoezL774oqJKK9P51isiItVXQZEz4evKHadO+NoqpC6Xd2paEsZCaRkSoDB2Bm4LZyXB6nXgCiAFWGWMWWCt3VDqnGBgMjDIWrvbGHPylPJ/ApKBBu6q8wQJs+DTh6CwZB2t7D3ONpxzQBs7diytW7fmgQceAODpp5+mfv36jBkzhiFDhnDo0CEKCwt57rnnGDJkyAnX7ty5k+uuu46kpCRyc3O566672LBhA506dTphbc3777+fVatWkZuby/Dhw3nmmWd49dVXSUtL49JLL6VRo0YsXryYiIgIVq9eTaNGjXjppZeYNm0aAPfccw8PP/wwO3fu5Oqrr+aiiy7ip59+IiwsjPnz5xMQcOJEfTt27GDkyJEUFRUxaNCgMustLi5m7NixLFy4EGMM9957L3/84x+Jj4/nz3/+M0eOHKFRo0a88847NG/e/Jz+bkVExHPyCotZuzvr+AD+NbsPkVfoAqBtk3oM7tGCuMgQ4iJDaB5U+UsgVWfubDmLA7Zaa7cDGGM+BIYAG0qdMxKYa63dDWCtPb4gozEmHLgWeB74c4VU9OXjsC/x9MdTVkFx/on7CnNh/oMQ/27Z1zTrBlef0iB43IgRI3j44YePh7NZs2bx1Vdf4e/vzyeffEKDBg3IyMigT58+DB48+LQ/SbzxxhvUrVuXhIQEEhISiImJOX7s+eefJyQkhOLiYgYOHEhCQgIPPfQQL730EosXL6ZRo0Yn3Cs+Pp63336bFStWYK2ld+/eXHLJJTRs2JAtW7Ywc+ZM3nzzTW6++WbmzJnDqFGjTrj+T3/6E/fffz933HEHr7/+epn1Tp06lR07drB27Vp8fHzIzMyksLCQP/7xj8yfP5/GjRvz0Ucf8eSTTx4PiSIiUnUdyS8iftchVu44yIrtmaxLyaKw2B6f8PXWuFb0jgwhNqJ8E77K6bkznIUBe0ptpwC9TzqnPeBrjFkC1AcmWWunlxx7BXisZP9pGWNGA6MBWrVqdX4VnxzMfmt/OURHR3PgwAHS0tJIT0+nYcOGtGrVisLCQp544gmWLl2Kl5cXqamp7N+/n2bNmpV5n6VLl/LQQ04rXlRUFFFRUcePzZo1i6lTp1JUVMTevXvZsGHDCcdP9sMPP3DDDTcQGBgIwLBhw/j+++8ZPHgwkZGR9OjRA4CePXuyc+fOU67/8ccfmTNnDgC33347Y8eOPeWcRYsWcd999+Hj4/wTCwkJISkpiaSkJK644goAiouL1WomIlJFZR0rYNXOkjC2I5P1JRO++ngZuoUH8fuLIukdGULP1iEEBVTfCV+rIneGs7KagE5eyNMH6AkMBAKAZcaY5Tih7YC1Nr5kTNppWWunAlPBWVvzjBWdoYULgJe7Ol2ZJwtqCXd9fuZrz2D48OHMnj2bffv2MWLECABmzJhBeno68fHx+Pr6EhERQV5e3hnvU1ar2o4dO3jxxRdZtWoVDRs25M477/zN+5xpPdU6dX79acfb2/uE7tPfquXkZ5x8jrWWLl26sGzZsjNeKyIile9ATh6rdvwaxjbuywGcCV+jWwbzhwEXEBcZSnSr4Bo14WtV5M6/3RSgZantcCCtjHMyrLVHgaPGmKVAdyAGGGyMuQbwBxoYY9631o7CnQY+deKYMwDfAGf/eRgxYgT33nsvGRkZfPfddwBkZ2fTpEkTfH19Wbx4Mbt27TrjPfr378+MGTO49NJLSUpKIiEhAYDDhw8TGBhIUFAQ+/fv58svv2TAgAEA1K9fn5ycnFO6Nfv378+dd97J448/jrWWTz75hPfee6/cH0+/fv348MMPGTVqFDNmzCjznCuvvJIpU6YwYMCA492aHTp0ID09nWXLltG3b18KCwvZvHkzXbp0KfezRUSkYqRm5R7voly5I5PtGc6Er3X9vOnZuiHXRTUnLjKUqPCgGj3ha1XkznC2CmhnjIkEUoEROGPMSpsPvGaM8QH8cLo9X7bWfgyMg+Nvcz7i9mAGvw76r+C3Nbt06UJOTg5hYWHHu/Fuu+02rr/+emJjY+nRowcdO3Y84z3uv/9+7rrrLqKioujRowdxcXEAdO/enejoaLp06UKbNm3o16/f8WtGjx7N1VdfTfPmzVm8ePHx/TExMdx5553H73HPPfcQHR1dZhdmWSZNmsTIkSOZNGkSN954Y5nn3HPPPWzevJmoqCh8fX259957efDBB5k9ezYPPfQQ2dnZFBUV8fDDDyuciYi4mbWWnQePHW8VW7E9k9QspyGigb8PcZEhjIhrSVxkKF1aNKhVE75WReZMXVznfXOn5esVnKk0pllrnzfG3AdgrZ1Scs6jwF2AC2e6jVdOuscAnHD2m1NpxMbG2tWrV5+wLzk5mU6dOlXARyOVRZ8zEZHymbc2lYkLN5GWlUuL4AAevaoDQ6PDcLksWw4cOR7GSk/42qieM+FrXEQIcZGhdGhWv1ZP+OpJxph4a+0pE5a6NZxVNoWzmkGfMxGR3zZvbSrj5iaSW1h8fJ+vt6FD0/qkZuUen/C1eZA/vSOdIBYXGcIFjQM1x1gVcbpwphF9IiIi1dDEhZtOCGYAhcWW5H053BgTRlxkKL0jQwhvqAlfqxuFMxERkWqmsNh1fMzYyVwuywvDu1dyRVKRasWIv5rUdVvT6XMlInJmSanZDH7tx9MebxGs2firuxofzvz9/Tl48KC+6VcD1loOHjyIv7+/p0sREaly8gqLmfDlRoa8/iMZR/K5q18EASdNcRHg682jV3XwUIVSUWp8t2Z4eDgpKSmkp6d7uhQpB39/f8LDwz1dhohIlbJi+0Een5vIjoyj3BLbkieu6URQXV+6hweX+bamVG81Ppz5+voSGRnp6TJERETOWk5eIRO+3MiMFbtpFVKXGff0pl/bXycWHxodpjBWA9X4cCYiIlIdfbtxP09+ksT+w3ncc1Ekf76yPXX99G27NtBnWUREpAo5eCSfZz/bwPyf02jftB6Tb7uQ6FYNPV2WVCKFMxERkSrAWsuCdWk88+kGcvIKefjydjwwoC1+PjX+3T05icKZiIiIh6Vl5fLXeUl8u/EAPVoG88LwKNo3re/pssRDFM5EREQ8xOWyfLByNxO+3Eixy/K36zpz54URWuuyllM4ExER8YDt6Ud4fG4iK3dkclHbRvxzWDdahtT1dFlSBSiciYiIVKKiYhdv/bCDl/+7mTo+XrwwPIqbeoZr/Us5TuFMRESkkqxPy2bsnASSUg9zVZem/H1IV5o00KoociKFMxERETfLKyzmf7/dwpTvttOwrh9v3BbD1d2ae7osqaIUzkRERNxo1c5Mxs5JYHv6UW7qGc6T13YiuK6fp8uSKkzhTERExA2O5Bfxwlcbmb5sF+ENA5j++zj6t2/s6bKkGlA4ExERqWCLNx3gybmJ7D2cx139Injkyg4E1tG3XCkf/UsRERGpIIeOFvD3zzYwd20qbZvUY/Z9F9KztZZekrOjcCYiInKerLV8lrCXpxesJzu3kIcGtuMPl15AHR9vT5cm1ZDCmYiIyHnYl53HX+clsSh5P1HhQbx/T286NW/g6bKkGlM4ExEROQfWWj5ctYd/fJ5MocvFk9d04q5+Efh4a6FyOT8KZyIiImdpZ8ZRxs1NZNn2g/RtE8qEG7vROjTQ02VJDaFwJiIiUk5FxS6m/biDf3+9GT9vLyYM68YtvVpq6SWpUApnIiIi5ZC89zBj5ySQkJLN5Z2a8tzQrjQL0tJLUvEUzkRERM4gv6iY17/dyuQl2wgK8OW1kdFc2625WsvEbRTORERETiN+VyZj5ySy9cARhkWH8bfrOtMwUEsviXspnImIiJzkaH4RExdu4t1lO2kRFMA7d/ViQIcmni5LagmFMxERkVKWbk5n3NxE0rJzuaNPax4d1JF6WnpJKpH+tYmIiABZxwp47vNkZsen0KZxIB+P6UtsRIiny5JayK3hzBgzCJgEeANvWWsnlHHOAOAVwBfIsNZeYozxB5YCdUpqnG2tHe/OWkVEpHay1vJl0j6emr+eQ8cKePDStjx4WVv8fbX0kniG28KZMcYbeB24AkgBVhljFlhrN5Q6JxiYDAyy1u42xvzSoZ8PXGatPWKM8QV+MMZ8aa1d7q56RUSk9jlwOI+/zU9i4fr9dA1rwLu/70WXFkGeLktqOXe2nMUBW6212wGMMR8CQ4ANpc4ZCcy11u4GsNYeKPndAkdKzvEt+WXdWKuIiNQi1lo+Xp3C3z/fQEGRi3FXd+TuiyK19JJUCe4MZ2HAnlLbKUDvk85pD/gaY5YA9YFJ1trpcLzlLR5oC7xurV1R1kOMMaOB0QCtWrWqyPpFRKQG2n3wGOM+SeDHrQeJiwzhXzdGEdlISy9J1eHOcFbW7Hwnt375AD2BgUAAsMwYs9xau9laWwz0KOn6/MQY09Vam3TKDa2dCkwFiI2NVeuaiIiUqdhlebtk6SVvL8PzN3Tl1l6t8PLSZLJStbgznKUALUtthwNpZZyTYa09Chw1xiwFugObfznBWptV0rI2CDglnImIiPyWTftyGDsngZ/3ZDGwYxOeu6ErzYMCPF2WSJncGc5WAe2MMZFAKjACZ4xZafOB14wxPoAfTrfny8aYxkBhSTALAC4H/uXGWkVEpAYqKHLx+uKtTF6ylfr+vrx6azTXR2npJana3BbOrLVFxpgHgYU4U2lMs9auN8bcV3J8irU22RjzFZAAuHCm20gyxkQB75aMO/MCZllrP3NXrSIiUvOs3X2IsXMS2Lz/CEN7tOCp67sQoqWXpBowzouRNUNsbKxdvXq1p8sQEREPOlZQxL+/3sy0H3fQrIE/z9/Qlcs6NvV0WSKnMMbEW2tjT96vFQJERKTG+HFrBo/PTWBPZi6j+rRi7KCO1Pf39XRZImdF4UxERKq97GOFPP/FBmatTiGyUSAfje5D7zahni5L5JwonImISLX2VdI+/jY/icyjBdw/4AL+NLCdll6Sak3hTEREqqUDOXk8vWA9XyTuo3PzBrx9Zy+6hmnpJan+FM5ERKRasdYyOz6F5z5PJrewmEev6sDo/m3w1dJLUkMonImISLWxJ/MYT3ySyPdbMugV0ZAJN0ZxQeN6ni5LpEIpnImISJVX7LJMX7aTiQs3YYC/D+nCbb1ba+klqZEUzkREpErbst9ZemnN7iwGdGjM8zd0IyxYSy9JzaVwJiIiVVJBkYsp323jtW+3EljHm5dv6c7QHmFaeklqPIUzERGpctbtyWLsnAQ27svh+u4tGH99ZxrVq+PpskQqhcKZiIhUGbkFxby8aDNvfb+dxvXr8OYdsVzRWUsvSe2icCYiIlXCT9syGDc3kV0HjzGydysev7ojDbT0ktRCCmciIuJR2bmFTPgymZkr99A6tC4z7+1D3wu09JLUXgpnIiLiMf/dsJ+/zkskPSefMf3b8PDl7Qnw09JLUrspnImISKWYtzaViQs3kZaVS9Mgf5o1qMPPe7Lp2Kw+b94RS1R4sKdLFKkSFM5ERMTt5q1NZdzcRHILiwHYl53Hvuw8runajEm3RmvpJZFS9L9BRETcbuLCTceDWWnrUrIVzEROov8RIiLidmlZuWe1X6Q2UzgTERG3+iZ5/2mPtdAyTCKnUDgTERG3cLkskxZt4e53VxMW7E8dnxO/5QT4evPoVR08VJ1I1aVwJiIiFS4nr5Ax78fz8qLNDIsOY9FfBvCvG6MICw7AAGHBAfxzWDeGRod5ulSRKkdva4qISIXaeuAIo99bza6Dxxh/fWfuvDACYwxDo8MUxkTKQeFMREQqzH837Od/PvqZOj5ezLinN33aaKZ/kbOlcCYiIufN5bJM+mYLk77ZQrewIKbc3pMwDfYXOScKZyIicl4O5xXy549+ZlHyAW6MCef5G7ri76slmETOlcKZiIics1/Gl+0+eIxnBnfhjr6tMcZ4uiyRak3hTEREzsnC9fv4y6x1x8eX9db4MpEKoXAmIiJnxeWyvLJoM69+u5Xu4UG8MaqnJpMVqUAKZyIiUm7Zuc74sm82HmB4z3CeG6rxZSIVTeFMRETKZcv+HEa/F8+ezGP8fUgXRvXR+DIRd3DrCgHGmEHGmE3GmK3GmMdPc84AY8zPxpj1xpjvSva1NMYsNsYkl+z/kzvrFBGRM/sqaR9DX/+RnLxCPri3D7f3jVAwE3ETt7WcGWO8gdeBK4AUYJUxZoG1dkOpc4KBycAga+1uY0yTkkNFwF+stWuMMfWBeGPMf0tfKyIi7udyWV5etJn//XYr3VsGM2VUDM2DNL5MxJ3c2a0ZB2y11m4HMMZ8CAwBSgeskcBca+1uAGvtgZLf9wJ7S/6cY4xJBsJOulZERNwoO7eQhz9cy+JN6dwcG86zQzS+TKQyuDOchQF7Sm2nAL1POqc94GuMWQLUByZZa6eXPsEYEwFEAyvKeogxZjQwGqBVq1YVULaIiGzen8Po6atJOZTL34d2ZVTvVurGFKkk7gxnZf0vtmU8vycwEAgAlhljlltrNwMYY+oBc4CHrbWHy3qItXYqMBUgNjb25PuLiMhZ+ippL3+ZtY4APx9mju5Dr4gQT5ckUjkSZsE3z0J2CgSFw8CnIOrmSi/DneEsBWhZajscSCvjnAxr7VHgqDFmKdAd2GyM8cUJZjOstXPdWKeIiADFLstL/93E64u30aNlMFNG9aRZkL+nyxKpHAmz4NOHoDDX2c7e42xDpQc0d4azVUA7Y0wkkAqMwBljVtp84DVjjA/gh9Pt+bJx2s7/AyRba19yY40iIgJkHyvkTx+tZcmmdEb0askzQ7pQx0fjy6QGc7ng0A7Yl+j8WvYaFOWdeE5hrtOSVlPCmbW2yBjzILAQ8AamWWvXG2PuKzk+xVqbbIz5CkgAXMBb1tokY8xFwO1AojHm55JbPmGt/cJd9YqI1Fab9uUw+r3VpGXl8vwNXbmtd2tPlyRSsQrzID0Z9ib8Gsb2J0HBEee48QZbXPa12SmVV2cJY23NGaYVGxtrV69e7ekyRESqjS8S9/LIx+sIrOPDlFEx9Gyt8WVSzR3LhH2lQti+REjf9Gv48qsHzbqd+KtxJ3gt1unKPFlQS/ifJLeUaoyJt9bGnrxfKwSIiNRCxS7Lv7/exOQl24hu5Ywva9pA48ukGnG5IGvXqUHscOqv59Rv4YSvDtf8GsQaRoJXGXPwD3zqxDFnAL4Bzv5KpnAmIlLLZB8r5KEP1/Ld5nRujWvF04M7a3yZVG1F+XAg+cQQtj8J8ksmcjBe0Kg9tO53YotYYKPyP+OXcWU1/G1NERGpYjbuO8zo6fHszc7lHzd0Y2RvzQ8pVcyxTCd4/RLC9iZAxiZwFTnHfQOhWVcnNP0Swpp0dlq5zlfUzR4JYydTOBMRqSU+T9jLo7PXUa+ODx+O7kvP1g09XZLUZtaWdEsmnvir9Lives1KuiUHlQSxqNN3S9YgCmciIjVcscsyceEmpny3jZ6tG/LGbTE00fgyqUxFBZC+8dQglp/tHDdeENoOWvaGXvf82iJWr8mZ71tDKZyJiNRgWccK+OPMtXy/JYPberdi/PVd8POp2a0O4mG5WaeGsPSN4Cp0jvvWhaZdoNvwX1vDmnQCv7qerbsKUTgTEamhkvceZvR7q9mfnc+EYd0YEafxZVKBrHW6IE8IYgmQtfvXc+o1dQJYu8t/DWIhbcBLL6CcicKZiEgN9Om6NB6bnUCDAB8+HNOHmFYaXybnobjQmSusdAjblwB5Jd2SGAhtC2GxEPt7J4g17Qb1m3q07OpK4UxEpAYpKnYxceEm/m/pdmJbN2TyqBia1Nf4MjkLedmwL+nEIJa+EYoLnOM+AU63ZJdhv7aGNe0MfoGerbsGUTgTEakhDh11xpf9sDWDUX1a8dR1Gl8mZ2CtM2HrCa1hiXBo56/n1G0EzaPggvudENYsCkIvULekmymciYjUABvSDjPmfWd82b9u7MYtvTS+rFZImFW+STQ45coAACAASURBVFOLCyFjS6nZ9Et+zz1UcoJxQleLaIi5oySIdXPGjBlTqR+SKJyJiFR7C9al8djsdQQH+PHRmD5Ea3xZ7ZAw68TlhrL3/LrdqP2JIexAMhTnO+f5+DuTtnYa7ASw5t2d7Tr1PPexyAkUzkREqqmiYhcvLNzE1KXb6RXRkNdv0/iyWuWbZ09cBxKc7U8f+nW7bqjTCtZ7zK+tYaFtwVvf/qsyfXZERKqhQ0cLeHDmGn7cepA7+rbmr9d21viy2iTv8Ikz6Z9s5CwnjNVvpm7JakjhTESkmlmfls2Y9+I5cDifF4ZHcXNsS0+XJJXl4DZYORXWzjj9OUEtof1VlVeTVDiFMxGRamT+z6mMnZNAcIAfs+7rS4+WwZ4uSdzNWtj5PSx/AzZ9CV4+0HWY0z35w0sndm36BjgvBUi1pnAmIlINFBW7mPDlRt76YQdxESG8flsMjevX8XRZ4k6FeZD4MayYAvuTnPFj/R+B2LuhQXPnnIYR5XtbU6oVhTMRkSou82gBD36whp+2HeTOCyN48tpO+HprfFmNlbMPVv0HVk+DYxnQpAsMfg263QS+J73wEXWzwlgNpHAmIlKFJaU648vSj+QzcXgUN2l8Wc2VttbpukyaC64i6HA19L4PIvtrUH8to3AmIlJFzVvrjC8LCfRj9n19iQrX+LIap7gINn3uhLLdy8CvHvS6G+JGO5PCSq2kcCYiUsUUFbv4xxcbmfbjDuIiQ5h8WwyN6ml8WY2SmwVrpsPKNyF7NwS3hqv+AdGjwD/I09WJhymciYhUIQeP5POHD9awfHumxpfVRBlbnQH+P38AhUeh9UUw6J9OF6bWq5QSCmciIlVE6fFl/76pOzf2DPd0SVIRrIXti52uyy1fg7efM7i/933OouIiJ1E4ExGpAuauSWHc3ERCA/2Yc9+FdAtX11a1V3AMEmfB8imQngyBjWHAOIj9PdRr4unqpApTOBMR8aDCYhf/+CKZt3/cSZ82Ibw+MoZQjS+r3g6nOWPJ4t+G3EPOepZDpzgTx/rocyu/TeFMRMRDMo7k84cZa1ixI5Pf94tk3DUdNb6sOkuJh+WTYcM8cBVDx2uhzwPQ+kJNhSFnReFMRMQDElKyuO+9eA4eLeClm7szLEbjy6ql4kJIXuCMJ0tZBXUaOGPJ4u51Zu8XOQcKZyIilWx2fApPfJJI43p1mHP/hXQN0/iyaudYJqx51+m+PJwKDSPh6hegx0ioU9/T1Uk1p3AmIlJJCotdPP95Mu/8tJO+bUJ5bWS0xpdVN+mbnFaydR9CUS5EXgLXvgTtrgQvdUlLxVA4ExGpBOk5zvxlK3dkcvdFkYy7uiM+Gl9WPbhcsO0bJ5Rt+wa86zjrWfa5H5p28XR1UgO5NZwZYwYBkwBv4C1r7YQyzhkAvAL4AhnW2ktK9k8DrgMOWGu7urNOERF3Wrcni/vejyfzaAGv3NKDodFhni5JyqPgKKyb6UyFcXAL1GsGl/4VYu+CwEaerk5qMLeFM2OMN/A6cAWQAqwyxiyw1m4odU4wMBkYZK3dbYwpPfHLO8BrwHR31Sgi4m4fr97Dk/OSNL6sOsnaA6vehPh3IC8bWkTDsDeh81Dw8fN0dVILuLPlLA7Yaq3dDmCM+RAYAmwodc5IYK61djeAtfbALwestUuNMRFurE9ExG0Ki138/bMNTF+2iwsvCOW1kTGEBOobe5VlLexZ6UyFkfwpYKHTYKfrsmVvTYUhlcqd4SwM2FNqOwXofdI57QFfY8wSoD4wyVqrljIRqdbSc5z5y1buzOTeiyMZO0jjy6qsogJnXrLlb0DaGmfR8b5/cKbCCG7l6eqklnJnOCvrxwxbxvN7AgOBAGCZMWa5tXZzuR9izGhgNECrVvqPJCKe9fMeZ/6yrNwCJo3owZAeGl9WJR09CPHTYOVbcGQfhLaDa/8N3W8Fv0BPVye1nDvDWQrQstR2OJBWxjkZ1tqjwFFjzFKgO1DucGatnQpMBYiNjT05/ImIVJpZq/bw13lJNGngjC/r0kLjy6qc/RtgxRuQMAuK8uCCy2DIa3DBQE2FIVWGO8PZKqCdMSYSSAVG4IwxK20+8Joxxgfww+n2fNmNNYmIVLiCIhfPfrae95fvpl/bUF67NYaGGl9WdbhcsOVrZzzZju/AJ8BpIet9HzTp6OnqRE7htnBmrS0yxjwILMSZSmOatXa9Mea+kuNTrLXJxpivgATAhTPdRhKAMWYmMABoZIxJAcZba//jrnpFRM7FgZw8Hnh/Dat3HWJM/zY8elUHjS+rKvJz4OeZTktZ5nao3wIGjoeed0LdEE9XJ3Jaxtqa0xMYGxtrV69e7ekyRKSWWLv7EPe9H092biEvDO/O4O4tPF2SABza6SyrtGY65B+G8F7OW5edBoO3r6erEznOGBNvrY09eb9WCBAROQcfrdrN3+atp2lQHebe34/OLRp4uqTazVrY9ZPTSrbxczBe0HkI9L4fWvbydHUiZ0XhTETkLBQUuXjm0/XMWLGbi9s14tUR0Rpf5klF+ZA01xlPti8BAhpCv4eh1z0QpDdlpXpSOBMRKacDh/O4f8Ya4ncdYswlbXjsqo54e2lyUo84kg6rp8Gqt+DoAWjUAa57BaJuAb+6nq5O5LwonImInMa8talMXLiJtKxcQuvVIb+wiCIXvDYymuuiNL7MI/YmwIopkPgxFBdAuyud8WRtLtUs/lJjlDucGWNaA+2stYuMMQGAj7U2x32liYh4zry1qYybm0huYTEAGUfyMcAjV3VQMKtsrmLY9KUTynZ+D751IeYOZyqMRu08XZ1IhStXODPG3IszC38IcAHOhLJTcGb2FxGpcSYu3HQ8mP3CAh+s2M0fLm3rmaJqm7zDsPZ9J5Rl7YKglnDF3yHmdmdsmUgNVd6Wsz/gLGS+AsBau8UY08RtVYmIeFhaVu5Z7ZcKdHAbrJwKa2dAQQ607ANXPAsdrwNvjcaRmq+8/8rzrbUFpqQ/v2RG/5ozQZqIyEmaNKjD/sP5p+xvERzggWpqAWudLsvlbzhdmF4+0HWY03UZFuPp6kQqVXnD2XfGmCeAAGPMFcADwKfuK0tExHNcLkugnw9wYjgL8PXm0as6eKaomiBhFnzzLGSnQFA4DHzKmRg28WOn63J/EtQNhf6PQOzd0KC5pysW8YhyrRBgjPEC7gauBAzOkkxv2Sq2vIBWCBCRivDesp38bf56RsS15PvNGaRl5dIiOIBHr+rA0GjNnXVOEmbBpw9BYaluYS8f8PaHwiPQpIvz1mW34eCr1kmpHc53hYAAnLUx3yy5mXfJvmMVV6KIiOelHDrGhC83cnG7Rvzzhm4YTc9QMb559sRgBuAqAq8iuGMBRPbXVBgiJcq7Ou83OGHsFwHAooovR0TEc6y1jJubCMA/hymYVajslLL3F+VDm0sUzERKKW8487fWHvllo+TPmoJZRGqUj+NT+H5LBo9f3ZHwhvoSVyF2L4f3hnHad8iCwiu1HJHqoLzh7Kgx5vjrMsaYnoDeJxeRGmP/4Tz+/tkG4iJDuK13a0+XU71ZCzuWwjvXwbSrYO/P0HkY+Jw0lsw3wHkpQEROUN4xZw8DHxtj0kq2mwO3uKckEZHKZa3lyU8SKSx28cKNUXhpvcxzYy1s+wa+mwh7lkO9pnDl8xB7F/gFlv22ZtTNnq5apMopVziz1q4yxnQEOuC8rbnRWlvo1spERCrJgnVpLEo+wF+v7UREo0BPl1P9WAubv4KlEyE1HhqEwTUvQvTt4Ov/63lRNyuMiZTDGcOZMeYya+23xphhJx1qZ4zBWjvXjbWJiLjdwSP5PPPpBnq0DOaufpGeLqd6cblg46dOKNuXCMGt4LpXoMdI8Knj6epEqq3fajnrD3wLXM+JozlNybbCmYhUa+MXrOdIXhEvDI/CW92Z5eMqhvWfwNIXIT0ZQi6AIZOdVjFvX09XJ1Lt/VY4yzHG/BlIwgljv3zlqlKTz4qInIuvkvbxWcJeHrmyPe2b1vd0OVVfcREkzoLv/w0Ht0LjjjDsLWeZJS9vT1cnUmP8VjirV/J7B6AXMB8noF0PLHVjXSIibpV1rIC/zU+ic/MGjLnkAk+XU7UVFcC6D+D7lyBrFzTtBje96yy95FXel/5FpLzOGM6stc8AGGO+BmKstTkl208DH7u9OhERN/n7Z8kcOlrAO3f1wtdbAaNMhXmw9j344RU4nAItomHQBOhwtSaNFXGj8k6l0QooKLVdAERUeDUiIpVgyaYDzFmTwoOXtqVLiyBPl1P1FByD+Hfgx0lwZB+07A3XT4K2AxXKRCpBecPZe8BKY8wnOOPNbgDedVtVIiJukpNXyBNzE2nbpB5/HNjW0+VULfk5sOo/sOw1OJoOERfDsKla91KkkpV3nrPnjTFfAheX7LrLWrvWfWWJiLjHhC83su9wHnPuv5A6PhrEDkBuFqycCssnQ+4huOAy6P8YtO7r6cpEaqXytpxhrV0DrHFjLSIibrVs20FmrNjNvRdHEt2qoafL8bxjmbD8DVjxf5CfDe0HQf9HITzW05WJ1GrlDmciItXZsYIixs5JICK0Ln++ooOny/GsI+lO1+Wqt6DgCHS63gllzbt7ujIRQeFMRGqJf3+9md2Zx/hwdB8C/Gppd2bOPvjxVVg9DYryoMsN0P8RaNrF05WJSCkKZyJS48XvOsS0H3dwe5/W9GkT6ulyKl92ijMdxprp4CpyZvK/+C/QqJ2nKxORMiiciUiNlldYzGOz19EiKICxV3f0dDmVK3MH/PAy/PwBYJ01Ly/6Hwhp4+nKROQMFM5EpEZ79ZstbEs/yvTfx1GvTi35kpex1VliKeEjZ1mlnr+Dfg9DcEtPVyYi5eDWr1TGmEHAJMAbeMtaO6GMcwYArwC+QIa19pLyXisiciZJqdn839Lt3NQznP7tG3u6HPc7kOwsRr5+Lnj7Qdxo6PcQNGjh6cpE5Cy4LZwZY7yB14ErgBRglTFmgbV2Q6lzgoHJwCBr7W5jTJPyXisiciYFRS4e+XgdoYF+/PXazp4ux732JsDSiZC8AHwDoe+DcOEfoV4TT1cmIufAnS1nccBWa+12AGPMh8AQoHTAGgnMtdbuBrDWHjiLa0VETmvKd9vYuC+HN++IJaiur6fLcY/UePhuImz+Euo0gIsfgT4PQGAtfOlBpAZxZzgLA/aU2k4Bep90TnvA1xizBKgPTLLWTi/ntSIiZdq0L4f//XYLg7u34IrOTT1dTsXbvRy+ewG2fQP+wTDgCeg9BgKCPV2ZiFQAd4azshZis2U8vycwEAgAlhljlpfzWuchxowGRgO0atXqnIsVkZqhqNjFY7PXUd/fl/HX16DuTGth5/dOKNv5PdQNhYHjodc94N/A09WJSAVyZzhLAUq/GhQOpJVxToa19ihw1BizFOhezmsBsNZOBaYCxMbGlhngRKT2mPbjDtalZPO/t0YTWq+Op8s5f9Y6LWTfTYQ9y6FeU7jyeYi9C/wCPV2diLiBO8PZKqCdMSYSSAVG4IwxK20+8Joxxgfww+m6fBnYWI5rRUROsD39CP/+ejNXdm7KdVHNPV3O+bEWNn/lDPRPjYcGYXD1RIi5HXwDPF2diLiR28KZtbbIGPMgsBBnOoxp1tr1xpj7So5PsdYmG2O+AhIAF86UGUkAZV3rrlpFpPpzuSxj5yRQx8eL54Z2xZiyRkdUAy4XbPzUCWX7EiG4FVz3ijOBrE8NaAkUkd/k1nnOrLVfAF+ctG/KSdsTgYnluVZE5HTeW76LVTsP8eJN3WnSwN/T5Zw9VzGs/8SZpyw9GUIugCGTnaWWvGvo26YiUqZaMl22iNRkezKP8a+vNnJJ+8bcGBPm6XLOTnERJM5yZvQ/uBUadYBhbzmLknvrS7RIbaT/+SJSrVlrGTc3EQP8Y1i36tOdWVQA6z6A71+CrF3QtCvc9C50GgxeXp6uTkQ8SOFMRKq1Wav38MPWDJ4b2pWw4GowUL4wD9a+Bz+8AodToEU0DJoA7QcplIkIoHAmItXYvuw8nvssmT5tQhgZV8XnOSw4BvHvwI+T4Mg+aNkbrp8EbQdCdWntE5FKoXAmItWStZYnP0mk0OXiXzdG4eVVRQNOfg6s+g8sew2OpkPExTBsKkT2VygTkTIpnIlItbRgXRrfbDzAX6/tROvQKjgZa142rJgKy1+H3EPQ5lK45DFofaGnKxORKk7hTESqnfScfMYvWE90q2Du6hfp6XJOdCwTlr8BK/4P8rOh3VVOKAuP9XRlIlJNKJyJSLXz9IL1HMsvZuLwKLyrSnfmkXSn63LVW1BwBDpeB/0fhRY9PF2ZiFQzCmciUq18lbSXzxP38uhVHWjbpL6ny4GcffDjq7B6GhTlOfOT9X8EmnbxdGUiUk0pnIlItZF1rIC/zltPlxYNGN2/jWeLyU5xpsNYMx1cRdDtJrj4L9C4vWfrEpFqT+FMRKqNZz/bQNaxAt79fS98vT00J1jmDvjhZfj5A8BC91vh4j9DiIfDoojUGApnIlItLN54gLlrUnnosrZ0aRFUOQ9NmAXfPOu0ktVvCsGRkLISvLwh5g646GFnYXIRkQqkcCYiVd7hvEKe+CSR9k3r8YfL2lbOQxNmwacPQWGus52zz/l1wUAY8ho0aFE5dYhIraO1QkSkyvvnFxvZfziPF4Z3p46Pd+U89Jtnfw1mpWVsVjATEbdSOBORKu2nrRnMXLmbey5uQ4+WwZX34Ow9p9mfUnk1iEitpHAmIlXWsYIixs5NILJRIH++ohLfgtz05emPBYVXXh0iUispnIlIlTVx4Sb2ZObyrxuj8PetpO7MLYtg1h0Q3Bp8Ak485hsAA5+qnDpEpNZSOBORKmn1zkze+Wknv+vbmrjIkMp56PYl8NFt0LgDjF4Cg1+FoJaAcX6//lWIurlyahGRWktva4pIlZNXWMxjcxJoERTAY4M6Vs5Dd/0EM2+FhpFw+3yoG+IEMYUxEalkCmciUuVM+mYL29OP8t7dcQTWqYQvU3tWwYyboEEY/G4BBIa6/5kiIqehbk0RqVISUrKYunQ7t8S25OJ2jd3/wLS18P6NENjYCWb1mrj/mSIiZ6BwJiJVRkGRi8dmJ9Conh9PXNvJ/Q/clwjTh4J/EPzuU81fJiJVgro1RaTKmLxkKxv35fDWHbEEBfi692EHNsL0IeAX6LSYBbd07/NERMpJLWciUiVs3HeY1xdvZUiPFlzeual7H5axFaYPBi8fuGMBhES693kiImdBLWci4nFFxU53ZgN/X8Zf38W9D8vcDu9eD65iuPNzaFRJa3WKiJSTwpmIeNxbP+wgISWb10fGEBLo574HZe2GdwdDUS787jNoUknTdIiInAV1a4qIR21LP8JL/93MoC7NuKZbM/c96HCa02KWdxhu/wSadXXfs0REzoNazkTEY1wuy9jZCQT4evPs0C4YY9zzoJz9TjA7mgG3z4MW0e55johIBVDLmYh4zPRlO1m96xBPXdeZJvX93fOQoxnO4P/DaXDbbGjZyz3PERGpIGo5ExGP2JN5jH99tYkBHRozLCbMPQ85lunMY3ZoJ4ycBa37uuc5IiIVyK0tZ8aYQcaYTcaYrcaYx8s4PsAYk22M+bnk11Oljv3JGJNkjFlvjHnYnXWKSOWy1jJ2TgLeXoZ/3NDNPd2Zednw/jDI2AQjZkCbSyr+GSIibuC2ljNjjDfwOnAFkAKsMsYssNZuOOnU76211510bVfgXiAOKAC+MsZ8bq3d4q56RaTyfLhqDz9tO8jzN3SlRXBAxT8gPwfeH+6sAHDL+9D28op/hoiIm7iz5SwO2Gqt3W6tLQA+BIaU89pOwHJr7TFrbRHwHXCDm+oUkUq0NzuX5z9Ppm+bUG7t1ariH1BwFD64BVLjYfjb0OHqin+GiIgbuTOchQF7Sm2nlOw7WV9jzDpjzJfGmF9mn0wC+htjQo0xdYFrAK2tIlLNWWt5Ym4ixS7LhBu74eVVwd2Zhbkw81bYvQyGTYXOgyv2/iIilcCdLwSU9VXXnrS9BmhtrT1ijLkGmAe0s9YmG2P+BfwXOAKsA4rKfIgxo4HRAK1aueGncBGpMPN+TmXxpnSeuq4zrUMDK/bmRfnw0e2wYykMnQzdhlfs/UVEKok7W85SOLG1KxxIK32CtfawtfZIyZ+/AHyNMY1Ktv9jrY2x1vYHMoEyx5tZa6daa2OttbGNGzd2x8chIhXgQE4eTy/YQM/WDfndhREVe/PiQvj4Ltj6X7j+FegxsmLvLyJSidwZzlYB7YwxkcYYP2AEsKD0CcaYZqbkNS1jTFxJPQdLtpuU/N4KGAbMdGOtIuJm4+evJ7ewmH/dGIV3RXZnFhfBnLth0+dw9UToeWfF3VtExAPc1q1prS0yxjwILAS8gWnW2vXGmPtKjk8BhgP3G2OKgFxghLX2l67POcaYUKAQ+IO19pC7ahUR9/oicS9fJu3jsUEdaNukXsXd2FUM8+6DDfPhyueg9+iKu7eIiIeYX7NQ9RcbG2tXr17t6TJEpJRDRwu44uXvaB4UwCcPXIiPdwU12LtcsOCP8PP7cNnfoP8jFXNfEZFKYoyJt9bGnrxfKwSIiFs9+9kGso4V8t7dvSsumFkLX/zFCWaXjFUwE5EaRWtriojbfJO8n0/WpvLApW3p1LxBxdzUWvhqHKyeBv0ehgHjKua+IiJVhMKZiLjF4bxCnvwkiQ5N6/PgpW0r5qbWwqLxsOIN6H0/XP40uGPpJxERD1K3poi4xT+/SOZATh7/d3tP/Hwq6OfAxf+AHydB7N0w6J8KZiJSI6nlTEQq3A9bMpi5cg/39m9D95bBFXPTpRNh6QsQPQqueVHBTERqLIUzEalQR/OLeHxuAm0aBfI/l7evmJv+9L/w7XMQdQtc/yp46UuXiNRc6tYUkQo1ceEmUrNymTWmL/6+3ud/wxX/B1//FToPhSGTwasC7ikiUoXpx08RqTCrdmbyzk87+V3fCHpFhJz/DVe/DV8+Bh2uhRvfAm/9PCkiNZ/CmYhUiLzCYsbOTiC8YQCPXtXh/G/48wfw2f9AuyvhprfB2/f87ykiUg3ox1ARqRAvL9rM9oyjzLinN4F1zvNLS+JsmP8HaHMJ3Pwe+NSpmCJFRKoBtZyJyHlbtyeLN5duZ0SvlvRr2+j8brZhPswdDa36woiZ4OtfMUWKiFQTCmcicl4Kilw8NjuBJvX9eeLaTud3s01fwuzfQ1hPGPkR+NWtmCJFRKoRdWuKyHl5ffFWNu3P4T+/i6WB/3mMC9uyCGbdAc2iYNRsqFO/4ooUEalG1HImIucsee9hXl+8lRuiwxjYqem532j7EvjoNmjcAW6fC/5BFVajiEh1o3AmIuekqNjFo7PXEVzXl6eu63zuN9r1E8y8FRpGwu3zIaBhxRUpIlINqVtTRM7J1O+3k5R6mMm3xdAw0O/cbrJnFcy4CRqEwe8WQGBoxRYpIlINqeVMRM7a1gNHeGXRFq7u2oxrujU/t5ukrYX3b4TAxk4wq9ekYosUEammFM5E5KwUuyyPzV5HXT9vnhnS5dxusi8Rpg91xpb97lNo0KJiixQRqcYUzkTkrLz7007W7M5i/PWdaVL/HOYgO7ARpg8Bv0CnxSy4ZcUXKSJSjSmciUi57Tp4lBcWbuTSDo0Z2iPs7G+QsRWmDwYvH6fFLCSy4osUEanm9EKAiJSLy2V5fE4ivl5e/GNYN4wxZ3eDzB3w7vXgKoY7P4fQC9xTqIhINaeWMxEpl5mrdrNs+0GeuLYTzYMCzu7irN3w7mAoyoU75kOTju4pUkSkBlA4E5HflJaVyz+/2Ei/tqGM6HWWY8QOpzktZnnZcPs8aNbVPUWKiNQQ6tYUkTOy1vLEJ4kUuywThkWdXXdmzn4nmB09CHfMgxY93FeoiEgNoZYzETmjuWtSWbIpnccGdaBlyFksRH40w3kr83Aa3PYxhMe6r0gRkRpELWcicloHcvJ49rMNxLZuyO/6RpT/wmOZzjxmh3bAyFnQuq/bahQRqWkUzkSkTNZa/jYvidzCYv41PAovr3J2Z+Zlw/vDIGMT3DoT2lzi3kJFRGoYdWuKSJm+SNzHwvX7+fMV7bmgcb3yXZSfA+8Pd1YAuPk9aHu5e4sUEamBFM5E5BSZRwt4an4S3cKCuOeick4UW3AUPrgFUuNh+NvQYZB7ixQRqaHUrSkip3jm0/Uczitkxk298fEux89whbkw8//bu/PwKusz/+PvO3uALCCELQmLIpRN0IAFZ6gt1gVFrVqrtVYdl6vbOLbTOjrzG+vYsTNdrpnamXE6LWrttC6UUrUu1Wo76gijRAggm6JAEsKOSVgC2e7fH99DFggQJOc8J8nndV3nIud5nuNz57kk3Lm/3/v7vRbKF8MVP4Pxl8Y/SBGRHkqVMxFp5+XV23i6rIqvfvI0xg3JPf4HGg/Ck9fDhtfgsgdh0lXxD1JEpAeLa3JmZhea2TozW29md3Vw/lwzqzGzstjrnjbnvm5mq8zsHTN73Mw+wg7LInIiauoa+LunVjJuSA5fOfe043+gqQF+fROs/wPM/RFMuTb+QYqI9HBxS87MLBX4D+AiYDxwrZmN7+DS1919Sux1X+yzw4HbgRJ3nwikAtfEK1YRCb773Bp27q3nB1edQUbacX48NDXCb26Bdc/BnB/CWTcmJEYRkZ4unpWz6cB6d//A3euBJ4DLTuDzaUC2maUBfYCqOMQoIjGvv7eDJ0sruPXPRzOpMO/YFzc3wVNfhtVPwfn3w/RbExOkiEgvEM/kbDhQ0eZ9ZezY4WaY2XIze8HMJgC4+2bgh0A5sAWocfeXOrqJmd1mZqVmVrpjx46u/Q5Eeom9Bxu56zcrGT2oL3ecN+bYFzc3wzO3w8r58Km/h5lfS0yQIiK9RDyTs45WrPTD3i8FRrj7GcC/AU8BmFl/QpVtFDAM6GtmX+joaJcEHwAAFQhJREFUJu7+U3cvcfeSQYMGdVnwIr3J93+/lqqaOr5/5WSy0lOPfqE7PP/XUPZL+MTfwKxvJi5IEZFeIp7JWSVQ1OZ9IYcNTbp7rbvvjX39PJBuZgOB84AN7r7D3RuAhcDMOMYq0mu9tWE3v1i8iRtnjqRk5ICjX+gOv78bSh+Gc+6Ac+9OXJAiIr1IPJOzJcAYMxtlZhmECf3PtL3AzIaYmcW+nh6LZxdhOPPjZtYndn42sCaOsYr0SnX1Tdy5YDlFA7L51gVjj36hO7z8bXjzP+HjX4Hz7gXr5HZOIiJyQuK2CK27N5rZ14AXCd2WD7v7KjP7Uuz8T4CrgC+bWSNQB1zj7g68aWYLCMOejcAy4KfxilWkt/rXl99l4679/OqWs+mTcYwfB3/6LrzxAJTcDBd8V4mZiEgcWciFeoaSkhIvLS2NOgyRbqGsoporHnyDz00r5p+umHT0C1/7AfzxH2Hq9TD3x5CitatFRLqCmb3t7iWHH9dPWZFe6GBjGM4cnJvF3XPGHf3CRf8WErPJn4O5DygxExFJAO2tKdIL/ccf1/Putr08cuM0crPSO77ozf+Cl/4fjL88bMuUcowuThER6TL6NVikl1lVVcOD//M+V0wdzifHFXR8Uekj8MKdMO4SuHIepOr3OBGRRFFyJtKLNDQ1c+eCFeT3yeCeuR3tpgaUPQbPfh3GnA9XPQypR6msiYhIXCg566wV8+FfJ8K9+eHPFfOjjkjkhP30tQ9YVVXLP14+gfw+GUdesHIBPP1VGP0JuPq/IS0z8UGKiPRyGqvojBXz4Xe3Q0NdeF9TEd4DTL46uri6saeWbeYHL66jqrqOYflhja3Lp3a0u5d0lfXb9/DAy+9x8aShXDhx6JEXrH4aFt4GxTPhmschPSvxQYqIiCpnnfLKfa2J2SENdeG4nLCnlm3m7oUr2VxdhwObq+u4e+FKnlq2OerQeqymZudbC1bQJzOVey+dcOQF616ABX8BhSXw+Scho0/igxQREUDJWefUVB7leAVUV3R8To7q+79fS11DU7tjdQ1N3PfsalZUVrN9zwGam3vO+nvJ4JE3NrCsvJp7505gUM5hQ5XvvQzzvwhDJsN1v4bMftEEKSIigIY1OyevMCRiHXlgMpx+EUy7GUZ/UutAHaa52flg5z7KKqopq/iQsopqqmoOdHjt7n31XPrvbwCQlmIMzs1iSF54DY19PTQvu+VYQU4m6al63sezcec+fvjSOmaPK+CyKcPan/zgVXjyOhg0Fq5fCFl50QQpIiItlJx1xux72s85A0jPhtnfhr3bYekvYN1zMODUkKRN+Txk948u3gjt2nswloi1vvYcaASgX2Yakwvz6JeZxt6DjUd8dlBOJt/9zCS21tSxpeYAW2sOsKXmAKuranllzTYONDS3u94MBvXLZGhea+I2ODerzfssBudmkZXee9fnam527lq4gvSUFO7/zCSs7bZLmxbB49fAgNFw/dO99v9ZEZFko+SsMw5N+n/lvjDEmVcYErZDx8+9K0ymfutn8OLfwivfgUlXwbRbYNiU6OKOswMNTayqqmFZeTXLK2soq/iQit0hgU1NMcYOzuGSycOYWpTPlOJ8Th3Uj9QUa5lz1nZoMzs9lb+b8zE+PX5wh/dyd2rrGtlS2z5x21ZzgC21B9iwcx+L3t/Vkgi2NaBvRvukrYMqXL/MnvlX4bG3yvm/D3bzvSsnMSSvzQT/iiXwq89C7nD44tPQ95ToghQRkXa0t2ZX27IcljwEK38NDfuhcFpI0sZf3q2735qbnQ279lFW3loRW7OllsbY3LBheVlMKc5nSlE+U4r6M3F47jE30o5Xt+a+g41srW1N3g6vwm2rPcCuffVHfC4nM611CDUviyF52Qw5rAqXl53evvKU5DZX13H+v7zK1OL+/PfN01tjr1oGj14GfQbATc9D7rBj/4dERCQujra3ppKzeKmrhuWPw5J5sGs99DklbBxd8hfQf0TU0R3Xrr0HWV5ZTVl5NcsqqlleUU1trCrVNyOVyYX5LcnY1KJ8CnK7T+J5oKGJ7bUH2VJTx9baAy3J29ZYFW5rTR3b9xzk8L8aWekpsaHTzJaq29C8rFgSl83gvEwG9s0kJSX6BM7dueGRJZRu3M2Ld8yiaECs+3LrSnh0LmTmwI3PQ35RtIGKiPRiSs6i0twMG14NSdq658EdTr8Apt0Kp34qKRoIwvBkLcvbzBMr370fgBSD0wfnMLVNVey0gjA82ZM1NDWzY8/Bo1bhttaGKlxDU/u/P+mpRkFO+4rboSrcofcFOZmkxbmR4delFXxrwQr+4dIJ3DBzZDi4fS38fA6kZYWKWf+RcY1BRESOTclZMqipDHsWLn0U9u2A/qNiDQTXhSGmBHB3NrR0T7YOTx5KMobmZcWSsPCaVJh3zOHJ3qy52dm1rz6WvNW1JHItyVxtOH54I0OKheaHkLR1XIUryM084UaGtkPFACMH9uGVb5wbKnk714fEDOCmF+CUU7vkGYiIyEen5CyZNNbDmmdCNa18cahkTLwyzE0bfmaX3mr3vnqWV4ShybLY8GRNXQMAfTJSmVyYx5Si/mF4sjifwd1oeLI7cHdq6hraDZ+2rcJtix0/WiPD4fPeBudmtUvm+sYaGTpqsshMS+F7V07m8hH18MgcaKqHG5+DgnEJ+/5FROTolJwlq60rQwPBivnQsA+GnxWStAmfCct1nICDjU2srqptVxXbtKv98GRLVaw4nzEFOT1+eLK72HuwsU3Vra4laWtbhdvdUSNDVhpD87LYtGs/Bxubjzh/Zt4eFmbdD/V74IZnYcjERHw7IiLSCUrOkt2BGlj+RKim7Xw3rDl1qIFgwKgjLnd3Nu7aT1nFhyyvqGFZRTVrqmqpbwr/QA/JzWpJwqYU5TNpeF5LlUW6p6M1MmypqePFVduOuH4wu3ky4zuMzD4ANzzTo5d1ERHpjpScdRfusOG1kKStfQ68GcZ8mj2TbqA0/SzKKveE4cnKaqr3tw5PThqex5Ti0Dk5pah/+zWtpMc755//yObq1kWSB1HNExnfYUhKNX1veTbsmSkiIknlaMmZSinJxoyDxX/GmvQzeHfgV8ld/RjT1j/DKe+9xKnNg3iz+Tz297+YC8aPaKmKjSnoF/fuP0lu37pgLP/72we5gycYZjtpIpUm4K0/+zmzlJiJiHQrSs4i5u5s2rWf5ZXVLIst8Lq6zfDk4NxLOWvUtVyeVcbZOxdy1/bHYf9CSL0Cht0KQ4rCPkbSq12e+gaXpM8jrSnsW5pCEykpGcwq6HgfUxERSV4a1kyw6v317SbsL6+o5sPY8GR2eiqTCvNiQ5NhvtjQvMOaAratDkOeK56E+r0wdEpoIJh4JWT0ieA7kkjV74fNpfDEdXCw9sjzeUXw9XcSH5eIiByX5pxFoL6xmTVb2ndPbti5DwjFrjEF/VoWdp1SlM/pg09gePJAbUjQlsyDHWshKx+mfiE0EGgNq56rrhoq3gyblm9aFLZiam44xgcM7q1OWHgiItJ5Ss5O0vH2gnR3KnbXsaziw5ZEbFVVLfWx5Q0KcjKZUpTPGbHtjiYV5pGTlX7ygbnDpjfCputrn4XmRjh1Nky/FcacDykntpCpJJk926B8EWxaHJKxbe8ADilpMOxMGDEDRpwDz34dajcf+XlVzkREkpaSs5PQ0QKfWekp3DBzJH3S08JyFpU1LetQZaWnMHl4fpuNwPMZmpcV/02za7eE3Qfe/jns2QJ5xVByE5z5Reg7ML73lpPnDh9uDAsTb3ojJGS73w/n0vtA4TQYMTO8hpe0H8ZeMR9+dzs0tHZskp4Nc38Mk69O6LchIiKdo+TsJBy+TEFbZnDaoH7t1hQbOzgn2u7JpoawDMeSebDxdUjNCIvaTrsl/AOvBoLk0NwchqTLY0OUmxbDnqpwLisfime0JmNDz4DU41RaV8yHV+4L24TlFcLse5SYiYgkMSVnJ2HUXc9xtKe08t7zu2Z4Ml62r4XSh6Ds8bBK/JDJIUmb9Fk1ECRaUwNsWdGajJUvhroPw7mcoe2TsUEfgxQtjyIi0pNpnbOTMCw/u8PK2fD87OROzCDsozjnB6GKsmJ+qKb97nb4w9+HDddLboaBp0UdZc/UUAeVpa3DlBVLwhZdAANGw9iLY8nYDOg/ShVNEREBVDnrlI7mnGWnp/JPV0xq1xTQLbiHZOGtn4XN15sbYfQnQzXt9AshVfn6R3agBsrfDIlY+WLYvDTWSWkweEJIxA5Vx3KGRB2tiIhETJWzk3AoATtWt2a3YdY6dLZnGyz9Bbz9CDx5HeQWQsmNcOYN0K8g6kiT397trUtalC+CrW07KafCjK9A8UwoPjvslSoiItIJca2cmdmFwANAKjDP3f/5sPPnAk8DG2KHFrr7fWY2FniyzaWjgXvc/UfHul+yrXPWbTQ1wrsvhGrahlchJR3GXxaW4yg6W8NtECqO1ZtiS1rEKmO71odzadlQNC0saVE8IzRdaD6fiIgcR8IbAswsFXgX+DRQCSwBrnX31W2uORf4prtfcpz/zmbgbHffdKx7KjnrAjvejTUQPBZWnB88sbWBILNf1NElTnMz7FzXuqRF+eLWdcSy8kJF7NAaY53ppBQRETlMFMOa04H17v5BLIAngMuA1cf81JFmA+8fLzGTLjLodLjoe+0bCJ69A/5wD0z5fGggGHR61FF2vaZG2Lq8dUmL8kWtnZT9hrQOBauTUkRE4iyeydlwoKLN+0rg7A6um2Fmy4EqQhVt1WHnrwEeP9pNzOw24DaA4uLikwpY2sjoGxawPevGsF3Qknmw5CF48ycwahZMuxXGzum+DQQNdbD57dY5YxVvddBJGZu8r05KERFJoHgOa34WuMDdb4m9vx6Y7u5/2eaaXKDZ3fea2RzgAXcf0+Z8BiFpm+Du2453Tw1rxtne7aGBoPQRqK2EnGGtOxAke/fhoU7KQ1shbX67fSfloS7K4hmQOzTqaEVEpBeIYlizEihq876QkGi1cPfaNl8/b2YPmtlAd98ZO3wRsLQziZkkQL8CmPVNOOcOeO/FUE370/3w6vfgY5eGuWkjZiZHlelQJ+WhNca2rQJvbu2k/PiXYxP41UkpIiLJJZ7J2RJgjJmNIkzovwb4fNsLzGwIsM3d3cymAynArjaXXMsxhjQlIqlpMO7i8Nq5HkofhrJfwqqFUDAept0Mkz8HmTmJiccdqstbl7TYtOjITspZd4bEsbAkDNmKiIgkqXgvpTEH+BFhKY2H3f1+M/sSgLv/xMy+BnwZaATqgG+4+6LYZ/sQ5qyNdveaztxPw5oRqt8P7ywIy3FsXQEZOXDGNaGaVjCua+/V0knZZhukdp2Uh4YoY3tSpmV07f1FRES6gPbWlMRwD1sWLfkZrPotNNXDyD8PSdq4iz/akhMtnZSL2+xJuTuc6zekdUmL4hmhcqdOShER6QaUnEni7dvZ2kBQUx429z7rxrADwcbX4ZX7oKYS8grD0h2Trw6fa+mkjM0Xq1wC9XvDuf6jWpe0KJ4ROiuTYY6biIjICVJyJtFpboL3XgoNBOtfBgwsBbx1r1JSM+DU2WFtsaqloeIGUDChdXPw4pnqpBQRkR5De2tKdFJSYexF4bXrffivT0D9nvbXNNWHLaSGl8DZXwoJWdHZ0GdANDGLiIhERMmZJNYpp7YOUR7B4NZXEhqOiIhIstHMaUm8vMITOy4iItKLKDmTxJt9D6Rntz+Wnh2Oi4iI9HJKziTxJl8Nc38MeUWAhT/n/ri1W1NERKQX05wzicbkq5WMiYiIdECVMxEREZEkouRMREREJIkoORMRERFJIkrORERERJKIkjMRERGRJKLkTERERCSJKDkTERERSSJKzkRERESSiJIzERERkSSi5ExEREQkiZi7Rx1DlzGzHcCmON9mILAzzvfoTfQ8u56eadfTM+1aep5dT8+0ayXqeY5w90GHH+xRyVkimFmpu5dEHUdPoefZ9fRMu56eadfS8+x6eqZdK+rnqWFNERERkSSi5ExEREQkiSg5O3E/jTqAHkbPs+vpmXY9PdOupefZ9fRMu1akz1NzzkRERESSiCpnIiIiIklEyVknmdmFZrbOzNab2V1Rx9PdmdnDZrbdzN6JOpaewsyKzOxPZrbGzFaZ2V9FHVN3ZmZZZvaWmS2PPc9/iDqmnsDMUs1smZk9G3UsPYGZbTSzlWZWZmalUcfTE5hZvpktMLO1sZ+nMxIeg4Y1j8/MUoF3gU8DlcAS4Fp3Xx1pYN2Ymc0C9gK/cPeJUcfTE5jZUGCouy81sxzgbeBy/X/60ZiZAX3dfa+ZpQP/C/yVu/9fxKF1a2b2DaAEyHX3S6KOp7szs41AibtrjbMuYmaPAq+7+zwzywD6uHt1ImNQ5axzpgPr3f0Dd68HngAuizimbs3dXwN2Rx1HT+LuW9x9aezrPcAaYHi0UXVfHuyNvU2PvfTb7Ekws0LgYmBe1LGIdMTMcoFZwEMA7l6f6MQMlJx11nCgos37SvSPniQxMxsJTAXejDaS7i02BFcGbAf+4O56nifnR8CdQHPUgfQgDrxkZm+b2W1RB9MDjAZ2AI/Eht/nmVnfRAeh5KxzrINj+g1akpKZ9QN+A9zh7rVRx9OduXuTu08BCoHpZqYh+I/IzC4Btrv721HH0sOc4+5nAhcBX41NGZGPLg04E/hPd58K7AMSPs9cyVnnVAJFbd4XAlURxSJyVLG5Ub8BfuXuC6OOp6eIDWv8D3BhxKF0Z+cAl8bmSD0BfMrMfhltSN2fu1fF/twO/JYwDUc+ukqgsk2VfAEhWUsoJWedswQYY2ajYpMDrwGeiTgmkXZiE9gfAta4+79EHU93Z2aDzCw/9nU2cB6wNtqoui93v9vdC919JOFn6B/d/QsRh9WtmVnfWPMPsaG38wF1wJ8Ed98KVJjZ2Nih2UDCm6rSEn3D7sjdG83sa8CLQCrwsLuvijisbs3MHgfOBQaaWSXwbXd/KNqour1zgOuBlbF5UgB/6+7PRxhTdzYUeDTWrZ0CzHd3Lf8gyWQw8NvwexlpwGPu/vtoQ+oR/hL4VawY8wFwU6ID0FIaIiIiIklEw5oiIiIiSUTJmYiIiEgSUXImIiIikkSUnImIiIgkESVnIiIiIklEyZmIiIhIElFyJiIiIpJElJyJiIiIJJH/DyjDIFGwlrG+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(solver.dice_history, marker='o', label=\"training dice\")\n",
    "plt.plot(solver.val_dice_history, marker='o', label=\"validation dice\")\n",
    "plt.ylabel('dice')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 1530.418829,
   "end_time": "2020-11-03T15:02:50.208412",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-11-03T14:37:19.789583",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0dd7d45ff8e1440a826c07800f1edf65": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3276f86b7614463d98060032e7672194",
       "max": 87306240.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_3f599df17a204317b35c9316f36e30c4",
       "value": 87306240.0
      }
     },
     "26485954cc9f499ea29230011b875efd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_40ebc2d8cc694759958c2fdb3f519b0a",
       "placeholder": "​",
       "style": "IPY_MODEL_9a93810ac9a9414780bf93ead02bce9f",
       "value": " 83.3M/83.3M [00:06&lt;00:00, 14.4MB/s]"
      }
     },
     "2c8fbe8c699b4b74bff5d61dfb998fd5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3276f86b7614463d98060032e7672194": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3f599df17a204317b35c9316f36e30c4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "40ebc2d8cc694759958c2fdb3f519b0a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9a93810ac9a9414780bf93ead02bce9f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "abb65ff0b90a41fcbeab9422e72df8fd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_0dd7d45ff8e1440a826c07800f1edf65",
        "IPY_MODEL_26485954cc9f499ea29230011b875efd"
       ],
       "layout": "IPY_MODEL_2c8fbe8c699b4b74bff5d61dfb998fd5"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}