Mesa_FireCLR_Training_PS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/mlenv/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "from torchvision import models\n",
    "import torchvision.transforms.functional as TF\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.backends.cudnn as cudnn\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataloader import default_collate\n",
    "from torch.utils.data import Dataset\n",
    "from torchsummary import summary\n",
    "from tqdm import tqdm\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from skimage import io, transform\n",
    "import random\n",
    "import argparse\n",
    "\n",
    "from models.cnn_simclr import CNN_SimCLR \n",
    "from models.simclr import SimCLR\n",
    "from utils import augmentation\n",
    "from utils import ContrastiveLearning\n",
    "from utils import load_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 4382607 tiles from the two fires.\n"
     ]
    }
   ],
   "source": [
    "# load training data\n",
    "# East Troublesome and McFarland Fires, selected Planetscope images with 4 bands\n",
    "# These images are tiled and normalized into numpy arr based on the fixed max and min values for each band\n",
    "\n",
    "fdir = \"/home/data/PS_Training/\"\n",
    "flist = glob.glob(fdir+\"*.npy\")\n",
    "\n",
    "print(\"There are %d tiles from the two fires.\" %len(flist))\n",
    "\n",
    "band_dict = {1: \"Band 1 - Blue\", 2: \"Band 2 - Green\", 3: \"Band 3 - Red\", 4: \"Band 4 - NIR\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABwAAAALICAYAAAB4lORhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAABOmElEQVR4nO3dfbxtd10f+M/XnMTwaHi4piEh3FhSaNoOYK8By9FSIhSEazItQ7EWo5OZW6vewmAHgr5kwOoUOjOAXq02CjVWntIATS5WhYmJzm01mvAcIhICSEJCwkMg4AMNfOePvS4eLvfm7Hv23mefs877/Xrt19l7Pe3vOr919t7rfPbvt6q7AwAAAAAAAIzDNyy7AAAAAAAAAGB+BIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABGD0qupJVXXLsus4rKq6qh657DoAAAC2uq12PgcA24UAEIClqKqPVtWfV9UXquqzVfUbVfXwJdXyr6vqfVV1T1W9dMZtPamqvjLs1xeq6taqetmcSgUAAFi6rXI+V1XfXFVvqKpPVNXnquq/VtXjZ9zmA6rqlcM+frGq/rSqLp91uwCw2QSAACzT3u6+f5LTknwyyYEl1XFTkhcm+Y05be8T3X3/Yd9Wk1xUVRfMadsAAABbwVY4n7t/kj9K8neTPDjJpUl+o6ruv5GNVdU3JvmdJH8nyTOTPDDJ30zyxiRPP8Y6Kxt5LgBYNAEgAEvX3X+R5PIk5xyeVlXPqKp3VdXnq+rja3vmVdXuYRjNC4dvY36qqn5izfz7VNWvDt9E/UCSb1vn+S/t7t9McvcC9u0jSf7b2n1bq6quqar/Zc3jH6iqQ2seP7qq3lFVn6mqD1bVs+ddIwAAwEYt83yuu2/u7ld2923d/eXuviTJSUketcHdeW6SM5Jc0N3vH7b5xe6+vLvX7kNX1Y9U1YeSfGiY9syqendV3VVV/62q/oc1yz+sqt5cVXdW1Ueq6l+umffSqrqsqn6tqu6uqhuqas8G6weArxIAArB0VXXfJP8kyR+smfzFJN+f5JQkz0jyL47Si241kxO785K8pKr+5jD9/0jy14fbP0xy4aJqX09VnZ3kifnafZt23fsleUeS1yf55iTPSfLvquqoYSIAAMBm20rnc1X12EwCwJuOczcO+64kv93dX5xi2QuSPD7JOVX1uCSvTfLPkzwkyb9PcmVVfWNVfUOSg0nek+T0TPb3+VX1D9ds63sy6WV4SpIrk/z8BusHgK8SAAKwTP+5qu5K8rkkT0nyfx2e0d3XdPf7uvsr3f3eJG9I8vePWP9l3f3n3f2eTE6mHjNMf3aSn+nuz3T3x5P83KJ35AgPG771+fkkf5Lk2iSH1lnnaJ6Z5KPd/R+6+57ufleSNyf5n+ZYKwAAwEZsqfO5qnpgkv84bPdzG9ynhya5fc02H3v43K6qPnjEsv9mqPHPk+xL8u+7+9qh1+ClSf4yyRMy6cG4q7t/qru/1N03J/nlTL7gedih7v4v3f3lYR8eEwCYkQAQgGW6oLtPSXJykh9N8rtV9deSpKoeX1VXD0OkfC7JD2VyMrbW7Wvu/1km139Ikocl+fiaeR+bV8HDRe4P3848xmKf6O5TuvuBmXyD888zuRbF8XpEkscPJ5x3DSfX35fkr22kdgAAgDnaMudzVXWfTHrZ/UF3/5t7WW6987lPZ3JNwyRJd7972Md/lOQbj1h2bY2PSPJjR5y7PXzYl0fkr74kenjejyc5dc36R/4uTnZtQQBmJQAEYOmGb0i+JcmXMxkGJpkMe3llkod39zcl+aUkNeUmb8vkZOuwYwV1G6n1/mtufzrF8p/LZF/2HmORLya575rHa8O9jyf53SFMPHy7f3f/iw3vAAAAwBwt+3yuqr4xyX9OcksmQ3DeW63rnc9dleSpw+UY1tNr7n88k16La8/d7tvdbxjmfeSIeQ/o7u+e4jkAYMMEgAAsXU2cn+RBSW4cJj8gyWe6+y+q6twk//Q4NnlZkhdX1YOq6owk+9d5/hOr6uRM3hdXqurkqjrh+PfkqNu+fyZDu9xwjEXeneQfVdV9q+qRSS5aM+9tSf5GVT13qPHEqvq2NdfGAAAAWKplns9V1YlJLs9k1JULu/srG9qJv/JrmQSQb62qv11VJwzninvWWe+Xk/zQ0POxqup+VfWMqnpAkj9McndVvaiq7jNs829X1bfNWCsA3CsBIADLdLCqvpDk80l+JpMTtsNB2Q8n+amqujvJSzI5CZzWyzIZJuYjSd6eyTUU7s0vZ3LC+L1JfmK4/9zjeL4jPezwsDJDHQ/OZOjOo3lVki8l+WQmw4S+7vCM7r47yVMzCRA/kcmwMK/I1w89AwAAsNm2wvnc38vk2ulPTXLXmuE9v+P4dmWiu/8iyT9I8oEkv5HJvn0wk+v4Pfte1rsuyf+a5OeTfDbJTUl+YJj35aHGxw779Kkkv5LkmzZSIwBMq7p7/aUAAAAAAACAbUEPQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACOysplP9tCHPrR37969mU8JAAAwWtdff/2nunvXMp7b+R0AAMB8zfMcb1MDwN27d+e6667bzKcEAAAYrar62LKe2/kdAADAfM3zHM8QoAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAI7Ky7AIAYGz2Hjg00/oH96/OqRIAAID5meVcx3kOAGwuPQABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZkZdkFAAAAAMBOsffAoQ2ve3D/6hwrAQDGTA9AAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBGZKgCsqlOq6vKq+uOqurGqvr2qHlxV76iqDw0/H7ToYgEAAAAAAIB7N20PwJ9N8lvd/egkj0lyY5KLk1zV3WcnuWp4DAAAAAAAACzRugFgVX1Tku9M8pok6e4vdfddSc5Pcumw2KVJLlhMiQAAAAAAAMC0VqZY5qwkdyb5D1X1mCTXJ3leklO7+7ZhmduTnHq0latqX5J9SXLmmWfOXDAAbIa9Bw4tu4SlmHW/D+5fnVMlAAAAAMBGTTME6EqSb03yi939uCRfzBHDfXZ3J+mjrdzdl3T3nu7es2vXrlnrBQAAYEZVdUJVvauq3jY8Pquqrq2qm6rqTVV10rJrBAAAYOOmCQBvSXJLd187PL48k0Dwk1V1WpIMP+9YTIkAAADM2fMyubb7Ya9I8qrufmSSzya5aClVAQAAMBfrBoDdfXuSj1fVo4ZJ5yX5QJIrk1w4TLswyRULqRAAAIC5qaozkjwjya8MjyvJkzP5smfiGu8AAADb3jTXAEyS/UleNwwDc3OSH8wkPLysqi5K8rEkz15MiQAAAMzRq5O8MMkDhscPSXJXd98zPL4lyelHW9E13gHYKNcbB4DNNVUA2N3vTrLnKLPOm2s1AAAALExVPTPJHd19fVU96XjX7+5LklySJHv27DnqdeABAABYvml7AAIAALD9PTHJ91TVdyc5OckDk/xsklOqamXoBXhGkluXWCMAAAAzWvcagAAAAIxDd7+4u8/o7t1JnpPkd7r7+5JcneRZw2Ku8Q4AALDNCQABAAB4UZIXVNVNmVwT8DVLrgcAAIAZGAIUgFGa9QLzADB23X1NkmuG+zcnOXeZ9QAAADA/egACAAAAAADAiOgBCAAAAADbwKwjnRzcvzqnSnaWWX7vfucALIsegAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEVlZdgEAjNfeA4c2vO7B/atzrAQAADjSLJ/XE5/Z2T5mPdYBYDvSAxAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRFaWXQAAAAAAwBjtPXBopvUP7l+dUyUA7DR6AAIAAAAAAMCICAABAAAAAABgRAwBCsCWNOswKQAAAAAAO5UegAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIhrAAIAAABsUzv12tmz7vfB/atzqmR72anHy3bmWAdgo/QABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABgRASAAAAAAAACMyMqyCwAAAAAAYP72Hji04XUP7l/dts89q+1cO8BhegACAAAAAADAiAgAAQAAAAAAYEQMAQoAzI1hUgAAAABg+aYKAKvqo0nuTvLlJPd0956qenCSNyXZneSjSZ7d3Z9dTJkAAAAAAADANI5nCNB/0N2P7e49w+OLk1zV3WcnuWp4DAAAAAAAACzRLNcAPD/JpcP9S5NcMHM1AAAAAAAAwEymDQA7ydur6vqq2jdMO7W7bxvu357k1LlXBwAAAAAAAByXqa4BmGS1u2+tqm9O8o6q+uO1M7u7q6qPtuIQGO5LkjPPPHOmYgEAAAAAAIB7N1UPwO6+dfh5R5K3Jjk3ySer6rQkGX7ecYx1L+nuPd29Z9euXfOpGgAAAAAAADiqdQPAqrpfVT3g8P0kT03y/iRXJrlwWOzCJFcsqkgAAAAAAABgOtMMAXpqkrdW1eHlX9/dv1VVf5Tksqq6KMnHkjx7cWUCAAAAAAAA01g3AOzum5M85ijTP53kvEUUBcD87D1waMPrHty/OsdKAAAAAADYDNP0AAQAAAAAWJpZvtwKADvRutcABAAAAAAAALYPASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEVlZdgEAAAAAbD97Dxza8LoH96/OsRIAAI6kByAAAAAAAACMiAAQAAAAAAAARkQACAAAsINU1clV9YdV9Z6quqGqXjZMP6uqrq2qm6rqTVV10rJrBQAAYGMEgAAAADvLXyZ5cnc/Jsljkzytqp6Q5BVJXtXdj0zy2SQXLa9EAAAAZiEABAAA2EF64gvDwxOHWyd5cpLLh+mXJrlg86sDAABgHlaWXQAAAACbq6pOSHJ9kkcm+YUkH05yV3ffMyxyS5LTj7LeviT7kuTMM8/cnGJhi9t74NBM6x/cvzqnSgDma9bXNwCWSw9AAACAHaa7v9zdj01yRpJzkzx6yvUu6e493b1n165diywRAACAGQgAAQAAdqjuvivJ1Um+PckpVXV4lJgzkty6rLoAAACYjQAQAABgB6mqXVV1ynD/PkmekuTGTILAZw2LXZjkiqUUCAAAwMxcAxAAAGBnOS3JpcN1AL8hyWXd/baq+kCSN1bVTyd5V5LXLLNIAAAANk4ACAAAsIN093uTPO4o02/O5HqAAAAAbHOGAAUAAAAAAIAR0QMQAAAAgB1l74FDG1734P7VOVYCALAYegACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAirgEIsMXNcm0Kdh7HCwAAAACgByAAAAAAAACMiAAQAAAAAAAARsQQoAAAAAAwJcPuAwDbgR6AAAAAAAAAMCICQAAAAAAAABiRqQPAqjqhqt5VVW8bHp9VVddW1U1V9aaqOmlxZQIAAAAAAADTOJ4egM9LcuOax69I8qrufmSSzya5aJ6FAQAAAAAAAMdvqgCwqs5I8owkvzI8riRPTnL5sMilSS5YQH0AAAAAAADAcZi2B+Crk7wwyVeGxw9Jcld33zM8viXJ6Udbsar2VdV1VXXdnXfeOUutAAAAAAAAwDrWDQCr6plJ7uju6zfyBN19SXfv6e49u3bt2sgmAAAAAAAAgCmtTLHME5N8T1V9d5KTkzwwyc8mOaWqVoZegGckuXVxZQIAAAAAAADTWLcHYHe/uLvP6O7dSZ6T5He6+/uSXJ3kWcNiFya5YmFVAgAAAAAAAFOZ9hqAR/OiJC+oqpsyuSbga+ZTEgAAAAAAALBR0wwB+lXdfU2Sa4b7Nyc5d/4lAQAAAAAAABs1Sw9AAAAAAAAAYIs5rh6AAAAAAMzP3gOHll0CAAAjpAcgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGZGXZBQCwde09cGjZJexIO/X3Put+H9y/OqdKAAAAAGB70wMQAAAAAAAARkQACAAAAAAAACNiCFAAAABgR9upQ7ADADBeegACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjMjKsgsAAJiHvQcObXjdg/tX51gJAAAAyzTL+SHAWOgBCAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZkZdkFAAAs294Dh5b6/Af3ry71+QEAAAAYFz0AAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjMjKsgsAAAAAmMXeA4eWXQLHSZsBACzWuj0Aq+rkqvrDqnpPVd1QVS8bpp9VVddW1U1V9aaqOmnx5QIAAAAAAAD3ZpohQP8yyZO7+zFJHpvkaVX1hCSvSPKq7n5kks8muWhhVQIAAAAAAABTWTcA7IkvDA9PHG6d5MlJLh+mX5rkgkUUCAAAwPxU1cOr6uqq+sAwysvzhukPrqp3VNWHhp8PWnatAAAAbMw0PQBTVSdU1buT3JHkHUk+nOSu7r5nWOSWJKcvpEIAAADm6Z4kP9bd5yR5QpIfqapzklyc5KruPjvJVcNjAAAAtqGpAsDu/nJ3PzbJGUnOTfLoaZ+gqvZV1XVVdd2dd965sSoBAACYi+6+rbvfOdy/O8mNmXyh8/xMRndJjPICAACwrU0VAB7W3XcluTrJtyc5papWhllnJLn1GOtc0t17unvPrl27ZqkVAACAOaqq3Ukel+TaJKd2923DrNuTnLqsugAAAJjNugFgVe2qqlOG+/dJ8pRMviF6dZJnDYtdmOSKBdUIAADAnFXV/ZO8Ocnzu/vza+d1d2dy7fcj1zHCCwAAwDYwTQ/A05JcXVXvTfJHSd7R3W9L8qIkL6iqm5I8JMlrFlcmAAAA81JVJ2YS/r2uu98yTP5kVZ02zD8tk2vAfw0jvAAAAGwPK+st0N3vzWRImCOn35zJ9QABRm/vgUMbXvfg/tU5VgIAMJuqqky+wHljd79yzawrMxnd5eUxygsAAMC2tm4ACAAAwKg8Mclzk7yvqt49TPvxTIK/y6rqoiQfS/Ls5ZQHAADArASAAAAAO0h3H0pSx5h93mbWAgAAwGJMcw1AAAAAAAAAYJvQAxAAAAAAAHa4vQcOzbT+wf2rc6oEmAcBIMCCzfrhCQAAAAAAjochQAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGZGXZBQAAAAAAwGF7DxxadgkA254egAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEVlZdgEAAAAAew8cWnYJADCzWd/PDu5fnVMlwE6nByAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGZGXZBQBshr0HDi27BIBjmuU16uD+1TlWAgAAAMAYrBsAVtXDk/xaklOTdJJLuvtnq+rBSd6UZHeSjyZ5dnd/dnGlAgAAAFuVL90BAMDWMc0QoPck+bHuPifJE5L8SFWdk+TiJFd199lJrhoeAwAAAAAAAEu0bgDY3bd19zuH+3cnuTHJ6UnOT3LpsNilSS5YUI0AAAAAAADAlKbpAfhVVbU7yeOSXJvk1O6+bZh1eyZDhB5tnX1VdV1VXXfnnXfOUisAAAAAAACwjqkDwKq6f5I3J3l+d39+7bzu7kyuD/h1uvuS7t7T3Xt27do1U7EAAAAAAADAvZsqAKyqEzMJ/17X3W8ZJn+yqk4b5p+W5I7FlAgAAAAAAABMa90AsKoqyWuS3Njdr1wz68okFw73L0xyxfzLAwAAAAAAAI7HyhTLPDHJc5O8r6rePUz78SQvT3JZVV2U5GNJnr2QCgEAAAAAAICprRsAdvehJHWM2efNtxyAY9t74NCySwAAAAAAgC1vqmsAAgAAAAAAANuDABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBFZWXYBAAAAAABAsvfAoWWXAIyEHoAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCIrCy7AAAAlmfvgUMbXvfg/tU5VgIAAADAvOgBCAAAAAAAACMiAAQAAAAAAIAREQACAADsIFX12qq6o6rev2bag6vqHVX1oeHng5ZZIwAAALMRAAIAAOwsv5rkaUdMuzjJVd19dpKrhscAAABsUyvLLgAAgJ1p74FDM61/cP/qnCqBnaW7f6+qdh8x+fwkTxruX5rkmiQv2ryqAAAAmCc9AAEAADi1u28b7t+e5NSjLVRV+6rquqq67s4779y86gAAADguAkAAAAC+qrs7SR9j3iXdvae79+zatWuTKwMAAGBaAkAAAAA+WVWnJcnw844l1wMAAMAMBIAAAABcmeTC4f6FSa5YYi0AAADMSAAIAACwg1TVG5L8fpJHVdUtVXVRkpcneUpVfSjJdw2PAQAA2KZWll0AAAAAm6e7v/cYs87b1EIAAABYGD0AAQAAAAAAYEQEgAAAAAAAADAihgAFjsveA4c2vO7B/atzrASAZLbXZQAAAADGSQ9AAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAI7Ky7AKAnWPvgUPLLgGAOfK6DgAAwDws8/zy4P7VpT03LJIegAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIhrAMIO43pNAAAAAAAwbuv2AKyq11bVHVX1/jXTHlxV76iqDw0/H7TYMgEAAAAAAIBpTDME6K8medoR0y5OclV3n53kquExAAAAAAAAsGTrBoDd/XtJPnPE5POTXDrcvzTJBfMtCwAAAAAAANiIaXoAHs2p3X3bcP/2JKfOqR4AAAAAAABgBiuzbqC7u6r6WPOral+SfUly5plnzvp0AACQJNl74NCG1z24f3WOlQAAAABsLRvtAfjJqjotSYafdxxrwe6+pLv3dPeeXbt2bfDpAAAAAAAAgGlsNAC8MsmFw/0Lk1wxn3IAAAAAAACAWawbAFbVG5L8fpJHVdUtVXVRkpcneUpVfSjJdw2PAQAAAAAAgCVb9xqA3f29x5h13pxrAQAAAAAAAGa00SFAAQAAAAAAgC1IAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIisLLsAWJa9Bw5teN2D+1fnWAkAAAAAAMD86AEIAAAAAAAAIyIABAAAAAAAgBExBCgAAAAw02USAAB8ltgYl6piUQSAsAHLfDPzog4AAAAAANwbQ4ACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYkZVlFwAbtffAoWWXAABsU7N+jji4f3VOlQDMl/MkAIDj4/yQsdIDEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIivLLgAAAAC2kr0HDs20/sH9q3OqBACArW7Wz46wKHoAAgAAAAAAwIjoAcjS+GYEALAT6VkEAAAALJoegAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZkZdkFAMdn74FDyy4BAHY878cAAAAs26znpgf3r86pErYiPQABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABgR1wCcg+08zu52rh0AYCfy+W1jZvm97dTfGQAAANuXHoAAAAAAAAAwInoAAgAAMCqz9pTd7s8PAADTWObn1llHWjHKy/r0AAQAAAAAAIAREQACAAAAAADAiBgCdLDMrq7beXiY7Vw7AADHZ9bPfjtlmBUAAABYtpl6AFbV06rqg1V1U1VdPK+iAAAA2HzO8QAAAMZhwwFgVZ2Q5BeSPD3JOUm+t6rOmVdhAAAAbB7neAAAAOMxSw/Ac5Pc1N03d/eXkrwxyfnzKQsAAIBN5hwPAABgJGa5BuDpST6+5vEtSR5/5EJVtS/JvuHhF6rqgzM850730CSfWnYR7BiONzaT443N5HhjM225463+5c587lls8bofMcdtrXuOt4PO77bc3+4Opz22Fu2xtWiPrUV7bC3aY+vRJlvLUtvDuenXOdweczvHmyUAnEp3X5LkkkU/z05QVdd1955l18HO4HhjMzne2EyONzaT442x2Snnd/52txbtsbVoj61Fe2wt2mNr0R5bjzbZWrTH1rKI9phlCNBbkzx8zeMzhmkAAABsP87xAAAARmKWAPCPkpxdVWdV1UlJnpPkyvmUBQAAwCZzjgcAADASGx4CtLvvqaofTfLbSU5I8truvmFulXE0ox9qhy3F8cZmcryxmRxvbCbHG9uGc7yv4W93a9EeW4v22Fq0x9aiPbYW7bH1aJOtRXtsLXNvj+rueW8TAAAAAAAAWJJZhgAFAAAAAAAAthgBIAAAAAAAAIyIAHCJquppVfXBqrqpqi4+yvxvrKo3DfOvrardw/SnVNX1VfW+4eeT16zzd4fpN1XVz1VVbeIusYUt6Hi7Ztjmu4fbN2/iLrFFzXCsnbvmWHpPVf2P026TnWtBx9tHh9e8d1fVdZu4O2xxGz3e1sw/s6q+UFX/atptArPz2WRrmXd7VNXDq+rqqvpAVd1QVc/b5F3a9hbxNzLMP6Gq3lVVb9ukXRmFBb1mnVJVl1fVH1fVjVX17Zu4S9vagtrjfxter95fVW+oqpM3cZe2NZ/Ht5Z5t4f39Nks4u9jmO79fAMW9Hp1/O/n3e22hFuSE5J8OMm3JDkpyXuSnHPEMj+c5JeG+89J8qbh/uOSPGy4/7eT3LpmnT9M8oQkleQ3kzx92fvqtvzbAo+3a5LsWfb+uW2d24zH2n2TrAz3T0tyR5KVabbptjNvizjehscfTfLQZe+f29a6zXK8rZl/eZL/lORfTbtNNze32W4+m2yt24La47Qk3zpMf0CSP9Eey22TNeu9IMnrk7xt2fu5XW4L/Hx7aZL/Zbh/UpJTlr2v2+G2oNes05N8JMl9hnmXJfmBZe/rdrjN0h5r5vs8vrXbw3v6FmqPNdO9n2+R9tjI+7kegMtzbpKbuvvm7v5SkjcmOf+IZc7PpFGTSYOfV1XV3e/q7k8M029Icp8hMT4tyQO7+w96chT8WpILFr4nbAdzP942pWq2o1mOtT/r7nuG6Scn6ePYJjvTIo43OJYNH29JUlUXZPLPnhuOc5vAbHw22Vrm3h7dfVt3v3O4f3eSGzP5BzvTWcjnqao6I8kzkvzKQqsfn7m3R1V9U5LvTPKaJOnuL3X3XYvdjdFY1PnGSib/W1nJJCj8RJiGz+Nby9zbw3v6TBbx9+H9fOPm3h4bfT8XAC7P6Uk+vubxLfn6F7SvLjN8aPhckoccscw/TvLO7v7LYflb1tkmO9MijrfD/sMwpMVPHn6RYkeb6VirqsdX1Q1J3pfkh4b502yTnWkRx1syOTl/e02GPd63wPrZXjZ8vFXV/ZO8KMnLNrBNYDY+m2wti3rvzjB/dyYjmFy7iOJHalFt8uokL0zylYVVPk6LaI+zktyZybn7u6rqV6rqfovdjdGYe3t0961J/u8kf5rktiSf6+63L3QvxsPn8a1lEe3xVd7Tj9ui2uPV8X6+EYtojw29nwsAt7Gq+ltJXpHkny+7FsbvGMfb93X330nyHcPtucuojfHo7mu7+28l+bYkLy7XQmCB7uV4W+3ub03y9CQ/UlXfubQiGYuXJnlVd39h2YUAx8dnk63l3tpj+GfJm5M8v7s/v6wad5qjtUlVPTPJHd19/ZLL23GO8TeykuRbk/xidz8uyReTuM7ZJjjG38eDMun1cVaShyW5X1X9s2XWuUO8ND6PbyUvzb20h/f0TffSHKU9vJ8vzUtz9L+PDb2fr8y9PKZ1a5KHr3l8xjDtaMvcMgwL8E1JPp18tfvtW5N8f3d/eM3yZ6yzTXamRRxvGb65lu6+u6pen0n35l9b1E6wLcx0rB3W3TdW1RcyXHdyim2yMy3ieLtuzWvbHVX11kxe235vMbvANjLL8fb4JM+qqn+b5JQkX6mqv0hy/RTbBGbjs8nWspD37qo6MZN/FL6uu9+yqOJHahFt8sQk31NV353J0IcPrKpf724hx/oW0R63JLmluw/3ork8AsBpLaI9zkryke6+M0mq6i1J/l6SX1/IHoyLz+Nby9zbo7t/3nv6hi3i7+P0eD/fqEW0x+XZwPu5HoDL80dJzq6qs6rqpEwu9HjlEctcmeTC4f6zkvxOd3dVnZLkN5Jc3N3/9fDC3X1bks9X1ROGoRi/P8kVC94Ptoe5H29VtVJVDx3un5jkmUnev9jdYBuY5Vg7a3jDS1U9Ismjk3x0ym2yM839eKuq+1XVA4bp90vy1HhtY2LDx1t3f0d37+7u3ZkMofJ/dvfPT7lNYDY+m2wti3jvrkyuhXJjd79yU/ZiXObeJt394u4+Y3jfe86wvH8WTmcR7XF7ko9X1aOGdc5L8oFF78hILOI95E+TPKGq7ju8fp2XyXXOWJ/P41vL3NvDe/pM5t4e3s9nsoj22ND7uR6AS9Ld91TVjyb57SQnJHltd99QVT+VSe+DKzN5wfuPVXVTks9kcqAkyY8meWSSl1TVS4ZpT+3uO5L8cJJfTXKfJL853NjhFnG8ZdLN+LeH8O+EJP9vkl/etJ1iS5rxWFtNcnFV/fdMxhb/4e7+VJIcbZubumNsSYs43qrqW5K8dXLekZUkr+/u39rcPWMrmvF4O65tLnRHYIfx2WRrWdB792omlyJ4X1W9e1j2x7v7v2zenm1fi/obYWMW2B77k7xu+CfkzUl+cPP2avtaUHt8qqouT/LOJPckeVeSSzZ1x7Ypn8e3lkW0RyY9yL2nb8CC2oMNWmB7HPf7eXX3RvcDAAAAAAAA2GIMAQoAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRASAAOwYVfWkqrpl2XVsRFV9tKq+a9l1AAAAbCXb+TwPABZJAAjAUg3B1p9X1Req6rNV9RtV9fAl1XJ1Vd1ZVZ+vqvdU1fkzbOtJVfWVYb/urqoPVtUPzrNeAACArWgrneetqenvV1VX1U/PsI0fGLbxwiOm31JVTxruv7Sqfn3NvK6qLw6/i1ur6pVVdcJGawCAaQkAAdgK9nb3/ZOcluSTSQ4sqY7nJTmtux+YZF+SX6+q02bY3ieG/Xpgkv8tyS9X1aPmUCcAAMBWt1XO81JVJyb52STXzmFzn0nywqp6wHGs85jhd/H3k/yTJP/zHOoAgHslAARgy+juv0hyeZJzDk+rqmdU1buGXnkfr6qXrpm3e/g25YVV9adV9amq+ok18+9TVb86fOP0A0m+bZ3nf29333P4YZITk8z8LdWe+C+ZnCj+D0Nt31BVF1fVh6vq01V1WVU9eE3tz62qjw3zfuJY2wYAANjKln2eN/ixJG9P8sdz2KUbk/x+khcc74rdfVOS/5rksXOoAwDulQAQgC2jqu6bybch/2DN5C8m+f4kpyR5RpJ/UVUXHLHqapJHJTkvyUuq6m8O0/+PJH99uP3DJBdOUcPbquovMvlm6DVJrtvY3nzNNr+hqr4nyUOT3DRM3p/kgky+AfqwJJ9N8gvD8uck+cUkzx3mPSTJGbPWAQAAsNmWfZ5XVY/IpMfdT82yH0f4ySTPX/slzmlU1aOTfEf+6rwQABZGAAjAVvCfq+quJJ9L8pQk/9fhGd19TXe/r7u/0t3vTfKGTEKztV7W3X/e3e9J8p4kjxmmPzvJz3T3Z7r740l+br1CuvuZSR6Q5LuTvL27vzLDfj1s2K8/T/LWJC/o7ncN834oyU909y3d/ZdJXprkWVW1kuRZSd7W3b83zPvJJLPUAQAAsNm2ynnezyX5ye7+wuy79NX6353kHUleNOUq76yqL2bSe/CaJP9uXrUAwLEIAAHYCi7o7lOSnJzkR5P8blX9tSSpqsdX1dVVdWdVfS6T4OyhR6x/+5r7f5bk/sP9hyX5+Jp5H5ummO7+7939m0meOvTc+zrDBdwP3848xqY+MezXAzM56XzymnmPSPLWqrprOCm+McmXk5x6ZN3d/cUkn56mdgAAgC1i6ed5VbU3yQO6+03rFVtVZ649z1tv+SQvyaTn4qlTLPutmdT/T5I8Psn9plgHAGYiAARgy+juL3f3WzIJwlaHya9PcmWSh3f3NyX5pSQ15SZvy9dew+9YQd2xrGQyrMzRar3/mtuf3ttGhl58L0ryd9YMa/PxJE/v7lPW3E7u7luPrHsYMuchx1k7AADA0i35PO+8JHuq6vaquj2TAO75VXXFUer807XneesV0d1/nOQtSaa6ZvtwbfjLMrl+4EumWQcAZiEABGDLqInzkzwokx5xyWQ4zs90919U1blJ/ulxbPKyJC+uqgdV1RmZXHfvWM/96Kp6+nBB+ROr6p8l+c4kv7uxvfla3f2lJP9P/upE75eS/MxwPYpU1a5h35Pk8iTPrKrVqjopk2tVeM8GAAC2nWWe52VyOYW/keSxw+3KJL+c5AePayeO7WXDtk45jnVenuR/PdwbEgAWxT8TAdgKDg5DrHw+yc8kubC7bxjm/XCSn6qquzMJzy47ju2+LJPhYD6S5O1J/uO9LFuZXIfvjiR3Jnlekn/S3e88judbz2uTnDkMQ/OzmZx8vn3Ytz/IZCiYDPv+I5l8K/a2JJ9Ncssc6wAAAFi0pZ/ndffd3X374Vsm12f/Ynd/5vh356jb/8jw/FMP6dnd70vye0n+93nUAADHUt297BoAAAAAAACAOdEDEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIisbOaTPfShD+3du3dv5lMCAACM1vXXX/+p7t61jOd2fgcAADBf8zzH29QAcPfu3bnuuus28ykBAABGq6o+tqzndn4HAAAwX/M8xzMEKAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiKwsuwAAFmvvgUMbXvfg/tU5VgIAALA1OE8CAMZOD0AAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAI7Ky7AIAAAAAYLvYe+DQTOsf3L86p0oAAI5ND0AAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIisLLsAAAAAAGB9ew8cmmn9g/tX51QJALDV6QEIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAB2mKo6oareVVVvGx6fVVXXVtVNVfWmqjpp2TUCAACwcQJAAACAned5SW5c8/gVSV7V3Y9M8tkkFy2lKgAAAOZCAAgAALCDVNUZSZ6R5FeGx5XkyUkuHxa5NMkFSykOAACAuRAAAgAA7CyvTvLCJF8ZHj8kyV3dfc/w+JYkpx9txaraV1XXVdV1d95558ILBQAAYGMEgAAAADtEVT0zyR3dff1G1u/uS7p7T3fv2bVr15yrAwAAYF5Wll0AAAAAm+aJSb6nqr47yclJHpjkZ5OcUlUrQy/AM5LcusQaAQAAmJEegAAAADtEd7+4u8/o7t1JnpPkd7r7+5JcneRZw2IXJrliSSUCAAAwBwJAAAAAXpTkBVV1UybXBHzNkusBAABgBoYABQAA2IG6+5ok1wz3b05y7jLrAQAAYH6mCgCr6qNJ7k7y5ST3dPeeqnpwkjcl2Z3ko0me3d2fXUyZAAAAAMB2tffAoZnWP7h/dU6VAMDOcDxDgP6D7n5sd+8ZHl+c5KruPjvJVcNjAAAAAAAAYIlmuQbg+UkuHe5fmuSCmasBAAAAAAAAZjJtANhJ3l5V11fVvmHaqd1923D/9iSnHm3FqtpXVddV1XV33nnnjOUCAAAAAAAA92aqawAmWe3uW6vqm5O8o6r+eO3M7u6q6qOt2N2XJLkkSfbs2XPUZQAAAAAAAID5mKoHYHffOvy8I8lbk5yb5JNVdVqSDD/vWFSRAAAAAAAAwHTWDQCr6n5V9YDD95M8Ncn7k1yZ5MJhsQuTXLGoIgEAAAAAAIDpTDME6KlJ3lpVh5d/fXf/VlX9UZLLquqiJB9L8uzFlQkAAAAAAABMY90AsLtvTvKYo0z/dJLzFlEUAAAAAAAAsDFTXQMQAAAAAAAA2B4EgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBEBIAAAAAAAAIzIyrILAAAAAAAWb++BQzOtf3D/6pwqAQAWTQ9AAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEVpZdALBz7D1waKb1D+5fnVMlwBjN8hrj9QUAAACAMdEDEAAAAAAAAEZEAAgAAAAAAAAjYghQAAAAAI6bIdgBALYuPQABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABiRqQPAqjqhqt5VVW8bHp9VVddW1U1V9aaqOmlxZQIAAAAAAADTOJ4egM9LcuOax69I8qrufmSSzya5aJ6FAQAAAAAAAMdvZZqFquqMJM9I8jNJXlBVleTJSf7psMilSV6a5BcXUCMAAAAAI7L3wKGZ1j+4f3VOlQCLMsvfub9xgNlN2wPw1UlemOQrw+OHJLmru+8ZHt+S5PSjrVhV+6rquqq67s4775ylVgAAAAAAAGAd6waAVfXMJHd09/UbeYLuvqS793T3nl27dm1kEwAAAAAAAMCUphkC9IlJvqeqvjvJyUkemORnk5xSVStDL8Azkty6uDIBAAAAAACAaazbA7C7X9zdZ3T37iTPSfI73f19Sa5O8qxhsQuTXLGwKgEAAJiLqjq5qv6wqt5TVTdU1cuG6WdV1bVVdVNVvamqTlp2rQAAAGzMtNcAPJoXJXlBVd2UyTUBXzOfkgAAAFigv0zy5O5+TJLHJnlaVT0hySuSvKq7H5nks0kuWl6JAAAAzGKaIUC/qruvSXLNcP/mJOfOvySAo9t74NCG1z24f3WOlQAAbF/d3Um+MDw8cbh1kicn+afD9EuTvDTJL252fQAAAMzuuAJAAAAAtr+qOiHJ9UkemeQXknw4yV3DNd6T5JYkpx9lvX1J9iXJmWeeuTnFAhzFLF8QBeDoZn1t9QV82FpmGQIUAACAbai7v9zdj01yRiYjuzx6yvUu6e493b1n165diywRAACAGQgAAQAAdqjuvivJ1Um+PckpVXV4lJgzkty6rLoAAACYjQAQAABgB6mqXVV1ynD/PkmekuTGTILAZw2LXZjkiqUUCAAAwMxcAxBgi9up17Yw7jwALMxpSS4drgP4DUku6+63VdUHkryxqn46ybuSvGaZRQIAALBxAkAAAIAdpLvfm+RxR5l+cybXAwQAAGCbMwQoAAAAAAAAjIgegAAAAADAurbrJSp28iUmZtn37bzfAOgBCAAAAAAAAKMiAAQAAAAAAIARMQQowIJt1yFSAAAAAADYnvQABAAAAAAAgBERAAIAAAAAAMCIGAIUAAAAYJua5ZIDB/evzrESGK9l/p25rAgAG6UHIAAAAAAAAIyIHoAAwNz4BjoAAAAALJ8egAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZkZdkFAGwHew8cWnYJAAAAAAAwFT0AAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjsrLsAgAAAABgp9h74NCyS9iW/N7YKRzrwLzoAQgAAAAAAAAjIgAEAAAAAACAETEEKADM2azDdRzcvzqnSgAAAACAnUgPQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARsQ1AAEAAAAA2DL2Hjg00/oH96/OqRKOx6ztNgttDl9PD0AAAAAAAAAYET0AAYAtYZnfFAQAAACAMdEDEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEXENQAAAAIAdyDWYgbGa9fXt4P7VpT03wLwIAIEdwYcvmI6/lY1Z5sklAAAAABzJEKAAAAAAAAAwIusGgFV1clX9YVW9p6puqKqXDdPPqqprq+qmqnpTVZ20+HIBAAAAAACAezNND8C/TPLk7n5MkscmeVpVPSHJK5K8qrsfmeSzSS5aWJUAAAAAAADAVNYNAHviC8PDE4dbJ3lyksuH6ZcmuWARBQIAAAAAAADTW5lmoao6Icn1SR6Z5BeSfDjJXd19z7DILUlOP8a6+5LsS5Izzzxz1noB2Eb2Hji0bZ/74P7VOVUCAAAAALC5pgoAu/vLSR5bVackeWuSR0/7BN19SZJLkmTPnj29gRoBAAAARmmZX5oD4Oi8NgNjMM01AL+qu+9KcnWSb09ySlUdDhDPSHLrfEsDAAAAAAAAjte6AWBV7Rp6/qWq7pPkKUluzCQIfNaw2IVJrlhQjQAAAAAAAMCUphkC9LQklw7XAfyGJJd199uq6gNJ3lhVP53kXUles8A6AQAAAAAAgCmsGwB293uTPO4o029Ocu4iigIANsZ1CgAAAACA47oGIAAAAAAAALC1CQABAAAAAABgRKa5BiAAAAAAAMfJZRoAWBY9AAEAAAAAAGBE9AAEjotvrgHM3yyvrQf3r86xEgAAAADGQA9AAAAAAAAAGBEBIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBFZWXYBAGxdew8cWnYJO5LfOwAAAAAwCz0AAQAAAAAAYEQEgAAAADtIVT28qq6uqg9U1Q1V9bxh+oOr6h1V9aHh54OWXSsAAAAbIwAEAADYWe5J8mPdfU6SJyT5kao6J8nFSa7q7rOTXDU8BgAAYBsSAAIAAOwg3X1bd79zuH93khuTnJ7k/CSXDotdmuSCpRQIAADAzFaWXQAAAMuz98ChDa97cP/qHCsBlqGqdid5XJJrk5za3bcNs25PcupRlt+XZF+SnHnmmZtUJQAAAMdLD0AAAIAdqKrun+TNSZ7f3Z9fO6+7O0kfuU53X9Lde7p7z65duzapUgAAAI6XABAAAGCHqaoTMwn/Xtfdbxkmf7KqThvmn5bkjmXVBwAAwGwEgAAAADtIVVWS1yS5sbtfuWbWlUkuHO5fmOSKza4NAACA+XANQAAAgJ3liUmem+R9VfXuYdqPJ3l5ksuq6qIkH0vy7OWUBwAAwKwEgADAjrf3wKFllwCwabr7UJI6xuzzNrMWAAAAFsMQoAAAAAAAADAiegACAAAAAPA1jJQCsL3pAQgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGJGVZRcAAAAAAACwHe09cGim9Q/uX51TJfC1BIAAcBSzfngDAAAAAFgWQ4ACAAAAAADAiAgAAQAAAAAAYEQEgAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEVpZdAAAAG7f3wKFllwAAAADAFiMABAAAAAAAtq1Zvhx7cP/qHCuBrWPdIUCr6uFVdXVVfaCqbqiq5w3TH1xV76iqDw0/H7T4cgEAAAAAAIB7M801AO9J8mPdfU6SJyT5kao6J8nFSa7q7rOTXDU8BgAAAAAAAJZo3QCwu2/r7ncO9+9OcmOS05Ocn+TSYbFLk1ywoBoBAAAAAACAKU3TA/Crqmp3kscluTbJqd192zDr9iSnHmOdfVV1XVVdd+edd85SKwAAAAAAALCOqQPAqrp/kjcneX53f37tvO7uJH209br7ku7e0917du3aNVOxAAAAAAAAwL2bKgCsqhMzCf9e191vGSZ/sqpOG+afluSOxZQIAAAAAAAATGtlvQWqqpK8JsmN3f3KNbOuTHJhkpcPP69YSIUAAAAAAAAjtPfAoQ2ve3D/6hwrYWzWDQCTPDHJc5O8r6rePUz78UyCv8uq6qIkH0vy7IVUCAAAAAAAAExt3QCwuw8lqWPMPm++5QAAAAAAAACzmOoagAAAAAAAAMD2IAAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEVpZdALC59h44tOwSABiJWd9TDu5fnVMlADCbWd7TvJ8BALAV6QEIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCICAABAAAAAABgRASAAAAAAAAAMCIryy4AAAAAYLvae+DQsksAAICvowcgAAAAAAAAjIgAEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEVlZdgGwHc16kfeD+1fnVAkAAAAAAMDX0gMQAAAAAAAARkQPQAAAAGBbm3WUFgAAGBs9AAEAAAAAAGBEBIAAAAAAAAAwIoYABQBgx5l1qLiD+1fnVAkAAADA/OkBCAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiLgGIAAAALB0s16fFQAA+CsCQNhmnBQDMBazvqcd3L86p0oAAAAAxsUQoAAAAAAAADAiAkAAAIAdpKpeW1V3VNX710x7cFW9o6o+NPx80DJrBAAAYDYCQAAAgJ3lV5M87YhpFye5qrvPTnLV8BgAAIBtSgAIAACwg3T37yX5zBGTz09y6XD/0iQXbGZNAAAAzNfKsguAZdl74NCySwAAgK3i1O6+bbh/e5JTl1kMAAAAsxEAAgAA8FXd3VXVR5tXVfuS7EuSM888c1PrYuvzJUsAANg6DAEKAADAJ6vqtCQZft5xtIW6+5Lu3tPde3bt2rWpBQIAADA9ASAAAABXJrlwuH9hkiuWWAsAAAAzEgACAADsIFX1hiS/n+RRVXVLVV2U5OVJnlJVH0ryXcNjAAAAtinXAAQAgOM0y3WuDu5fnWMlcPy6+3uPMeu8TS0EAACAhdEDEAAAAAAAAEZEAAgAAAAAAAAjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEVpZdAAAAML29Bw7NtP7B/atzqgQAAADYqvQABAAAAAAAgBFZNwCsqtdW1R1V9f410x5cVe+oqg8NPx+02DIBAAAAAACAaUzTA/BXkzztiGkXJ7mqu89OctXwGAAAAAAAAFiydQPA7v69JJ85YvL5SS4d7l+a5IL5lgUAAAAAAABsxMoG1zu1u28b7t+e5NRjLVhV+5LsS5Izzzxzg08H47L3wKFllwAA2573UwAAAICjm2YI0HvV3Z2k72X+Jd29p7v37Nq1a9anAwAAAAAAAO7FRgPAT1bVaUky/LxjfiUBAAAAAAAAG7XRAPDKJBcO9y9McsV8ygEAAAAAAABmsW4AWFVvSPL7SR5VVbdU1UVJXp7kKVX1oSTfNTwGAAAAAAAAlmxlvQW6+3uPMeu8OdcCAAAAAACwafYeOLTsEmAhNjoEKAAAAAAAALAFCQABAAAAAABgRASAAAAAAAAAMCICQAAAAAAAABiRlWUXwPa2zAukHty/urTnBgDYrmb5/ObzFwAAAGwPAkAAAABgqV/wBABg8+ngM26GAAUAAAAAAIAREQACAAAAAADAiAgAAQAAAAAAYERcA5Bty/UpAAAAAAAAvp4egAAAAAAAADAiAkAAAAAAAAAYEUOAAgAAwBYx66UODu5fnVMlAACwdc3yuXmnfGbWAxAAAAAAAABGRAAIAAAAAAAAI2IIUAAA2ESzDu8HAAAAsB49AAEAAAAAAGBEBIAAAAAAAAAwIgJAAAAAAAAAGBHXAAQAAGBUZr3W5sH9q3OqZPO5zigAAJDoAQgAAAAAAACjIgAEAAAAAACAEREAAgAAAAAAwIgIAAEAAAAAAGBEBIAAAAAAAAAwIivLLoDZ7T1waMPrHty/OsdKAAAAAAAAWDYBIAAAAAAAAJtmlo5NTMcQoAAAAAAAADAiAkAAAAAAAAAYEQEgAAAAAAAAjIhrAAIAAFOZ9RoNB/evzqmS4zdL7cusGwAAADZCD0AAAAAAAAAYET0AAQAAYI3t3NsVAICdY9bPrYybHoAAAAAAAAAwIgJAAAAAAAAAGBFDgA52alfZnbrfAAAAAAAAY6UHIAAAAAAAAIyIABAAAAAAAABGRAAIAAAAAAAAIyIABAAAAAAAgBERAAIAAAAAAMCIrCy7AAAAYGfYe+DQsksAAACAHUEPQAAAAAAAABiRmQLAqnpaVX2wqm6qqovnVRQAAACbzzkeAADAOGw4AKyqE5L8QpKnJzknyfdW1TnzKgwAAIDN4xwPAABgPGbpAXhukpu6++bu/lKSNyY5fz5lAQAAsMmc4wEAAIzEygzrnp7k42se35Lk8UcuVFX7kuwbHn6hqj44w3OyHA9N8qllF8FCaeOdQTvvDNp5/LTxzqCdt5D6lwvZ7Lza+BFz2MZh657jOb+bzoKOme3IaxnbjWOW7cYxy3bjmGVLOI7P68s4Zud2jjdLADiV7r4kySWLfh4Wp6qu6+49y66DxdHGO4N23hm08/hp451BO4/fdm1j53ccj+16nLNzOWbZbhyzbDeOWbab7X7MzjIE6K1JHr7m8RnDNAAAALYf53gAAAAjMUsA+EdJzq6qs6rqpCTPSXLlfMoCAABgkznHAwAAGIkNDwHa3fdU1Y8m+e0kJyR5bXffMLfK2EoM8TN+2nhn0M47g3YeP228M2jn8dtybewcjwXYcsc5rMMxy3bjmGW7ccyy3WzrY7a6e9k1AAAAAAAAAHMyyxCgAAAAAAAAwBYjAAQAAAAAAIAREQDuMFX1tKr6YFXdVFUXH2X+N1bVm4b511bV7mH6U6rq+qp63/DzycP0+1bVb1TVH1fVDVX18k3eJY5i3u18xLpXVtX7N2E3uBeLaOOqOqmqLqmqPxn+pv/xJu4SR7Ggdv7eYfp7q+q3quqhm7hLHMUM7XxuVb17uL2nqv7HabfJ5pp3G1fVw6vq6qr6wPD563mbvEscxSL+lof5J1TVu6rqbZu0K3BUC3q/+ujwueTdVXXdJu4OO8BGj9k188+sqi9U1b+adpswiwUds15nWZgZPhvsrqo/X/P54JfWrPN3h2P2pqr6uaqqTdwlRm5Bx+w1wzYPz/vmTdyl9XW32w65JTkhyYeTfEuSk5K8J8k5Ryzzw0l+abj/nCRvGu4/LsnDhvt/O8mtw/37JvkHw/2Tkvx/SZ6+7H3dybdFtPOa9f5Rktcnef+y93Mn3xbVxkleluSnh/vfkOShy97XnXxb0Gv2SpI7Drdtkn+b5KXL3tedfJuxne+bZGW4f9rQtivTbNNt27fxaUm+dZj+gCR/oo3H185r1nvB8PnrbcveT7ede1vUMZ7koz5zui3iNssxu2b+5Un+U5J/Ne023dw2elvEMTtM8zrrtpDbjJ8NducY/1tM8odJnpCkkvxm/J/ZbU63BR6z1yTZs+z9O9ZND8Cd5dwkN3X3zd39pSRvTHL+Ecucn+TS4f7lSc6rqurud3X3J4bpNyS5T1V9Y3f/WXdfnSTDNt+Z5IyF7wn3Zu7tnCRVdf9M/gH10wvfA9azkDZO8j8n+TdJ0t1f6e5PLXQvWM8i2rmG2/2Gb9E9MMknwjLN0s5/1t33DNNPTtLHsU02z9zbuLtv6+53DvfvTnJjktMXvB/cu0X8LaeqzkjyjCS/stDqYX0LOcZhgTZ8zCZJVV2Q5COZfJY+nm3CRi3imIVFmumYPZqqOi3JA7v7D3qSrPxakgvmXjk71dyP2e1AALiznJ7k42se35Kv/2fRV5cZTtI+l+QhRyzzj5O8s7v/cu3Eqjolyd4kV82vZDZgUe38r5P8P0n+bN4Fc9zm3sbD32+S/OuqemdV/aeqOnXulXM85t7O3f3fk/yLJO/LJPg7J8lr5l86x2Gmdq6qx1fVDZm06Q8N86fZJptnEW38VcOQJI9Lcu0iimdqi2rnVyd5YZKvLKxymM6ijvFO8vaaDFm+b4H1s/Ns+Jgdvvz6okxGSDnebcJGLeKYTbzOsjiz/s/irJoMc/+7VfUda5a/ZZ1twkYt4pg97D8Mw3/+5FYLDAWAHJeq+ltJXpHknx8xfSXJG5L8XHffvIzamJ8j27mqHpvkr3f3W5dZF/NzlL/llUx67/637v7WJL+f5P9eUnnMyVH+lk/MJAB8XJKHJXlvkhcvrUBm1t3XdvffSvJtSV5cVScvuybm697aePhnz5uTPL+7P7+sGpnd0dq5qp6Z5I7uvn7J5cHM7uW1bHX47Pn0JD9SVd+5tCLhr7w0yau6+wvLLgSm9NIc+5j1OstWdFuSM7v7cRmGu6+qBy65Jrg393bMfl93/50k3zHcnrukGo9KALiz3Jrk4WsenzFMO+oyQ6j3TUk+PTw+I8lbk3x/d3/4iPUuSfKh7n71/MvmOC2inb89yZ6q+miSQ0n+RlVds6D6Wd8i2vjTmfTufMvw+D8l+dZFFM/UFtHOj02S7v7wMJzGZUn+3oLqZzoztfNh3X1jki9kuObjFNtk8yyijQ8H+m9O8rrufktYtkW08xOTfM/w+euNSZ5cVb++iOJhCgt5LevuW4efd2TyueXcBdTOzjTLMfv4JP92eP19fpIfr6ofnXKbsFGLOGa9zrJIGz5mhxGKPp0kw5fdPpzkbwzLr720lNdZ5mkRx+za19m7M7l2+5Z6nRUA7ix/lOTsqjqrqk7K5EKWVx6xzJVJLhzuPyvJ73R3D8MD/kaSi7v7v65doap+OpM/hucvsHamN/d27u5f7O6HdffuJKtJ/qS7n7TY3eBeLKKNO8nBJE8aJp2X5AML2wOmsYjX7FuTnFNVu4bHT8nk2mEszyztfNbwgTRV9Ygkj07y0Sm3yeaZexsPQ4q8JsmN3f3KTdkL1jP3du7uF3f3GcPnr+cMy/+zzdgZOIpFvJbdr6oeMEy/X5KnJnn/JuwLO8OGj9nu/o7u3j28/r46yf/Z3T8/5TZho+Z+zHqdZcFm+Wywq6pOSJKq+pYkZye5ubtvS/L5qnrCcM7z/Umu2IydYUeY+zFbVStV9dBh+olJnpkt9jq7suwC2Dzdfc/wDaDfTnJCktd29w1V9VNJruvuKzP5Z9J/rKqbknwmkz+EJPnRJI9M8pKqeskw7alJTkryE0n+OMk7J6/N+fnu/pXN2i++1iLaefimGFvEAtv4RcM6r05yZ5If3LSd4ussqJ0/UVUvS/J7VfXfk3wsyQ9s3l5xpBnbeTXJxUNbfiXJD3f3p5LkaNvc1B3jqxbRxlW1msmwIu+rqncPy/54d/+Xzdsz1lrU3zJsFQt6LfuWJG8dziFXkry+u39rc/eMsZrxmD2ubS50R9gxFnHMJjk1XmdZkBmP2e9M8lNrPhv8UHd/Zpj3w0l+Ncl9kvzmcIOZLeKYHb5c8dtD+HdCkv83yS9v7p7du5p0+gAAAAAAAADGwBCgAAAAAAAAMCICQAAAAAAAABgRASAAAAAAAACMiAAQAAAAAAAARkQACAAAAAAAACMiAAQAAAAAAIAREQACAAAAAADAiPz/rMI3mTewqbcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1800x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Distribution of the normalized bands (random sample)\n",
    "\n",
    "test = np.load(\"/home/data/PS_Training/McFarland_PlanetScope20210825_0200_0423.npy\")\n",
    "\n",
    "fig, ax = plt.subplots(2,2, figsize=(25,10), tight_layout=True)\n",
    "for j in range(0,4):\n",
    "    img_b = test[j,:,:]\n",
    "    ax[j//2,j%2].hist(img_b.flatten(), alpha=0.8,bins=50)\n",
    "    ax[j//2,j%2].set_title(band_dict[j+1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data augmentation setting\n",
    "# transform fuction\n",
    "# Check the parameters in the transform functions if you need to change any of them\n",
    "\n",
    "list_trans =[transforms.RandomHorizontalFlip(p=0.5),\n",
    "            augmentation.FixedRotationTransform(angles=[90,180,270]),\n",
    "            transforms.GaussianBlur(kernel_size=(1, 3), sigma=(0.0001, 0.001)),\n",
    "            transforms.RandomResizedCrop(32,scale=(0.4,0.9),interpolation = TF.InterpolationMode.BICUBIC)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup hyperparameters\n",
    "batch_size = 1024\n",
    "workers = 20\n",
    "arch = \"Simple_CNN\"\n",
    "num_trans = 4\n",
    "out_dim = 128\n",
    "n_views = 2\n",
    "temperature = 0.07\n",
    "log_every_n_steps = 100\n",
    "checkpoint_every_n_steps = 1000\n",
    "learning_rate = 0.0003\n",
    "weight_decay = 1e-4\n",
    "epochs = 100\n",
    "\n",
    "args_dict = {\"device\":torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"), \"arch\": arch,\n",
    "        \"gpu_index\":0,\"n_views\": n_views,\"temperature\": temperature,\"log_every_n_steps\": log_every_n_steps, \n",
    "        \"checkpoint_every_n_steps\": checkpoint_every_n_steps,\n",
    "        \"batch_size\": batch_size, \"out_dim\": out_dim, \"fp16_precision\":True, \"seed\": 64, #1st: 21; 2nd: 15; 3nd: 64\n",
    "        \"weight_decay\":weight_decay,\"lr\": learning_rate, \"epochs\":epochs, \"workers\":workers}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data augmentation for the contrastive learning\n",
    "transformed_all_img_dataset = load_data.Load_Transformed_Dataset(flist, transform = ContrastiveLearning.ContrastiveLearningViewGenerator(ContrastiveLearning.ContrastiveLearningDataset(num_trans,list_trans).get_trans_func(),n_views=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the transformed data\n",
    "train_loader = DataLoader(transformed_all_img_dataset, \n",
    "                          batch_size, shuffle=True, \n",
    "                          num_workers=workers, \n",
    "                          pin_memory=True, drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training on gpu\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "cudnn.deterministic = True\n",
    "cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CNN_SimCLR(n_channels=4,tile_size=32,out_dim=256)\n",
    "model= nn.DataParallel(model) #Distribute the model on GPUs, parallel computing\n",
    "model.to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(),learning_rate , weight_decay=weight_decay)\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader), eta_min=0, last_epoch=-1)\n",
    "\n",
    "simclr = SimCLR(model=model, optimizer=optimizer, scheduler=scheduler, args_dict=args_dict)\n",
    "simclr.train(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "module.encoder.conv.0.0.weight\n",
      "torch.Size([32, 4, 3, 3])\n",
      "module.encoder.conv.0.0.bias\n",
      "torch.Size([32])\n",
      "module.encoder.conv.0.2.weight\n",
      "torch.Size([32, 32, 3, 3])\n",
      "module.encoder.conv.0.2.bias\n",
      "torch.Size([32])\n",
      "module.encoder.conv.1.0.weight\n",
      "torch.Size([256, 32, 3, 3])\n",
      "module.encoder.conv.1.0.bias\n",
      "torch.Size([256])\n",
      "module.encoder.conv.1.2.weight\n",
      "torch.Size([256, 256, 3, 3])\n",
      "module.encoder.conv.1.2.bias\n",
      "torch.Size([256])\n",
      "module.projection.0.weight\n",
      "torch.Size([256, 256])\n",
      "module.projection.0.bias\n",
      "torch.Size([256])\n",
      "module.projection.2.weight\n",
      "torch.Size([128, 256])\n",
      "module.projection.2.bias\n",
      "torch.Size([128])\n"
     ]
    }
   ],
   "source": [
    "# print name and shape of each layer\n",
    "for name, param in model.named_parameters():\n",
    "    print(name)\n",
    "    print(param.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.12 ('mlenv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "58a3192e8a7933dc19dfe2cbd46a519cde9c99cb96266d2437ea59eff5c83e98"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}