metric-learning-pipeline-only-text-sbert.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011016,
     "end_time": "2021-04-15T20:01:21.706125",
     "exception": false,
     "start_time": "2021-04-15T20:01:21.695109",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# About this Notebook\n",
    "\n",
    "After carefull considerations and doing a lot of experiments with tfidf and Bert-Based models , I strongly feel Bert-based models might do better if trained and used in the right way. [This](https://www.kaggle.com/c/shopee-product-matching/discussion/231510) dicussion thread talks about the usage of different approaches for text and discusses why BERT-base model might be better.\n",
    "\n",
    "I started with normal Hugging Face BERT type models , but I found Sentence transformers pre-trained models a better idea . As sentence transformer models were already trained in a siamese fashion especially for information retreival and semantic similarity tasks it's much better idea to start with them and then fine-tune it on our data. I have used <b> paraphrase-xlm-r-multilingual-v1 </b> from sentence transformers , one can try with other very good models also . I have uploaded all models for offline use [here](https://www.kaggle.com/tanulsingh077/sentence-transformer-models)\n",
    "\n",
    "One more additional thing which has come as a result of experimentation is to train with full data instead of splitting and then saving models on eval set. To avoid overfitting one can use strong regularizers like using fully connected layer on top of bert output , weight decay,etc\n",
    "\n",
    "This is the training notebook , you can find the inference notebook [here](https://www.kaggle.com/tanulsingh077/reaching-0-612-with-text-only-shopee)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:21.732702Z",
     "iopub.status.busy": "2021-04-15T20:01:21.732041Z",
     "iopub.status.idle": "2021-04-15T20:01:25.414275Z",
     "shell.execute_reply": "2021-04-15T20:01:25.413686Z"
    },
    "papermill": {
     "duration": 3.698282,
     "end_time": "2021-04-15T20:01:25.414500",
     "exception": false,
     "start_time": "2021-04-15T20:01:21.716218",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import math\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# Visuals and CV2\n",
    "import cv2\n",
    "\n",
    "# albumentations for augs\n",
    "import albumentations\n",
    "from albumentations.pytorch.transforms import ToTensorV2\n",
    "\n",
    "from sklearn.model_selection import KFold, train_test_split\n",
    "\n",
    "#torch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "from torch.optim import Adam\n",
    "from torch.optim.lr_scheduler import _LRScheduler\n",
    "from torch.optim import Adam, lr_scheduler\n",
    "\n",
    "import transformers\n",
    "from transformers import AdamW\n",
    "from transformers import get_linear_schedule_with_warmup,get_cosine_schedule_with_warmup\n",
    "from transformers import get_cosine_with_hard_restarts_schedule_with_warmup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.010071,
     "end_time": "2021-04-15T20:01:25.435085",
     "exception": false,
     "start_time": "2021-04-15T20:01:25.425014",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:25.463168Z",
     "iopub.status.busy": "2021-04-15T20:01:25.462588Z",
     "iopub.status.idle": "2021-04-15T20:01:31.975116Z",
     "shell.execute_reply": "2021-04-15T20:01:31.974637Z"
    },
    "papermill": {
     "duration": 6.530222,
     "end_time": "2021-04-15T20:01:31.975251",
     "exception": false,
     "start_time": "2021-04-15T20:01:25.445029",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "658c1534acfb433bbdaf466aa50a18c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/541 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5df3c4bb4b842f399133b45d6cbce76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07fa6f765ed34c4bb5851efa6ce25b0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/150 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3249cc6b3bd94483a44f2a8856bcf65c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/188 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "NUM_WORKERS = 4\n",
    "TRAIN_BATCH_SIZE = 32\n",
    "EPOCHS = 5\n",
    "SEED = 2020\n",
    "LR = 5e-5\n",
    "\n",
    "device = torch.device('cuda')\n",
    "\n",
    "################################################# MODEL ####################################################################\n",
    "\n",
    "transformer_model = 'sentence-transformers/paraphrase-xlm-r-multilingual-v1'\n",
    "TOKENIZER = transformers.AutoTokenizer.from_pretrained(transformer_model)\n",
    "\n",
    "################################################ Metric Loss and its params #######################################################\n",
    "loss_module = 'arcface'#'softmax'\n",
    "s = 30.0\n",
    "m = 0.5 \n",
    "ls_eps = 0.0\n",
    "easy_margin = False\n",
    "\n",
    "############################################################################################################################\n",
    "model_params = {\n",
    "    'n_classes':11014,\n",
    "    'model_name':transformer_model,\n",
    "    'pooling':'clf',\n",
    "    'use_fc':False,\n",
    "    'fc_dim':512,\n",
    "    'dropout':0.0,\n",
    "    'loss_module':loss_module,\n",
    "    's':30.0,\n",
    "    'margin':0.50,\n",
    "    'ls_eps':0.0,\n",
    "    'theta_zero':0.785\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011415,
     "end_time": "2021-04-15T20:01:31.999796",
     "exception": false,
     "start_time": "2021-04-15T20:01:31.988381",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.029197Z",
     "iopub.status.busy": "2021-04-15T20:01:32.028688Z",
     "iopub.status.idle": "2021-04-15T20:01:32.032334Z",
     "shell.execute_reply": "2021-04-15T20:01:32.031915Z"
    },
    "papermill": {
     "duration": 0.020902,
     "end_time": "2021-04-15T20:01:32.032497",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.011595",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class AverageMeter(object):\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "    \n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "    \n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.059877Z",
     "iopub.status.busy": "2021-04-15T20:01:32.059190Z",
     "iopub.status.idle": "2021-04-15T20:01:32.061547Z",
     "shell.execute_reply": "2021-04-15T20:01:32.062035Z"
    },
    "papermill": {
     "duration": 0.017881,
     "end_time": "2021-04-15T20:01:32.062150",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.044269",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fetch_loss():\n",
    "    loss = nn.CrossEntropyLoss()\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011665,
     "end_time": "2021-04-15T20:01:32.085756",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.074091",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.116743Z",
     "iopub.status.busy": "2021-04-15T20:01:32.116068Z",
     "iopub.status.idle": "2021-04-15T20:01:32.118428Z",
     "shell.execute_reply": "2021-04-15T20:01:32.118802Z"
    },
    "papermill": {
     "duration": 0.021219,
     "end_time": "2021-04-15T20:01:32.118920",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.097701",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ShopeeDataset(Dataset):\n",
    "    def __init__(self, csv):\n",
    "        self.csv = csv.reset_index()\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.csv.shape[0]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        row = self.csv.iloc[index]\n",
    "        \n",
    "        text = row.title\n",
    "        \n",
    "        text = TOKENIZER(text, padding='max_length', truncation=True, max_length=64, return_tensors=\"pt\")\n",
    "        input_ids = text['input_ids'][0]\n",
    "        attention_mask = text['attention_mask'][0]  \n",
    "        \n",
    "        return input_ids, attention_mask, torch.tensor(row.label_group)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01169,
     "end_time": "2021-04-15T20:01:32.142602",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.130912",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Metric Learning Losses\n",
    "\n",
    "https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/blob/master/src/modeling/metric_learning.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.177301Z",
     "iopub.status.busy": "2021-04-15T20:01:32.174221Z",
     "iopub.status.idle": "2021-04-15T20:01:32.179558Z",
     "shell.execute_reply": "2021-04-15T20:01:32.179967Z"
    },
    "papermill": {
     "duration": 0.025332,
     "end_time": "2021-04-15T20:01:32.180090",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.154758",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ArcMarginProduct(nn.Module):\n",
    "    r\"\"\"Implement of large margin arc distance: :\n",
    "        Args:\n",
    "            in_features: size of each input sample\n",
    "            out_features: size of each output sample\n",
    "            s: norm of input feature\n",
    "            m: margin\n",
    "            cos(theta + m)\n",
    "        \"\"\"\n",
    "    def __init__(self, in_features, out_features, s=30.0, m=0.50, easy_margin=False, ls_eps=0.0):\n",
    "        super(ArcMarginProduct, self).__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.s = s\n",
    "        self.m = m\n",
    "        self.ls_eps = ls_eps  # label smoothing\n",
    "        self.weight = Parameter(torch.FloatTensor(out_features, in_features))\n",
    "        nn.init.xavier_uniform_(self.weight)\n",
    "\n",
    "        self.easy_margin = easy_margin\n",
    "        self.cos_m = math.cos(m)\n",
    "        self.sin_m = math.sin(m)\n",
    "        self.th = math.cos(math.pi - m)\n",
    "        self.mm = math.sin(math.pi - m) * m\n",
    "\n",
    "    def forward(self, input, label):\n",
    "        # --------------------------- cos(theta) & phi(theta) ---------------------------\n",
    "        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n",
    "        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n",
    "        phi = cosine * self.cos_m - sine * self.sin_m\n",
    "        if self.easy_margin:\n",
    "            phi = torch.where(cosine > 0, phi, cosine)\n",
    "        else:\n",
    "            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n",
    "        # --------------------------- convert label to one-hot ---------------------------\n",
    "        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n",
    "        one_hot = torch.zeros(cosine.size(), device='cuda')\n",
    "        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n",
    "        if self.ls_eps > 0:\n",
    "            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n",
    "        # -------------torch.where(out_i = {x_i if condition_i else y_i) -------------\n",
    "        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n",
    "        output *= self.s\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011813,
     "end_time": "2021-04-15T20:01:32.204034",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.192221",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.240916Z",
     "iopub.status.busy": "2021-04-15T20:01:32.240138Z",
     "iopub.status.idle": "2021-04-15T20:01:32.242722Z",
     "shell.execute_reply": "2021-04-15T20:01:32.242328Z"
    },
    "papermill": {
     "duration": 0.026395,
     "end_time": "2021-04-15T20:01:32.242822",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.216427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ShopeeNet(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                 n_classes,\n",
    "                 model_name='bert-base-uncased',\n",
    "                 pooling='mean_pooling',\n",
    "                 use_fc=False,\n",
    "                 fc_dim=512,\n",
    "                 dropout=0.0,\n",
    "                 loss_module='softmax',\n",
    "                 s=30.0,\n",
    "                 margin=0.50,\n",
    "                 ls_eps=0.0,\n",
    "                 theta_zero=0.785):\n",
    "        \"\"\"\n",
    "        :param n_classes:\n",
    "        :param model_name: name of model from pretrainedmodels\n",
    "            e.g. resnet50, resnext101_32x4d, pnasnet5large\n",
    "        :param pooling: One of ('SPoC', 'MAC', 'RMAC', 'GeM', 'Rpool', 'Flatten', 'CompactBilinearPooling')\n",
    "        :param loss_module: One of ('arcface', 'cosface', 'softmax')\n",
    "        \"\"\"\n",
    "        super(ShopeeNet, self).__init__()\n",
    "\n",
    "        self.transformer = transformers.AutoModel.from_pretrained(transformer_model)\n",
    "        final_in_features = self.transformer.config.hidden_size\n",
    "        \n",
    "        self.pooling = pooling\n",
    "        self.use_fc = use_fc\n",
    "    \n",
    "        if use_fc:\n",
    "            self.dropout = nn.Dropout(p=dropout)\n",
    "            self.fc = nn.Linear(final_in_features, fc_dim)\n",
    "            self.bn = nn.BatchNorm1d(fc_dim)\n",
    "            self.relu = nn.ReLU()\n",
    "            self._init_params()\n",
    "            final_in_features = fc_dim\n",
    "\n",
    "        self.loss_module = loss_module\n",
    "        if loss_module == 'arcface':\n",
    "            self.final = ArcMarginProduct(final_in_features, n_classes,\n",
    "                                          s=s, m=margin, easy_margin=False, ls_eps=ls_eps)\n",
    "        else:\n",
    "            self.final = nn.Linear(final_in_features, n_classes)\n",
    "\n",
    "    def _init_params(self):\n",
    "        nn.init.xavier_normal_(self.fc.weight)\n",
    "        nn.init.constant_(self.fc.bias, 0)\n",
    "        nn.init.constant_(self.bn.weight, 1)\n",
    "        nn.init.constant_(self.bn.bias, 0)\n",
    "\n",
    "    def forward(self, input_ids,attention_mask, label):\n",
    "        feature = self.extract_feat(input_ids,attention_mask)\n",
    "        if self.loss_module == 'arcface':\n",
    "            logits = self.final(feature, label)\n",
    "        else:\n",
    "            logits = self.final(feature)\n",
    "        return logits\n",
    "\n",
    "    def extract_feat(self, input_ids,attention_mask):\n",
    "        x = self.transformer(input_ids=input_ids,attention_mask=attention_mask)\n",
    "        \n",
    "        features = x[0]\n",
    "        features = features[:,0,:]\n",
    "\n",
    "        if self.use_fc:\n",
    "            features = self.dropout(features)\n",
    "            features = self.fc(features)\n",
    "            features = self.bn(features)\n",
    "            features = self.relu(features)\n",
    "\n",
    "        return features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.012879,
     "end_time": "2021-04-15T20:01:32.268435",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.255556",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.300489Z",
     "iopub.status.busy": "2021-04-15T20:01:32.299845Z",
     "iopub.status.idle": "2021-04-15T20:01:32.302247Z",
     "shell.execute_reply": "2021-04-15T20:01:32.302689Z"
    },
    "papermill": {
     "duration": 0.022028,
     "end_time": "2021-04-15T20:01:32.302803",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.280775",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_fn(dataloader,model,criterion,optimizer,device,scheduler,epoch):\n",
    "    model.train()\n",
    "    loss_score = AverageMeter()\n",
    "    \n",
    "    tk0 = tqdm(enumerate(dataloader), total=len(dataloader))\n",
    "    for bi,d in tk0:\n",
    "        \n",
    "        batch_size = d[0].shape[0]\n",
    "\n",
    "        input_ids = d[0]\n",
    "        attention_mask = d[1]\n",
    "        targets = d[2]\n",
    "\n",
    "        input_ids = input_ids.to(device)\n",
    "        attention_mask = attention_mask.to(device)\n",
    "        targets = targets.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        output = model(input_ids,attention_mask,targets)\n",
    "        \n",
    "        loss = criterion(output,targets)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        loss_score.update(loss.detach().item(), batch_size)\n",
    "        tk0.set_postfix(Train_Loss=loss_score.avg,Epoch=epoch,LR=optimizer.param_groups[0]['lr'])\n",
    "        \n",
    "        if scheduler is not None:\n",
    "                scheduler.step()\n",
    "        \n",
    "    return loss_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.012287,
     "end_time": "2021-04-15T20:01:32.327518",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.315231",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.359240Z",
     "iopub.status.busy": "2021-04-15T20:01:32.358750Z",
     "iopub.status.idle": "2021-04-15T20:01:32.616415Z",
     "shell.execute_reply": "2021-04-15T20:01:32.615409Z"
    },
    "papermill": {
     "duration": 0.276631,
     "end_time": "2021-04-15T20:01:32.616555",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.339924",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('../input/shopee-product-matching/train.csv')\n",
    "data['filepath'] = data['image'].apply(lambda x: os.path.join('../input/shopee-product-matching/', 'train_images', x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.646831Z",
     "iopub.status.busy": "2021-04-15T20:01:32.645802Z",
     "iopub.status.idle": "2021-04-15T20:01:32.655375Z",
     "shell.execute_reply": "2021-04-15T20:01:32.655755Z"
    },
    "papermill": {
     "duration": 0.026376,
     "end_time": "2021-04-15T20:01:32.655885",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.629509",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "data['label_group'] = encoder.fit_transform(data['label_group'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.690956Z",
     "iopub.status.busy": "2021-04-15T20:01:32.690306Z",
     "iopub.status.idle": "2021-04-15T20:01:32.693182Z",
     "shell.execute_reply": "2021-04-15T20:01:32.692784Z"
    },
    "papermill": {
     "duration": 0.024659,
     "end_time": "2021-04-15T20:01:32.693303",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.668644",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run():\n",
    "    # Defining DataSet\n",
    "    train_dataset = ShopeeDataset(\n",
    "        csv=data\n",
    "    )\n",
    "        \n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        train_dataset,\n",
    "        batch_size=TRAIN_BATCH_SIZE,\n",
    "        pin_memory=True,\n",
    "        drop_last=True,\n",
    "        num_workers=NUM_WORKERS\n",
    "    )\n",
    "    \n",
    "    # Defining Device\n",
    "    device = torch.device(\"cuda\")\n",
    "    \n",
    "    # Defining Model for specific fold\n",
    "    model = ShopeeNet(**model_params)\n",
    "    model.to(device)\n",
    "    \n",
    "    #DEfining criterion\n",
    "    criterion = fetch_loss()\n",
    "    criterion.to(device)\n",
    "        \n",
    "    # Defining Optimizer with weight decay to params other than bias and layer norms\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.0001},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},\n",
    "            ]  \n",
    "    \n",
    "    optimizer = AdamW(optimizer_parameters, lr=LR)\n",
    "    \n",
    "    #Defining LR SCheduler\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, \n",
    "        num_warmup_steps=len(train_loader)*2, \n",
    "        num_training_steps=len(train_loader)*EPOCHS\n",
    "    )\n",
    "        \n",
    "    # THE ENGINE LOOP\n",
    "    best_loss = 10000\n",
    "    for epoch in range(EPOCHS):\n",
    "        train_loss = train_fn(train_loader, model,criterion, optimizer, device,scheduler=scheduler,epoch=epoch)\n",
    "        \n",
    "        if train_loss.avg < best_loss:\n",
    "            best_loss = train_loss.avg\n",
    "            torch.save(model.state_dict(),f'sentence_transfomer_xlm_best_loss_num_epochs_{EPOCHS}_{loss_module}.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-15T20:01:32.722583Z",
     "iopub.status.busy": "2021-04-15T20:01:32.721754Z",
     "iopub.status.idle": "2021-04-15T20:25:01.386553Z",
     "shell.execute_reply": "2021-04-15T20:25:01.387249Z"
    },
    "papermill": {
     "duration": 1408.681724,
     "end_time": "2021-04-15T20:25:01.387553",
     "exception": false,
     "start_time": "2021-04-15T20:01:32.705829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e607585ea28046d9aa1c4edcfe17fff2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1070/1070 [04:23<00:00,  4.06it/s, Epoch=0, LR=2.5e-5, Train_Loss=23.9]\n",
      "100%|██████████| 1070/1070 [04:23<00:00,  4.06it/s, Epoch=1, LR=5e-5, Train_Loss=21.3]\n",
      "100%|██████████| 1070/1070 [04:23<00:00,  4.06it/s, Epoch=2, LR=3.33e-5, Train_Loss=17.9]\n",
      "100%|██████████| 1070/1070 [04:23<00:00,  4.06it/s, Epoch=3, LR=1.67e-5, Train_Loss=15]\n",
      "100%|██████████| 1070/1070 [04:24<00:00,  4.05it/s, Epoch=4, LR=1.56e-8, Train_Loss=13.4]\n"
     ]
    }
   ],
   "source": [
    "run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 2.906755,
     "end_time": "2021-04-15T20:25:07.092742",
     "exception": false,
     "start_time": "2021-04-15T20:25:04.185987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1434.597915,
   "end_time": "2021-04-15T20:25:11.611764",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-04-15T20:01:17.013849",
   "version": "2.2.2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02eeff0a1bb0404f85f6d88769f9f38e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "039cdc48ea9b4819bb01d25daa345f34": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_19fa477af22244fcb11f49811d73bc41",
       "placeholder": "​",
       "style": "IPY_MODEL_f7e09a613d3e473d8c429f21b92a7c79",
       "value": "Downloading: 100%"
      }
     },
     "07fa6f765ed34c4bb5851efa6ce25b0c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_cedcdc6155d2474797d5c0cc55841761",
        "IPY_MODEL_0befa2c1ddef40a18592cd7f4472063b",
        "IPY_MODEL_60fc93eab0924dd9ae545430bb69685c"
       ],
       "layout": "IPY_MODEL_9cdc390822804b28b9c0b568ffb11acf"
      }
     },
     "0befa2c1ddef40a18592cd7f4472063b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_67f1331ffa33408ebdd0eeabed459511",
       "max": 150.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a26c2adcde094bc0a707c232815b8333",
       "value": 150.0
      }
     },
     "186a428b5bff417589660bcbc3924893": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1951b4b09b894e2188e41a753e1c0f6e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "19fa477af22244fcb11f49811d73bc41": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1ac289eb81704487a4d6d3de742f1694": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e06270aae5e54cc9ac6722a94cda3d47",
       "max": 541.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c9e275f0c29b42cab4dbd0fbf6dd85ea",
       "value": 541.0
      }
     },
     "1dbc24d6968e48a78f9d8bb890c840c5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_33b1dcf875024d01a08027207f5f3b35",
       "placeholder": "​",
       "style": "IPY_MODEL_e33b42ccbc3b4452bfa8d6ce456be49f",
       "value": " 5.07M/5.07M [00:01&lt;00:00, 5.49MB/s]"
      }
     },
     "2912a11e775046abb495aff8e8fb68b8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d9d09cd3928b406ab2607e4e075e5b60",
       "max": 5069051.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_8ee88ae86025440ba692841f8867d91e",
       "value": 5069051.0
      }
     },
     "2d2e7d3f0dad4d3b877e908e0764b7f4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3249cc6b3bd94483a44f2a8856bcf65c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5e8caf5e45a84b1d84e9380338fb7291",
        "IPY_MODEL_4b3d7aa9651d4af0979b903aebb3a5a4",
        "IPY_MODEL_c7822fb5e7a44aa6874a55ac345c85bd"
       ],
       "layout": "IPY_MODEL_3a62f99ad90a44f1ba9c45f045e0145b"
      }
     },
     "33b1dcf875024d01a08027207f5f3b35": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "33b389a3520e494890661bcf288020b3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "364b956fba0a485f85105e7820988bfd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3a62f99ad90a44f1ba9c45f045e0145b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4ae42600edde4c648814ca96176762ce": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6e89e2a5e2644e3fa983280dc0c8520e",
       "placeholder": "​",
       "style": "IPY_MODEL_ec8dcf6ea1a945ad8b453e5a4efcdca3",
       "value": "Downloading: 100%"
      }
     },
     "4b3d7aa9651d4af0979b903aebb3a5a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2d2e7d3f0dad4d3b877e908e0764b7f4",
       "max": 188.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_891345834db04fe2880a687da6c4886d",
       "value": 188.0
      }
     },
     "5e8caf5e45a84b1d84e9380338fb7291": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b603a6934d9e44ccb35777b5f5b8f88e",
       "placeholder": "​",
       "style": "IPY_MODEL_67144955a13c49c28bd2a61852bd1ae5",
       "value": "Downloading: 100%"
      }
     },
     "60fc93eab0924dd9ae545430bb69685c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fe00f63bef2c4c878407bbef4cd92e13",
       "placeholder": "​",
       "style": "IPY_MODEL_f9662120032040608a509da6b004c2a3",
       "value": " 150/150 [00:00&lt;00:00, 5.48kB/s]"
      }
     },
     "63543794ef704220ae8d78b5d003ffaa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "658c1534acfb433bbdaf466aa50a18c7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_8ab0c8dbb78f46e3845566194e122c75",
        "IPY_MODEL_1ac289eb81704487a4d6d3de742f1694",
        "IPY_MODEL_f246f4c22d12407d94a4d4721a98e6b1"
       ],
       "layout": "IPY_MODEL_7fb26f80a5f84c18a42b8047d26a7b1a"
      }
     },
     "67144955a13c49c28bd2a61852bd1ae5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "67f1331ffa33408ebdd0eeabed459511": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6e89e2a5e2644e3fa983280dc0c8520e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6f9c30569a72464c87e18577836a05cb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7fb26f80a5f84c18a42b8047d26a7b1a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "891345834db04fe2880a687da6c4886d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "8a3aae1518b54de5acbca9010ac51954": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_de29e4fac44949cea7d1a51c8e543f58",
       "max": 1112256686.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ad56c042ca934c8dad70a56b4b63d436",
       "value": 1112256686.0
      }
     },
     "8ab0c8dbb78f46e3845566194e122c75": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_02eeff0a1bb0404f85f6d88769f9f38e",
       "placeholder": "​",
       "style": "IPY_MODEL_d9d7e6d18d0a4b14aae1869f961fbadc",
       "value": "Downloading: 100%"
      }
     },
     "8ee88ae86025440ba692841f8867d91e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9cdc390822804b28b9c0b568ffb11acf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a26c2adcde094bc0a707c232815b8333": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ad56c042ca934c8dad70a56b4b63d436": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "b603a6934d9e44ccb35777b5f5b8f88e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ba9982c2882042ac97a814bdee797ab4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c5df3c4bb4b842f399133b45d6cbce76": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_039cdc48ea9b4819bb01d25daa345f34",
        "IPY_MODEL_2912a11e775046abb495aff8e8fb68b8",
        "IPY_MODEL_1dbc24d6968e48a78f9d8bb890c840c5"
       ],
       "layout": "IPY_MODEL_63543794ef704220ae8d78b5d003ffaa"
      }
     },
     "c7822fb5e7a44aa6874a55ac345c85bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d7b0b1537f334e61b5f1d4cfac1030f7",
       "placeholder": "​",
       "style": "IPY_MODEL_ba9982c2882042ac97a814bdee797ab4",
       "value": " 188/188 [00:00&lt;00:00, 6.92kB/s]"
      }
     },
     "c9e275f0c29b42cab4dbd0fbf6dd85ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "cedcdc6155d2474797d5c0cc55841761": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1951b4b09b894e2188e41a753e1c0f6e",
       "placeholder": "​",
       "style": "IPY_MODEL_f758d3bfc26d41749129eda5809072b0",
       "value": "Downloading: 100%"
      }
     },
     "d11a1dd975b94bb18d5a15b704ac94f8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d2d922616688421a9bc6d57f18a6b9bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_186a428b5bff417589660bcbc3924893",
       "placeholder": "​",
       "style": "IPY_MODEL_33b389a3520e494890661bcf288020b3",
       "value": " 1.11G/1.11G [00:54&lt;00:00, 23.4MB/s]"
      }
     },
     "d7b0b1537f334e61b5f1d4cfac1030f7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d9d09cd3928b406ab2607e4e075e5b60": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d9d7e6d18d0a4b14aae1869f961fbadc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "de29e4fac44949cea7d1a51c8e543f58": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e06270aae5e54cc9ac6722a94cda3d47": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e33b42ccbc3b4452bfa8d6ce456be49f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e607585ea28046d9aa1c4edcfe17fff2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4ae42600edde4c648814ca96176762ce",
        "IPY_MODEL_8a3aae1518b54de5acbca9010ac51954",
        "IPY_MODEL_d2d922616688421a9bc6d57f18a6b9bd"
       ],
       "layout": "IPY_MODEL_364b956fba0a485f85105e7820988bfd"
      }
     },
     "ec8dcf6ea1a945ad8b453e5a4efcdca3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f246f4c22d12407d94a4d4721a98e6b1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d11a1dd975b94bb18d5a15b704ac94f8",
       "placeholder": "​",
       "style": "IPY_MODEL_6f9c30569a72464c87e18577836a05cb",
       "value": " 541/541 [00:00&lt;00:00, 19.3kB/s]"
      }
     },
     "f758d3bfc26d41749129eda5809072b0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f7e09a613d3e473d8c429f21b92a7c79": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f9662120032040608a509da6b004c2a3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "fe00f63bef2c4c878407bbef4cd92e13": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}