XNLI_XLMR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "XNLI_XLM-R.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "rtjif02VH4Ya",
        "Wb2VUo2zH6oV",
        "2iry8Q2MH9Rp"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b1320f30df004a0a9252f1da772aa7cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_99d1a54b1734416e9ac66bf8d317b5f3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b49514fb3b6849078a2380b4b13d4063",
              "IPY_MODEL_b9c91fab701c431f81a18e44871a73f9"
            ]
          }
        },
        "2fe9e7083dfa4299b14894dab7ec3541": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bd63178b54a14bf08ba264231c386324",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5a02046bc04d4e99ade1f88b349d7afe",
              "IPY_MODEL_ad0784c543194989b867996077f5a5e2"
            ]
          }
        },
        "bd63178b54a14bf08ba264231c386324": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a02046bc04d4e99ade1f88b349d7afe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85eaf13a6aee4214b543228221f34e06",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2539,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2539,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8cb8c79146be4332adfb7d25dba4a96c"
          }
        },
        "ad0784c543194989b867996077f5a5e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b47f5fd14bc64b06836f56a4c0666bd1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.54k/2.54k [00:00&lt;00:00, 4.32kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83affa4dac9040028dbaf6e6da0ad279"
          }
        },
        "85eaf13a6aee4214b543228221f34e06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8cb8c79146be4332adfb7d25dba4a96c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b47f5fd14bc64b06836f56a4c0666bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83affa4dac9040028dbaf6e6da0ad279": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5zpByhG3BOH",
        "colab_type": "text"
      },
      "source": [
        "## Install & import *requirements*, Workspace configuration"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-xQY5AKpl6S6",
        "colab_type": "text"
      },
      "source": [
        "Install all requirements needed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ttpkHGfJ5xyz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install torch==1.4.0 nlp==0.3.0 pyarrow==0.16.0 transformers==3.0.2 scikit-learn==0.22.2 numpy==1.18.2 tqdm==4.43.0 six==1.15.0 tabulate==0.8.7 pkbar==0.4 nltk==3.2.5 tabulate==0.8.7\n",
        "exit()  # to restart the kernel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ltt8lCQ0l81P",
        "colab_type": "text"
      },
      "source": [
        "Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "53Eta-Ev5kwH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "import os\n",
        "import pickle\n",
        "import datetime\n",
        "import random\n",
        "import numpy as np\n",
        "from pathlib import Path\n",
        "import logging\n",
        "import importlib\n",
        "import warnings\n",
        "import time\n",
        "\n",
        "import nlp\n",
        "import nltk\n",
        "import torch\n",
        "from tabulate import tabulate\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from torch.utils.data import Dataset\n",
        "from torch.optim import Adam\n",
        "from torch.nn import CrossEntropyLoss\n",
        "from torch.utils.data import DataLoader\n",
        "from tqdm.autonotebook import tqdm\n",
        "\n",
        "import torch.nn as nn\n",
        "from torch.nn.modules.module import _addindent\n",
        "from nltk.tokenize import RegexpTokenizer \n",
        "\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
        "from transformers import (AutoTokenizer, AutoModel,\n",
        "                          AdamW, get_linear_schedule_with_warmup,\n",
        "                          get_cosine_schedule_with_warmup)\n",
        "\n",
        "import pkbar\n",
        "import shutil"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i00LIEO8l_Yc",
        "colab_type": "text"
      },
      "source": [
        "- Configure workspace\n",
        "    1. Ignore warnings \n",
        "    2. Fix seed\n",
        "    3. Download nltk needed packages\n",
        "    4. Configure logger\n",
        "    5. Create & Remove folders\n",
        "\n",
        "- Utilites \n",
        "    1. save & load pickles\n",
        "    2. Ensure directory existing, create if not existing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MC-leKX55sAs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def manage_workspace_directories():\n",
        "    dir_path = os.getcwd()\n",
        "    folders = ['model', 'runs']\n",
        "    folders_to_remove = ['sample_data']\n",
        "\n",
        "    for folder in folders:\n",
        "        if not os.path.exists(folder):\n",
        "            os.mkdir(os.path.join(dir_path, folder))\n",
        "    \n",
        "    for folder in folders_to_remove:\n",
        "        if os.path.exists(folder):\n",
        "            shutil.rmtree(folder)\n",
        "\n",
        "\n",
        "def configure_workspace(SEED=1873337):\n",
        "    \"\"\"\n",
        "    Configure seed for reproducability, configure logging messages\n",
        "    \"\"\"\n",
        "    warnings.filterwarnings(\"ignore\")\n",
        "    random.seed(SEED)\n",
        "    np.random.seed(SEED)\n",
        "    os.environ['PYTHONHASHSEED'] = str(SEED)\n",
        "    torch.manual_seed(SEED)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    nltk.download('punkt', quiet=True)\n",
        "    logging.basicConfig(format=\"%(levelname)s - %(asctime)s: %(message)s\",\n",
        "                        datefmt='%H:%M:%S', level=logging.WARNING)\n",
        "    manage_workspace_directories()\n",
        "\n",
        "\n",
        "def save_pickle(save_to, save_what):\n",
        "    \"\"\" Save data into pickle format\"\"\"\n",
        "    with open(save_to, mode='wb') as f:\n",
        "        pickle.dump(save_what, f)\n",
        "\n",
        "\n",
        "def load_pickle(load_from):\n",
        "    \"\"\" Load data from presaved pickle file\"\"\"\n",
        "    with open(load_from, 'rb') as f:\n",
        "        return pickle.load(f)\n",
        "\n",
        "\n",
        "def ensure_dir(path):\n",
        "    \"\"\"\n",
        "    Makes sure direcctory exists and if not it creates it\n",
        "    \"\"\"\n",
        "    if not os.path.exists(path):\n",
        "        os.makedirs(path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BlSpRK4P2eeE",
        "colab_type": "text"
      },
      "source": [
        "## Dataset Parser"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jBHWFwm2mbIR",
        "colab_type": "text"
      },
      "source": [
        "- Create data parser object inherting from `torch.utils.data.Dataset`\n",
        "    - Creates labels vocabulary\n",
        "    - load Pretrained language model tokenizer `AutoTokenizer.from_pretrained('xlm-roberta-large')`\n",
        "    - Encode dataset using `encode_plus()` pass to it a premise to be paired with a hypothesis as NLI is a sequence-pair classification\n",
        "    - Pad sequences as per batch (not used) as sequences are padded/trimmed per batch\n",
        "    - Decode predictions  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M9YVMAoC5wRM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PTMDatasetParser(Dataset):\n",
        "    \"\"\"\n",
        "    XLMRDatasetParser identical to XLMRDatasetParser but without passing to the model the languages sequences\n",
        "    it builds the label2idx and idx2label, uses XLM Tokenizer, lowers all tokens,\n",
        "    indexes the dataset and pads per batch to feed model sequences of the same length\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, _device, data_, model_=\"xlm-roberta-large\"):\n",
        "        super(PTMDatasetParser).__init__()\n",
        "        self.encoded_data = []\n",
        "        self.device = _device\n",
        "        self.languages, self.premises, self.hypotheses, self.labels = data_\n",
        "        self.model_name = model_\n",
        "        self.label2idx = {'entailment': 0, 'neutral': 1, 'contradiction': 2}\n",
        "        self.idx2label = {0: 'entailment', 1: 'neutral', 2: 'contradiction'}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.premises)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        if self.encoded_data is None:\n",
        "            raise RuntimeError(\"Dataset is not indexed yet.\\\n",
        "                                To fetch raw elements, use get_element(idx)\")\n",
        "        return self.encoded_data[idx]\n",
        "\n",
        "    def get_element(self, idx):\n",
        "        return self.languages[idx], self.premises[idx], self.hypotheses[idx], self.labels[idx]\n",
        "\n",
        "    def encode_dataset(self):\n",
        "        \"\"\" Indexing and encoding the dataset using XLMTokenizer whilst lowering all tokens\n",
        "        Tokenizer takes the sentence A (premises) and sentence B (hypotheses) adds to them the special tokens\n",
        "        [CLS] & [SEP], it pads sentences to max length of 128 and truncates the ones with lengthier number of tokens\n",
        "            - Original Sequence (inputs_ids) is changed to its corresponding IDs\n",
        "            - Attention masks helps the model differntiate between [PAD] and original tokens\n",
        "            - Token types helps the model know which sentence is Sentence A and which is the sentence B\n",
        "        \"\"\"\n",
        "\n",
        "        tokenizer = AutoTokenizer.from_pretrained(self.model_name, do_lower_case=True)\n",
        "        for lang_, premise_, hypothesis_, labels_ in tqdm(\n",
        "                zip(self.languages, self.premises, self.hypotheses, self.labels),\n",
        "                leave=False, total=len(self.premises),\n",
        "                desc=f'Encoding dataset'):\n",
        "            encoded_dict = tokenizer.encode_plus(text=premise_,\n",
        "                                                 text_pair=hypothesis_,\n",
        "                                                 add_special_tokens=True,\n",
        "                                                 return_token_type_ids=True,\n",
        "                                                 return_attention_mask=True,\n",
        "                                                 max_length=128,\n",
        "                                                 truncation=True,\n",
        "                                                 pad_to_max_length=True,\n",
        "                                                 return_tensors='pt')\n",
        "            self.encoded_data.append({'premises_hypotheses': torch.squeeze(encoded_dict[\"input_ids\"]),\n",
        "                                      'attention_mask': torch.squeeze(encoded_dict[\"attention_mask\"]),\n",
        "                                      'token_types': torch.squeeze(encoded_dict[\"token_type_ids\"]),\n",
        "                                      'outputs': torch.LongTensor([self.label2idx.get(labels_)])})\n",
        "\n",
        "    @staticmethod\n",
        "    def pad_batch(batch):\n",
        "        \"\"\" Pads sequences per batch with `padding_value=0`\n",
        "        Args: batch: List[dict]\n",
        "        Returns: dict of models inputs padded as per max len in batch\n",
        "        \"\"\"\n",
        "        # XLM-R tokenizer padding index is 2, hence it is the padding value in `padded_premises_hypotheses`\n",
        "        premises_hypotheses_batch = [sample[\"premises_hypotheses\"] for sample in batch]\n",
        "        padded_premises_hypotheses = pad_sequence(premises_hypotheses_batch, padding_value=2, batch_first=True)\n",
        "        mask = [sample[\"attention_mask\"] for sample in batch]\n",
        "        padded_mask = pad_sequence(mask, padding_value=0, batch_first=True)\n",
        "        # Token types is padded with 1 (unlike others padded with 0), padded with 1's due to\n",
        "        # that we are doing pair_sentences encoding so the other sentence is consisting of 1's\n",
        "        token_types_batch = [sample[\"token_types\"] for sample in batch]\n",
        "        padded_token_types = pad_sequence(token_types_batch, padding_value=1, batch_first=True)\n",
        "        outputs_batch = pad_sequence([sample[\"outputs\"] for sample in batch], batch_first=True)\n",
        "\n",
        "        return {\"premises_hypotheses\": padded_premises_hypotheses,\n",
        "                \"attention_mask\": padded_mask,\n",
        "                \"token_types\": padded_token_types,\n",
        "                \"outputs\": outputs_batch}\n",
        "\n",
        "    def decode_predictions(self, predictions):\n",
        "        \"\"\"\n",
        "        Flattens predictions list (if it is a list of lists)\n",
        "        and get the corresponding label name for each label index (label_stoi)\n",
        "        \"\"\"\n",
        "        if any(isinstance(el, list) for el in predictions):\n",
        "            return [self.idx2label.get(label) for tag in predictions for label in tag]\n",
        "        else:\n",
        "            predictions_ = [_e for e in predictions for _e in e]\n",
        "            return [self.idx2label.get(label) for tag in predictions_ for label in tag]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uctr9g-6m6Vf",
        "colab_type": "text"
      },
      "source": [
        "- Functions provided to parse data sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O36ecM-Y5tjM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "int2nli_label = {0: 'entailment', 1: 'neutral', 2: 'contradiction'}\n",
        "nli_label2int = {v: k for k, v in int2nli_label.items()}\n",
        "nli_labels = list(nli_label2int.keys())\n",
        "\n",
        "\n",
        "def read_mnli(dataset):\n",
        "    \"\"\"\n",
        "    READ MNLI Dataset returns 4 lists each of them is a list of strings\n",
        "    languages, premises, hypotheses, labels\n",
        "    \"\"\"\n",
        "    int2nli_label = {0: 'entailment', 1: 'neutral', 2: 'contradiction'}\n",
        "    languages = []\n",
        "    premises = []\n",
        "    hypotheses = []\n",
        "    labels = []\n",
        "\n",
        "    for sample in dataset:\n",
        "        languages.append('en')\n",
        "        premises.append(sample['premise'])\n",
        "        hypotheses.append(sample['hypothesis'])\n",
        "        labels.append(int2nli_label[sample['label']])\n",
        "\n",
        "    assert len(languages) == len(premises) == len(hypotheses) == len(labels)\n",
        "    return languages, premises, hypotheses, labels\n",
        "\n",
        "\n",
        "def read_xnli(dataset):\n",
        "    languages = []\n",
        "    premises = []\n",
        "    hypotheses = []\n",
        "    labels = []\n",
        "\n",
        "    for sample in dataset:\n",
        "\n",
        "        language2premise = {}\n",
        "        language2hypothesis = {}\n",
        "\n",
        "        # read premises\n",
        "        for language, premise in sample['premise'].items():\n",
        "            language2premise[language] = premise\n",
        "\n",
        "        # read hypotheses\n",
        "        for language, hypothesis in zip(sample['hypothesis']['language'], sample['hypothesis']['translation']):\n",
        "            language2hypothesis[language] = hypothesis\n",
        "\n",
        "        label = int2nli_label[sample['label']]\n",
        "\n",
        "        sample_languages = set(language2premise.keys()) & set(language2hypothesis.keys())\n",
        "        assert len(sample_languages) == len(language2premise) == len(language2hypothesis)\n",
        "\n",
        "        for language in sample_languages:\n",
        "            languages.append(language)\n",
        "            premises.append(language2premise[language])\n",
        "            hypotheses.append(language2hypothesis[language])\n",
        "            labels.append(label)\n",
        "\n",
        "    assert len(languages) == len(premises) == len(hypotheses) == len(labels)\n",
        "    return languages, premises, hypotheses, labels\n",
        "    \n",
        "\n",
        "def read_train():\n",
        "    \"\"\"\n",
        "    Read training dataset (MNLI)\n",
        "    \"\"\"\n",
        "    return read_mnli(nlp.load_dataset('multi_nli')['train'])\n",
        "\n",
        "\n",
        "def read_dev():\n",
        "    \"\"\"\n",
        "    Read validation dataset (MNLI)\n",
        "    \"\"\"\n",
        "    return read_mnli(nlp.load_dataset('multi_nli')['validation_matched'])\n",
        "\n",
        "\n",
        "def read_test():\n",
        "    \"\"\"\n",
        "    Read test dataset (XNLI)\n",
        "    \"\"\"\n",
        "    return read_xnli(nlp.load_dataset('xnli')['test'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5sfc-NXH2ius",
        "colab_type": "text"
      },
      "source": [
        "## Models"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M5NNv1him__5",
        "colab_type": "text"
      },
      "source": [
        "- Create the model using `AutoModel.from_pretrained('xlm-roberta-large')\n",
        "- forward method, passes the last hidden state from the model to dropout layer then a classifier layer\n",
        "    - Sequences\n",
        "    - Attention_mask: to help the model differentiate between padding tokens and original tokens\n",
        "    - Token types: to allow model differentiate between tokens from sentence A and that of sentence B \n",
        "- `save_(dir_path)` saves the model, and its state dict in `.pt` & `.pth` file extensions respectively.\n",
        "- `load_(dir_path)` load model's state_dict \n",
        "- `predict_sentences_(seq, mask, tokens_type)`\n",
        "- `print_summary()` display model's summary "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Sp86qi-7kXt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PTM_Model(nn.Module):\n",
        "    def __init__(self, hparams, model_name, freeze_model=False):\n",
        "        \"\"\"\n",
        "        Cross-lingual Language Model which is based on RoBERTa. It is a large multi-lingual language model,\n",
        "        trained on 2.5TB of filtered CommonCrawl data. It is a transformer-based masked language model on\n",
        "        100 languages.\n",
        "\n",
        "        RoBERTa is built on BERT, but, it modifies key hyperparameters, removes BERT's NSP training objective,\n",
        "        and train with larger learning rates, and larger batch sizes.\n",
        "\n",
        "        :param hparams: hyperparameters\n",
        "        :param model_name: by default it is set to \"xlm-roberta-large\"\n",
        "        :param freeze_bert: Boolean flag whether to freeze all model layers, or to fine tune the model given our task\n",
        "        \"\"\"\n",
        "        super(PTM_Model, self).__init__()\n",
        "        self.name = hparams.model_name\n",
        "        self._device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "        self.pretrained_model = AutoModel.from_pretrained(model_name)\n",
        "\n",
        "        if freeze_model:  # Freeze model Layer\n",
        "            for param in self.pretrained_model.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "        self.dropout = nn.Dropout(hparams.dropout)\n",
        "        classifier_input_dim = self.pretrained_model.config.hidden_size\n",
        "        self.classifier = nn.Linear(classifier_input_dim, hparams.num_classes)\n",
        "\n",
        "    def forward(self, sequences, attention_mask, tokens_type):\n",
        "        \"\"\"\n",
        "        XLM-RoBERTa is unlike XLM as it does not require `lang` tensors to understand which language\n",
        "        is used, and should be able to determine the correct language from the input ids.\n",
        "        XLM-RoBERTa takes 3 args as explained below.\n",
        "         Forward method takes\n",
        "        :param sequences: (inputs_ids) Indices of input sequence tokens in the vocabulary.\n",
        "        :param attention_mask: Mask to avoid performing attention on padding token indices.\n",
        "            Mask values selected in [0, 1]: 1 for tokens that are NOT MASKED, 0 for MASKED tokens.\n",
        "        :param tokens_type: Segment token indices to indicate first and second portions of the inputs.\n",
        "            Indices are selected in [0, 1]: 0 corresponds to a sentence A token, 1 corresponds to a sentence B token\n",
        "\n",
        "        XLM Output is\n",
        "            - last_hidden_state [batch_size, sequence_length, hidden_size] which represents the Sequence\n",
        "                of hidden-states at the output of the last layer of the model.\n",
        "            - other parameters that were not used here but can be found in the documentation of transformers package\n",
        "\n",
        "        :return: logits\n",
        "            Which is produced after taking the mean of XLM last_hidden_layer then passed to classifier layer\n",
        "\n",
        "        Note for tokenizer:\n",
        "        it is unlike bert's tokenizer it uses BPE as a tokenizer and a different pre-training scheme.\n",
        "\n",
        "        \"\"\"\n",
        "\n",
        "        outputs = self.pretrained_model(input_ids=sequences,\n",
        "                                        attention_mask=attention_mask,\n",
        "                                        token_type_ids=tokens_type)\n",
        "        last_hidden_state = outputs[0]\n",
        "        sentence_embeddings = torch.mean(last_hidden_state, dim=1)\n",
        "        o = self.dropout(sentence_embeddings)\n",
        "        logits = self.classifier(o)\n",
        "        return logits\n",
        "\n",
        "    def save_(self, dir_path):\n",
        "        \"\"\"\n",
        "        Saves model and its state dict into the given dir path\n",
        "        Args: dir_path (str)\n",
        "        \"\"\"\n",
        "        torch.save(self, f'{dir_path}.pt')\n",
        "        torch.save(self.state_dict(), f'{dir_path}.pth')\n",
        "\n",
        "    def load_(self, path):\n",
        "        \"\"\"\n",
        "        Loads the model and its state dictionary\n",
        "        Args: path (str): [Model's state dict is located]\n",
        "        \"\"\"\n",
        "        state_dict = torch.load(path) if self._device == 'cuda' else torch.load(path, map_location=self._device)\n",
        "        self.load_state_dict(state_dict)\n",
        "\n",
        "    def predict_sentence_(self, seq, mask, tokens_type):\n",
        "        predicted_labels = []\n",
        "        self.eval()\n",
        "        with torch.no_grad():\n",
        "            predictions = self(seq, mask, tokens_type)\n",
        "            _, argmax = torch.max(predictions, dim=-1)\n",
        "            predicted_labels.extend(argmax.tolist())\n",
        "        return predicted_labels\n",
        "\n",
        "    def print_summary(self, show_weights=False, show_parameters=False):\n",
        "        \"\"\"\n",
        "        Summarizes torch model by showing trainable parameters and weights.\n",
        "        \"\"\"\n",
        "        tmpstr = self.__class__.__name__ + ' (\\n'\n",
        "        for key, module in self._modules.items():\n",
        "            # if it contains layers let call it recursively to get params and weights\n",
        "            if type(module) in [\n",
        "                torch.nn.modules.container.Container,\n",
        "                torch.nn.modules.container.Sequential\n",
        "            ]:\n",
        "                modstr = self.print_summary()\n",
        "            else:\n",
        "                modstr = module.__repr__()\n",
        "            modstr = _addindent(modstr, 2)\n",
        "\n",
        "            params = sum([np.prod(p.size()) for p in module.parameters()])\n",
        "            weights = tuple([tuple(p.size()) for p in module.parameters()])\n",
        "\n",
        "            tmpstr += '  (' + key + '): ' + modstr\n",
        "            if show_weights:\n",
        "                tmpstr += ', weights={}'.format(weights)\n",
        "            if show_parameters:\n",
        "                tmpstr += ', parameters={}'.format(params)\n",
        "            tmpstr += '\\n'\n",
        "\n",
        "        tmpstr = tmpstr + ')'\n",
        "        print(f'========== {self.name} Model Summary ==========')\n",
        "        print(tmpstr)\n",
        "        num_params = sum(p.numel()\n",
        "                         for p in self.parameters() if p.requires_grad)\n",
        "        print(f\"Number of parameters: {num_params:,}\")\n",
        "        print('==================================================')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b7oTkk1E2gVp",
        "colab_type": "text"
      },
      "source": [
        "### Hyperparameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hUiaKdCaoWX5",
        "colab_type": "text"
      },
      "source": [
        "Set model's hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UmMVO9Z-RDey",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class HyperParameters:\n",
        "    \"\"\"\n",
        "    Hyperparameters configuration class where all vars are defined\n",
        "    \"\"\"\n",
        "    def __init__(self, model_name_, vocab, label_vocab, embeddings_, batch_size_):\n",
        "        self.model_name = model_name_\n",
        "        self.vocab_size = len(vocab) if vocab else \"Using AutoTokenizer's vocab\"\n",
        "        self.num_classes = len(label_vocab)\n",
        "        self.hidden_dim = 128\n",
        "        self.bidirectional = True\n",
        "        self.embedding_dim = 300\n",
        "        self.num_layers = 1\n",
        "        self.dropout = 0.3\n",
        "        self.embeddings = embeddings_\n",
        "        self.batch_size = batch_size_\n",
        "\n",
        "    def _print_info(self):\n",
        "        \"\"\"\n",
        "        prints summary of model's hyperparameters\n",
        "        \"\"\"\n",
        "        print(\"========== Hyperparameters ==========\",\n",
        "            f\"Name: {self.model_name.replace('_', ' ')}\",\n",
        "            f\"Vocab Size: {self.vocab_size}\",\n",
        "            f\"Tags Size: {self.num_classes}\",\n",
        "            f\"Embeddings Dim: {self.embedding_dim}\",\n",
        "            f\"Hidden Size: {self.hidden_dim}\",\n",
        "            f\"BiLSTM: {self.bidirectional}\",\n",
        "            f\"Layers Num: {self.num_layers}\",\n",
        "            f\"Dropout: {self.dropout}\",\n",
        "            f\"Pretrained_embeddings: {False if self.embeddings is None else True}\",\n",
        "            f\"Batch Size: {self.batch_size}\", sep='\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iMDjr1Jy2kXD",
        "colab_type": "text"
      },
      "source": [
        "## Trainer "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K8ynVWBmoZNp",
        "colab_type": "text"
      },
      "source": [
        "- Create the trainer object with optimizer and loss function\n",
        "- load xnli metric from `nlp`\n",
        "- allow gradients clipping\n",
        "- save model with the current best `val_loss`\n",
        "- show progress bar whilst training\n",
        "- save the model after training \n",
        "- evaluate model after training epoch, fetch `val_loss`, `val_acc`\n",
        "- save & load checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_h3fsz_zOiv3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from random import randint\n",
        "\n",
        "\n",
        "class PTMTrainer:\n",
        "    def __init__(self, model, loss_function, optimizer,\n",
        "                 epochs, verbose, writer, _device):\n",
        "        \"\"\"\n",
        "        Trainer object requires model, criterion (loss function), optimizer\n",
        "        verbose level and number of epochs\n",
        "        \"\"\"\n",
        "        self.model = model\n",
        "        self.loss_function = loss_function\n",
        "        self.optimizer = optimizer\n",
        "        self._verbose = verbose\n",
        "        self._epochs = epochs\n",
        "        self.writer = writer\n",
        "        self.device = _device\n",
        "\n",
        "    def train(self, train_dataset, valid_dataset, save_to=None):\n",
        "        \"\"\"\n",
        "        Trains the model, while keeping track of the best model trained so far\n",
        "        and saves it if there is an improvement in the validation loss\n",
        "        Applies Gradients clipping if their norm increased above the 1.0\n",
        "        Writes the model loss, val_Loss plots in tensorboard.\n",
        "        Activates early stopping if needed with 5 epochs as patience to prevent overfitting\n",
        "\n",
        "        It uses pkbar package to have a progress bar keras like,\n",
        "        the progress bar shows the loss, val_loss during training as well as acc & val_acc\n",
        "        In order to compute the accuracy, we make use from nlp metric library for XNLI tasks. the accuracy is computed\n",
        "        from model's predictions during training and the original labels, the predictions are fetched as the argmax of\n",
        "        the logits tensor.\n",
        "\n",
        "        Model takes 3 arguments (from the sequences padded as per batch) to do the forward pass:\n",
        "        1. Sequence (inputs_ids)\n",
        "        2. Attention masks\n",
        "        3. Token types\n",
        "\n",
        "        Args:\n",
        "            train_dataset (DataLoader)\n",
        "            valid_dataset (DataLoader)\n",
        "            save_to (str, optional): Save model to this path. Defaults to None.\n",
        "\n",
        "        Returns:\n",
        "            float: average training loss\n",
        "            float: average training acc\n",
        "        \"\"\"\n",
        "        metric = nlp.load_metric('xnli')\n",
        "        train_loss, train_acc, best_val_loss = 0.0, 0.0, float(1e4)\n",
        "\n",
        "        for epoch in range(1, self._epochs + 1):\n",
        "            print(f'Epoch {epoch}/{self._epochs}:')\n",
        "            kbar = pkbar.Kbar(target=len(train_dataset))\n",
        "\n",
        "            epoch_acc, epoch_loss = 0.0, 0.0\n",
        "            self.model.train()\n",
        "            for batch_idx, sample in enumerate(train_dataset):\n",
        "                seq = sample[\"premises_hypotheses\"].to(self.device)\n",
        "                mask = sample[\"attention_mask\"].to(self.device)\n",
        "                token_types = sample[\"token_types\"].to(self.device)\n",
        "                labels = sample[\"outputs\"].to(self.device)\n",
        "                labels_ = labels.view(-1)\n",
        "\n",
        "                self.optimizer.zero_grad()\n",
        "                logits = self.model(seq, mask, token_types)\n",
        "                _, preds = torch.max(logits, dim=-1)\n",
        "                acc_ = metric.compute(preds, labels_)['accuracy']\n",
        "\n",
        "                sample_loss = self.loss_function(logits, labels_)\n",
        "                sample_loss.backward()\n",
        "                clip_grad_norm_(self.model.parameters(), 1.)  # Gradient Clipping\n",
        "                self.optimizer.step()\n",
        "                epoch_loss += sample_loss.tolist()\n",
        "                epoch_acc += acc_.tolist()\n",
        "\n",
        "                if self._verbose > 0:\n",
        "                    kbar.update(batch_idx, values=[(\"loss\", sample_loss.item()), (\"acc\", acc_.item())])\n",
        "\n",
        "            avg_epoch_loss = epoch_loss / len(train_dataset)\n",
        "            avg_epoch_acc = epoch_acc / len(train_dataset)\n",
        "            train_loss += avg_epoch_loss\n",
        "            train_acc += avg_epoch_acc\n",
        "\n",
        "            valid_loss, val_acc = self.evaluate(valid_dataset)\n",
        "            kbar.add(1,\n",
        "                     values=[(\"loss\", train_loss), (\"acc\", train_acc), (\"val_loss\", valid_loss), (\"val_acc\", val_acc)])\n",
        "            if self.writer:\n",
        "                self.writer.set_step(epoch, 'train')\n",
        "                self.writer.add_scalar('loss', avg_epoch_loss)\n",
        "                self.writer.set_step(epoch, 'valid')\n",
        "                self.writer.add_scalar('val_loss', valid_loss)\n",
        "\n",
        "            is_best = valid_loss <= best_val_loss\n",
        "            if is_best:\n",
        "                logging.info(\"Model Checkpoint saved\")\n",
        "                best_val_loss = valid_loss\n",
        "                model_dir = os.path.join(os.getcwd(), 'model',\n",
        "                                         f'{self.model.name}_ckpt_best')\n",
        "                self.model.save_(model_dir)\n",
        "        avg_epoch_loss = train_loss / self._epochs\n",
        "        avg_epoch_acc = train_acc / self._epochs\n",
        "\n",
        "        if save_to is not None:\n",
        "            self.model.save_(save_to)\n",
        "        return avg_epoch_loss, avg_epoch_acc\n",
        "\n",
        "    def evaluate(self, valid_dataset):\n",
        "        metric = nlp.load_metric('xnli')\n",
        "        valid_acc, valid_loss = 0.0, 0.0\n",
        "        self.model.eval()\n",
        "        with torch.no_grad():\n",
        "            for sample in tqdm(valid_dataset, desc='Evaluating',\n",
        "                               leave=False, total=len(valid_dataset)):\n",
        "                seq = sample[\"premises_hypotheses\"].to(self.device)\n",
        "                mask = sample[\"attention_mask\"].to(self.device)\n",
        "                token_types = sample[\"token_types\"].to(self.device)\n",
        "                labels = sample[\"outputs\"].to(self.device)\n",
        "                labels_ = labels.view(-1)\n",
        "                logits = self.model(seq, mask, token_types)\n",
        "                _, preds = torch.max(logits, dim=-1)\n",
        "                val_acc = metric.compute(preds, labels)['accuracy']\n",
        "                sample_loss = self.loss_function(logits, labels_)\n",
        "                valid_loss += sample_loss.tolist()\n",
        "                valid_acc += val_acc.tolist()\n",
        "        return valid_loss / len(valid_dataset), valid_acc / len(valid_dataset)\n",
        "\n",
        "    def save_checkpoint(self, filename):\n",
        "        \"\"\" Saves model's and optimizer's checkpoints to continue training from a certain checkpoint \"\"\"\n",
        "        state = {\"model\": self.model.state_dict(),\n",
        "                 \"optimizer\": self.optimizer.state_dict()}\n",
        "        torch.save(state, filename)\n",
        "\n",
        "    def load_checkpoint(self, filename):\n",
        "        \"\"\" loads model's and optimizer's checkpoints to continue training from a certain checkpoint \"\"\"\n",
        "        checkpoint = torch.load(filename)\n",
        "        self.model.load_state_dict(checkpoint['state_dict'])\n",
        "        self.optimizer.load_state_dict(checkpoint['optimizer'])\n",
        "        # now individually transfer the optimizer parts...\n",
        "        for state in self.optimizer.state.values():\n",
        "            for k, v in state.items():\n",
        "                if isinstance(v, torch.Tensor):\n",
        "                    state[k] = v.to(self.device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VKC3XaP92mR9",
        "colab_type": "text"
      },
      "source": [
        "### Callbacks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mGWIUr5oqKUs",
        "colab_type": "text"
      },
      "source": [
        "Write train params (acc, loss, val_acc, val_loss) to tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "teDg2xknABPn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class WriterTensorboardX:\n",
        "    \"\"\"\n",
        "    Logs training process to tensorboard for visualization\n",
        "    \"\"\"\n",
        "    def __init__(self, writer_dir, logger, enable):\n",
        "        self.writer = None\n",
        "        ensure_dir(writer_dir)\n",
        "        if enable:\n",
        "            log_path = writer_dir\n",
        "            try:\n",
        "                self.writer = importlib.import_module('tensorboardX').SummaryWriter(log_path)\n",
        "            except ModuleNotFoundError:\n",
        "                os.system('pip install tensorboardX')\n",
        "                self.writer = importlib.import_module('tensorboardX').SummaryWriter(log_path)\n",
        "        self.step = 0\n",
        "        self.mode = ''\n",
        "\n",
        "        self.tensorboard_writer_ftns = ['add_scalar', 'add_scalars', 'add_image', 'add_audio', 'add_text', 'add_histogram', 'add_pr_curve', 'add_embedding']\n",
        "\n",
        "    def set_step(self, step, mode='train'):\n",
        "        self.mode = mode\n",
        "        self.step = step\n",
        "\n",
        "    def __getattr__(self, name):\n",
        "        \"\"\"\n",
        "        If visualization is configured to use:\n",
        "            return add_data() methods of tensorboard with additional information (step, tag) added.\n",
        "        Otherwise:\n",
        "            return blank function handle that does nothing\n",
        "        \"\"\"\n",
        "        if name in self.tensorboard_writer_ftns:\n",
        "            add_data = getattr(self.writer, name, None)\n",
        "            def wrapper(tag, data, *args, **kwargs):\n",
        "                if add_data is not None:\n",
        "                    add_data('{}/{}'.format(self.mode, tag), data, self.step, *args, **kwargs)\n",
        "            return wrapper\n",
        "        else:\n",
        "            # default action for returning methods defined in this class, set_step() for instance.\n",
        "            try:\n",
        "                attr = object.__getattr__(name)\n",
        "            except AttributeError:\n",
        "                raise AttributeError(\"type object 'WriterTensorboardX' has no attribute '{}'\".format(name))\n",
        "            return attr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x2UubP__j3bC",
        "colab_type": "text"
      },
      "source": [
        "## Evaluate Performance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OE3HD-D2PZkx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def compute_metrics(languages, predicted_labels, labels):\n",
        "    \"\"\"\n",
        "    Computes accuracy of the predicted labels (y_hat) vs labels (y)\n",
        "    \"\"\"\n",
        "    labels = [nli_label2int[label] for label in labels]\n",
        "    predicted_labels = [nli_label2int[predicted_label] for predicted_label in predicted_labels]\n",
        "\n",
        "\n",
        "    metric = nlp.load_metric('xnli', experiment_id=101)\n",
        "    headers = ('', 'accuracy', '# samples')\n",
        "    table = []\n",
        "    table.append(('overall', metric.compute(predicted_labels, labels)['accuracy'], len(labels)))\n",
        "\n",
        "    # per language\n",
        "    for evaluation_language in sorted(set(languages)):\n",
        "        evaluation_language_predicted_labels = [predicted_label for predicted_label, language in zip(predicted_labels, languages) if language == evaluation_language]\n",
        "        evaluation_language_labels = [label for label, language in zip(labels, languages) if language == evaluation_language]\n",
        "        evaluation_language_accuracy = metric.compute(evaluation_language_predicted_labels, evaluation_language_labels)['accuracy']\n",
        "        table.append((evaluation_language, evaluation_language_accuracy, len(evaluation_language_labels)))\n",
        "    print(tabulate(table, headers=headers, tablefmt=\"pretty\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zpomMxmf2vGD",
        "colab_type": "text"
      },
      "source": [
        "## Main"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qsHtM5mMHv4t",
        "colab_type": "text"
      },
      "source": [
        "### Parse and pickle dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TdWcYY0HuqiL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "\n",
        "def print_summary(summary_data, verbose=False):\n",
        "    premises, dev_premises, test_premises, word2idx, label2idx = summary_data\n",
        "    # To clear out cell from unwanted downloading and indexing progress bars\n",
        "    clear_output()\n",
        "    if verbose:\n",
        "        print(\"\\n=============Data Summary======================\",\n",
        "              f\"train_x length: {len(premises)} sentences\",\n",
        "              f\"dev_x length: {len(dev_premises)} sentences\",\n",
        "              f\"test_x length: {len(test_premises)} sentences\",\n",
        "              f\"Vocab size: {len(word2idx)}\",\n",
        "              f\"Labels vocab size: {len(label2idx)}\",\n",
        "              \"===============================================\\n\", sep=\"\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ae-wGoz861we",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "configure_workspace()\n",
        "device_ = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "save_path = os.path.join(os.getcwd(), \"model\", \"word_stoi.pkl\")\n",
        "model_name_ = \"xlm-roberta-large\"\n",
        "\n",
        "data = read_train()\n",
        "train_dataset = PTMDatasetParser(device_, data, model_name_)\n",
        "train_dataset.encode_dataset()\n",
        "save_path = os.path.join(os.getcwd(), \"model\", \"train_dataset.pkl\")\n",
        "save_pickle(save_to=save_path, save_what=train_dataset)\n",
        "print(\"Processed and encoded Training dataset\")\n",
        "\n",
        "dev_data = read_dev()\n",
        "dev_dataset = PTMDatasetParser(device_, dev_data, model_name_)\n",
        "dev_dataset.encode_dataset()\n",
        "dev_save_path = os.path.join(os.getcwd(), \"model\", \"dev_dataset.pkl\")\n",
        "save_pickle(save_to=dev_save_path, save_what=dev_dataset)\n",
        "print(\"Processed and encoded Validation dataset\")\n",
        "\n",
        "test_data = read_test()\n",
        "test_dataset = PTMDatasetParser(device_, test_data, model_name_)\n",
        "test_dataset.encode_dataset()\n",
        "test_save_path = os.path.join(os.getcwd(), \"model\", \"test_dataset.pkl\")\n",
        "save_pickle(save_to=test_save_path, save_what=test_dataset)\n",
        "print(\"Processed and encoded Testing dataset\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MC797EShHytl",
        "colab_type": "text"
      },
      "source": [
        "### Hyperparams & DataLoades"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kwZHYcKjYdlF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set Hyperparameters\n",
        "pretrained_embeddings_ = None\n",
        "batch_size = 8\n",
        "\n",
        "name_ = 'XLM-R_Seq_CLS'\n",
        "hp = HyperParameters(name_, None, train_dataset.label2idx,\n",
        "                     pretrained_embeddings_, batch_size)\n",
        "hp._print_info()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASF3digQYgp8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Prepare data loaders\n",
        "train_dataset_ = DataLoader(dataset=train_dataset, batch_size=batch_size,\n",
        "                            collate_fn=PTMDatasetParser.pad_batch,\n",
        "                            shuffle=True)\n",
        "dev_dataset_ = DataLoader(dataset=dev_dataset,\n",
        "                            batch_size=batch_size,\n",
        "                            collate_fn=PTMDatasetParser.pad_batch)\n",
        "test_dataset_ = DataLoader(dataset=test_dataset,\n",
        "                            batch_size=batch_size,\n",
        "                            collate_fn=PTMDatasetParser.pad_batch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V53yRq7-H1Us",
        "colab_type": "text"
      },
      "source": [
        "### Build and train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zTpytVwtYi6F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = PTM_Model(hp, model_name_, freeze_model=False).to(device_)\n",
        "model.print_summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RmD7MpCCyI3p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "log_path = os.path.join(os.getcwd(), 'runs', hp.model_name)\n",
        "writer_ = WriterTensorboardX(log_path, logger=logging, enable=True)\n",
        "epochs_ = 1\n",
        "\n",
        "optimizer_ = AdamW(model.parameters(), lr=5e-6)\n",
        "\n",
        "trainer = PTMTrainer(model=model, writer=writer_,\n",
        "                       epochs=epochs_, _device=device_,\n",
        "                       loss_function=CrossEntropyLoss(),\n",
        "                       optimizer=optimizer_, verbose=True)\n",
        "\n",
        "save_to_ = os.path.join(os.getcwd(), 'model', f\"{model.name}_model\")\n",
        "            \n",
        "try:\n",
        "    _, _ = trainer.train(train_dataset_, dev_dataset_, save_to=save_to_)\n",
        "except (KeyboardInterrupt, RuntimeError) as e: \n",
        "    print(f\"\\nSaving model weights & optimizer's state, please wait...\")\n",
        "    model.save_(save_to_)\n",
        "    torch.save(optimizer_.state_dict(), os.path.join(os.getcwd(), 'model', f'{model.name}_optimizer_ckpt.pt'))\n",
        "    print(f\"Model & Optimizer states are saved\")\n",
        "\n",
        "   \n",
        "print(f\"Saving trainer's checkpoint\")\n",
        "trainer.save_checkpoint(filename=f\"{model.name}_retrain_ckpt.pt\")\n",
        "print(f\"A checkpoint is saved\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rtjif02VH4Ya",
        "colab_type": "text"
      },
      "source": [
        "### Save or Load models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3IvrX7TWwnCt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save_checkpoint({'state_dict': model.state_dict(), 'optimizer' : optimizer_.state_dict()}, filename=f\"{model.name}_ckpt_retrain.pth.tar\")\n",
        "\n",
        "# torch.save(model.state_dict(), os.path.join(os.getcwd(), 'model', f'{model.name}_ckpt.pth'))\n",
        "# torch.save(model.state_dict(), os.path.join(os.getcwd(), 'model', f'{model.name}_ckpt.pth'))\n",
        "# torch.save(optimizer_.state_dict(), os.path.join(os.getcwd(), 'model', f'{model.name}_optimizer_ckpt.pt'))\n",
        "\n",
        "# model.load_state_dict(torch.load(os.path.join(os.getcwd(), 'model', f'{model.name}_model.pth')))\n",
        "# optimizer_.load_state_dict(torch.load(os.path.join(os.getcwd(), 'model', f'{model.name}_optimizer_ckpt.pt')))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wb2VUo2zH6oV",
        "colab_type": "text"
      },
      "source": [
        "### Predict and compute scores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGTQHqSOjGgh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "referenced_widgets": [
            "b1320f30df004a0a9252f1da772aa7cd",
            "99d1a54b1734416e9ac66bf8d317b5f3",
            "b49514fb3b6849078a2380b4b13d4063",
            "b9c91fab701c431f81a18e44871a73f9"
          ]
        },
        "outputId": "dbb74ec2-65c0-44fe-b0f2-296bc56ccfb7"
      },
      "source": [
        "predicted_labels = []\n",
        "pbar = pkbar.Pbar(name='Predicting on Test dataset', target=len(test_dataset_))\n",
        "\n",
        "for batch_idx, sample in enumerate(test_dataset_):\n",
        "    seq = sample[\"premises_hypotheses\"].to(device_)\n",
        "    mask = sample[\"attention_mask\"].to(device_)\n",
        "    labels = sample[\"outputs\"].to(device_)\n",
        "    token_types = sample[\"token_types\"].to(device_)\n",
        "    batch_predicted_labels = model.predict_sentence_(seq, mask, token_types)\n",
        "    predicted_labels.extend(batch_predicted_labels)\n",
        "    pbar.update(batch_idx)\n",
        "\n",
        "save_pickle(os.path.join(os.getcwd(), \"model\", \"predicted_labels.pkl\"), predicted_labels)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1320f30df004a0a9252f1da772aa7cd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Predicting on Testing dataset', max=9394.0, style=Progres…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cw2LdlYKxSrA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "save_pickle(os.path.join(os.getcwd(), \"model\", \"predicted_labels.pkl\"), predicted_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTfjI4dMp7Vj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predicted_labels = load_pickle(\"/content/xlm_predicted.pkl\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpKwNcAp9xg2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427,
          "referenced_widgets": [
            "2fe9e7083dfa4299b14894dab7ec3541",
            "bd63178b54a14bf08ba264231c386324",
            "5a02046bc04d4e99ade1f88b349d7afe",
            "ad0784c543194989b867996077f5a5e2",
            "85eaf13a6aee4214b543228221f34e06",
            "8cb8c79146be4332adfb7d25dba4a96c",
            "b47f5fd14bc64b06836f56a4c0666bd1",
            "83affa4dac9040028dbaf6e6da0ad279"
          ]
        },
        "outputId": "e248f2e5-b82a-4839-c149-f053588e3001"
      },
      "source": [
        "decoded_predicted_labels = [train_dataset.idx2label.get(tag_) for tag in predicted_labels for tag_ in tag]\n",
        "assert len(decoded_predicted_labels) == len(test_dataset.labels)\n",
        "compute_metrics(test_dataset.languages, decoded_predicted_labels, test_dataset.labels)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2fe9e7083dfa4299b14894dab7ec3541",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2539.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "+---------+--------------------+-----------+\n",
            "|         |      accuracy      | # samples |\n",
            "+---------+--------------------+-----------+\n",
            "| overall | 0.7965003326679974 |   75150   |\n",
            "|   ar    | 0.8283433133732535 |   5010    |\n",
            "|   bg    | 0.7914171656686627 |   5010    |\n",
            "|   de    | 0.7722554890219561 |   5010    |\n",
            "|   el    | 0.7223552894211577 |   5010    |\n",
            "|   en    | 0.8457085828343314 |   5010    |\n",
            "|   es    | 0.7167664670658682 |   5010    |\n",
            "|   fr    | 0.8301397205588822 |   5010    |\n",
            "|   hi    | 0.7842315369261477 |   5010    |\n",
            "|   ru    | 0.8181636726546906 |   5010    |\n",
            "|   sw    | 0.881437125748503  |   5010    |\n",
            "|   th    | 0.7856287425149701 |   5010    |\n",
            "|   tr    | 0.8245508982035928 |   5010    |\n",
            "|   ur    | 0.7992015968063872 |   5010    |\n",
            "|   vi    | 0.7592814371257485 |   5010    |\n",
            "|   zh    | 0.7880239520958083 |   5010    |\n",
            "+---------+--------------------+-----------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jmmft7bmvNY1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import seaborn as sn\n",
        "%matplotlib inline\n",
        "\n",
        "\n",
        "def pprint_confusion_matrix(conf_matrix):\n",
        "    df_cm = pd.DataFrame(conf_matrix)\n",
        "    fig = plt.figure(figsize=(15, 10))\n",
        "    axes = fig.add_subplot(111)\n",
        "    sn.set(font_scale=1.4)  # for label size\n",
        "    sn.heatmap(df_cm, annot=True, annot_kws={\"size\": 13})  # font size\n",
        "    axes.set_xlabel('Predicted labels')\n",
        "    axes.set_ylabel('True labels')\n",
        "    axes.set_title('Confusion Matrix')\n",
        "    axes.xaxis.set_ticklabels(['Entailment', 'Neutral', 'Contradiction'])\n",
        "    axes.yaxis.set_ticklabels(['Entailment', 'Neutral', 'Contradiction'])\n",
        "    plt.savefig(\"XLM-R_Seq_CLS_model_confusion_matrix.png\")\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dcHbo3ZyvUG2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 624
        },
        "outputId": "f0e70095-0a41-4977-a743-4bbfd7bac748"
      },
      "source": [
        "pprint_confusion_matrix(confusion_matrix(test_dataset.labels, decoded_predicted_labels, normalize='true'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzIAAAJgCAYAAABRBFZlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd5hVxfnA8e/Q69KUWFCKmmMvGOvPXmKLNRg1JrYkmphYYuwajZrYEpWIPfaKUTGoMRoblhgLNhTwaFQQBQsKLE0Edn5/3Lt47+6ye3fl7u6B7+d5znP3zJk5M+eSyL68M3NCjBFJkiRJypI2LT0ASZIkSWosAxlJkiRJmWMgI0mSJClzDGQkSZIkZY6BjCRJkqTMMZCRJEmSlDkGMpLURCGELiGEa0IIU0IIMYQwtAx9TAgh3Lyk75tlfieSJDCQkZRxIYTlQwgXhhDGhhBmhxDmhBDG5MtWKnP3JwBHAn8DfgrcVub+mk0IYUA+OIshhN8vps4N1XWa2MfuIYQ/fKuBSpKWWcEXYkrKqhDCYOBfQA/gLuAloApYHzgA+DLG+N0y9v840CfGuFEZ++gIVMUY55erj8X0OwD4APgK+CDGuHYd4/oU6Ah0ijGGJvRxBfDrxrZtqe9EktS6tGvpAUhSU4QQegD/yJ9uHGMcW+P66cApZR5GX+DLcnYQY5xXzvuX4GFgvxDCRjHG1wrK9wC6AyOBfcs9iBBCIBcwzW0F34kkqRVwapmkrDoKWAX4Xc0gBiDGOCPGeHphWQjhhyGE0SGEuSGEL0IId4UQVq1R5+YQwlchhJVDCP8IIcwKIXweQvhLCKFtvs52+elU6wHbFkzBGhBCOKz65xr33S5fvl1B2eohhL/n19jMCyFMDiHcF0JYsaBOrfUgIYTlQgjXhRA+yY/1rRDCL2rUqZ4admoI4RchhPfyfbwcQtikEd/zy8C7wME1yg8GRgGTazYIIewVQngwhPBRvs+JIYQ/hxA6FdS5Gfh1/udYcAwoKLsmhHBACOFNYB65LFut7yT/ZzYvhLBejXHck//zW60RzytJyggzMpKyai9y057+XkrlEMJPyK1heQU4DVgeOBbYKp9tmFpQvQ3wCLmpaicCOwG/A94DrgbGk1sTcwEwC/hTvt3npQ4+hNAeeBToDFwJTAFWBHYFVsqf19WuE/AUsGa+3XvAPsB1IYQ+McYLazQ5AOgGXAtE4GRgRAhhUCOmZt0F/CyEcHKMsSqfDduDXCBS17S6w8kFHpcDM4DNgd+SCzwPzNe5Nv+cO5P7LqsVfofbAEOAK4BPgLcXM77jgB2AW0MIm8YY54cQDs63PTrG+F6JzylJypIYo4eHh0fmDnJTul4vsW57cr8IjwM6F5RvR+6X+78UlN2cLzurxj1eBUbXKHsLGFWj7LB8+wE1yqv72i5/vkH+fEgDY58A3Fxwfmy+3aEFZW2Bx8kFdn3yZQPy9aYCvQrq7pUv/0ED/Va3PxVI8j9vn792RL6vnuSCjFijbZc67nc6ufVLqxSU1WpbcC3m62/Y0HeSL9sxX/88YGVgGvDvlv7fqYeHh4dH+Q6nlknKqgpgZol1vwd8B7g6xji3ujDGOIpchmaPOtr8rcb5s8Cgxg9zsSrzn7uEELo2ot0e5LIWt1cXxBgXAkPJLbzfqUb9+2KM0wrOn81/lvwsMcaUXCD343zRj4F/xhinL6b+HIAQQpsQQo8QwnLAc0AABpfaL/B8jPH1Esf4BLnA6FTggXxfRzSiL0lSxhjISMqqSnKLzUvRP/+Z1nFtPLnsQ6H5McaaU7umAb1KHl0DYowfAJcCPwemhhAeDyEcF0Lo00DT/sD/8sFLofH5zwE1yj+s0W91UNPYZ7kTGJJfw7J9/rxOIYR1QwgPk5t2N51c4PV0/nKPRvTZ2Clhp5CbkjcY+G2M8aNGtpckZYiBjKSsGg8kIYQOZbh31bdou7g97dvWqhjj74B1gXPz1y8B3g4hrF2z7rdQM+Cp1tjtkoeTy4LdRC4T9lCdN82tn3kKWBs4g9xUtp3JTbmDxv29M7fhKkXWJ7fuBnIbMUiSlmIGMpKy6gGgE7B/CXUn5j+TOq6tSW7NxZJSnfHoWaO8f82KADHGsTHGC2KM25PLJPQktzB+cSYCq1fvoFZgzfznhMYNtzQxxo+BZ8it9RkRF78F8vbAcuTW8FwWY3wwxvg4dexuxuKDvkYLIXQGbiWXxbkCOC6EsM2Sur8kqfUxkJGUVdcCHwOXhBDWqnkxhNA9hFC9m9hoci9vPKrGFsBbk1s/U2d2oYmqp0Mt+iU6H3QcWWN8FSGEmjtHjieXhagZBBV6iNyOa9XrVQghtCG3c9c8cov+y+Us4BzgsnrqVGeAFmV88uM7oY66s/PXl8SUvQuB1YBDye00Nx64qZHrjyRJGeL2y5IyKcY4PYSwD7kXNr4aQriT3DtPqshN1zqI3M5mZ8TcdrwnkfsX+2dDCLfzzfbLHwMXLcFxjQ0hvABcEELonR/DgdT+7+0OwJUhhHvJrd0J5LZK7g7cXU8XfyMXFN0QQtgIeJ/c9ss7AqfFGL9YUs9SU4zxWb7ZLGBx/gN8AdwSQhgGzCe3DXK3OuqOzn9eEUL4F7AAeDDGOLsx48q/m+cY4KIY4wv5skOAF4G/AL9qzP0kSdlgICMps2KMo0MI65J7x8ue5IKXQO4FjteRe49Jdd3bQghzyL1D5iJgDrkg6JRY/A6ZJeFgchmjU8ktdr+B3LqRxwrqvAH8C9gd+AW57YzHAvvEGEcu7sYxxq9CCNuTe4fNweSyN/8Djowx1txprdnFGL8MIexBbr3POeQW/N9H7v07Y2pUH0Fut7WD+ObPbiD5TE0pQgjdya3bGQucXTCOV0MIfwT+EEIYEWN8bHH3kCRlU4hxiU1RliRJkqRm4RoZSZIkSZljICNJkiQpcwxkJEmSJGWOgYwkSZKkzGm1u5bNn/q+uxBILaDzSlu39BCkZdLAHiu09BCkZda7n78SGq7V8prz9+P2yw1q9d+JGRlJkiRJmWMgI0mSJClzWu3UMkmSJEkFqha29AhaFTMykiRJkjLHjIwkSZKUBbGqpUfQqpiRkSRJkpQ5ZmQkSZKkLKgyI1PIjIwkSZKkzDEjI0mSJGVAdI1METMykiRJkjLHjIwkSZKUBa6RKWJGRpIkSVLmmJGRJEmSssA1MkXMyEiSJEnKHAMZSZIkSZnj1DJJkiQpC6oWtvQIWhUzMpIkSZIyx4yMJEmSlAUu9i9iRkaSJElS5piRkSRJkrLAF2IWMSMjSZIkKXPMyEiSJEkZEF0jU8SMjCRJkqTMMSMjSZIkZYFrZIqYkZEkSZKUOWZkJEmSpCxwjUwRMzKSJEmSMseMjCRJkpQFVQtbegStihkZSZIkSZljRkaSJEnKAtfIFDEjI0mSJClzDGQkSZIkZY5TyyRJkqQs8IWYRczISJIkScocMzKSJElSFrjYv4gZGUmSJEmZY0ZGkiRJygLXyBQxIyNJkiQpc8zISJIkSRkQ48KWHkKrYiAjSZIkqcmSJFkDGAZsBcwFhgOnpGk6p4F2XYHfA/sDKwIfA7cBF6Zp+nVD/RrISJIkSVnQCnctS5KkJ/AUMBEYAvQFLgWWBw5soPnVwD7AGcBbwKbAeUAv4LcN9W0gI0mSJKmpjiIXeGyYpulUgCRJFgB3JElyXpqmY+tqlCRJO3KZmIvTNB2WL34qSZL+wI8pIZBxsb8kSZKUBVVVzXeUbnfgieogJu8+YB6wWz3tArmkyowa5dPz1xpkRkaSJElSkfyUsZ51XJqepun0gvO1gBsLK6RpOi9JkveANRd3/zRN5ydJcitwTJIk/wHGApsAvyC33qZBZmQkSZKkLIhVzXfA8cAHdRzH1xhVL3JZlJqmAb0beKKjgCeAF4CZwJPAbWmanlvK12FGRpIkSVJNQ4Gb6yivK2hpqguAPchlYd4BNgfOTpLkkzRNL26osYGMJEmSlAVVzfcemfz0sVKClmnUPQWtF/D24holSbIucCKwd5qmD+SLn0mSpD1wbpIkV6dpOrO+jp1aJkmSJKmpxpNbJ7NIkiQdgdWoJ5AB1s5/vl6j/DWgI9CvoY4NZCRJkiQ11cPAjkmS9Cko25dcMPJwPe0m5j83rlG+MRALri+WU8skSZKkLGiFL8QErgWOAUYmSXIe37wQ8+40TcdVV0qS5Abg0DRNq+OP0cBLwDVJkvQF3gU2A04DbkzTdE5DHZuRkSRJktQk+bU0OwCzgBHAZcDdwBE1qrbNH9XtFgJ7Av8gF7z8Ezgc+Au5wKhBIcb4LYdfHvOnvt86ByYt5TqvtHVLD0FaJg3ssUJLD0FaZr37+SslvYCxpX31wt3N9vtxp80PaPXfiRkZSZIkSZnjGhlJkiQpC1rnGpkWY0ZGkiRJUuaYkZEkSZKyoMqMTCEzMpIkSZIyx4yMJEmSlAVmZIqYkZEkSZKUOWZkJEmSpAyIcWFLD6FVMSMjSZIkKXPMyEiSJElZ4BqZImZkJEmSJGWOGRlJkiQpC6IZmUJmZCRJkiRljoGMJEmSpMxxapkkSZKUBS72L2JGRpIkSVLmmJGRJEmSssDF/kXMyEiSJEnKHDMykiRJUha4RqaIGRlJkiRJmWNGRpIkScoC18gUMSMjSZIkKXPMyEiSJElZ4BqZImZkJEmSJGWOGRlJkiQpC8zIFDEjI0mSJClzzMhIkiRJWeCuZUXMyEiSJEnKHDMykiRJUha4RqaIGRlJkiRJmWMgI0mSJClznFomSZIkZYGL/YuYkZEkSZKUOQYyatDChQv5yxXXs/UeB7DpTvtx/Ol/ZNr0GYutf9Od97Lr/oez6U77sfsBP2P4iIcWXXvl9bfYZKd9i44NttmDfQ/5VXM8itSqtWnThosuOJMpH49h2hcpf7/7Ovr06bXY+rt8fzveeP1JZs74H6+/9gQ777TNomtrrDGIu4dfx8QPRjPti5Q3Xn+SIw4/qKj9E4/dw+yZ7zP9y3cWHXvsvlPZnk/KkjZt2nDK2cfx4vjHee2DZ7jipovp1btnnXW/s8LyXH3rJYx69SHe/fwV9hqyW606vz3tV4x88g7GfvwCN997VbmHr6VVVVXzHRlgIKMGXX/733nquRe487qhPPGP2wA47by/1Fn3qWdf4Kobbueis0/mpcdHcP7vT+SSK6/n+ZdeBWDjDdfl5cfvX3S8+O/76Lvccuy5yw7N9jxSa3XKyb9hz712YcutfkD/gd8D4JabLq+z7sCBq3LP36/noouvoPdya3HRxVdw7z030L9/PwB69ezBqFH/YfMt96BXn4Sjjz6Fiy/6PfvsU/wL1p/O/ys9e3930fHPhx8v70NKGXHUcYex427bMmTXQ9l6/d0B+PNV59ZZtypGnhv1Aif88gymfPxJnXU+nPARf73wGu6+bUTZxiwtawxk1KB7Rz7CEQfvzyorr0j3bl054egjeO6F0Uz+5NNadT/8eDLfXX0QG6y7FgAbrrsW311tIOn/3q/z3s/892Wmfvkl++y+c1mfQcqCn//sYP785yv54IMPqaycyamn/Yldd92BVVdduVbdQ366P6++OoY77xzB/Pnzueuu+3nttTc55Kf7A/DSy69x9TW3MGVK7v+n/3n+ZR799yi23WaLZn0mKasO+Ol+XDfsFiZN/JhZM2dx8TmXs+2O/8dK/VaoVffzT6dyx4338OpLb7BwYd3/kn3fXQ/y5L+fZdoX08s9dC3NYlXzHRlQ1kAmhDCwlDK1XpUzZzHl089YJ1l9Udmq/VaiW9cupO9+UKv+bjtuy+zZc3h1zFiqqqp45fW3mDDpY7ba/Ht13v/v/3iYnbfbit696k7XS8uKHj0q6N+/H6++9uaisvffn8iMGZWsv/7ateqvv/7avPLqm0Vlr772Zp11ATp37sRmmw5mzJhxReXHHvNzPvvkLd54/UlOOfk3tGvnHjBS94purLzKiox9Y/yisg8nfMTMylmsuc53W3BkkgqV+2+s+4DBNcruBTYuc79aQmbPmQtAt25di8q7d+vGrDlzatXv3asnO2+/FUcccyoxH82fcuxRrDFoQK26Uz75jOdeGM0Nl1+w5AcuZUz37t0AmDGjsqh8+vRKKiq6167frRuVlcV1Z8yoZJ21k1p127Rpwy03X85HH03mttvvXVR+xpkX8nb6PyorZ7LJ9zbk1luGUVHRjTPOvHBJPJKUWV3zf+fNrJxVVF45Yybdunetq4nUPDKydqW5lCUjE0JYM4TwQ6BHCGG/guMwoFM97Y4MIYwOIYy+/ta7yjE0NVLXLp0BmDVrdlH5zFmz6NalS63619x8Jw8/Nor7br6C159+iPtuvopb776f+x58tFbd+x58hAGrrswmG61fnsFLGTJzZu4Xph49KorKe/asoLJyZu36s2ZRUVFct0ePCipnFtdt164dd9x+FSuu8B322udQFixYsOjaCy++wvTpM6iqquLFl17lD+f+hR8f9MMl9UhSZs3O/53XvaJbUXlFj+7Mmjm7riaSWkC5ppYlwA+AnsCeBcdg4BeLaxRjvC7G+L0Y4/d+fshBi6umZlTRvRsrfqcv497536KySR9PYdbsOXx39dqzBMel/2PHbbZgtYH9CSGw+qD+7LjNljz9nxeL6i1YsJARD/2b/ffevezPIGXBjBmVTJz4ERttuO6isoEDV6VHjwrefHN8rfpjxoxj8EbrFpVttOG6RVPHOnbsyH333EDf5fuw6+4H1RkQFaqqqiKE8C2fRMq+mZWz+HjSFNZZf81FZav0X5nuFd1Ix73bgiPTMs9dy4qUJZCJMY6MMR4O/CDGeHjBcWyM8fly9KnyGbL3rtx4+z18NPkTZs2ezWVX38j/bbYxK6/4nVp1N1pvbZ585r9MnPQxAO9N+JAnnnmetQvW2ACM+s8LVM6cxd67udWrVO36G+7gpJN+zYABq9C9ezcuOP8MHn30KSZO/KhW3dtuv5eNN96AAw7Ym3bt2nHAAXszePD63HrbPQB07dqFfz54Ox06tGePPX/K7NnFU0F79Khgj913omvXXGZ1ww3X4azf/4577nmg/A8qZcDdt43gF8ccSr9VV6Jbt66cdNaxPPPk83w8aUqd9Tt07ECHjh0IIdC+fTs6dOxA27ZtF11v1y5X1q5dW9q0aZOr36F9cz2OtFQKMcby3TyE5cllYAZQsB4nxnhEQ23nT32/fANToyxcuJBLr76RkQ8/ztdfz2eLTTbiD6ccS6+ePXjo0Sc558/DePnx+4FcpmXY327hX48/zbQZlfSo6M4u22/N8b86nPYFi4iPOuFMlu/Tmz+ecUJLPZYWo/NKW7f0EJZZbdq04cLzz+CQQ35Ex44dePyJZ/jlr07miy+mcdBB+3L1lRfRs/c3C413+f52XHzxWQwauCrvf/AhJ574Bx57/BkAfvrT/bnphqHMmTOXqoJ/WbvjzhH8+jenstxyvRl5/y2suebqtGnThimffMZdd43gwouuYP78+c3+7IKBPWrvhqWW06ZNG04661j2O3BPOnRsz/OjXuTM3/2JaV9OZ68f7sa5l5zOhgO++e/lu5+/Uusel198LcP+fB0AFw37A/sduGfR9Y8+nMz2G+9Zq52a37ufv5KJdPTcu89ptt+POx9wdqv/TsodyDwPPAu8AiysLo8x3tdQWwMZqWUYyEgtw0BGajkGMrVlIZAp965lXWKMp5S5D0mSJGnpl5G1K82l3C/EfCiE4GpuSZIkSUtUuTMyxwGnhxC+Br4GAhBjjBX1N5MkSZJUxIxMkbIGMjHG2m9xkyRJkqRvqayBTMi9kOBgYGCM8bwQwirAijHGl8rZryRJkrTUiWZkCpV7jcxVwBbAj/Pns4Ary9ynJEmSpKVcudfIbBZjHBxCeA0gxjgthNChzH1KkiRJWsqVO5CZH0JoC0RY9IJMc2KSJElSY7nYv0i5A5nLgfuBviGEPwFDgDPL3KckSZKkZpIkyRrAMGArYC4wHDglTdM59bQZAHxQz21XStN0Sn39lnvXsjtCCK8AO5LbenmfGOP4cvYpSZIkLZVibOkR1JIkSU/gKWAiuaRFX+BSYHngwHqaTiG3lr6m4cCXDQUxUP6MDMCnwLP5vjqHEAbHGF9thn4lSZIklddRQC9gwzRNpwIkSbIAuCNJkvPSNB1bV6M0TecBLxSWJUmyFtCf3KyuBpV7++XzgMOA98ivk8l/7lDOfiVJkqSlTutcI7M78ER1EJN3H3AjsBtQZyCzGD8BFgJ3lVK53BmZHwGrxRi/LnM/kiRJkpaQ/JSxnnVcmp6m6fSC87XIBS2LpGk6L0mS94A1G9FfIPfKlidLmVYG5X+PzFvU/QVIkiRJaoyqquY74Hhyi/FrHsfXGFUvYDq1TQN6N+LptgIGALeX2qDcGZkLgNdCCG8B86oLY4x7lblfSZIkSU03FLi5jvK6gpYl4WBgDjCi1AblDmRuAS4C3sT3x0iSJElNF5vv1+n89LFSgpZp1D0Dqxfwdil9JUnSAdgfGJmm6axSx1juQGZOjLGkXQckSZIkZc54cutkFkmSpCOwGnBTiffYndw0tJKnlUH518g8G0K4IISwRQhhcPVR5j4lSZKkpU6sis12NMLDwI5JkvQpKNsX6Ji/VoqDgc+Afzem43JnZDbKf25eUOb2y5IkSdLS4VrgGGBkkiTn8c0LMe9O03RcdaUkSW4ADk3TtCj+SJKkB/AD4G9pmi5oTMdlDWRijNuX8/6SJEnSMqMVvkcmTdPpSZLsQO4lliOAucBw4OQaVdvmj5p+CHSikdPKAEKMjUodlXbTEE6o73qM8dKG7jF/6vtLfmCSGtR5pa1begjSMmlgjxVaegjSMuvdz18JLT2GUsy55rhm+/24yy//2uq/k3JlZLqX6b6SJEnSsqkZdy3LgrIEMjHGc8pxX0mSJEmCMgUyIYSTY4wXhxCGkVvcXyTGeGw5+pUkSZK0bCjX1LLx+c/RZbq/JEmStGxp3LbIS71yTS17MP95SznuL0mSJGnZVtbtl0MIywOnAGuT21YNgBij75GRJEmSGqMVbr/cktqU+f53kJtmNhA4B5gAvFzmPiVJkiQt5cqakQH6xBhvCCEcF2N8Gng6hGAgI0mSJDWWGZki5Q5k5uc/p4QQ9gAmA73L3KckSZKkpVy5A5k/hhB6AL8DhgEVwPFl7lOSJEla+kR3LStU7kBmWoxxBjAD2B4ghPB/Ze5TkiRJ0lKu3IHMMGBwCWWSJEmS6uMamSJlCWRCCFsAWwLLhxBOKLhUAbQtR5+SJEmSlh3lysh0ALrl79+9oLwSGFKmPiVJkqSlV5VrZAqVJZAp2Gr55hjjxHL0IUmSJGnZVe41Mh1DCNcBAwr7ijHuUOZ+JUmSpKVLdI1MoXIHMvcA1wDXAwvL3JckSZKkZUS5A5kFMcary9yHJEmStPRzjUyRNmW+/4MhhKNDCCuGEHpXH2XuU5IkSdJSrtwZmUPznycVlEVgUJn7lSRJkrQUK2sgE2McWM77S5IkScuK6Asxi5RlalkI4eSCn/evce38cvQpSZIkadlRrjUyBxb8fFqNa7uWqU9JkiRp6VUVm+/IgHIFMmExP9d1LkmSJEmNUq41MnExP9d1LkmSJKkhvhCzSLkCmQ1CCJXksi+d8z+TP+9Upj4lSZIkLSPKEsjEGNuW476SJEnSMisja1eaS7lfiClJkiRJS1y5X4gpSZIkaUnwPTJFzMhIkiRJyhwzMpIkSVIWuEamiBkZSZIkSZljRkaSJEnKAt8jU8SMjCRJkqTMMSMjSZIkZYFrZIqYkZEkSZKUOQYykiRJkjLHqWWSJElSBkRfiFnEjIwkSZKkzDEjI0mSJGWBi/2LmJGRJEmSlDlmZCRJkqQsMCNTxIyMJEmSpMwxIyNJkiRlQXTXskJmZCRJkiRljhkZSZIkKQtcI1PEjIwkSZKkzDEjI0mSJGVANCNTxIyMJEmSpMwxIyNJkiRlgRmZImZkJEmSJGWOGRlJkiQpC6p8j0whAxlJkiRJTZYkyRrAMGArYC4wHDglTdM5JbTtAZwDDAGWB6YAt6ZpelZDbQ1kJEmSJDVJkiQ9gaeAieSCkb7ApeSCkgMbaNsVeBqIwMnAZGAQsEopfRvISJIkSVnQOhf7HwX0AjZM03QqQJIkC4A7kiQ5L03TsfW0PRXoCaybpumsfNmoUjt2sb8kSZKkptodeKI6iMm7D5gH7NZA258D1xcEMY1iRkaSJEnKgmbMyOSnjPWs49L0NE2nF5yvBdxYWCFN03lJkrwHrFnP/QcAKwBTkyR5APg+8BXwAHBcmqbTGhqjGRlJkiRJNR0PfFDHcXyNer2A6dQ2Dehdz/1XyH/+GZgJ/AD4Hbkszl2lDNCMjCRJkpQBMTbrGpmhwM11lNcVtDRFdULlf8BP0jSNAEmSzADuSZJkkzRNX67vBgYykiRJkorkp4+VErRMo+4paL2AtxtoB7n1NYUR2hP5z3UBAxlJkiQp81rnrmXjya2TWSRJko7AasBN9bR7j9yGAIvTqaGOXSMjSZIkqakeBnZMkqRPQdm+QMf8tTqlafo18G9gpyRJQsGlnfOfrzTUsRkZSZIkKQtaZ0bmWuAYYGSSJOfxzQsx707TdFx1pSRJbgAOTdO0MP44B3geuCtJkpuA/sAFwKNpmr7UUMdmZCRJkiQ1SX4tzQ7ALGAEcBlwN3BEjapt80dh21eAXYGBwEjgj8BwYEgpfYdm3v2gZP16r9s6ByYt5d45f8eWHoK0TDrygkktPQRpmXX7xBGh4Votb8bhOzXb78c9bnq81X8nZmQkSZIkZY5rZCRJkqQsaJ1rZFqMGRlJkiRJmWNGRpIkScqCqpYeQOtiRkaSJElS5hjISJIkScocp5ZJkiRJGRBd7F/EjIwkSZKkzDEjI0mSJGWBGZkiZmQkSZIkZY4ZGUmSJCkL3H65iBkZSZIkSZljRkaSJEnKAHctK2ZGRpIkSVLmmJGRJEmSssA1MkXMyEiSJEnKHDMykiRJUga4RqaYGRlJkiRJmWNGRpIkScoC18gUMSMjSZIkKXPMyEiSJEkZEM3IFDEjI0mSJClzDGQkSZIkZY5TyyRJkqQscElbwEwAACAASURBVGpZETMykiRJkjLHjIwkSZKUAS72L2ZGRpIkSVLmmJGRJEmSssCMTBEzMpIkSZIyx4yMJEmSlAGukSlmRkaSJElS5piRkSRJkjLAjEwxMzKSJEmSMseMjCRJkpQBZmSKmZGRJEmSlDlmZCRJkqQsiKGlR9CqmJGRJEmSlDlmZCRJkqQMcI1MMTMykiRJkjLHQEaSJElS5ji1TJIkScqAWOVi/0JmZCRJkiRljhkZSZIkKQNc7F/MjIwkSZKkzDEjI0mSJGVA9IWYRczISJIkScocMzKSJElSBrhGppgZGUmSJEmZY0ZGkiRJygDfI1PMjIwkSZKkzDEjI0mSJGVAjC09gtbFQEaSJElSkyVJsgYwDNgKmAsMB05J03ROA+1GAdvWcWmTNE1HN9SvgYwkSZKUAa1xjUySJD2Bp4CJwBCgL3ApsDxwYAm3+A9wYo2y8aX0bSAjSZIkqamOAnoBG6ZpOhUgSZIFwB1JkpyXpunYBtpPT9P0haZ07GJ/SZIkKQNiVWi2oxF2B56oDmLy7gPmAbstyeevyYyMJEmSpCL5KWM967g0PU3T6QXnawE3FlZI03RekiTvAWuW0NW2SZLMIheXjAbOTtP0iVLGaEZGkiRJUk3HAx/UcRxfo14vYDq1TQN6N9DH0/n77Q78FAjAv5Mk2aGUAZqRkSRJkjKgmbdfHgrcXEd5XUFLk6RpenbheZIkDwBvAH8AnmyovYGMJEmSpCL56WOlBC3TqHsKWi/g7Ub2OS9JkpHAb0qpbyAjSZIkZUBr3H6Z3FbJaxUWJEnSEVgNuKmcHbtGRpIkSVJTPQzsmCRJn4KyfYGO+WslywdA+wAvl1LfjIwkSZKUATG2yozMtcAxwMgkSc7jmxdi3p2m6bjqSkmS3AAcmqZpu/z51sBJwP3ABGAF4DhgELl30zTIjIwkSZKkJsmvpdkBmAWMAC4D7gaOqFG1bf6oNgXoAJwPPApcTW5NznZpmo4qpW8zMpIkSVIGxKqWHkHd0jR9B9i1gTqHAYcVnP+voTYNMSMjSZIkKXMalZEJIfQCVokxjinTeCRJkiTVoap1rpFpMQ1mZEIIo0IIFSGE3sCrwN9CCJeWf2iSJEmSVLdSMjI9YoyVIYSfA7fGGM8OIZiRkSRJkppRK921rMWUskamXQhhReBHwENlHo8kSZIkNaiUjMy55LZEey7G+HIIYRDwbnmHJUmSJKlQrDIjU6jBQCbGeA9wT8H5+8APyzkoSZIkSarPYgOZEMIwIC7ueozx2LKMSJIkSVItcbG/mS+b6svIjG62UUiSJElSIyw2kIkx3lJ4HkLoEmOcU/4hSZIkSVL9SnmPzBYhhHHA2/nzDUIIV5V9ZJIkSZIWiVWh2Y4sKGX75aHALsAXADHGN4BtyjkoSZIkSapPKdsvE2OcFEJRZLawPMORJEmSVJcqX4hZpJRAZlIIYUsghhDaA8cB48s7LEmSJElavFICmV8CfwVWBiaTeznmr8s5KEmSJEnFohmZIqW8EHMqcHAzjEWSJEmSSlLKrmWDQggPhhA+DyF8FkIYGUIY1ByDkyRJkpQTY/MdWVDKrmV3An8HVgRWAu4B7irnoCRJkiSpPqWskekSY7yt4Pz2EMJJ5RqQJEmSpNrctazYYgOZEELv/I//CiGcCgwHInAA8HAzjE2SJEmS6lRfRuYVcoFLdeh3VMG1CJxWrkFJkiRJKuauZcUWG8jEGAc250DUerRp04bTz/4t+x+0Nx07duSZUc9zym/PYdqX0+usv92O/8fvzzuJ/v37MWHCJM49888889Tzi67vsNPWnHj6bxgwaFXmzJ7Lvx58jD+efQnz5n1ddJ/OXTrz2LMj6LfKigzou2FZn1HKgoVVkcufS3lg3Md8vaCKzfsvx5k7rUOvzh3qrP/lnHlc9kzKsx98zoKqKlbu0YVh+2xM326dABj90Zdc/mzK+1/OoqJTew7ZeCAHbti/OR9JyozQpg0HnvoTth6yPe07duDNZ1/nxtOuYda0mbXqbrD9YHY/cm9WXbM/bdq24aP0Q/5+8R2kL+deu7fCwBX50ck/YfXB36Vzty58MflzHrnhIUYNf7y5H0taqpSy2J8QwrohhB+FEA6pPso9MLWcXx//c76/2/bsufOP2WTdHQG4/JoL6qy7av9+/O2WoVx52fWsNWBzrrzseq6/dSj9VlkJgD7L9ea6W4cy/PYRrDNwS/bc+SC22GoTjjvxl7XudfpZxzNp4kflezApY256+X1GvfcZtx24BY/8YjsAznxkTJ115y1YyFH3vkz7tm24/7CteebonTh/1/Xp0j7371WTZ8zh2H+8wkEb9eeZo3fiwt03ZNhz7/DYO5801+NImbLn0fsyeOdNOXvvUzl2818A8Kuhx9VZt2uPbjx288P8btuj+dVGh/H8yGc56ZYz6b1in0XXx/33Tc7a82R+sc7B3HjaNRx0+qF8b9fNmu15tHRw17JipWy/fDYwLH9sD1wM7FXmcakFHXzoEK66/EY+nPgRM2fO4k9nX8r2O23Nyv1WrFV3/4P2Zswb4xhxz0PMn7+A++/9J2+OGc/+B+0NwIorfYdOnToy/PYRxBiZMvlTHn/0GdZeNym6z2ZbbMymW2zMVZff2CzPKGXBfW9O4rBNBtGvZxe6d2zP8VsnPD9hKpMr59aq++C4j5k5bwGn7bA2vTp3oE0IrLZcd7p1zAUyz02Yyqo9u7DbmivRJgTWX7EnO62xAveM+bC5H0vKhB0O+j4PXXM/n0/6lLkz5zD8/FvZYLvB9Fl5+Vp1n//HM4x+9EXmVM6hamEVT9z+KF/N/opBG6wOwHuvv8vjtz7C9M+mAfDO6LcZ8/RrrLX5us36TNLSppSMzBBgR+CTGOPhwAZAj7KOSi2moqI7/VZZiTGvj11UNnHCJCorZ9YKPgDWWue7vPnGuKKyt8aMZ+11cnXHvvk2Tz72DD857Ee0bduWlfutyM67bcejDz+5qH6nzp24eOgfOOm4s1kwf0GZnkzKlplfzeeTmV+xdt+KRWWr9OxCtw7teOfz2lNbXp70Jav26sLZ/36T7a5+gn1vfpbbX52w6HqMkZr/wFYVI+lnte8lLeu6VHRhuX7L88Gb7y0q++zDT5lTOZv+aw1osH2/ZFW6965g0tt1/0NBh04dWH2j7/Lh+AlLaMRaVlTF0GxHFpQSyMyNMVYBC0IIFcBnwCr1NQgh9K7vWBIDV3l07d4VgJmVs4rKK2fMpHv3brXqd+vWtc663fL3iTHy97tGcswJR/LelFd4ccxjjB3zNnffcf+i+qeddTyPPfp0UfAkLetm54P66oxKtW4d2zH769oB//S583l50pes+50ePHbk9vxx1/W54cX3eHj8ZAA2778cH3w5i4fGfcyCqipe+3gaT733aZ33kpZ1nbp2BmDuzDlF5XMq59C5e+d621b06cFx15zMw9eN5NMJU2pdD23a8Muhx/HllKk8d9+oJTZmaVlUSiAzOoTQE/gbuZ3MXgX+20CbV4DR+c+ax+jFNQohHBlCGB1CGD173pclDE1L2uyZswHoXlEctFT06M7MmbNq1Z81a3addWfl77PlVpsw9Mo/8bvfnMmgFQazYbIt3bp35bIr/wTAJpttxPY7bsVfLriiHI8jZVbX/NqWWfOKA41Z8xbQtUPtfVq6dmhL324d+fHgAbRv24Z1VujB7mutxKj3PgOgf6+uXPKDjbjztYnseM1TXPGfd9hr7X707Ny+/A8jZcxXs3PTNzt371JU3qWiC3Nn1p7aWa1n316cPvwc3nz2de6+6PZa19u2a8tvhv2Wnn178ZcjzmfhgoVLduBa6sUYmu3IggZfiBljPDr/4zUhhEeAihhj3atNv2nTpB3PYozXAdcB9Ou9bkaWGS1dKitn8tGkyay3wdqMeysFcgv6Kyq6M37sO7Xqjx/7DltstUlR2Trrrcl/nnkRgPU2XIfx497hycefBWDq519w56338tf85gFbb7cFK628Ai+NeQyAdu3b0a5dO8a8+ywn/OZMHn/06bI9q9Sade/UnhW6d2L8Z5Uk+ellH02fw6yvF/Dd5WpnR5PlKxj3aWWt8lDwd9HWg/qy9aC+i85Peug1Nu5nklyqaU7lHKZ+9DkD1h3Eh+MmALD8Kt+hS0VXPnx7Qp1tluu3PKfdeQ6jH32Ru/50S63r7Tu259irT6JTl05c9JNzmTfnqzI+gbRsWGxGJoQwuOYB9Aba5X8uSQihVwhh0xDCNtXHkhi4yueOW+7l6GOPYJVVV6Zb966c/offMuqJ5/ho0uRade8d/gAbbLgOe++3G+3atWPv/XZj/Q3W5p67RgLwystvsOZaa7DN9lsC0Kt3T358yBDefD23rua6q25h6032YJdth7DLtkNy62QWLGCXbYfw3NMvNN9DS63QD9dbhZtHv8/HM+Ywa94C/vpcypb9l2OlHl1q1d1znZWZ8dXX3P36RBZWRdLPK3n47cnssPp3FtUZ+8kM5i+sYu78hfz9jQ95fsJUjtx89eZ8JCkznrzr3+z5y31ZfpW+dO7WmQNP+yljRr3G1I8+r1V3xdVW5qx7z+e/DzxbZxDTsUsnTrrlTNq1b8fFh/7RIEZaQurLyFxSz7UI7NDQzUMIPweOA/oBrwObk5uW1mBbtZwrh15Pj54V/POJ4XTo0IFnR/2XY446FYB9h+zBhZeeTbLqpkBuI4BfHHo8vz/vJC4Zdh4TJ37Ezw85flHQM/rF1zjtxPM467wTWXmVlZj31TxeeH40Z5ycm1o2a+bsRdPQAL6cmtvRZcrkT5vzkaVW6fBNBlE5bz4/ufO/fL0w9x6ZP+62PgAPj5/MH58Yy/O/2RmAlSo6M2yfjfnL028z9Nl3WL5bR365+ersknyz2+DV/32XNyZPZ2GMrLdCD64bsimr9amd3ZEED151P117dOPcBy6mXYf2vPXcG1x1/FAAttxnG444/yh+vvbBAOz5y33pvWIfdj3iB+x6xA8W3ePG06/l+X88w6a7bc7aW6zHvLnzuPq1mxZd/8/9z3DTGdc274Mp07KyCL+5hFjGjaJDCG8CmwAvxBg3DCGsCZwfY9yvobZOLZNaxjvn79jSQ5CWSUdeMKmlhyAts26fOCITEcKLK+3XbL8fbza59X8nDa6R+Za+ijF+FUIghNAxxvh2CKH2Hr6SJEmS6uW/8hcrdyDzUX7Hs38Aj4UQpgETy9ynJEmSpKVcWQOZGOO++R//EEJ4ityLNB8pZ5+SJEnS0sg1MsUafI9MyPlJCOGs/PmqIYRNS2jXNoTwdvV5jPHpGOMDMcavv92QJUmSJC3rSnkh5lXAFsBB+fOZwJUNNYoxLgTSEMKqTR+eJEmSJPCFmDWVMrVssxjj4BDCawAxxmkhhA4l3r8XMDaE8BKwaI/dGONejR+qJEmSJOWUEsjMDyG0Jb9RQghheaCqxPv/vqkDkyRJkvSNUn8BX1aUEshcDtwP9A0h/AkYApxZ4v13jzGeUlgQQrgIeLpRo5QkSZKkAg2ukYkx3gGcDFwATAH2iTHeU+L9d66jbLfShydJkiQJIBKa7ciCBjMy+cX6c4AHC8tijB/W0+ZXwNHAaiGEMQWXugPPN324kiRJklTa1LJ/klsfE4BOwEAgBdapp82dwL/IZXFOLSifGWP8smlDlSRJkpZdVbGlR9C6NBjIxBjXKzwPIQwml22pr80MYEYI4ZQal7qFELrVl82RJEmSpIaUkpEpEmN8NYSwWYnVm5LNkSRJklRDVUbWrjSXUtbInFBw2gYYDEwu5eZNyeZIkiRJUkNKych0L/h5Abksy31N6ayR2RxJkiRJqlO9gUz+RZjdY4wnNuXm3yabI0mSJOkbWdkWubksNpAJIbSLMS4IIfzft7j/EsvmSJIkSVK1+jIyL5HLoLweQngAuAeYXX0xxjiioZvHGM8BCCF0iTHO+ZZjlSRJkpZZVS09gFamTQl1OgFfADsAPwD2zH82KISwRQhhHPB2/nyDEMJVTRyrJEmSJAH1Z2T65te4vMU3WyhXK/V1PEOBXYAHAGKMb4QQtmnKQCVJkqRlmWtkitUXyLQFukGd31jJ7xWNMU4KoegWC0ttK0mSJKl1S5JkDWAYsBUwFxgOnJKmaclLS5Ik2RcYAYxN03TdUtrUF8hMiTGeW2rnizEphLAlEEMI7YHjgPHf8p6SJEnSMqc1rpFJkqQn8BQwERgC9AUuBZYHDizxHl3IzeT6tDF917dGZknkrn4J/BpYGfgY2DB/LkmSJCn7jgJ6AXunafpImqa3AscCByRJsk6J9/g98D7wSGM6ri8js2NjblSXGONU4OBvex9JkiRpWdcaMzLA7sATaZpOLSi7D7gR2A0YW1/jJEnWJBf4bAY06t2Viw1kYoxfNuZGhUIIZ9VzOcYYz2vqvSVJkiSVV37KWM86Lk1P03R6wfla5IKWRdI0nZckyXvAmiV0dSVwfZqmbyVJ0qgxlrL9clPMruMA+BlwSpn6lCRJkpZakdBsB3A88EEdx/E1htULmE5t04De9T1PkiQHAusBZzfl+6hvalmTxRgvqf45hNCd3CL/w8ntYHDJ4tpJkiRJahWGAjfXUV5X0NJoSZJ0JxcXnF4jw1OysgQyACGE3sAJ5NbI3AIMjjFOK1d/kiRJ0tKsqhlfI5MPLkoJMKZR9xS0XsDb9bQ7A/gSGJGfxgbQAWiTP5+bpum8+jouSyATQvgzsB9wHbBejHFWOfqRJEmS1KLGk1sns0iSJB2B1YCb6mm3JrAu8EUd16YBvyWXFVqscq2R+R2wEnAmMDmEUJk/ZoYQKsvUpyRJkrTUqiI029EIDwM7JknSp6BsX6Bj/trinAlsX+N4FJiQ//nehjou1xqZcgVIkiRJklqPa4FjgJFJkpzHNy/EvDtN03HVlZIkuQE4NE3TdgBpmr5V80ZJkhwG9EvTdFQpHRtwSJIkSWqS/FqaHYBZwAjgMuBu4IgaVdvmjyWmbIv9JUmSJC05saUHsBhpmr4D7NpAncOAw0qoUzIzMpIkSZIyx4yMJEmSlAFVLT2AVsaMjCRJkqTMMSMjSZIkZUBVaMY3YmaAGRlJkiRJmWNGRpIkScqA1rprWUsxIyNJkiQpc8zISJIkSRngrmXFzMhIkiRJyhwzMpIkSVIGVLlpWREzMpIkSZIyx4yMJEmSlAFVmJIpZEZGkiRJUuaYkZEkSZIywPfIFDMjI0mSJClzDGQkSZIkZY5TyyRJkqQMcPvlYmZkJEmSJGWOGRlJkiQpA6paegCtjBkZSZIkSZljRkaSJEnKALdfLmZGRpIkSVLmmJGRJEmSMsBdy4qZkZEkSZKUOWZkJEmSpAxw17JiZmQkSZIkZY4ZGUmSJCkDzMgUMyMjSZIkKXPMyEiSJEkZEN21rIgZGUmSJEmZY0ZGkiRJygDXyBQzIyNJkiQpcwxkJEmSJGWOU8skSZKkDHBqWTEzMpIkSZIyx4yMJEmSlAGxpQfQypiRkSRJkpQ5ZmQkSZKkDKjyhZhFzMhIkiRJyhwzMpIkSVIGuGtZMTMykiRJkjLHjIwkSZKUAWZkipmRkSRJkpQ5ZmQkSZKkDPA9MsXMyEiSJEnKHDMykiRJUgb4HpliZmQkSZIkZY4ZGUmSJCkD3LWsmBkZSZIkSZljICNJkiQpc5xaJkmSJGWA2y8XMyMjSZIkKXNabUamc9uOLT0EaZm06sn/aukhSMukj0ff2NJDkNTKVbXSnEySJGsAw4CtgLnAcOCUNE3nNNDuSmAHoB+5hNPbwKVpmg4vpd9WG8hIkiRJat2SJOkJPAVMBIYAfYFLgeWBAxto3gW4GkiBAOwP3JUkSZs0Te9sqG8DGUmSJCkDWun2y0cBvYAN0zSdCpAkyQLgjiRJzkvTdOziGqZpeniNokeSJFkLOAxoMJBxjYwkSZKkptodeKI6iMm7D5gH7NaE+00FOpRS0YyMJEmSlAHNuUImP2WsZx2XpqdpOr3gfC2gaJFfmqbzkiR5D1izhH4C0BboDuwJfB/4SSljNCMjSZIkqabjgQ/qOI6vUa8XMJ3apgG9S+hnb2A+8CVwA3Bcmqb3ljJAMzKSJElSBjTzGpmhwM11lNcVtHwbo4BNyGV/dgOuSJJkQZqmNzTU0EBGkiRJUpH89LFSgpZp1D0FrRe57ZRL6Wd0/vTxJEk6AJcmSXJzmqYL62vr1DJJkiQpA6pC8x2NMJ7cOplFkiTpCKxGCYFMHV4BKsht31wvAxlJkiRJTfUwsGOSJH0KyvYFOuavNdZWQCW53cvq5dQySZIkKQOqmnXfspJdCxwDjEyS5Dy+eSHm3WmajquulCTJDcChaZq2y59vDZwI3E/uZZoV5HYt+xlwapqmCxrq2EBGkiRJUpOkaTo9SZIdgMuBEcBcYDhwco2qbfNHtUnA10B18DON3DS1fdI0HVlK3wYykiRJUga0ynwMkKbpO8CuDdQ5DDis4HwCsP+36dc1MpIkSZIyx0BGkiRJUuY4tUySJEnKgGZ+IWarZ0ZGkiRJUuaYkZEkSZIyoJVuv9xizMhIkiRJyhwzMpIkSVIGmI8pZkZGkiRJUuaYkZEkSZIywF3LipmRkSRJkpQ5ZmQkSZKkDHDXsmJmZCRJkiRljhkZSZIkKQPMxxQzIyNJkiQpc8zISJIkSRngrmXFzMhIkiRJyhwzMpIkSVIGRFfJFDEjI0n6//buPM6K6kz4+O8BFNkXReOCYlwuwQWNmKig4jZxGX2j4vJqJpo4ajZxiWt0FDWZ+Ia4LxONUUeHqDFo1JhRBxXFGEBRUEBvHONC3OICAooo9Hn/uNXQt2nobqS6qe7f1099+tapU3XOvR+L7uc+59SRJKlwDGQkSZIkFY5DyyRJkqQCcLJ/NTMykiRJkgrHjIwkSZJUADVO9q9iRkaSJElS4ZiRkSRJkgrAfEw1MzKSJEmSCseMjCRJklQAzpGpZkZGkiRJUuGYkZEkSZIKwHVkqpmRkSRJklQ4ZmQkSZKkAkjOkaliRkaSJElS4ZiRkSRJkgrAOTLVzMhIkiRJKhwzMpIkSVIBOEemmhkZSZIkSYVjICNJkiSpcBxaJkmSJBWAk/2rmZGRJEmSVDhmZCRJkqQCqElO9q/LjIwkSZKkwjEjI0mSJBWA+ZhqZmQkSZIkFY4ZGUmSJKkAaszJVDEjI0mSJKlwzMhIkiRJBZDMyFQxIyNJkiSpcMzISJIkSQVQ09odWM2YkZEkSZJUOGZkJEmSpALwqWXVzMhIkiRJKhwzMpIkSVIB+NSyagYykiRJklZaqVTaArgaGAYsAO4AziqXy5+s4JyewGnAfkAJ+ByYAvykXC4/25R2HVomSZIkaaWUSqXewGNAD2AE8GPg/wI3NXLqxsCJwDjgCOA7QEfgqVKp9NWmtG1GRpIkSSqA1fTxyycCfYDtyuXy+wClUmkRMKZUKl1cLpdnLOe8V4HN6mZtSqXSOOBvwElUApsVMiMjSZIkaWXtDzxSG8RkxgILqQwba1C5XP64/tCzcrn8KfAisEFTGjYjI0mSJBVASi032T8bMta7gUNzyuXynDr7X6HeMLJyubywVCq9AgxsZpvdgO2BW5tS34yMJEmSpPpOoTL8q/52Sr16fYA5LGs20LeZbf4U6Apc05TKZmQkSZKkAmjhBTGvAG5poLyhoOULK5VKR1EJkn5YLpf/tynnGMhIkiRJqpINH2tK0DKbhoeg9QFeakpbpVJpH+BmYHS5XL6uqX10aJkkSZJUADUtuDXDi1TmySxRKpU6A5vRhECmVCp9Dbgb+B1wVnMaNpCRJEmStLL+BOxVKpXWrlN2MNA5O7ZcpVLpK1mdPwPfLZfLzRo759AySZIkqQBSy86Raarrqaz7cm+pVLoYWBe4DLizXC7PrK1UKpV+AxxTLpc7ZfvrAg8BnwGjgR1KpVJt9YXlcvm5xho2kJEkSZK0Usrl8pxSqbQncBWVIWILgDuAM+tV7ZhttQYB/bPX4+rVfR0Y0FjbBjKSJElSAbTwU8uarFwu/xXYt5E6xwLH1tkfD8QXadc5MpIkSZIKx4yMJEmSVAAprZ4ZmdZiRkaSJElS4ZiRkSRJkgqgmeu7tHlmZCRJkiQVjhkZSZIkqQBW03VkWo0ZGUmSJEmFYyAjSZIkqXAcWiZJkiQVwOq6IGZrMSMjSZIkqXAMZNSoDh06cNYFJzPpxXE89+oTXHPzL+jTt3eDddf7Uj/+49ZLGf/sH3n5vSkcNGK/Zeqces73uffRMcx4cyK3/P66vLsvFUaHDh0YdfGZvPS3ibz25rPcfNvV9O3bZ7n199x7V56c9ACz3n2eCRP/yPA9h1Yd79ixI2f9ZCTPTX+M19+eytPTxrHXPrstc52uXbvw9LRxvPPhzFX+nqSiWry4hktv+h27H30yOx3+A07992uZ/dG85da/5e4H2f/4s9np8B/wzyecwx0PPLrk2Efz5nPs2Zew+7dOYefDf8j+x5/NDXfe7+KGaraUUottRWAgo0adePKx7LXf7ozY9xh23XZ/AEZfd1GDdWtS4snxEznte+fy9pvvNFjnjdf+zpWX/Io7b7s7tz5LRXTyaSew3wF78Y09D2Obr1QCjut+PbrBupsM6M8t/3UNV152PV/eaAeuvOx6/nPMtfTfeMMldS694iKG7zmUww4+jk3W344Dv3EUfy2/ssy1/u3C03njtb/n86akgvrN7//E+ElTGXPpefzPzb8E4CeX3dhg3ccmTeW6397Lz398PBN/dx0/O+04Lrv5Lv7y3AwAuqzVmfO+/y+Mu+WX/OV313LDxT/mgfGTGPvQEy32fqS2yEBGjTriXw7hhqv/k1mvv8n8efP5xYVXsfteQ9lgoy8tU/e9d99nzE138ezkaSxe3PCyTWNvv59HH57A7A/m5N11qVC+fewRXHX5r3n9tVnMmzufC8//BXvveHeVRQAAGEdJREFUsxsb9d9gmbpHHnUw06bO4K477+Pzzz/n97+7n+enzeTIow4GYPPNN+VbxxzGSd8/h/99+W8AvPPOP5j1xptV19l5lyHstPMQrrrihvzfoFQgYx96nO8cuh8bfakfPbp15bTvHMafn53OW/94f5m6s95+l9Km/Rk8cDMABg/cnC0HbET51VkArLnGGmy+yYas0Wnp1OQOEby2nC/8pOWpIbXYVgS5BzIRsUtEHBUR367d8m5Tq06Pnt3ZsP/6zJj24pKyN177O/PmzmfgVlu2Ys+ktqVnrx7033hDpk2dsaTstVdnMfejeWy99cBl6m+19UCmTZ1eVfb8tBlsldUdttvXmfvRPL55yH688NIEps4Yz+jLRtG9e7cl9bt0WYvLr/4pp550Los+X5TTO5OKZ+78T3j7vQ8ZtPkmS8r6r78u3bt2ofzqstnLfXf9GvM/WcBzM1+mpqaGKTP+yutvvcvQHbauqvejC69kx0O/x/7Hn83HCz5lxL675/5epLYs16eWRcRtwGbAVGBxVpyAW5dT/wTgBIB+3Tem11rr5Nk9NUG37I+eeXPnV5XP/Wge3Xt0a+gUSSuhNsCYO7d6DP5HH82lR8/uDdZv6L4cOHALAPqu3YeevXqwZWkzdt5xX7p17cotY67h4n8/h1NHngfAeaN+zEP//RhTn5vO0GFfy+NtSYX0yYJPAejetUtVeY9uXfj4kwXL1O/buyf7DB3CceeOJtVUvsk+8/gj2WKTjarqXXPBySxeXMP0l1/l8cnT6NPAvS2tiAtiVss7IzMEGJpS+kFK6aRsG7m8yimlG1JKQ1JKQwxiVg8fz/8YYJk/pHr26sH8eR+3RpekNml+dq/17NmjqrxXr57LBCy19Ru6L+fNm191vZ//9Armz/uYd999j6su/zX7HrAXAF/faQf23mc3LvnZlav8vUhF17XLWgDMrxe0zPt4Ad3qBTcA199xP//9+CTuunIUz/7hBu66ahS33fs/3P3whGXqduzYgcEDN6N7ty787Fdj8nkDUjuRdyAzHVh2IoUKY97c+bw562222nbp0Jb+m2xIj57dKc98uRV7JrUtcz+ax6w33mTbwYOWlG0yoD89e/VgxozyMvVnTH+JwYO3qirbdttBzJj+EgDTn68MB63/5Jna/d332IUNNlyfqTMfp/zqJG67/T/o1KkT5Vcn8Y1991il700qmp7du7J+v768+MrrS8r+/s57zP9kAVsO2GiZ+i/+7+vsufP2bLbxBkQEm2+yIXvutD2PT5663DYWL67hjbfezaX/artqUmqxrQjyDmTWAWZGxEMRcV/tlnObWsXuvO1ujj/pGDbaeAO6d+/GGeeP5IlHn+LNWW83WH/NzmuyZuc1iQjWWKMTa3Zek44dOy453qlTpaxTp4506NChUn/NNVrq7UirrVtvuZORpx7PxptsRPce3Tj/wtN5ZNwTy0zQB7jz9nsYvP3WHDLiADp16sQhIw5g2+224o7f3gPAX556hhnTX+Ksn4yka9curLNOX3508nE8cN/DAFx3zU18bft9GD70IIYPPYhTTjqXRYsWMXzoQTw+/qkWfd/S6ujQb+zOTWP/e0kAc/ktd7HLV7dmw/WWHTGy3aDNeXTic7yeBSZ/m/UWj058jkGbDwBg2kuvMHHaTD5d+BmLF9fwzPQyY+4fx7AdtmnJtyS1OZHnc6IjosFZbCmlxxs7d4t+OxQjFGwHOnTowBnnj+SQIw9kzc5r8NT4SZz3458x+8M5HHToflx06U/YbsCuS+q//N6UZa5x1S+u5+rRlaci/b+rR3HIkQdWHf/7G2+xxw4HLnOeWt7shctfJ0H56tChAxdcdAZHHn0Inddcg/GPPcVpI/+NDz+czYjDD+SXV1zEgA22X1J/z7135aKfnc0mA/rz+muzOO+cf2f8o39ecnyj/hsw+vIL2XmXIcybO5/773uIn466lE8aGOM/dNjXGHvfLXyp76BljqllvPnMTa3dBdWxeHENl99yF/c98hSfLfqcnbfbivN/+G369OrBA+MnctG1tzLprspaaIsWL+aa2+7hwQmTmT13Pr26d+Ofhg3h5GMOZY1OnXhmepnRN97J62+9QxCsu3YfDhi+E8eN2J+OHX2A7Oqg85bDorX70BS7brhXi/19POHNR1b7zyTXQAYgItYDdsx2J6eU/tGU8wxkpNZhICO1DgMZqfUYyCyrCIFMrl8DRMThwGTgMOBwYFJEjMizTUmSJKktch2Zark+fhk4F9ixNgsTEf2AccDvc25XkiRJUhuWdyDTod5Qsg9ogUU4JUmSpLamKJmSlpJ3IPNgRDwE3J7tHwH8Kec2JUmSJLVxuQYyKaUzIuJQYGhWdENK6Z4825QkSZLU9uWdkSGlNBYYm3c7kiRJUluW99OGiyaXQCYinkwpDYuIeVA1mC+AlFLqmUe7kiRJktqHXAKZlNKw7GePPK4vSZIktTdO9q+W9zoytzWlTJIkSZKaI+85MlvV3YmITsAOObcpSZIktTnJjEyVXDIyEXFONj9m24iYm23zgHeBe/NoU5IkSVL7kdccmZ8DP4+In6eUzsmjDUmSJKk98all1XKdIwNMjohetTsR0Tsivplzm5IkSZLauLwDmQtSSh/V7qSU5gAX5NymJEmS1ObUkFpsK4K8A5mGrp/7IpySJEmS2ra8g4pnIuIy4Nps/4fAlJzblCRJktoc58hUyzsjcxLwGXBnti2kEsxIkiRJ0krLNSOTUvoYODvPNiRJkqT2oChzV1pKLoFMRFyRUjolIu6HZT/xlNJBebQrSZIkqX3IKyNzW/bzlzldX5IkSWpXkhmZKnktiDkl+/l4HteXJEmS1L7lNbTsBRoYUlYrpbRtHu1KkiRJah/yGlr2z9nP2ieU1Q41+xYrCHAkSZIkNazGxy9XyWto2esAEbFPSmn7OofOiohn8UlmkiRJkr6AvNeRiYgYWmdnlxZoU5IkSWpzUgv+VwS5riMDHAfcFBG9gABmA9/NuU1JkiRJbVzeC2JOAQZngQwppY/ybE+SJElqq5wjUy3vjAwRcQCwFbBWRACQUroo73YlSZIktV25BjIR8SugK7AHcCMwApicZ5uSJElSW1SUuSstJe+J97uklL4NzE4pXQjsDGyZc5uSJEmS2ri8h5Z9mv38JCI2AD4A1s+5TUmSJKnNcY5MtbwDmfsjojcwGniWymKYv865TUmSJEktpFQqbQFcDQwDFgB3AGeVy+VPGjnvCOBw4OvAhsAZ5XL5l01tN7dAJiI6AI+klOYAYyPij8BaPrlMkiRJar7VcY5MqVTqDTwGvE5lPvy6wGVAP+DIRk4fAXwZ+CNwYnPbzi2QSSnVRMS1wPbZ/kJgYV7tSZIkSWpxJwJ9gO3K5fL7AKVSaREwplQqXVwul2es4NwjyuVyTXZOswOZvCf7PxIRh0btc5clSZIkrZSalFpsa4b9gUdqg5jMWCoJjP1WdGJtELOy8p4jcyJwGrAoIj4FAkgppZ45tytJkiRpJWVDxno3cGhOuVyeU2f/K8BNdSuUy+WFpVLpFWBgjl3MN5BJKfXI8/qSJElSe9HCc2ROAS5ooPxCYFSd/T7AnAbqzQb6rvpuLZX3gpiPpJT2aqxMkiRJ0mrlCuCWBsobClpaRS6BTESsBXQF1omIPlSGlAH0pPJoNUmSJEmrqWz4WFOCltk0PAStD/DSKu1UPXllZE6kko7aAJjC0kBmLnBNTm1KkiRJbVZKX2hufF5epDJPZolSqdQZ2Ay4Oc+Gc3lqWUrpypTSpsDpKaUvp5Q2zbbBKSUDGUmSJKlt+BOwV6lUWrtO2cFA5+xYbvKe7H91ROwCDKjbVkrp1jzblSRJktqamtVwQUzgeuAk4N5SqXQxSxfEvLNcLs+srVQqlX4DHFMulzvVKRsEDKpzrW1KpdIIgHK5/PvGGs57sv9tVNJKU4HFWXECDGQkSZKkgiuXy3NKpdKewFXA3cAC4A7gzHpVO2ZbXYdT/WS0b2cbLJ2aslyRmrfgTbNExIvAoLQSjWzRb4fVMuSU2rrZC+e1dhekdunNZ25qvJKkXHTeclghFm/fuO82Lfb38RsfvrDafya5zJGpYzrwpZzbkCRJktTO5Dq0DFgHmBkRk4GFtYUppYNybleSJElqU1bTOTKtJu9AZlTO15ckSZLUDuX91LLHI2I9YMesaHJK6R95tilJkiS1RXnObS+iXOfIRMThwGTgMCpPJZgUESPybFOSJElS25f30LJzgR1rszAR0Q8YBzT6XGhJkiRJS9WYkamS91PLOtQbSvZBC7QpSZIkqY3LOyPzYEQ8BNye7R8B/CnnNiVJkqQ2J/nUsiq5BDIRsTmwXkrpjIg4BBiWHfoLMCaPNiVJkiS1H3llZK4AzgFIKd0N3A0QEdtkxw7MqV1JkiSpTfKpZdXymq+yXkrphfqFWdmAnNqUJEmS1E7kFcj0XsGxLjm1KUmSJKmdyCuQeSYijq9fGBH/CkzJqU1JkiSpzaohtdhWBHnNkTkFuCcijmZp4DIEWBM4OKc2JUmSJLUTuQQyKaV3gV0iYg9g66z4gZTSo3m0J0mSJLV1Tvavlus6Mimlx4DH8mxDkiRJUvuT94KYkiRJklaBGjMyVfKa7C9JkiRJuTEjI0mSJBWAc2SqmZGRJEmSVDhmZCRJkqQCKMr6Li3FjIwkSZKkwjEjI0mSJBWAc2SqmZGRJEmSVDhmZCRJkqQCcB2ZamZkJEmSJBWOGRlJkiSpAJJPLatiRkaSJElS4RjISJIkSSoch5ZJkiRJBeBk/2pmZCRJkiQVjhkZSZIkqQBcELOaGRlJkiRJhWNGRpIkSSoAH79czYyMJEmSpMIxIyNJkiQVgHNkqpmRkSRJklQ4ZmQkSZKkAjAjU82MjCRJkqTCMSMjSZIkFYD5mGpmZCRJkiQVTjjWTnmIiBNSSje0dj+k9sZ7T2od3ntSyzMjo7yc0NodkNop7z2pdXjvSS3MQEaSJElS4RjISJIkSSocAxnlxXHCUuvw3pNah/ee1MKc7C9JkiSpcMzISJIkSSocAxlJkiRJhWMg085ExOKImFpnO7uR+sMjYpcmXPeg2mtFxKiIOH1V9Xk57X0zIgbl2YbU0iIiRcSldfZPj4hRK3mt3hHxg5U897WIWGdlzpVaS0R8KSLuiIhXImJKRPwpIrZcieucEhFdv2BfBkTE9Oz1kIi4qpH6P6m3/9QXaV9qLwxk2p8FKaXt6myXNFJ/ONBoIJNSuq8J11qVvgkYyKitWQgcsoqCiN5Ag4FMRHRaBdeXVhsREcA9wPiU0mYppR2Ac4D1VuJypwANBjIR0bG5F0spPZNSGtlItapAJqXU6O9dSQYyymTfwF4YEc9GxAsRMTAiBgDfA07Nsje7RsSBETEpIp6LiHERsV52/rERcU0D1x0fEZdHxDMR8WJE7BgRd0fEyxHx0zr1vhURk7N2rq/9ZRER8yPiZxExLSImRsR6WYboIGB0Vn+zlviMpBawiMqTj06tfyAi+kXE2Ih4OtuGZuVVGdCImJ7du5cAm2X3yOgsuzohIu4DZmZ1/5B9cz0jIlzMT0W2B/B5SulXtQUppWnAk9n//9Oz321HwJLRBuMj4vcR8VJEjImKkcAGwGMR8VhWd35EXBoR04CdI+L87B6cHhE3ZEEUEbFD9rtqGvDD2n5kbf0xe909Im7O+vJ8RBwaEZcAXbJ7dUxtm9nPaE7/c/+UpdWMgUz7U/uPZe12RJ1j76eUvgr8B3B6Suk14FfA5Vn2ZgLwJLBTSml74A7gzCa0+VlKaUh2rXup/AO/NXBsRKwdEV8BjgCGppS2AxYDR2fndgMmppQGA08Ax6eUngLuA87I+vXKF/lApNXMtcDREdGrXvmVVO7FHYFDgRsbuc7ZwCvZPXJGVvZV4OSUUu1wm+9m31wPAUZGxNqr5i1ILW5rYEoD5YcA2wGDgb2pfAG2fnZseyrZl0HAl6n8DroKeAvYI6W0R1avGzAppTQ4pfQkcE1KaceU0tZAF+Cfs3o3Aydlv6+W59+Aj1JK26SUtgUeTSmdzdLREkfXq9+s/q/oA5LaIocXtD8LsmChIXdnP6dQ+cezIRsBd2b/kK4JvNqENu/Lfr4AzEgpvQ0QEX8D+gPDgB2Ap7MvlLoA/8jO+Qz4Y51+7dOE9qTCSinNjYhbgZHAgjqH9gYG1fnStWdEdG/m5SenlOresyMj4uDsdX9gC+CDlei2tLoaBtyeUloMvBsRjwM7AnOp3A9/B4iIqcAAKl/W1bcYGFtnf4+IOJPK8LO+wIyImAD0Tik9kdW5DdivgWvtDRxZu5NSmt0C/ZfaLAMZ1bUw+7mY5f+/cTVwWUrpvogYDoxqxnVr6ryu3e8EBPCfKaVzGjj387R0saMV9UtqS64AnqXyDW+tDlSyoZ/WrRgRi6jOrq+1gut+XOe84VT+qNo5pfRJRIxv5FxpdTYDGNHMc+r+PlrR75dPs0CCiFgLuA4YklKaFZWHcbTWfdPU/kttlkPL1Jh5QI86+72AN7PXx6yiNh4BRkTEugAR0TciNmlmv6Q2I6X0IfA74Lg6xQ8DJ9XuRERtZvU1KkPGiIivAptm5Y3dI72A2VkQMxDYaZV0XmodjwKd6871iohtgTnAERHRMSL6AbsBkxu51orundqg5f0sIzoCIKU0B5gTEcOy4/WHiNX6H6rnz/TJXn4eEWs0UH/CSvRfajcMZNqf+nNkGnvS2P3AwVndXalkYO6KiCnA+6uiQymlmcB5wMMR8TyVf+jXX/FZ3AGcEZWHDjjZX23RpUDdp5eNBIZkE4RnUnkQB1SGvPSNiBnAj4C/AqSUPgD+nE0SHt3A9R8EOkXEi1QeDDAxp/ch5S7L3B8M7B2Vxy/PAH4O/BZ4HphGJdg5M6X0TiOXuwF4sHayf7125gC/BqYDDwFP1zn8HeDabJjX8ibe/xTok92X06g8pKC2zedrJ/vXcc9K9F9qN2LpqB1JkiRJKgYzMpIkSZIKx0BGkiRJUuEYyEiSJEkqHAMZSZIkSYVjICNJkiSpcAxkJKmZImJx9kjy6RFxV0R0/QLXuiUiRmSvb4yIQSuoOzwidlmJNl6LiHWaWl6vzvxmtjUqIk5vbh8lSWouAxlJar4FKaXtUkpbA5+xdE0XACJipVbYTin9a7au0vIMB5odyEiS1BYZyEjSFzMB2DzLlkyIiPuAmdlK3KMj4ulsEcsTAaLimogoR8Q4YN3aC0XE+IgYkr3eNyKejYhpEfFIRAygEjCdWrtAbUT0i4ixWRtPR8TQ7Ny1I+LhiJgRETey/MX5loiIP0TElOycE+oduzwrfyRbXZyI2CwiHszOmRARAxu45siImJm9/ztW7uOVJKlhK/WtoSRpSeZlP+DBrOirwNYppVezYOCjlNKOEdEZ+HNEPAxsD5SAQcB6wEzgpnrX7Udl9fDdsmv1TSl9GBG/AuanlH6Z1fstcHlK6cmI2JjKSuNfAS4AnkwpXRQRBwDHNeHtfDdrowvwdESMTSl9AHQDnkkpnRoR52fX/hGVlci/l1J6OSK+DlwH7FnvmmcDm6aUFkZE7yZ9qJIkNZGBjCQ1X5eImJq9ngD8hsqQr8kppVez8n8Ctq2d/wL0ArYAdgNuTyktBt6KiEcbuP5OwBO110opfbicfuwNDIpYknDpGRHdszYOyc59ICJmN+E9jYyIg7PX/bO+fgDUAHdm5f8F3J21sQtwV522OzdwzeeBMRHxB+APTeiDJElNZiAjSc23IKW0Xd2C7A/6j+sWASellB6qV2//VdiPDsBOKaVPG+hLk0XEcCpB0c4ppU8iYjyw1nKqp6zdOfU/gwYcQCWoOhA4NyK2SSktalbnJElaDufISFI+HgK+HxFrAETElhHRDXgCOCKbQ7M+sEcD504EdouITbNz+2bl84Aedeo9DJxUuxMRtYHFE8BRWdl+QJ9G+toLmJ0FMQOpZIRqdQBqs0pHURmyNhd4NSIOy9qIiBhc94IR0QHon1J6DDgra6N7I/2QJKnJDGQkKR83Upn/8mxETAeup5IFvwd4OTt2K/CX+iemlN4DTqAyjGsaS4d23Q8cXDvZHxgJDMkm089k6dPTLqQSCM2gMsTsjUb6+iDQKSJeBC6hEkjV+hj4WvYe9gQuysqPBo7L+jcD+D/1rtkR+K+IeAF4DrgqpTSnkX5IktRkkVJq7T5IkiRJUrOYkZEkSZJUOAYykiRJkgrHQEaSJElS4RjISJIkSSocAxlJkiRJhWMgI0mSJKlwDGQkSZIkFc7/B48GK5VAoYJqAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x720 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2iry8Q2MH9Rp",
        "colab_type": "text"
      },
      "source": [
        "### GDrive, Zip, TensorBoard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iNl-KgPzmK-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cmXWXHZVSI-o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip \"/content/drive/My Drive/7748_xlmr_model.zip\" -d \"/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C-Ati0Jat-2f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 154
        },
        "outputId": "9baca1d7-573e-4162-eeb1-610ad379a5e6"
      },
      "source": [
        "!zip -r \"/content/drive/My Drive/78_xlmr_model.zip\" \"/content/model/\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  adding: content/model/ (stored 0%)\n",
            "  adding: content/model/XLM-R_Seq_CLS_optimizer_ckpt.pt (deflated 47%)\n",
            "  adding: content/model/.ipynb_checkpoints/ (stored 0%)\n",
            "  adding: content/model/XLM-R_Seq_CLS_model.pt (deflated 22%)\n",
            "  adding: content/model/XLM-R_Seq_CLS_model.pth (deflated 22%)\n",
            "  adding: content/model/predicted_labels.pkl (deflated 81%)\n",
            "  adding: content/model/XLM-R_Seq_CLS_ckpt.pth (deflated 22%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I74XVV6MCjOv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!zip -r \"/content/drive/My Drive/7748_xlmr_model_runs.zip\" \"/content/runs/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYTUnXi6wSop",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir runs"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}