cnn/ex3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Analytics: 3rd Assignment (Part 2: Exercise 2)\n",
    "## MSc in Data Science (2023/2024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:31:50.676978: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-02-28 20:31:50.677012: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-02-28 20:31:50.678713: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-02-28 20:31:50.688948: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-02-28 20:31:52.197893: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import tasks.preprocessing, tasks.models, tasks.tuning, tasks.util\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "\n",
    "INPUT_DIR = \"input\"\n",
    "INPUT_MODEL_PATH = os.path.join(INPUT_DIR, \"models\")\n",
    "OUTPUT_DIR = \"output\"\n",
    "INTERMEDIATE_DIR = \"intermediate\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing with  /physical_device:GPU:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:31:54.222519: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:31:54.298808: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:31:54.299101: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.list_physical_devices(\"GPU\")\n",
    "print(\"Executing with \", gpus[0].name if len(gpus) != 0 else \"CPU\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "\n",
    "Acquiring and preprocessing our data with the goal of eventually acquiring a sufficient representation of our text is the most difficult and time-consuming task. We thus split it in distinct phases:\n",
    "\n",
    "* Original dataset acquisition and parsing\n",
    "* Qualitative analysis and preprocessing\n",
    "* Transformation for the NLP task\n",
    "\n",
    "Note that due to the relative custom code complexity, most of the code used in this section was developed and imported from python source files located in the `tasks` module. In-depth documentation and implementation details can be found in these files. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training dataset...\n",
      "\tReading data...\n",
      "\tParsing data...\n",
      "\tGetting words...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13cc323f88bf4fb0b0a978c042e3007a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12544 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting POS tags...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba8c7de7a7c54763af9ca5920af7d263",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12544 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting Sentence ids...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c80b27c090741efb7d0d687cb1ef597",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12544 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading validation dataset...\n",
      "\tReading data...\n",
      "\tParsing data...\n",
      "\tGetting words...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b7eb7f5a21e4f28bd4c06be239d090f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2001 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting POS tags...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8af2a3fc69dd4d5da607ef125897caaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2001 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting Sentence ids...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97d9e40d652f4683ba2e0c03f7511035",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2001 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading test dataset...\n",
      "\tReading data...\n",
      "\tParsing data...\n",
      "\tGetting words...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8cb85e36c144a2abfe2f8a3ca1bc859",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2077 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting POS tags...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c3b64f826b54be0a493f28b2154f604",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2077 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tGetting Sentence ids...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c256b08029094713b4bc8af3b6c288c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2077 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape: (207227, 3)\n",
      "Validation data shape: (25511, 3)\n",
      "Test data shape: {test_df.shape}\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading training dataset...\")\n",
    "train_df = tasks.preprocessing.conllu_to_pd(\n",
    "    \"input/UD_English-EWT/en_ewt-ud-train.conllu\"\n",
    ")\n",
    "print(\"Loading validation dataset...\")\n",
    "val_df = tasks.preprocessing.conllu_to_pd(\n",
    "    \"input/UD_English-EWT/en_ewt-ud-dev.conllu\"\n",
    ")\n",
    "print(\"Loading test dataset...\")\n",
    "test_df = tasks.preprocessing.conllu_to_pd(\n",
    "    \"input/UD_English-EWT/en_ewt-ud-test.conllu\"\n",
    ")\n",
    "\n",
    "print(\n",
    "    f\"Training data shape: {train_df.shape}\\nValidation data shape: {val_df.shape}\"\n",
    "    \"\\nTest data shape: {test_df.shape}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below we can see a preview of our parsed training dataset. Our preprocessing exploits pandas's ordering scheme in order to make sure the words are inserted in the order they appear in the sentence. This ordering will prove important later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>pos</th>\n",
       "      <th>sent_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>al</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zaman</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>:</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>american</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207222</th>\n",
       "      <td>on</td>\n",
       "      <td>ADP</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207223</th>\n",
       "      <td>my</td>\n",
       "      <td>PRON</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207224</th>\n",
       "      <td>car</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207225</th>\n",
       "      <td>)</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207226</th>\n",
       "      <td>.</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>207227 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           words    pos                                            sent_id\n",
       "0             al  PROPN  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "1              -  PUNCT  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "2          zaman  PROPN  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "3              :  PUNCT  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "4       american    ADJ  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "...          ...    ...                                                ...\n",
       "207222        on    ADP                                reviews-319816-0029\n",
       "207223        my   PRON                                reviews-319816-0029\n",
       "207224       car   NOUN                                reviews-319816-0029\n",
       "207225         )  PUNCT                                reviews-319816-0029\n",
       "207226         .  PUNCT                                reviews-319816-0029\n",
       "\n",
       "[207227 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As mentioned above, our dataset features words connected with punctuation such as \"don't\". These are normally treated as two words, with the first being their intuitive POS tag (\"do\" - AUX) and the second as part of the first (\"n't\" - PART).\n",
    "\n",
    "This dataset contains both the full words and their split versions, with only the latter featuring valid POS tags. The former are instead marked by a pseudo-tag (here \"_\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>pos</th>\n",
       "      <th>sent_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>don't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>704</th>\n",
       "      <td>won't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>don't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-blogspot.com_healingiraq_20040409053012...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1058</th>\n",
       "      <td>don't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-blogspot.com_healingiraq_20040409053012...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1078</th>\n",
       "      <td>doesn't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-blogspot.com_healingiraq_20040409053012...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207078</th>\n",
       "      <td>couldn't</td>\n",
       "      <td>_</td>\n",
       "      <td>reviews-319816-0025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207091</th>\n",
       "      <td>don't</td>\n",
       "      <td>_</td>\n",
       "      <td>reviews-319816-0025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207096</th>\n",
       "      <td>employees'</td>\n",
       "      <td>_</td>\n",
       "      <td>reviews-319816-0025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207164</th>\n",
       "      <td>i'm</td>\n",
       "      <td>_</td>\n",
       "      <td>reviews-319816-0027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207186</th>\n",
       "      <td>didn't</td>\n",
       "      <td>_</td>\n",
       "      <td>reviews-319816-0028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2613 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             words pos                                            sent_id\n",
       "176          don't   _  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "704          won't   _  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "868          don't   _  weblog-blogspot.com_healingiraq_20040409053012...\n",
       "1058         don't   _  weblog-blogspot.com_healingiraq_20040409053012...\n",
       "1078       doesn't   _  weblog-blogspot.com_healingiraq_20040409053012...\n",
       "...            ...  ..                                                ...\n",
       "207078    couldn't   _                                reviews-319816-0025\n",
       "207091       don't   _                                reviews-319816-0025\n",
       "207096  employees'   _                                reviews-319816-0025\n",
       "207164         i'm   _                                reviews-319816-0027\n",
       "207186      didn't   _                                reviews-319816-0028\n",
       "\n",
       "[2613 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "invalid_idx = train_df.pos == \"_\"\n",
    "train_df[invalid_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"don't won't doesn't haven't didn't others it's elena's women's children's i'm people's musharraf's sharon's hamas's right's cannot isn't one's let's reporter's he's that's pakistan's world's bush's military's sharif's can't couldn't\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join(train_df[invalid_idx].words.unique()[:30])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below we can see an example of a word being contained both times in the dataset, one in full with the pseudo-tag, and the other as split words with valid POS tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>pos</th>\n",
       "      <th>sent_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>don't</td>\n",
       "      <td>_</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>do</td>\n",
       "      <td>AUX</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>n't</td>\n",
       "      <td>PART</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     words   pos                                            sent_id\n",
       "176  don't     _  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "177     do   AUX  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "178    n't  PART  weblog-juancole.com_juancole_20051126063000_EN..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.iloc[176:179]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We thus remove the full words including the pseudo-tag from our datasets, ensuring that all target POS tags will be compliant with the UPOS scheme."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df[~invalid_idx]\n",
    "val_df = val_df[val_df.pos != \"_\"]\n",
    "test_df = test_df[test_df.pos != \"_\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Qualitative Analysis\n",
    "\n",
    "We analyze our dataset in two granualities: sentences and individual words. We begin by analyzing how many words are in each sentence, which will give us an idea on the size of context available for each word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def length_sentences(df: pd.DataFrame) -> float:\n",
    "    lengths = df.groupby([\"sent_id\"]).agg(lambda x: len(x))\n",
    "    return lengths.words\n",
    "\n",
    "\n",
    "train_length = length_sentences(train_df)\n",
    "val_length = length_sentences(val_df)\n",
    "test_length = length_sentences(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Figure saved to output/ex_2_dataset_stats.png\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHFCAYAAAAT5Oa6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABle0lEQVR4nO3deVxU5f4H8M+w7yOLMKIomIgguKRmagmmoihqeosUI7maZW6h4lapaKVXK5eLlWXuy9W6qZkaCqmoF1cUzQ3DUFxAXGAARdbn9wc/ThwWF0Rmhvm8X695vTzPec45zwPDzNdnVQghBIiIiIj0mIGmC0BERESkaQyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7DIiIiIhI7zEgIiIiIr3HgIiIiIj0HgMiIiIi0nsMiEjnrF69GgqFAmZmZrh69WqF835+fvD29tZAyYD9+/dDoVDgv//9r0ae/7SuXLmCvn37ws7ODgqFAmFhYZoukkxcXBwiIiKQmZmp6aLUGk2+fzUpNDQUrq6umi7GM9u1axciIiI0XQyqBgZEpLPy8vLwySefaLoYOm3ChAk4evQoVq5cicOHD2PChAmaLpJMXFwcZs+erVcBEem2Xbt2Yfbs2ZouBlUDAyLSWb1798bGjRtx+vRpTRel1uXm5qImtiE8e/YsXnrpJbz++ut4+eWX0aRJkxooHVFFBQUFKCws1HQxiKrEgIh01pQpU2Bvb4+pU6c+Mt+VK1egUCiwevXqCucUCoWseTsiIgIKhQJnzpzBm2++CaVSCTs7O0ycOBGFhYVITExE7969YW1tDVdXVyxYsKDSZz58+BATJ06ESqWCubk5fH19cerUqQr5Tpw4gf79+8POzg5mZmZo27YtfvzxR1me0i7CPXv2YPjw4ahfvz4sLCyQl5dXZZ1TUlLw9ttvw9HREaampvD09MRXX32F4uJiAH937SUlJeG3336DQqGAQqHAlStXqrznTz/9hI4dO0KpVMLCwgJNmzbF8OHDZXmysrIQHh4ONzc3mJiYoGHDhggLC8P9+/cr/NzHjh2LdevWwdPTExYWFmjdujV27Ngh+11MnjwZAODm5iaVcf/+/VKezZs3o1OnTrC0tISVlRV69epV4eccGhoKKysrJCUloU+fPrCysoKLiwsmTZpU4WeYl5eHOXPmwNPTE2ZmZrC3t0e3bt0QFxcn5RFC4JtvvkGbNm1gbm4OW1tbvPHGG/jrr79k9zp16hQCAwOl34GzszP69u2L69evV/kzLuvgwYN4+eWXYW5ujoYNG2LGjBkoKiqSyuDu7o5evXpVuC4nJwdKpRJjxoyp8t5vvvkmWrZsKUvr168fFAoFfvrpJynt5MmTUCgU+PXXX6W0s2fPYsCAAbC1tYWZmRnatGmDNWvWyO5V+v5at24dJk2ahIYNG8LU1BRJSUkASt7THh4e0ntz7dq1T/QzKbVx40Z06tQJVlZWsLKyQps2bbBixQpZnpUrV6J169YwMzODnZ0dBg4ciAsXLsjy+Pn5wc/Pr8L9y3fflX6GfPnll1i4cCHc3NxgZWWFTp064ciRI7Lrvv76awCQ3q+P+7siLSKIdMyqVasEAHH8+HGxZMkSAUD8/vvv0nlfX1/RsmVL6Tg5OVkAEKtWrapwLwBi1qxZ0vGsWbMEAOHh4SE+/fRTER0dLaZMmSIAiLFjx4oWLVqIf//73yI6Olr885//FADEzz//LF2/b98+AUC4uLiIAQMGiF9//VWsX79eNGvWTNjY2IjLly9Leffu3StMTEzEq6++KjZv3iyioqJEaGhohbKW1rdhw4bivffeE7/99pv473//KwoLCyv9+aSnp4uGDRuK+vXri2XLlomoqCgxduxYAUB88MEHQggh1Gq1OHz4sFCpVKJLly7i8OHD4vDhw+Lhw4eV3jMuLk4oFAoxePBgsWvXLrF3716xatUqERISIuW5f/++aNOmjXBwcBALFy4UMTExYsmSJUKpVIrXXntNFBcXy37urq6u4qWXXhI//vij2LVrl/Dz8xNGRkbSz+jatWti3LhxAoDYsmWLVEa1Wi2EEOLzzz8XCoVCDB8+XOzYsUNs2bJFdOrUSVhaWopz585Jzxo2bJgwMTERnp6e4ssvvxQxMTFi5syZQqFQiNmzZ0v5CgoKRLdu3YSRkZEIDw8Xu3btEtu3bxcfffSR+M9//iPlGzlypDA2NhaTJk0SUVFRYuPGjaJFixbCyclJpKWlCSGEyMnJEfb29qJ9+/bixx9/FLGxsWLz5s1i1KhR4vz585X+jEv5+voKe3t74ezsLP7973+L3bt3i/HjxwsAYsyYMVK+JUuWCIVCIS5duiS7/uuvvxYAZD+D8pYtWyYAiJs3b0p1t7a2Fubm5mLkyJFSvvnz5wsjIyORlZUlhBDi4sWLwtraWrzwwgti7dq1YufOnWLIkCECgJg/f750XenfQcOGDcUbb7whtm/fLnbs2CHu3r0rvZ/L/324uLiIJk2aPPJnI4QQM2bMEADEoEGDxE8//ST27NkjFi5cKGbMmCHlmTt3rgAghgwZInbu3CnWrl0rmjZtKpRKpezn5evrK3x9fSs8Y9iwYbKylH6GuLq6it69e4tt27aJbdu2CR8fH2FraysyMzOFEEIkJSWJN954QwCQ3q+P+rsi7cKAiHRO2YAoLy9PNG3aVLRv3176wq2JgOirr76S5WvTpo30xVyqoKBA1K9fXwwaNEhKK/0iePHFF2UBwJUrV4SxsbF49913pbQWLVqItm3bioKCAtmzAgMDRYMGDURRUZGsvu+8884T/XymTZsmAIijR4/K0j/44AOhUChEYmKilNakSRPRt2/fx97zyy+/FACkD/7KzJs3TxgYGIjjx4/L0v/73/8KAGLXrl1SGgDh5OQkfdEKIURaWpowMDAQ8+bNk9K++OILAUAkJyfL7pmSkiKMjIzEuHHjZOnZ2dlCpVKJoKAgKW3YsGECgPjxxx9lefv06SM8PDyk47Vr1woAYvny5VXW8fDhw5W+P65duybMzc3FlClThBBCnDhxQgAQ27Ztq/JeVfH19RUAxC+//CJLHzlypDAwMBBXr14VQgiRlZUlrK2txYcffijL5+XlJbp16/bIZyQlJQkAYu3atUIIIQ4dOiQAiClTpgg3NzcpX8+ePUXnzp2l48GDBwtTU1ORkpIiu19AQICwsLCQ3h+lfwddu3aV5SsqKhLOzs5V/n08LiD666+/hKGhoRg6dGiVeTIyMoS5ubno06ePLD0lJUWYmpqK4OBgKe1pAyIfHx/Zf0SOHTsmAMgC5jFjxgi2NegmdpmRTjMxMcFnn32GEydOVOhqehaBgYGyY09PTygUCgQEBEhpRkZGaNasWaUz3YKDg6FQKKTjJk2aoHPnzti3bx8AICkpCRcvXsTQoUMBAIWFhdKrT58+SE1NRWJiouye//jHP56o7Hv37oWXlxdeeuklWXpoaCiEENi7d+8T3aesDh06AACCgoLw448/4saNGxXy7NixA97e3mjTpo2sPr169arQ1QUA3bp1g7W1tXTs5OQER0fHSn+e5e3evRuFhYV45513ZM8yMzODr69vhWcpFAr069dPltaqVSvZs3777TeYmZlV6AYsX0eFQoG3335b9lyVSoXWrVtLz23WrBlsbW0xdepULFu2DOfPn39sncqytrZG//79ZWnBwcEoLi7GgQMHpDz//Oc/sXr1aqlLcu/evTh//jzGjh37yPu/8MILcHV1RUxMDAAgOjoaPj4+ePvtt5GcnIzLly8jLy8Phw4dQo8ePaTr9u7di+7du8PFxUV2v9DQUDx48ACHDx+WpZd/zyYmJuLmzZtV/n08TnR0NIqKih7ZHXj48GHk5uYiNDRUlu7i4oLXXnsNv//++2OfU5W+ffvC0NBQOm7VqhUAPNF7lrQfAyLSeYMHD8aLL76Ijz/+GAUFBTVyTzs7O9mxiYkJLCwsYGZmViH94cOHFa5XqVSVpt29excAcOvWLQBAeHg4jI2NZa/Ro0cDAO7cuSO7vkGDBk9U9rt371aa19nZWTr/tLp27Ypt27ZJQUijRo3g7e2N//znP1KeW7du4cyZMxXqY21tDSFEhfrY29tXeI6pqSlyc3MfW57Sn1+HDh0qPG/z5s0VnlXZ787U1FT2u7t9+zacnZ1hYFD1x+KtW7cghICTk1OF5x45ckR6rlKpRGxsLNq0aYOPPvoILVu2hLOzM2bNmvVE71EnJ6cKaaXvqbK/v3HjxiE7OxsbNmwAACxduhSNGjXCgAEDHvuM7t27S8FBTEwMevbsCR8fHzg5OSEmJgb/+9//kJubKwuInva9VT5v6fmq/j4e5/bt2wCARo0aVZmn9BlVlbM67/9S5d+zpqamAPBE71nSfkaaLgDRs1IoFJg/fz569uyJ77//vsL50i/C8gNon+WD8XHS0tIqTSv9QHVwcAAATJ8+HYMGDar0Hh4eHrLjsv+jfhR7e3ukpqZWSL9586bs2U9rwIABGDBgAPLy8nDkyBHMmzcPwcHBcHV1RadOneDg4ABzc3OsXLmy0uur+9xH3eu///1vjc2Mq1+/Pg4dOoTi4uIqgyIHBwcoFAocPHhQ+jIsq2yaj48PNm3aBCEEzpw5g9WrV2POnDkwNzfHtGnTHlmW0oCvrNL3VNkv5WbNmiEgIABff/01AgICsH37dsyePVvWilGV7t27Y8WKFTh27BiOHj0qLWHx2muvITo6GlevXoWVlRVefvll6ZqnfW+Vf8+Wlr2qv4/HqV+/PgDg+vXrFVqpyj+jqnKWLaOZmRnUanWFfOUDatIPbCGiOqFHjx7o2bMn5syZg5ycHNk5JycnmJmZ4cyZM7L0X3755bmV5z//+Y9sWvzVq1cRFxcnzWjx8PCAu7s7Tp8+jfbt21f6Ktud9DS6d++O8+fP4+TJk7L0tWvXQqFQoFu3btWuF1Dype/r64v58+cDgDSrKzAwEJcvX4a9vX2l9anOontV/Q+8V69eMDIywuXLl6v8+T2tgIAAPHz4sNLZiKUCAwMhhMCNGzcqfaaPj0+FaxQKBVq3bo1FixahXr16FX4vlcnOzsb27dtlaRs3boSBgQG6du0qS//www9x5swZDBs2DIaGhhg5cuQT1bd79+5QKBSYMWOG7L49evTAvn37EB0dja5du8LY2Fh2zd69e6UAqNTatWthYWEhC54q4+HhgQYNGlT59/E4/v7+MDQ0xLfffltlnk6dOsHc3Bzr16+XpV+/fl3q8ivl6uqKS5cuyf6zdPfu3ScqS1XYaqS72EJEdcb8+fPRrl07pKeny6YUl475WLlyJV544QW0bt0ax44dw8aNG59bWdLT0zFw4ECMHDkSarUas2bNgpmZGaZPny7l+e677xAQEIBevXohNDQUDRs2xL1793DhwgWcPHlSNv35aUyYMAFr165F3759MWfOHDRp0gQ7d+7EN998gw8++ADNmzd/6nvOnDkT169fR/fu3dGoUSNkZmZiyZIlMDY2hq+vLwAgLCwMP//8M7p27YoJEyagVatWKC4uRkpKCvbs2YNJkyahY8eOT/Xc0gBjyZIlGDZsGIyNjeHh4QFXV1fMmTMHH3/8Mf766y/07t0btra2uHXrFo4dOwZLS8unXhxvyJAhWLVqFUaNGoXExER069YNxcXFOHr0KDw9PTF48GB06dIF7733Hv75z3/ixIkT6Nq1KywtLZGamopDhw7Bx8cHH3zwAXbs2IFvvvkGr7/+Opo2bQohBLZs2YLMzEz07NnzsWWxt7fHBx98gJSUFDRv3hy7du3C8uXL8cEHH6Bx48ayvD179oSXlxf27dsnLbXwJBwdHeHt7Y09e/agW7dusLCwAFASEN27dw/37t3DwoULZdfMmjULO3bsQLdu3TBz5kzY2dlhw4YN2LlzJxYsWAClUvnIZxoYGODTTz/Fu+++K/19ZGZmIiIi4om6zFxdXfHRRx/h008/RW5uLoYMGQKlUonz58/jzp07mD17NurVq4cZM2bgo48+wjvvvIMhQ4bg7t27mD17NszMzDBr1izpfiEhIfjuu+/w9ttvY+TIkbh79y4WLFgAGxubJ/oZVqb0PTt//nwEBATA0NAQrVq1gomJSbXvSbVEc+O5iaqn7Cyz8oKDgwUA2SwzIUqmmb/77rvCyclJWFpain79+okrV65UOcvs9u3bsuuHDRsmLC0tKzyv/Iy20tk169atE+PHjxf169cXpqam4tVXXxUnTpyocP3p06dFUFCQcHR0FMbGxkKlUonXXntNLFu27InqW5WrV6+K4OBgYW9vL4yNjYWHh4f44osvpJlrpZ50ltmOHTtEQECAaNiwoTAxMRGOjo6iT58+4uDBg7J8OTk54pNPPhEeHh7CxMREKJVK4ePjIyZMmCBNSRdCVJhCXrY8w4YNk6VNnz5dODs7CwMDAwFA7Nu3Tzq3bds20a1bN2FjYyNMTU1FkyZNxBtvvCFiYmKkPFX97kp/12Xl5uaKmTNnCnd3d2FiYiLs7e3Fa6+9JuLi4mT5Vq5cKTp27CgsLS2Fubm5eOGFF8Q777wj/Y4vXrwohgwZIl544QVhbm4ulEqleOmll8Tq1asf/YMWf7+n9u/fL9q3by9MTU1FgwYNxEcffVRhRmKpiIgIAUAcOXLksfcva8KECQKA+Pzzz2Xp7u7uAoA4c+ZMhWv++OMP0a9fP6FUKoWJiYlo3bp1hRmcpX8HP/30U6XP/eGHH6SfcfPmzcXKlSsrzOx6lLVr14oOHToIMzMzYWVlJdq2bVuhDD/88INo1aqV9D4cMGBApUsRrFmzRnh6egozMzPh5eUlNm/eXOUssy+++KLC9eU/Q/Ly8sS7774r6tevLxQKRaWzJEk7KYSogeVuiYhIY9q3bw+FQoHjx49ruihEOotdZkREOigrKwtnz57Fjh07EB8fj61bt2q6SEQ6jQEREZEOOnnyJLp16wZ7e3vMmjULr7/+uqaLRKTT2GVGREREeo/T7omIiEjvMSAiIiIivceAiIiIiPQeB1U/oeLiYty8eRPW1tZPvIUCERERaZYQAtnZ2Y/dq5AB0RO6efNmlXvnEBERkXa7du3aIzcGZkD0hEr3lbp27dozLetOREREtScrKwsuLi6P3R+SAdETKu0ms7GxYUBERESkYx433IWDqomIiEjvMSAiIiIivceAiIiIiPQexxAREZHeKSoqQkFBgaaLQTXA2NgYhoaGz3wfBkRERKQ3hBBIS0tDZmampotCNahevXpQqVTPtE4gAyIiItIbpcGQo6MjLCwsuNCujhNC4MGDB0hPTwcANGjQoNr3YkBERER6oaioSAqG7O3tNV0cqiHm5uYAgPT0dDg6Ola7+4yDqomISC+UjhmysLDQcEmoppX+Tp9lXBgDIiIi0ivsJqt7auJ3yoCIiIiI9B4DIiIiItJ7DIiIiIi0mJ+fH8LCwjRdjDqPARERERHpPQZEREREWio0NBSxsbFYsmQJFAoFFAoFjIyM8OWXX8rynT17FgYGBrh8+TKAkkHG3377LQICAmBubg43Nzf89NNPsmtu3LiBt956C7a2trC3t8eAAQNw5cqV2qqa1mFApGGFhYU4d+6c7FVYWKjpYhERkRZYsmQJOnXqhJEjRyI1NRWpqamYPXs2Vq1aJcu3cuVKvPrqq3jhhRektBkzZuAf//gHTp8+jbfffhtDhgzBhQsXAAAPHjxAt27dYGVlhQMHDuDQoUOwsrJC7969kZ+fX6t11BYMiDQsMTERo77egfCfEhD+UwJGfb0DiYmJmi4WERFpAaVSCRMTE1hYWEClUkGlUmH48OFITEzEsWPHAJSsvbN+/XoMHz5cdu2bb76Jd999F82bN8enn36K9u3bIzIyEgCwadMmGBgY4IcffoCPjw88PT2xatUqpKSkYP/+/bVdTa3Alaq1gJVjIyidm2q6GEREpAMaNGiAvn37YuXKlXjppZewY8cOPHz4EG+++aYsX6dOnSocJyQkAADi4+ORlJQEa2trWZ6HDx9K3W76hgERERGRjnn33XcREhKCRYsWYdWqVXjrrbeeaAXu0gUMi4uL0a5dO2zYsKFCnvr169d4eXUBAyIiIiItZmJigqKiIllanz59YGlpiW+//Ra//fYbDhw4UOG6I0eO4J133pEdt23bFgDw4osvYvPmzXB0dISNjc3zrYCO4BgiIiIiLebq6oqjR4/iypUruHPnDoqLi2FoaIjQ0FBMnz4dzZo1q9A9BgA//fQTVq5ciUuXLmHWrFk4duwYxo4dCwAYOnQoHBwcMGDAABw8eBDJycmIjY3Fhx9+iOvXr9d2FbUCAyIiIiItFh4eDkNDQ3h5eaF+/fpISUkBAIwYMQL5+fkVBlOXmj17NjZt2oRWrVphzZo12LBhA7y8vACUbIZ64MABNG7cGIMGDYKnpyeGDx+O3NxcvW0xYpcZERGRFmvevDkOHz5cIT01NRVGRkaybrGynJ2dsWfPnirvq1KpsGbNmhorp65jQERERKRD8vLycO3aNcyYMQNBQUFwcnLSdJHqBHaZERER6ZD//Oc/8PDwgFqtxoIFCzRdnDqDLUREREQ6JDQ0FKGhoY/MI4SoncLUIRptITpw4AD69esHZ2dnKBQKbNu2rUKeCxcuoH///lAqlbC2tsbLL78sDSgDSpoOx40bBwcHB1haWqJ///4VRshnZGQgJCQESqUSSqUSISEhyMzMfM61IyIiIl2h0YDo/v37aN26NZYuXVrp+cuXL+OVV15BixYtsH//fpw+fRozZsyAmZmZlCcsLAxbt27Fpk2bcOjQIeTk5CAwMFC2ZkNwcDASEhIQFRWFqKgoJCQkICQk5LnXj4iIiHSDRrvMAgICEBAQUOX5jz/+GH369JH1kTZt+vcWF2q1GitWrMC6devQo0cPAMD69evh4uKCmJgY9OrVCxcuXEBUVBSOHDmCjh07AgCWL1+OTp06ITExER4eHs+pdkRERKQrtHZQdXFxMXbu3InmzZujV69ecHR0RMeOHWXdavHx8SgoKIC/v7+U5uzsDG9vb8TFxQEADh8+DKVSKQVDAPDyyy9DqVRKeSqTl5eHrKws2YuIiIjqJq0NiNLT05GTk4N//etf6N27N/bs2YOBAwdi0KBBiI2NBQCkpaXBxMQEtra2smudnJyQlpYm5XF0dKxwf0dHRylPZebNmyeNOVIqlXBxcanB2hEREZE20dqAqLi4GAAwYMAATJgwAW3atMG0adMQGBiIZcuWPfJaIYS0gR0A2b+rylPe9OnToVarpde1a9eqWRMiIiLSdlo77d7BwQFGRkbSMuOlPD09cejQIQAlq2zm5+cjIyND1kqUnp6Ozp07S3lu3bpV4f63b99+5GJWpqamMDU1rYmqEBGRlktJScGdO3dq5VkODg5o3LhxrTyrMq6urggLC0NYWJjGyqCNtDYgMjExQYcOHZCYmChLv3TpEpo0aQIAaNeuHYyNjREdHY2goCAAJUuZnz17VhqI3alTJ6jVahw7dgwvvfQSAODo0aNQq9VS0ERERPorJSUFLVp4Ijf3Qa08z9zcAhcvXniqoMjPzw9t2rTB4sWLn/n5x48fh6Wl5TPfp67RaECUk5ODpKQk6Tg5ORkJCQmws7ND48aNMXnyZLz11lvo2rUrunXrhqioKPz666/Yv38/AECpVGLEiBGYNGkS7O3tYWdnh/DwcPj4+Eizzjw9PdG7d2+MHDkS3333HQDgvffeQ2BgIGeYERER7ty5g9zcB+g4fBZsGrg+12dlpV7B0ZWzcefOnRptJRJCoKioCEZGj/9ar1+/fo09ty7R6BiiEydOoG3btmjbti0AYOLEiWjbti1mzpwJABg4cCCWLVuGBQsWwMfHBz/88AN+/vlnvPLKK9I9Fi1ahNdffx1BQUHo0qULLCws8Ouvv8LQ0FDKs2HDBvj4+MDf3x/+/v5o1aoV1q1bV7uVJSIirWbTwBV2jT2e66s6AVdoaChiY2OxZMkSKBQKKBQKrF69GgqFArt370b79u1hamqKgwcP4vLlyxgwYACcnJxgZWWFDh06ICYmRnY/V1dXWUuTQqHADz/8gIEDB8LCwgLu7u7Yvn37M/40dY9GW4j8/Pweu7z48OHDMXz48CrPm5mZITIyEpGRkVXmsbOzw/r166tdTiIiIk1ZsmQJLl26BG9vb8yZMwcAcO7cOQDAlClT8OWXX6Jp06aoV68erl+/jj59+uCzzz6DmZkZ1qxZg379+iExMfGRLVKzZ8/GggUL8MUXXyAyMhJDhw7F1atXYWdnVyt11AZaO8uMiIiISoaHmJiYwMLCAiqVCiqVSuoFmTNnDnr27IkXXngB9vb2aN26Nd5//334+PjA3d0dn332GZo2bfrYFp/Q0FAMGTIEzZo1w9y5c3H//n0cO3asNqqnNRgQERER6aj27dvLju/fv48pU6bAy8sL9erVg5WVFS5evCjbA7QyrVq1kv5taWkJa2trpKenP5cyayutnWVGREREj1Z+ttjkyZOxe/dufPnll2jWrBnMzc3xxhtvID8//5H3MTY2lh0rFAppPUB9wYCIiIhIy5mYmMg2La/KwYMHERoaioEDBwIomc195cqV51y6uoFdZkRERFrO1dUVR48exZUrV3Dnzp0qW2+aNWuGLVu2ICEhAadPn0ZwcLDetfRUF1uIiIiIULJGkLY+Izw8HMOGDYOXlxdyc3OxatWqSvMtWrQIw4cPR+fOneHg4ICpU6dyc/InxICIiIj0moODA8zNLXB05exaeZ65uQUcHBye6prmzZvj8OHDsrTQ0NAK+VxdXbF3715Z2pgxY2TH5bvQKlv+JjMz86nKVxcwICIiIr3WuHFjXLx4QW/2MqPKMSAiIiK917hxYwYpeo6DqomIiEjvMSAiIiIivceAiIiIiPQeAyIiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B7XISIiIr2XkpJSpxdmdHV1RVhYGMLCwgCU7Ga/detWvP7665Xmv3LlCtzc3HDq1Cm0adOm2s+tqfvUBgZERESk11JSUuDZwgMPch/WyvMszM1w4WKiRheCTE1Nha2tbY3eMzQ0FJmZmdi2bZuU5uLigtTU1KfeqkQTGBAREZFeu3PnDh7kPsT699rAs4HVc33WhdQcvP19Au7cuaPRgEilUtXKcwwNDWvtWc+KY4iIiIgAeDawwouuyuf6qk7A9d1336Fhw4YoLi6Wpffv3x/Dhg3D5cuXMWDAADg5OcHKygodOnRATEzMI++pUChkLTnHjh1D27ZtYWZmhvbt2+PUqVOy/EVFRRgxYgTc3Nxgbm4ODw8PLFmyRDofERGBNWvW4JdffoFCoYBCocD+/ftx5coVKBQKJCQkSHljY2Px0ksvwdTUFA0aNMC0adNQWFgonffz88P48eMxZcoU2NnZQaVSISIi4ql/bk+LAREREZEWe/PNN3Hnzh3s27dPSsvIyMDu3bsxdOhQ5OTkoE+fPoiJicGpU6fQq1cv9OvXDykpKU90//v37yMwMBAeHh6Ij49HREQEwsPDZXmKi4vRqFEj/Pjjjzh//jxmzpyJjz76CD/++CMAIDw8HEFBQejduzdSU1ORmpqKzp07V3jWjRs30KdPH3To0AGnT5/Gt99+ixUrVuCzzz6T5VuzZg0sLS1x9OhRLFiwAHPmzEF0dPTT/uieCrvMiIiItJidnR169+6NjRs3onv37gCAn376CXZ2dujevTsMDQ3RunVrKf9nn32GrVu3Yvv27Rg7duxj779hwwYUFRVh5cqVsLCwQMuWLXH9+nV88MEHUh5jY2PMnj1bOnZzc0NcXBx+/PFHBAUFwcrKCubm5sjLy3tkF9k333wDFxcXLF26FAqFAi1atMDNmzcxdepUzJw5EwYGJe00rVq1wqxZswAA7u7uWLp0KX7//Xf07Nnz6X54T4EtRERERFpu6NCh+Pnnn5GXlwegJIgZPHgwDA0Ncf/+fUyZMgVeXl6oV68erKyscPHixSduIbpw4QJat24NCwsLKa1Tp04V8i1btgzt27dH/fr1YWVlheXLlz/xM8o+q1OnTlAoFFJaly5dkJOTg+vXr0tprVq1kl3XoEEDpKenP9WznhYDIiIiIi3Xr18/FBcXY+fOnbh27RoOHjyIt99+GwAwefJk/Pzzz/j8889x8OBBJCQkwMfHB/n5+U90byHEY/P8+OOPmDBhAoYPH449e/YgISEB//znP5/4GWWfVTYYKvv8sunGxsayPAqFosIYqprGLjMiIiItZ25ujkGDBmHDhg1ISkpC8+bN0a5dOwDAwYMHERoaioEDBwIAcnJycOXKlSe+t5eXF9atW4fc3FyYm5sDAI4cOSLLc/DgQXTu3BmjR4+W0i5fvizLY2JigqKiosc+6+eff5YFRnFxcbC2tkbDhg2fuMzPA1uIiIiIdMDQoUOxc+dOrFy5UmodAoBmzZphy5YtSEhIwOnTpxEcHPxUrSnBwcEwMDDAiBEjcP78eezatQtffvmlLE+zZs1w4sQJ7N69G5cuXcKMGTNw/PhxWR5XV1ecOXMGiYmJuHPnDgoKCio8a/To0bh27RrGjRuHixcv4pdffsGsWbMwceJEafyQprCFiIiICCVrBGnzM1577TXY2dkhMTERwcHBUvqiRYswfPhwdO7cGQ4ODpg6dSqysrKe+L5WVlb49ddfMWrUKLRt2xZeXl6YP38+/vGPf0h5Ro0ahYSEBLz11ltQKBQYMmQIRo8ejd9++03KM3LkSOzfvx/t27dHTk4O9u3bB1dXV9mzGjZsiF27dmHy5Mlo3bo17OzsMGLECHzyySfV/rnUFIV4ks5DQlZWFpRKJdRqNWxsbGrsvufOnUP4TwlQOjcFAKhv/oUv32yDli1b1tgziIgIePjwIZKTk+Hm5gYzMzMpXR9Xqq5rqvrdAk/+/c0WIiIi0muNGzfGhYuJdXovM3o8BkRERKT3GjduzCBFz3FQNREREek9jQZEBw4cQL9+/eDs7FxhX5Xy3n//fSgUCixevFiWnpeXh3HjxsHBwQGWlpbo37+/bHEnoGSJ85CQECiVSiiVSoSEhCAzM7PmK0REREQ6SaMB0f3799G6dWssXbr0kfm2bduGo0ePwtnZucK5sLAwbN26FZs2bcKhQ4eQk5ODwMBA2VoIwcHBSEhIQFRUFKKiopCQkICQkJAarw8RERHpJo2OIQoICEBAQMAj89y4cQNjx47F7t270bdvX9k5tVqNFStWYN26dejRowcAYP369XBxcUFMTAx69eqFCxcuICoqCkeOHEHHjh0BAMuXL0enTp2QmJgIDw+P51M5IiIi0hlaPYaouLgYISEhmDx5cqXT0OPj41FQUAB/f38pzdnZGd7e3oiLiwMAHD58GEqlUgqGAODll1+GUqmU8lQmLy8PWVlZshcRERHVTVodEM2fPx9GRkYYP358pefT0tJgYmICW1tbWbqTkxPS0tKkPI6OjhWudXR0lPJUZt68edKYI6VSCRcXl2eoCREREWkzrQ2I4uPjsWTJEqxevbrCRnCPU37zuMqur2yDubKmT58OtVotva5du/ZUZSAiIiLdobXrEB08eBDp6emydSGKioowadIkLF68GFeuXIFKpUJ+fj4yMjJkrUTp6eno3LkzAEClUuHWrVsV7n/79m04OTlV+XxTU1OYmprWYI2IiEhbpaSkcGFGPae1AVFISIg0ULpUr169EBISgn/+858AgHbt2sHY2BjR0dEICgoCAKSmpuLs2bNYsGABAKBTp05Qq9U4duwYXnrpJQDA0aNHoVarpaCJiIj0V0pKClp4tkDug9xaeZ65hTkuXrj4VEGRn58f2rRpU2HpmeoKDQ1FZmbmI5e70TcaDYhycnKQlJQkHScnJyMhIQF2dnZo3Lgx7O3tZfmNjY2hUqmkmWFKpRIjRozApEmTYG9vDzs7O4SHh8PHx0cKpjw9PdG7d2+MHDkS3333HQDgvffeQ2BgIGeYERER7ty5g9wHueg6pSuULsrn+iz1NTUOLDiAO3fusJVIy2g0IDpx4gS6desmHU+cOBEAMGzYMKxevfqJ7rFo0SIYGRkhKCgIubm56N69O1avXg1DQ0Mpz4YNGzB+/HhpNlr//v0fu/YRERHpF6WLEg7uDpouRgWhoaGIjY1FbGwslixZAqCkAeHBgwcIDw/HgQMHYGlpCX9/fyxatAgODiV1+O9//4vZs2cjKSkJFhYWaNu2LX755Rd88cUXWLNmDYC/x9ju27cPfn5+GqmfttBoQOTn5wchxBPnv3LlSoU0MzMzREZGIjIyssrr7OzssH79+uoUkYiISKOWLFmCS5cuwdvbG3PmzAFQMqbW19cXI0eOxMKFC5Gbm4upU6ciKCgIe/fuRWpqKoYMGYIFCxZg4MCByM7OxsGDByGEQHh4OC5cuICsrCysWrUKQMn3pL7T2jFEREREVDI8xMTEBBYWFlCpVACAmTNn4sUXX8TcuXOlfCtXroSLiwsuXbqEnJwcFBYWYtCgQWjSpAkAwMfHR8prbm6OvLw86X7EgIiIiEjnxMfHY9++fbCysqpw7vLly/D390f37t3h4+ODXr16wd/fH2+88UaFdfvob1q7DhERERFVrri4GP369UNCQoLs9eeff6Jr164wNDREdHQ0fvvtN3h5eSEyMhIeHh5ITk7WdNG1FgMiIiIiLWdiYiLbtPzFF1/EuXPn4OrqimbNmslelpaWAEoGTHfp0gWzZ8/GqVOnYGJigq1bt1Z6P2KXGREREYCSKfHa+gxXV1ccPXoUV65cgZWVFcaMGYPly5djyJAhmDx5MhwcHJCUlIRNmzZh+fLlOHHiBH7//Xf4+/vD0dERR48exe3bt+Hp6Sndb/fu3UhMTIS9vT2USiWMjY1rsqo6hwERERHpNQcHB5hbmOPAggO18jxzC3NpavyTCg8Px7Bhw+Dl5YXc3FwkJyfjf//7H6ZOnYpevXohLy8PTZo0Qe/evWFgYAAbGxscOHAAixcvRlZWFpo0aYKvvvoKAQEBAICRI0di//79aN++PXJycjjtHgyIiIhIzzVu3BgXL1zU6q07mjdvjsOHD1dI37JlS6X5PT09ERUVVeX96tevjz179jxVGeo6BkRERKT3GjduzJWj9RwHVRMREZHeY0BEREREeo8BEREREek9jiGqAwoLC5GYmCgde3h4wMiIv1oioso8zR6apBtq4nfKb806IDExEaO+3gErx0bISb+OZWOAli1barpYRERapXSdnQcPHsDc3FzDpaGa9ODBAwB4prWUGBDVEVaOjaB0bqrpYhARaS1DQ0PUq1cP6enpAAALCwsoFAoNl4qehRACDx48QHp6OurVqwdDQ8Nq34sBERER6Y3S3d1LgyKqG+rVqyf9bquLAREREekNhUKBBg0awNHREQUFBZouDtUAY2PjZ2oZKsWAiIiI9I6hoWGNfIlS3cFp90RERKT3GBARERGR3mNARERERHqPARERERHpPQZEREREpPcYEBEREZHeY0BEREREeo8BEREREek9BkRERESk9xgQERERkd5jQERERER6jwERERER6T0GRERERKT3GBARERGR3tNoQHTgwAH069cPzs7OUCgU2LZtm3SuoKAAU6dOhY+PDywtLeHs7Ix33nkHN2/elN0jLy8P48aNg4ODAywtLdG/f39cv35dlicjIwMhISFQKpVQKpUICQlBZmZmLdSQiIiIdIFGA6L79++jdevWWLp0aYVzDx48wMmTJzFjxgycPHkSW7ZswaVLl9C/f39ZvrCwMGzduhWbNm3CoUOHkJOTg8DAQBQVFUl5goODkZCQgKioKERFRSEhIQEhISHPvX5ERESkG4w0+fCAgAAEBARUek6pVCI6OlqWFhkZiZdeegkpKSlo3Lgx1Go1VqxYgXXr1qFHjx4AgPXr18PFxQUxMTHo1asXLly4gKioKBw5cgQdO3YEACxfvhydOnVCYmIiPDw8nm8liYiISOvp1BgitVoNhUKBevXqAQDi4+NRUFAAf39/KY+zszO8vb0RFxcHADh8+DCUSqUUDAHAyy+/DKVSKeUhIiIi/abRFqKn8fDhQ0ybNg3BwcGwsbEBAKSlpcHExAS2trayvE5OTkhLS5PyODo6Vrifo6OjlKcyeXl5yMvLk46zsrJqohpERESkhXSihaigoACDBw9GcXExvvnmm8fmF0JAoVBIx2X/XVWe8ubNmycNwlYqlXBxcale4YmIiEjraX1AVFBQgKCgICQnJyM6OlpqHQIAlUqF/Px8ZGRkyK5JT0+Hk5OTlOfWrVsV7nv79m0pT2WmT58OtVotva5du1ZDNSIiIiJto9UBUWkw9OeffyImJgb29vay8+3atYOxsbFs8HVqairOnj2Lzp07AwA6deoEtVqNY8eOSXmOHj0KtVot5amMqakpbGxsZC8iIiKqmzQ6hignJwdJSUnScXJyMhISEmBnZwdnZ2e88cYbOHnyJHbs2IGioiJpzI+dnR1MTEygVCoxYsQITJo0Cfb29rCzs0N4eDh8fHykWWeenp7o3bs3Ro4cie+++w4A8N577yEwMJAzzIiIiAiAhgOiEydOoFu3btLxxIkTAQDDhg1DREQEtm/fDgBo06aN7Lp9+/bBz88PALBo0SIYGRkhKCgIubm56N69O1avXg1DQ0Mp/4YNGzB+/HhpNlr//v0rXfuIiIiI9JNGAyI/Pz8IIao8/6hzpczMzBAZGYnIyMgq89jZ2WH9+vXVKiMRERHVfVo9hoiIiIioNjAgIiIiIr3HgIiIiIj0HgMiIiIi0nsMiIiIiEjvMSAiIiIivceAiIiIiPQeAyIiIiLSewyIiIiISO9pdKVq0k6FhYVITEyUjj08PGBkxLcKERHVXfyWowoSExMx6usdsHJshJz061g2BmjZsqWmi0VERPTcMCCiSlk5NoLSuammi0FERFQrOIaIiIiI9B4DIiIiItJ7DIiIiIhI7zEgIiIiIr3HgIiIiIj0HgMiIiIi0nsMiIiIiEjvMSAiIiIivceAiIiIiPQeAyIiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7DIiIiIhI7zEgIiIiIr3HgIiIiIj0HgMiIiIi0nsMiIiIiEjvMSAiIiIivafRgOjAgQPo168fnJ2doVAosG3bNtl5IQQiIiLg7OwMc3Nz+Pn54dy5c7I8eXl5GDduHBwcHGBpaYn+/fvj+vXrsjwZGRkICQmBUqmEUqlESEgIMjMzn3PtiIiISFdoNCC6f/8+WrdujaVLl1Z6fsGCBVi4cCGWLl2K48ePQ6VSoWfPnsjOzpbyhIWFYevWrdi0aRMOHTqEnJwcBAYGoqioSMoTHByMhIQEREVFISoqCgkJCQgJCXnu9SMiIiLdYKTJhwcEBCAgIKDSc0IILF68GB9//DEGDRoEAFizZg2cnJywceNGvP/++1Cr1VixYgXWrVuHHj16AADWr18PFxcXxMTEoFevXrhw4QKioqJw5MgRdOzYEQCwfPlydOrUCYmJifDw8KidyhIREZHW0toxRMnJyUhLS4O/v7+UZmpqCl9fX8TFxQEA4uPjUVBQIMvj7OwMb29vKc/hw4ehVCqlYAgAXn75ZSiVSilPZfLy8pCVlSV7ERERUd2ktQFRWloaAMDJyUmW7uTkJJ1LS0uDiYkJbG1tH5nH0dGxwv0dHR2lPJWZN2+eNOZIqVTCxcXlmepDRERE2ktrA6JSCoVCdiyEqJBWXvk8leV/3H2mT58OtVotva5du/aUJSciIiJdobUBkUqlAoAKrTjp6elSq5FKpUJ+fj4yMjIemefWrVsV7n/79u0KrU9lmZqawsbGRvYiIiKiuklrAyI3NzeoVCpER0dLafn5+YiNjUXnzp0BAO3atYOxsbEsT2pqKs6ePSvl6dSpE9RqNY4dOyblOXr0KNRqtZSHiIiI9JtGZ5nl5OQgKSlJOk5OTkZCQgLs7OzQuHFjhIWFYe7cuXB3d4e7uzvmzp0LCwsLBAcHAwCUSiVGjBiBSZMmwd7eHnZ2dggPD4ePj48068zT0xO9e/fGyJEj8d133wEA3nvvPQQGBnKGGREREQHQcEB04sQJdOvWTTqeOHEiAGDYsGFYvXo1pkyZgtzcXIwePRoZGRno2LEj9uzZA2tra+maRYsWwcjICEFBQcjNzUX37t2xevVqGBoaSnk2bNiA8ePHS7PR+vfvX+XaR0RERKR/NBoQ+fn5QQhR5XmFQoGIiAhERERUmcfMzAyRkZGIjIysMo+dnR3Wr1//LEUlIiKiOkxrxxARERER1ZZqBURNmzbF3bt3K6RnZmaiadOmz1woIiIiotpUrYDoypUrsr3CSuXl5eHGjRvPXCgiIiKi2vRUY4i2b98u/Xv37t1QKpXScVFREX7//Xe4urrWWOGIiIiIasNTBUSvv/46gJLBzsOGDZOdMzY2hqurK7766qsaKxwRERFRbXiqgKi4uBhAyaKJx48fh4ODw3MpFBEREVFtqta0++Tk5JouBxEREZHGVHsdot9//x2///470tPTpZajUitXrnzmghERERHVlmoFRLNnz8acOXPQvn17NGjQ4LG7zxMRERFps2oFRMuWLcPq1asREhJS0+UhIiIiqnXVWocoPz+fO8UTERFRnVGtgOjdd9/Fxo0ba7osRERERBpRrS6zhw8f4vvvv0dMTAxatWoFY2Nj2fmFCxfWSOGIiIiIakO1AqIzZ86gTZs2AICzZ8/KznGANREREemaagVE+/btq+lyEBEREWlMtcYQEREREdUl1Woh6tat2yO7xvbu3VvtAhERERHVtmoFRKXjh0oVFBQgISEBZ8+erbDpKxEREZG2q1ZAtGjRokrTIyIikJOT80wFIiIiIqptNTqG6O233+Y+ZkRERKRzajQgOnz4MMzMzGrylkRERETPXbW6zAYNGiQ7FkIgNTUVJ06cwIwZM2qkYERERES1pVoBkVKplB0bGBjAw8MDc+bMgb+/f40UjIiIiKi2VCsgWrVqVU2Xg4iIiEhjqhUQlYqPj8eFCxegUCjg5eWFtm3b1lS5iIiIiGpNtQKi9PR0DB48GPv370e9evUghIBarUa3bt2wadMm1K9fv6bLSURERPTcVGuW2bhx45CVlYVz587h3r17yMjIwNmzZ5GVlYXx48fXdBmJiIiInqtqtRBFRUUhJiYGnp6eUpqXlxe+/vprDqomIiIinVOtFqLi4mIYGxtXSDc2NkZxcfEzF4qIiIioNlUrIHrttdfw4Ycf4ubNm1LajRs3MGHCBHTv3r3GCkdERERUG6oVEC1duhTZ2dlwdXXFCy+8gGbNmsHNzQ3Z2dmIjIys6TISERERPVfVCohcXFxw8uRJ7Ny5E2FhYRg/fjx27dqF+Ph4NGrUqMYKV1hYiE8++QRubm4wNzdH06ZNMWfOHFm3nBACERERcHZ2hrm5Ofz8/HDu3DnZffLy8jBu3Dg4ODjA0tIS/fv3x/Xr12usnERERKTbniog2rt3L7y8vJCVlQUA6NmzJ8aNG4fx48ejQ4cOaNmyJQ4ePFhjhZs/fz6WLVuGpUuX4sKFC1iwYAG++OILWSvUggULsHDhQixduhTHjx+HSqVCz549kZ2dLeUJCwvD1q1bsWnTJhw6dAg5OTkIDAxEUVFRjZWViIiIdNdTBUSLFy/GyJEjYWNjU+GcUqnE+++/j4ULF9ZY4Q4fPowBAwagb9++cHV1xRtvvAF/f3+cOHECQEnr0OLFi/Hxxx9j0KBB8Pb2xpo1a/DgwQNs3LgRAKBWq7FixQp89dVX6NGjB9q2bYv169fjjz/+QExMTI2VlYiIiHTXUwVEp0+fRu/evas87+/vj/j4+GcuVKlXXnkFv//+Oy5duiQ9/9ChQ+jTpw8AIDk5GWlpabKp/qampvD19UVcXByAktW0CwoKZHmcnZ3h7e0t5alMXl4esrKyZC8iIiKqm55qHaJbt25VOt1eupmREW7fvv3MhSo1depUqNVqtGjRAoaGhigqKsLnn3+OIUOGAADS0tIAAE5OTrLrnJyccPXqVSmPiYkJbG1tK+Qpvb4y8+bNw+zZs2usLkRERKS9nqqFqGHDhvjjjz+qPH/mzBk0aNDgmQtVavPmzVi/fj02btyIkydPYs2aNfjyyy+xZs0aWT6FQiE7FkJUSCvvcXmmT58OtVotva5du1b9ihAREZFWe6qAqE+fPpg5cyYePnxY4Vxubi5mzZqFwMDAGivc5MmTMW3aNAwePBg+Pj4ICQnBhAkTMG/ePACASqUCgAotPenp6VKrkUqlQn5+PjIyMqrMUxlTU1PY2NjIXkRERFQ3PVVA9Mknn+DevXto3rw5FixYgF9++QXbt2/H/Pnz4eHhgXv37uHjjz+uscI9ePAABgbyIhoaGkrT7t3c3KBSqRAdHS2dz8/PR2xsLDp37gwAaNeuHYyNjWV5UlNTcfbsWSmPNikuLkJSUhLOnTuHc+fOobCwUNNFIiIiqvOeagyRk5MT4uLi8MEHH2D69OkQQgAo6bLq1asXvvnmm0e2ujytfv364fPPP0fjxo3RsmVLnDp1CgsXLsTw4cOl54aFhWHu3Llwd3eHu7s75s6dCwsLCwQHBwMomf02YsQITJo0Cfb29rCzs0N4eDh8fHzQo0ePGitrTbl/JxWfbb8CB5cc5KRfx7IxQMuWLTVdLCIiojrtqTd3bdKkCXbt2oWMjAwkJSVBCAF3d/cKg5ZrQmRkJGbMmIHRo0cjPT0dzs7OeP/99zFz5kwpz5QpU5Cbm4vRo0cjIyMDHTt2xJ49e2BtbS3lWbRoEYyMjBAUFITc3Fx0794dq1evhqGhYY2XuSZYOjSE0rmppotBRESkN6q12z0A2NraokOHDjVZlgqsra2xePFiLF68uMo8CoUCERERiIiIqDKPmZkZIiMjua0IERERVapaW3cQERER1SUMiIiIiEjvMSAiIiIivceAiIiIiPQeAyIiIiLSewyIiIiISO9Ve9o90ZMqLCxEYmKidOzh4QEjI771iIhIe/BbiZ67xMREjPp6B6wcG3H1bSIi0koMiKhWWDk24urbRESktTiGiIiIiPQeAyIiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7DIiIiIhI7zEgIiIiIr3HgIiIiIj0HgMiIiIi0nvcukNHld0wNSkpCUIIDZeIiIhIdzEg0lFlN0y9dTEeNk24WSoREVF1sctMh5VumGph56TpohAREek0BkRERESk9xgQERERkd7jGCI9UnYgNgB4eHjAyIhvASIiIn4b6pGyA7Fz0q9j2RigZUsOxiYiImJAVMcUFxchKSlJlla2Jah0IDYRERH9jQFRHXP/Tio+234FDi45AIDstKuYHNASzZo143pFREREVWBAVAdZOjSUWoGy06/js+2n4eCSw/WKiIiIqsBZZnqgNEDiekVERESVYwuRjig/Q4zdX0RERDVH61uIbty4gbfffhv29vawsLBAmzZtEB8fL50XQiAiIgLOzs4wNzeHn58fzp07J7tHXl4exo0bBwcHB1haWqJ///64fv16bVflmZTOEAv/KQHhPyVg9n9i8fBhXrXvVzr4+ty5c9KrsLCwBktMRESkO7Q6IMrIyECXLl1gbGyM3377DefPn8dXX32FevXqSXkWLFiAhQsXYunSpTh+/DhUKhV69uyJ7OxsKU9YWBi2bt2KTZs24dChQ8jJyUFgYCCKioo0UKvqK50hVhPdXyWDr09LAdaor3fIWqCIiIj0iVZ3mc2fPx8uLi5YtWqVlObq6ir9WwiBxYsX4+OPP8agQYMAAGvWrIGTkxM2btyI999/H2q1GitWrMC6devQo0cPAMD69evh4uKCmJgY9OrVq1brpE3KDr4mIiLSZ1rdQrR9+3a0b98eb775JhwdHdG2bVssX75cOp+cnIy0tDT4+/tLaaampvD19UVcXBwAID4+HgUFBbI8zs7O8Pb2lvJUJi8vD1lZWbIXERER1U1aHRD99ddf+Pbbb+Hu7o7du3dj1KhRGD9+PNauXQsASEtLAwA4Ocm7j5ycnKRzaWlpMDExga2tbZV5KjNv3jwolUrp5eLiUpNVIyIiIi2i1QFRcXExXnzxRcydOxdt27bF+++/j5EjR+Lbb7+V5VMoFLJjIUSFtPIel2f69OlQq9XS69q1a9WvCBEREWk1rQ6IGjRoAC8vL1map6cnUlJSAAAqlQoAKrT0pKenS61GKpUK+fn5yMjIqDJPZUxNTWFjYyN7ERERUd2k1QFRly5dKsx8unTpEpo0aQIAcHNzg0qlQnR0tHQ+Pz8fsbGx6Ny5MwCgXbt2MDY2luVJTU3F2bNnpTxERESk37R6ltmECRPQuXNnzJ07F0FBQTh27Bi+//57fP/99wBKusrCwsIwd+5cuLu7w93dHXPnzoWFhQWCg4MBAEqlEiNGjMCkSZNgb28POzs7hIeHw8fHR5p1RkRERPpNqwOiDh06YOvWrZg+fTrmzJkDNzc3LF68GEOHDpXyTJkyBbm5uRg9ejQyMjLQsWNH7NmzB9bW1lKeRYsWwcjICEFBQcjNzUX37t2xevVqGBoaaqJaREREpGW0OiACgMDAQAQGBlZ5XqFQICIiAhEREVXmMTMzQ2RkJCIjI59DCeuG0pWrAW4LQkRE+kfrAyJ9VptBSsnK1Vfg4JKDWxfjYdOk5XN7FhERkbZhQKTFajtIKV25Ojtdt/Z5IyIielZaPcuM/g5SnnXvMiIiIqoaAyIiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7nHavYYWFhbhz7TJycx8CADLTrsHeyl7DpSIiItIvDIi0wDsPN6NxrjkA4GheBv6HNpotEBERkZ5hQKRhRkZGeM3LAS0b/r33WpxCocESERER6R8GRPRIZbcPKeXh4QEjI751iIio7uC3Gj1S2e1DACAn/TqWjQFatuReZ0REVHcwIKLHKt0+hIiIqK7itHsiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7DIiIiIhI73HaPT2V8gs1cpFGIiKqC/hNRk+l7EKNXKSRiIjqCgZE9NS4UCMREdU1DIio2th9RkREdQW/veoCIZBx4y8AQGbaNdhb2dfKY9l9RkREdQUDojrA0MoOfVOWo6OdLY7mZeB/aFNrz2b3GRER1QUMiOoAhYEhOr5giwEvqgAAcQqFhktERESkWxgQUY0oP54I4JgiIiLSHfy2ohpRdjwRAGSnXcXkgJZo1qwZkpKSIISosWcVFhYiMTFROmbgRUREz4rfIlRjyo4nyk6/js+2n4aDSw5uXYyHTZOaG2ydmJiIUV/vgJVjIw7mJiKiGsGAiJ6b0gApO/16jd/byrERB3MTEVGN0am9zObNmweFQoGwsDApTQiBiIgIODs7w9zcHH5+fjh37pzsury8PIwbNw4ODg6wtLRE//79cf16zX9JExERkW7SmYDo+PHj+P7779GqVStZ+oIFC7Bw4UIsXboUx48fh0qlQs+ePZGdnS3lCQsLw9atW7Fp0yYcOnQIOTk5CAwMRFFRUW1Xg4iIiLSQTgREOTk5GDp0KJYvXw5bW1spXQiBxYsX4+OPP8agQYPg7e2NNWvW4MGDB9i4cSMAQK1WY8WKFfjqq6/Qo0cPtG3bFuvXr8cff/yBmJgYTVWJiIiItIhOBERjxoxB37590aNHD1l6cnIy0tLS4O/vL6WZmprC19cXcXFxAID4+HgUFBTI8jg7O8Pb21vKU5m8vDxkZWXJXkRERFQ3af2g6k2bNuHkyZM4fvx4hXNpaWkAACcnJ1m6k5MTrl69KuUxMTGRtSyV5im9vjLz5s3D7Nmzn7X4REREpAO0uoXo2rVr+PDDD7F+/XqYmZlVmU9RbmVmIUSFtPIel2f69OlQq9XS69q1a09XeCIiItIZWh0QxcfHIz09He3atYORkRGMjIwQGxuLf//73zAyMpJahsq39KSnp0vnVCoV8vPzkZGRUWWeypiamsLGxkb2IiIiorpJqwOi7t27448//kBCQoL0at++PYYOHYqEhAQ0bdoUKpUK0dHR0jX5+fmIjY1F586dAQDt2rWDsbGxLE9qairOnj0r5SEiIiL9ptVjiKytreHt7S1Ls7S0hL29vZQeFhaGuXPnwt3dHe7u7pg7dy4sLCwQHBwMAFAqlRgxYgQmTZoEe3t72NnZITw8HD4+PhUGaRMREZF+0uqA6ElMmTIFubm5GD16NDIyMtCxY0fs2bMH1tbWUp5FixbByMgIQUFByM3NRffu3bF69WoYGhpqsORERESkLXQuINq/f7/sWKFQICIiAhEREVVeY2ZmhsjISERGRj7fwtFjFRcXISkpSZbGzVmJiEjT+C1Eter+nVR8tv0KHFxyAICbsxIRkVZgQES1rnTTVyIiIm2h1bPMiIiIiGoDAyIiIiLSewyIiIiISO8xICIiIiK9x4CIiIiI9B4DIiIiItJ7DIiIiIhI7zEgIiIiIr3HhRlJo8pv5cFtPIiISBP4zUMaVXYrD27jQUREmsKAiDSOW3kQEZGmcQwRERER6T0GRERERKT3GBARERGR3uMYIg0rLCzExZR7eJCbCwBIvJmJe2Z/AQAy067B3spek8UjIiLSCwyItMBCQwtYGVoBAG4ZPMCwh5vRPLcejuZl4H9oo9nCERER6QEGRBpmZGQE59bOsHW1ldL8irLRoZkjACBOodBU0YiIiPQGxxARERGR3mNARERERHqPXWaklQoLC5GYmChL47YeRET0vPDbhbRG2X3NkpKS8EXURVg7NQIAbutBRETPFQMi0hpl9zW7dTEeNk1acksPIiKqFRxDRFqldF8zCzsnTReFiIj0CAMiIiIi0nvsMqOKhEDGDa6WTURE+oMBUR0ghMCfqWocTzLAn6lqCDfxTPcztLJD35Tl6Ghny9WyiYhILzAgqiPWGJrhN0NrpBvmwu4Z76UwMETHF2wx4EUVAK6WTUREdR8DIl1VtlvrZjIcmjmgSecmAICihwxgiIiIngYDIh1Vtlvrzzw1dioaabpIREREOkurZ5nNmzcPHTp0gLW1NRwdHfH6669XWL1YCIGIiAg4OzvD3Nwcfn5+OHfunCxPXl4exo0bBwcHB1haWqJ///64fv16bValxpXt1vL1sIWBgVb/KomIiLSaVn+LxsbGYsyYMThy5Aiio6NRWFgIf39/3L9/X8qzYMECLFy4EEuXLsXx48ehUqnQs2dPZGdnS3nCwsKwdetWbNq0CYcOHUJOTg4CAwNRVFSkiWoRERGRltHqLrOoqCjZ8apVq+Do6Ij4+Hh07doVQggsXrwYH3/8MQYNGgQAWLNmDZycnLBx40a8//77UKvVWLFiBdatW4cePXoAANavXw8XFxfExMSgV69etV4venrlt/UQ4tlm0hEREZWl1S1E5anVagCAnV3JPKrk5GSkpaXB399fymNqagpfX1/ExcUBAOLj41FQUCDL4+zsDG9vbymPTvj/QdRpl88j7fJ5ZNz4C8V6FBSUbOtxGuE/JWD2f2Lx8GGepotERER1iFa3EJUlhMDEiRPxyiuvwNvbGwCQlpYGAHBykm/z4OTkhKtXr0p5TExMYGtrWyFP6fWVycvLQ17e31+6WVlZNVKP6io7iBoAjmZkQNX4WSfY65bSbT2y03V7/BcREWkfnQmIxo4dizNnzuDQoUMVzinKrZMjhKiQVt7j8sybNw+zZ8+uXmFrSGFxman1qVfRoWk9aW0gADAy4PR6IiKimqATAdG4ceOwfft2HDhwAI0a/T29XKUqCQ7S0tLQoEEDKT09PV1qNVKpVMjPz0dGRoaslSg9PR2dO3eu8pnTp0/HxIkTpeOsrCy4uLjUWJ2ehAJASN6PcM9V6mWLUG0pLCysMHvRw8MDRkY68edBREQ1QKvHEAkhMHbsWGzZsgV79+6Fm5ub7LybmxtUKhWio6OltPz8fMTGxkrBTrt27WBsbCzLk5qairNnzz4yIDI1NYWNjY3sVdsMDRTw9SiZWt/xBVu2CD0niYmJGPX1DoT/lIDwnxIw6usdFQIkIiKq27T6v8BjxozBxo0b8csvv8Da2loa86NUKmFubg6FQoGwsDDMnTsX7u7ucHd3x9y5c2FhYYHg4GAp74gRIzBp0iTY29vDzs4O4eHh8PHxkWad1SXFRcVSNxvAzVmflJVjIyidm2q6GEREpCFaHRB9++23AAA/Pz9Z+qpVqxAaGgoAmDJlCnJzczF69GhkZGSgY8eO2LNnD6ytraX8ixYtgpGREYKCgpCbm4vu3btj9erVMDQ0rK2q1B7F391sALg5KxER0RPQ6oDoSdaaUSgUiIiIQERERJV5zMzMEBkZicjIyBosnXYyMDCAr4ctOjRzlNL0aXPW8uOBOBaIiIieBL8pSKeVXbARKFm08Yuoi7B2aoSc9OtYNgZo2bKlBktIRES6gAFRHVd26n5G6jUY/v96SnVlbFHJgo1X4OCSAwC4dTEeNk1acjwQERE9FQZEWqa4qBgXrmcCAP5MVcPZut4z3a/s1P3DD+/BztAAHrn16tTYotIFGwFw0UYiIqoWBkTaRgEsNrSCraE10g1z4fuMtyudul86psjZxkD6tz6NLSIiInoUBkRaxsDAAI4+jnD2dC5JKMqunQeLMl1rN/5Csa3+7JNGRETEgEhPlR1bBABZajX6qH9EJzvbOrsqNmegERFRVfhtoKfKji0CgKPqDPRrbYdOzUu2PKkLq2JzBhoRET0pBkR6qvzYIqBuBEFlcQYaERE9KQZEVKdVNQOtbOtRUlLSEy0CSkREdRcDIqqgWAjZ1H/h9ozBQpkB29qy/lHZ1qPSliMiItJfDIh0VNmg5cL1TIgmVjV2b5W1AWbeNoXj/0/9f9bh1YZWduibshwd7Wy1av2j0tajml67qPzgbYADuImItB0/oXWULGh5mIsGNjX3qzQyUMDRwxFNOjcBABQ9fLaxRQoDQ3R8wRYDXlQBqPvrHyUmJmLU1ztg5dgIADiAm4hIBzAg0lHlgxYDQwMNl4jKsnJsxMHbREQ6hAGRjijbRQbUfDdZlc8tKpatV6QtY4CIiIhqEgMiLVZ2X7NMdRYW5pnB0dAaAGq8m6xKinLrFWnRGKDnpfz6RRz/Q0RU9/FTXpuV3dcsPxcNXmqARt6NpNO10U1mYGBQYb2iuj4GqOwMNI7/ISLSDwyItFj5fc2qEwCVn0LvbF3vmcpUdssPWfeZEHWqa63s+kVERFT3MSCq48pPofd9xvuV3fKjbPdZ2an1QN3sWuN0eiKiuouf5HVM2XFHAPDnTTXqezRCk85N/v/cDUChqHZrUdktPwqLBXbeTIZCoUBm6lV0aFpPmloP1I2utfIrWpfuhQZwOj0RUV3CgKiuKTPuCCg3+LrsmKRyrUXVWehR1lqUkQFV42ddwlH7VLaiNbvSiIjqHgZEdUz5cUfA32OPKpwrypbyVGehx/IbxJbdHLbsWKOMG3+h2LYG9wqr5a1AqlrRmrPRiIjqDn56E4CaX+jxebYeactWIJyNRkRUdzAg0lPlxxrV9EKPZVuPioXAnzfVMDJQ1Mhmsdq0FUhp61H5sUZC1GCLGBERPXcMiPTVo8Ya1TBZd5wiF4r/H4gNABmp12CYlQVAt6fqVzbWqKZwdhsR0fPHT1Q99aixRmVVmLVWjdlpZbvjiouL0ffi3ytfH354D3aGBvDIrYe4hxn47aayZNaaDgZHVY01elbcLJaI6PljQESPVr4lSZELxxvqak/dr2zla2cbA3Ro5oiiYoH6tyqucUTcLJaI6HljQESPVL4lqbi4GItvqkum7pcJji7dzERm1t8tTNUJlqpa4wiQd6cJIfBnqhrHkwxqZEzS81Z2fFFhYSEASN1d7PoiItIO/CSmp1I2QCobHN0yeABjA2PYlbYkPeOq2GVnqQEVV75eY2iG3/5/PaVnnr/2nLcdKT++yNCyHhxcXnhk11fZcUMcpE1E9PwxIKJqK996ZGxr/HdLUplVsaszg638GkeyVbFvJsOhmYO0+va9K882SLs2th0pO77IyNqhwsy0UqUtRmXHDdX0IG0iIqqIARE9H2VXxX7EDLayK2SX7XYr3wWXlKrG24rNaJ5bD3/mqbFT0Uh6TtmWpKoGaQNVb0arqW1HyrYcAUB22lVMDmiJZs2aISkpCZb1Hz9Iu/wMNHbBERFVDz856bko33pU1UKPZafkl+12q9AFZ5yLlS+YoYO7E44nGeA3g79X336SQdoAqtyMtsa3HXmKLrjSliMAyE6/js+2n36qqftlW5KetAsOYOBERFSeXn0ifvPNN/jiiy+QmpqKli1bYvHixXj11Vc1XSy98qjNZwF5t9uzdsE9stutXKtQ2W1HntWzdME9ydT98otAlrYklVd+HFLpxrSctk9EVJHeBESbN29GWFgYvvnmG3Tp0gXfffcdAgICcP78eTRu3FjTxdMf1V0QsoouuPIBVtmutvIz3araTqRstx0AJN7MxD2zZ9grTWEAOwsDONuUlMPO4tm2QSnf4nTnz1P49Loj6jeWtySVH5NUNggquzHto8YulfWoBSGftMWJLVNEpCv05pNp4cKFGDFiBN59910AwOLFi7F79258++23mDdvnoZLpz+edEHIx10nXVMuwCrb1VZ2WQAAuJymhqNVSaBia67Apf/fTiRTnYWFeWZwLL0HHsD32ko0fGCBszfu4w/zknFIZQdsA/IB3LJ/30zGGkXJLDgAstW5y19TfPcuHj58WHKcchkqr4rBV6UtTg7TK7QklR+TVDYIelS+smOXyi4LUDagAuQLQpbtqit7PYAq71G+ZapssFTTyxEwECOip6UXnxD5+fmIj4/HtGnTZOn+/v6Ii4vTUKmoJlQWYJV2tZVdFgBAlWOU0vNz0eClBmjkXfLFX1xcjMNKE9g2ssUtg1vwvb4SDXMtcPZGDmxMFWj8wBIAZMdl/33j3gOce9VD6gYsLCiEd1xpgPV3vmv37iPa1ArCriQIKi64h4yb9aXAqTRYUqddha25ovIWJyGQfuUSHj58iHspl2FWrz5yc0sCrLy8fNy6kljmXJlAzMQChhmZJcfXr+DTXwqlFqfSZQHKBlQlP5e/W5YuXryI3Id5MMx9iIxbNxG+LAk2jg0BAJkpF2FgZg0bx4bITElEow7+lbZMlW/BKn3uowKsqv5dmi8pKQlGRkZISUnB5vMPnqqL8HlvkfKoAPBJn1XTgR4DR6K/6cU7/86dOygqKoKTk5Ms3cnJCWlpaZVek5eXh7y8POlYrVYDALLKtBDUhJycHNy7cg8FuQUlz7mhhlGWEYxgVOW/me/J8mWnZsPExgTmtuYAADMbMxhZG8Hc1lz2b1MrU2Rdz0K6YToAyK4ztTHFngdWsLGxQYaFgIG5AZQ2JYO0yx6X/XdmJuCQlYv0CyX3y0nLwR6LivfIzAQcPO1R360+AKAwrxAvXvoBDdQWuHHvPnbnmUDx0AbZN7LwnYMZNl0vAgBk3i2GYdEfuJ+Viay0m3hwcwvw0Ab3b6nxyr1CNFJbAIDsHmXPlU0HgPvpWRAm/wBMLJCVcQ+G+cUQxiX/vp//B3LUGSXPTT6L8QfzYGnvhJzbqeh0fz88GljgYup9WJko0Mio5LkXs+/DKq/kODH7AU4lNcD9rEzZ9QCQdSMJlg2a4aEwlD03M/kSxi/+Q5bP0MwalvZOVf4bAO7fvYWXs/fCo4EFElMf4HbjIXgoDPEw4zZWrlxZ4e+/vFu3bmHbmVswVToAAPLUd/B6K6fHXvekyt6/fNmf9Fll71ET5avp+xE9i1GjRj2X+5Z+bz92PTehB27cuCEAiLi4OFn6Z599Jjw8PCq9ZtasWQIAX3zxxRdffPFVB17Xrl17ZKygFy1EDg4OMDQ0rNAalJ6eXuX/hqZPn46JEydKx8XFxbh37x7s7e2ldW2qKysrCy4uLrh27RpsbGye6V7aTB/qyTrWDaxj3cA61g01XUchBLKzs+Hs7PzIfHoREJmYmKBdu3aIjo7GwIEDpfTo6GgMGDCg0mtMTU1hamoqS6tXr16NlsvGxqbOvqHL0od6so51A+tYN7COdUNN1lGpVD42j14ERAAwceJEhISEoH379ujUqRO+//57pKSkPLc+SyIiItIdehMQvfXWW7h79y7mzJmD1NRUeHt7Y9euXWjSpImmi0ZEREQapjcBEQCMHj0ao0eP1nQxYGpqilmzZlXokqtr9KGerGPdwDrWDaxj3aCpOiqEeNw8NCIiIqK67Rn3FCAiIiLSfQyIiIiISO8xICIiIiK9x4CIiIiI9B4DIg345ptv4ObmBjMzM7Rr1w4HDx7UdJGqbd68eejQoQOsra3h6OiI119/vcIGmUIIREREwNnZGebm5vDz88O5c+c0VOJnM2/ePCgUCoSFhUlpdaV+N27cwNtvvw17e3tYWFigTZs2iI+Pl87rej0LCwvxySefwM3NDebm5mjatCnmzJmD4uJiKY+u1fHAgQPo168fnJ2doVAosG3bNtn5J6lPXl4exo0bBwcHB1haWqJ///64fv16Ldbi0R5Vx4KCAkydOhU+Pj6wtLSEs7Mz3nnnHdy8eVN2D12uY3nvv/8+FAoFFi9eLEuvC3W8cOEC+vfvD6VSCWtra7z88stISUmRzj/vOjIgqmWbN29GWFgYPv74Y5w6dQqvvvoqAgICZL90XRIbG4sxY8bgyJEjiI6ORmFhIfz9/XH//n0pz4IFC7Bw4UIsXboUx48fh0qlQs+ePZGdna3Bkj+948eP4/vvv0erVq1k6XWhfhkZGejSpQuMjY3x22+/4fz58/jqq69kq7Prej3nz5+PZcuWYenSpbhw4QIWLFiAL774ApGRkVIeXavj/fv30bp1ayxdurTS809Sn7CwMGzduhWbNm3CoUOHkJOTg8DAQBQVFdVWNR7pUXV88OABTp48iRkzZuDkyZPYsmULLl26hP79+8vy6XIdy9q2bRuOHj1a6RYUul7Hy5cv45VXXkGLFi2wf/9+nD59GjNmzICZmZmU57nX8Zl3TqWn8tJLL4lRo0bJ0lq0aCGmTZumoRLVrPT0dAFAxMbGCiGEKC4uFiqVSvzrX/+S8jx8+FAolUqxbNkyTRXzqWVnZwt3d3cRHR0tfH19xYcffiiEqDv1mzp1qnjllVeqPF8X6tm3b18xfPhwWdqgQYPE22+/LYTQ/ToCEFu3bpWOn6Q+mZmZwtjYWGzatEnKc+PGDWFgYCCioqJqrexPqnwdK3Ps2DEBQFy9elUIUXfqeP36ddGwYUNx9uxZ0aRJE7Fo0SLpXF2o41tvvSX9LVamNurIFqJalJ+fj/j4ePj7+8vS/f39ERcXp6FS1Sy1Wg0AsLOzAwAkJycjLS1NVmdTU1P4+vrqVJ3HjBmDvn37okePHrL0ulK/7du3o3379njzzTfh6OiItm3bYvny5dL5ulDPV155Bb///jsuXboEADh9+jQOHTqEPn36AKgbdSzrSeoTHx+PgoICWR5nZ2d4e3vrZJ2Bks8ghUIhtW7WhToWFxcjJCQEkydPRsuWLSuc1/U6FhcXY+fOnWjevDl69eoFR0dHdOzYUdatVht1ZEBUi+7cuYOioiI4OTnJ0p2cnJCWlqahUtUcIQQmTpyIV155Bd7e3gAg1UuX67xp0yacPHkS8+bNq3CuLtQPAP766y98++23cHd3x+7duzFq1CiMHz8ea9euBVA36jl16lQMGTIELVq0gLGxMdq2bYuwsDAMGTIEQN2oY1lPUp+0tDSYmJjA1ta2yjy65OHDh5g2bRqCg4OlTUHrQh3nz58PIyMjjB8/vtLzul7H9PR05OTk4F//+hd69+6NPXv2YODAgRg0aBBiY2MB1E4d9WrrDm2hUChkx0KICmm6aOzYsThz5gwOHTpU4Zyu1vnatWv48MMPsWfPHllfdnm6Wr9SxcXFaN++PebOnQsAaNu2Lc6dO4dvv/0W77zzjpRPl+u5efNmrF+/Hhs3bkTLli2RkJCAsLAwODs7Y9iwYVI+Xa5jZapTH12sc0FBAQYPHozi4mJ88803j82vK3WMj4/HkiVLcPLkyacur67UsXRiw4ABAzBhwgQAQJs2bRAXF4dly5bB19e3ymtrso5sIapFDg4OMDQ0rBDNpqenV/hfnK4ZN24ctm/fjn379qFRo0ZSukqlAgCdrXN8fDzS09PRrl07GBkZwcjICLGxsfj3v/8NIyMjqQ66Wr9SDRo0gJeXlyzN09NTGuyv679HAJg8eTKmTZuGwYMHw8fHByEhIZgwYYLU8lcX6ljWk9RHpVIhPz8fGRkZVebRBQUFBQgKCkJycjKio6Ol1iFA9+t48OBBpKeno3HjxtJn0NWrVzFp0iS4uroC0P06Ojg4wMjI6LGfQc+7jgyIapGJiQnatWuH6OhoWXp0dDQ6d+6soVI9GyEExo4diy1btmDv3r1wc3OTnXdzc4NKpZLVOT8/H7GxsTpR5+7du+OPP/5AQkKC9Grfvj2GDh2KhIQENG3aVKfrV6pLly4Vlku4dOkSmjRpAkD3f49AyYwkAwP5R56hoaH0v9O6UMeynqQ+7dq1g7GxsSxPamoqzp49qzN1Lg2G/vzzT8TExMDe3l52XtfrGBISgjNnzsg+g5ydnTF58mTs3r0bgO7X0cTEBB06dHjkZ1Ct1LFGhmbTE9u0aZMwNjYWK1asEOfPnxdhYWHC0tJSXLlyRdNFq5YPPvhAKJVKsX//fpGamiq9Hjx4IOX517/+JZRKpdiyZYv4448/xJAhQ0SDBg1EVlaWBktefWVnmQlRN+p37NgxYWRkJD7//HPx559/ig0bNggLCwuxfv16KY+u13PYsGGiYcOGYseOHSI5OVls2bJFODg4iClTpkh5dK2O2dnZ4tSpU+LUqVMCgFi4cKE4deqUNMPqSeozatQo0ahRIxETEyNOnjwpXnvtNdG6dWtRWFioqWrJPKqOBQUFon///qJRo0YiISFB9hmUl5cn3UOX61iZ8rPMhND9Om7ZskUYGxuL77//Xvz5558iMjJSGBoaioMHD0r3eN51ZECkAV9//bVo0qSJMDExES+++KI0RV0XAaj0tWrVKilPcXGxmDVrllCpVMLU1FR07dpV/PHHH5or9DMqHxDVlfr9+uuvwtvbW5iamooWLVqI77//XnZe1+uZlZUlPvzwQ9G4cWNhZmYmmjZtKj7++GPZF6eu1XHfvn2V/v0NGzZMCPFk9cnNzRVjx44VdnZ2wtzcXAQGBoqUlBQN1KZyj6pjcnJylZ9B+/btk+6hy3WsTGUBUV2o44oVK0SzZs2EmZmZaN26tdi2bZvsHs+7jgohhKiZtiYiIiIi3cQxRERERKT3GBARERGR3mNARERERHqPARERERHpPQZEREREpPcYEBEREZHeY0BEREREeo8BERHRU1i9ejXq1aun6WIQUQ1jQERERER6jwEREVEl8vPzNV0EIqpFDIiISCf9+uuvqFevnrRbfUJCAhQKBSZPnizlef/99zFkyBAAwM8//4yWLVvC1NQUrq6u+Oqrr2T3c3V1xWeffYbQ0FAolUqMHDkSQEkXWePGjWFhYYGBAwfi7t27sutOnz6Nbt26wdraGjY2NmjXrh1OnDjxPKtORM8BAyIi0kldu3ZFdnY2Tp06BQCIjY2Fg4MDYmNjpTz79++Hr68v4uPjERQUhMGDB+OPP/5AREQEZsyYgdWrV8vu+cUXX8Db2xvx8fGYMWMGjh49iuHDh2P06NFISEhAt27d8Nlnn8muGTp0KBo1aoTjx48jPj4e06ZNg7Gx8XOvPxHVLG7uSkQ6q127dggODsakSZMwcOBAdOjQAbNnz8adO3dw//59NGjQABcuXMCnn36K27dvY8+ePdK1U6ZMwc6dO3Hu3DkAJS1Ebdu2xdatW6U8wcHByMjIwG+//SalDR48GFFRUcjMzAQA2NjYIDIyEsOGDaudShPRc8EWIiLSWX5+fti/fz+EEDh48CAGDBgAb29vHDp0CPv27YOTkxNatGiBCxcuoEuXLrJru3Tpgj///BNFRUVSWvv27WV5Lly4gE6dOsnSyh9PnDgR7777Lnr06IF//etfuHz5cg3XkohqAwMiItJZfn5+OHjwIE6fPg0DAwN4eXnB19cXsbGxUncZAAghoFAoZNdW1jhuaWn52DzlRURE4Ny5c+jbty/27t0LLy8vWSsTEekGBkREpLNKxxEtXrwYvr6+UCgU8PX1xf79+2UBkZeXFw4dOiS7Ni4uDs2bN4ehoWGV9/fy8sKRI0dkaeWPAaB58+aYMGEC9uzZg0GDBmHVqlU1UDsiqk0MiIhIZymVSrRp0wbr16+Hn58fgJIg6eTJk7h06ZKUNmnSJPz+++/49NNPcenSJaxZswZLly5FeHj4I+8/fvx4REVFYcGCBbh06RKWLl2KqKgo6Xxubi7Gjh2L/fv34+rVq/jf//6H48ePw9PT83lVmYieEwZERKTTunXrhqKiIin4sbW1hZeXF+rXry8FJi+++CJ+/PFHbNq0Cd7e3pg5cybmzJmD0NDQR9775Zdfxg8//IDIyEi0adMGe/bswSeffCKdNzQ0xN27d/HOO++gefPmCAoKQkBAAGbPnv28qktEzwlnmREREZHeYwsRERER6T0GRERERKT3GBARERGR3mNARERERHqPARERERHpPQZEREREpPcYEBEREZHeY0BEREREeo8BEREREek9BkRERESk9xgQERERkd5jQERERER67/8Avu3qcHns3yIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "stats_df = pd.DataFrame({\"words\": pd.concat([train_length, val_length, test_length], ignore_index=True),\n",
    "                        \"type\": len(train_length)*[\"train\"] +\n",
    "                         len(val_length)*[\"validation\"] + \n",
    "                         len(test_length)*[\"test\"]})\n",
    "\n",
    "sns.histplot(x=\"words\", \n",
    "             hue=\"type\", \n",
    "             data=stats_df, \n",
    "             multiple=\"stack\")\n",
    "\n",
    "plt.title(\"Number of sentences by word count\")\n",
    "tasks.util.save_plot(\"ex_2_dataset_stats.png\", OUTPUT_DIR)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we can see from the graph above, there is a sizable portion of our sentences that feature very few words. In order to make the RNN training more efficient, we choose to discard sentences with very few words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exclude_small_sentences(conllu_df: pd.DataFrame, min_len: int) -> pd.DataFrame:\n",
    "    assert 1 <= min_len\n",
    "\n",
    "    length_df = length_sentences(conllu_df)\n",
    "    valid_length_df = length_df[length_df >= min_len]\n",
    "    valid_ids = set(valid_length_df.index)\n",
    "    return conllu_df[conllu_df.sent_id.isin(valid_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_SENTENCE_LENGTH = 5\n",
    "\n",
    "train_df = exclude_small_sentences(train_df, MIN_SENTENCE_LENGTH)\n",
    "val_df = exclude_small_sentences(val_df, MIN_SENTENCE_LENGTH)\n",
    "test_df = exclude_small_sentences(test_df, MIN_SENTENCE_LENGTH)\n",
    "\n",
    "train_length = length_sentences(train_df)\n",
    "val_length = length_sentences(val_df)\n",
    "test_length = length_sentences(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    10539.000000\n",
       "mean        18.967170\n",
       "std         11.782365\n",
       "min          5.000000\n",
       "25%         10.000000\n",
       "50%         16.000000\n",
       "75%         24.000000\n",
       "max        159.000000\n",
       "Name: words, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_length.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1538.000000\n",
       "mean       15.607932\n",
       "std        10.050704\n",
       "min         5.000000\n",
       "25%         8.000000\n",
       "50%        13.000000\n",
       "75%        20.000000\n",
       "max        75.000000\n",
       "Name: words, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_length.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1535.000000\n",
       "mean       15.512052\n",
       "std        10.332400\n",
       "min         5.000000\n",
       "25%         8.000000\n",
       "50%        13.000000\n",
       "75%        20.000000\n",
       "max        81.000000\n",
       "Name: words, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_length.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 15967\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(set(train_df.words))\n",
    "print(f\"Vocabulary size: {vocab_size}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total word count:\n",
      "Training: 199895\n",
      "Validation: 24005\n",
      "Testing: 23811\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total word count:\\nTraining: {train_df.shape[0]}\"\n",
    "      f\"\\nValidation: {val_df.shape[0]}\"\n",
    "      f\"\\nTesting: {test_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total sentence count:\n",
      "Training: 10539\n",
      "Validation: 1538\n",
      "Testing: 1535\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total sentence count:\\nTraining: {len(set(train_df.sent_id))}\"\n",
    "      f\"\\nValidation: {len(set(val_df.sent_id))}\"\n",
    "      f\"\\nTesting: {len(set(test_df.sent_id))}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorization\n",
    "\n",
    "In order to make the RNN training more efficient, we choose to discard sentences with very few words. We also set a window size equal to the 90\\% percentile of sentence word count, meaning tht 90\\% of our windows will fully fit the training sentences. The rest will be automatically split into more sentences, and as such don't need to be excluded from the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_SEQUENCE_LENGTH = int(np.quantile(train_length, 0.9))\n",
    "MAX_SEQUENCE_LENGTH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will be using a combination of the `keras.preprocessing.Tokenizer` and `keras.utils.pad_sequences` utilities to create custom windows of words to be fed to our model, since it uses Time Distributed outputs. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(tokenizer, max_seq_len, values):\n",
    "    tokens = tokenizer.texts_to_sequences(values)\n",
    "    padded_tokens = keras.utils.pad_sequences(\n",
    "        tokens, maxlen=max_seq_len, padding=\"pre\", truncating=\"post\"\n",
    "    )\n",
    "    return padded_tokens\n",
    "\n",
    "\n",
    "def decode(tokenizer, encoded_sequence):\n",
    "    return np.array(\n",
    "        [\n",
    "            tokenizer.index_word[str(x[-1])]\n",
    "            for x in np.argmax(encoded_sequence, axis=2)\n",
    "        ]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encode X\n",
    "word_tokenizer = keras.preprocessing.text.Tokenizer(filters=\"\")\n",
    "word_tokenizer.fit_on_texts(train_df.words.values)\n",
    "\n",
    "train_data = encode(word_tokenizer, MAX_SEQUENCE_LENGTH, train_df.words.values)\n",
    "val_data = encode(word_tokenizer, MAX_SEQUENCE_LENGTH, val_df.words.values)\n",
    "test_data = encode(word_tokenizer, MAX_SEQUENCE_LENGTH, test_df.words.values)\n",
    "\n",
    "tag_tokenizer = keras.preprocessing.text.Tokenizer()\n",
    "tag_tokenizer.fit_on_texts(train_df.pos.values)\n",
    "\n",
    "# start label counting from 0, since to_categorical assumes argmax = number_of_categories\n",
    "tag_tokenizer.word_index = {\n",
    "    key: value - 1 for key, value in tag_tokenizer.word_index.items()\n",
    "}\n",
    "\n",
    "# start label counting from 0, since to_categorical assumes argmax = number_of_categories\n",
    "tag_tokenizer.index_word = {\n",
    "    str(int(key) - 1): value for key, value in tag_tokenizer.index_word.items()\n",
    "}\n",
    "\n",
    "y_train = keras.utils.to_categorical(\n",
    "    encode(tag_tokenizer, MAX_SEQUENCE_LENGTH, train_df.pos.values)\n",
    ")\n",
    "y_valid = keras.utils.to_categorical(\n",
    "    encode(tag_tokenizer, MAX_SEQUENCE_LENGTH, val_df.pos.values)\n",
    ")\n",
    "y_test = keras.utils.to_categorical(\n",
    "    encode(tag_tokenizer, MAX_SEQUENCE_LENGTH, test_df.pos.values)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'noun': 0,\n",
       " 'punct': 1,\n",
       " 'verb': 2,\n",
       " 'pron': 3,\n",
       " 'adp': 4,\n",
       " 'det': 5,\n",
       " 'adj': 6,\n",
       " 'aux': 7,\n",
       " 'propn': 8,\n",
       " 'adv': 9,\n",
       " 'cconj': 10,\n",
       " 'part': 11,\n",
       " 'sconj': 12,\n",
       " 'num': 13,\n",
       " 'sym': 14,\n",
       " 'intj': 15,\n",
       " 'x': 16}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check that counting is continuous starting from 0\n",
    "tag_tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(199895, 34, 17)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# input shape\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>BaselineLabelClassifier()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">BaselineLabelClassifier</label><div class=\"sk-toggleable__content\"><pre>BaselineLabelClassifier()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "BaselineLabelClassifier()"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tasks.models import BaselineLabelClassifier\n",
    "\n",
    "\n",
    "x_base_train = train_df.words\n",
    "x_base_valid = val_df.words\n",
    "x_base_test = test_df.words\n",
    "\n",
    "y_base_train = train_df.pos\n",
    "y_base_valid = val_df.pos\n",
    "y_base_test = test_df.pos\n",
    "\n",
    "base_cls = BaselineLabelClassifier()\n",
    "base_cls.fit(X=x_base_train, y=y_base_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ADJ       0.88      0.89      0.89     12854\n",
      "         ADP       0.88      0.67      0.76     17732\n",
      "         ADV       0.61      0.83      0.70      9995\n",
      "         AUX       0.88      0.78      0.83     12746\n",
      "       CCONJ       0.98      0.99      0.99      6656\n",
      "         DET       0.96      0.95      0.95     16228\n",
      "        INTJ       0.65      0.86      0.74       572\n",
      "        NOUN       0.88      0.90      0.89     34011\n",
      "         NUM       0.99      0.88      0.93      3753\n",
      "        PART       0.71      0.89      0.79      5734\n",
      "        PRON       0.90      0.95      0.92     18479\n",
      "       PROPN       0.89      0.83      0.86     11289\n",
      "       PUNCT       0.99      0.99      0.99     22574\n",
      "       SCONJ       0.64      0.42      0.50      3836\n",
      "         SYM       0.89      0.83      0.86       668\n",
      "        VERB       0.83      0.89      0.86     22363\n",
      "           X       0.82      0.58      0.68       405\n",
      "\n",
      "    accuracy                           0.87    199895\n",
      "   macro avg       0.85      0.83      0.83    199895\n",
      "weighted avg       0.88      0.87      0.87    199895\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "\n",
    "training_preds = base_cls.predict(x_base_train)\n",
    "print(classification_report(y_base_train, training_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ADJ       0.88      0.82      0.85      1666\n",
      "         ADP       0.89      0.67      0.76      2013\n",
      "         ADV       0.62      0.83      0.71      1131\n",
      "         AUX       0.89      0.78      0.83      1521\n",
      "       CCONJ       0.99      0.98      0.99       726\n",
      "         DET       0.96      0.95      0.96      1870\n",
      "        INTJ       0.66      0.73      0.69        86\n",
      "        NOUN       0.67      0.89      0.77      3909\n",
      "         NUM       0.96      0.55      0.70       485\n",
      "        PART       0.70      0.90      0.78       632\n",
      "        PRON       0.91      0.95      0.93      2118\n",
      "       PROPN       0.88      0.48      0.62      1753\n",
      "       PUNCT       0.99      0.98      0.99      2794\n",
      "       SCONJ       0.60      0.44      0.51       382\n",
      "         SYM       0.82      0.86      0.84       101\n",
      "        VERB       0.80      0.82      0.81      2544\n",
      "           X       0.00      0.00      0.00        80\n",
      "\n",
      "    accuracy                           0.83     23811\n",
      "   macro avg       0.78      0.74      0.75     23811\n",
      "weighted avg       0.84      0.83      0.82     23811\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_preds = base_cls.predict(x_base_test)\n",
    "print(classification_report(y_base_test, test_preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model we use is the pre-trained optimal model used in the previous assignment. We follow the same preprocessing and caching steps as in the previous assignment. Since the model is not trained again, we use only a subset of the original training data (25,000 windows) in order to save on scare main-memory resources. We consider this a representative sample for comparison with other classifiers due to the sample size (law of large numbers).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exclude_small_sentences(conllu_df: pd.DataFrame, min_len: int) -> pd.DataFrame:\n",
    "    assert 1 <= min_len\n",
    "\n",
    "    length_df = length_sentences(conllu_df)\n",
    "    valid_length_df = length_df[length_df >= min_len]\n",
    "    valid_ids = set(valid_length_df.index)\n",
    "    return conllu_df[conllu_df.sent_id.isin(valid_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "WINDOW_SIZE = 5\n",
    "# training data are used exclusively for training accuracy, thus\n",
    "# we only need a small, representative sample\n",
    "TRAINING_LIM = 25000\n",
    "VALID_LIM = 25000\n",
    "TEST_LIM = 10000\n",
    "SEED = 42\n",
    "PAD_TOKEN = \"<PAD>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>pos</th>\n",
       "      <th>sent_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>al</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zaman</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>:</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>american</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>weblog-juancole.com_juancole_20051126063000_EN...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207222</th>\n",
       "      <td>on</td>\n",
       "      <td>ADP</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207223</th>\n",
       "      <td>my</td>\n",
       "      <td>PRON</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207224</th>\n",
       "      <td>car</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207225</th>\n",
       "      <td>)</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207226</th>\n",
       "      <td>.</td>\n",
       "      <td>PUNCT</td>\n",
       "      <td>reviews-319816-0029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>199895 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           words    pos                                            sent_id\n",
       "0             al  PROPN  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "1              -  PUNCT  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "2          zaman  PROPN  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "3              :  PUNCT  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "4       american    ADJ  weblog-juancole.com_juancole_20051126063000_EN...\n",
       "...          ...    ...                                                ...\n",
       "207222        on    ADP                                reviews-319816-0029\n",
       "207223        my   PRON                                reviews-319816-0029\n",
       "207224       car   NOUN                                reviews-319816-0029\n",
       "207225         )  PUNCT                                reviews-319816-0029\n",
       "207226         .  PUNCT                                reviews-319816-0029\n",
       "\n",
       "[199895 rows x 3 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_filt = exclude_small_sentences(train_df, MIN_SENTENCE_LENGTH)\n",
    "val_df_filt = exclude_small_sentences(val_df, MIN_SENTENCE_LENGTH)\n",
    "test_df_filt = exclude_small_sentences(test_df, MIN_SENTENCE_LENGTH)\n",
    "\n",
    "train_df_filt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File ‘input/fasttext/cc.en.300.bin.gz’ already there; not retrieving.\n",
      "\n",
      "Skipping model file\n"
     ]
    }
   ],
   "source": [
    "# download and unzip only if the download and unzipped files do not exist \n",
    "!wget -nc -P input/fasttext https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.bin.gz\n",
    "\n",
    "![ -f \"input/fasttext/cc.en.300.bin\" ] && echo \"Skipping model file\" || gzip --decompress --keep --force \"input/fasttext/cc.en.300.bin.gz\"   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading embedding model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "import fasttext\n",
    "\n",
    "\n",
    "print(\"Loading embedding model...\")\n",
    "fasttext_model = fasttext.load_model(\"input/fasttext/cc.en.300.bin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempting to load intermediate calculations...\n",
      "Loaded cached calculations.\n"
     ]
    }
   ],
   "source": [
    "(\n",
    "    x_train_mlp,\n",
    "    x_valid_mlp,\n",
    "    x_test_mlp,\n",
    "    y_train_mlp,\n",
    "    y_valid_mlp,\n",
    "    y_test_mlp,\n",
    "    lb_mlp,\n",
    ") = tasks.preprocessing.mlp_input(\n",
    "    train_df_filt,\n",
    "    val_df_filt,\n",
    "    test_df_filt,\n",
    "    embed_model=fasttext_model,\n",
    "    intermediate_dir=INTERMEDIATE_DIR,\n",
    "    train_lim=TRAINING_LIM,\n",
    "    val_lim=VALID_LIM,\n",
    "    test_lim=TEST_LIM,\n",
    "    window_size=WINDOW_SIZE,\n",
    "    seed=SEED,\n",
    "    pad_token=PAD_TOKEN,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:32:53.482720: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.486395: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.486805: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.715424: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.716861: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.717394: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
      "2024-02-28 20:32:53.717622: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1929] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3445 MB memory:  -> device: 0, name: NVIDIA GeForce GTX 960M, pci bus id: 0000:01:00.0, compute capability: 5.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 320)               480320    \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 320)               0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 17)                5457      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 485777 (1.85 MB)\n",
      "Trainable params: 485777 (1.85 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mlp = keras.saving.load_model(os.path.join(INPUT_MODEL_PATH, \"mlp_model.keras\"))\n",
    "mlp.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:32:55.442481: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 150000000 exceeds 10% of free system memory.\n",
      "2024-02-28 20:32:56.022116: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 150000000 exceeds 10% of free system memory.\n",
      "2024-02-28 20:32:57.045813: I external/local_tsl/tsl/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [==============================] - 3s 3ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ADJ       0.90      0.92      0.91      1634\n",
      "         ADP       0.90      0.92      0.91      2246\n",
      "         ADV       0.93      0.85      0.89      1287\n",
      "         AUX       0.93      0.97      0.95      1580\n",
      "       CCONJ       0.99      1.00      1.00       847\n",
      "         DET       0.97      0.97      0.97      2041\n",
      "        INTJ       0.99      0.81      0.89        91\n",
      "        NOUN       0.91      0.94      0.93      4260\n",
      "         NUM       0.97      0.97      0.97       433\n",
      "        PART       0.80      0.97      0.87       686\n",
      "        PRON       0.97      0.97      0.97      2311\n",
      "       PROPN       0.92      0.83      0.87      1424\n",
      "       PUNCT       1.00      1.00      1.00      2765\n",
      "       SCONJ       0.79      0.66      0.72       501\n",
      "         SYM       0.93      0.90      0.91        77\n",
      "        VERB       0.92      0.90      0.91      2754\n",
      "           X       0.97      0.54      0.69        63\n",
      "\n",
      "    accuracy                           0.93     25000\n",
      "   macro avg       0.93      0.89      0.90     25000\n",
      "weighted avg       0.93      0.93      0.93     25000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    classification_report(\n",
    "        lb_mlp.inverse_transform(y_train_mlp),\n",
    "        lb_mlp.inverse_transform(mlp.predict(x_train_mlp)),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 23/313 [=>............................] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:33:00.604969: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 60000000 exceeds 10% of free system memory.\n",
      "2024-02-28 20:33:00.707999: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 60000000 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 1s 2ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ADJ       0.90      0.88      0.89       721\n",
      "         ADP       0.86      0.89      0.87       862\n",
      "         ADV       0.87      0.80      0.83       486\n",
      "         AUX       0.90      0.92      0.91       632\n",
      "       CCONJ       0.99      1.00      1.00       303\n",
      "         DET       0.96      0.97      0.97       796\n",
      "        INTJ       1.00      0.77      0.87        31\n",
      "        NOUN       0.84      0.90      0.87      1657\n",
      "         NUM       0.88      0.84      0.86       197\n",
      "        PART       0.68      0.86      0.76       253\n",
      "        PRON       0.95      0.94      0.94       888\n",
      "       PROPN       0.86      0.74      0.80       715\n",
      "       PUNCT       0.99      0.98      0.99      1143\n",
      "       SCONJ       0.64      0.54      0.59       153\n",
      "         SYM       0.78      0.78      0.78        46\n",
      "        VERB       0.86      0.87      0.87      1082\n",
      "           X       0.27      0.11      0.16        35\n",
      "\n",
      "    accuracy                           0.89     10000\n",
      "   macro avg       0.84      0.81      0.82     10000\n",
      "weighted avg       0.89      0.89      0.89     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    classification_report(\n",
    "        lb_mlp.inverse_transform(y_test_mlp),\n",
    "        lb_mlp.inverse_transform(mlp.predict(x_test_mlp)),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 34, 300)           4996200   \n",
      "                                                                 \n",
      " bidirectional (Bidirection  (None, 34, 600)           1083600   \n",
      " al)                                                             \n",
      "                                                                 \n",
      " layer_normalization (Layer  (None, 34, 600)           1200      \n",
      " Normalization)                                                  \n",
      "                                                                 \n",
      " bidirectional_1 (Bidirecti  (None, 34, 600)           1623600   \n",
      " onal)                                                           \n",
      "                                                                 \n",
      " time_distributed (TimeDist  (None, 34, 17)            10217     \n",
      " ributed)                                                        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 7714817 (29.43 MB)\n",
      "Trainable params: 2718617 (10.37 MB)\n",
      "Non-trainable params: 4996200 (19.06 MB)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "rnn_model = keras.saving.load_model(\n",
    "    os.path.join(INPUT_MODEL_PATH, \"rnn_model.keras\")\n",
    ")\n",
    "rnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:33:14.474995: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:454] Loaded cuDNN version 8904\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6247/6247 [==============================] - 218s 34ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-28 20:36:50.222861: W external/local_tsl/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 462157240 exceeds 10% of free system memory.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         adj       0.90      0.92      0.91     12854\n",
      "         adp       0.89      0.85      0.87     17732\n",
      "         adv       0.91      0.79      0.85      9995\n",
      "         aux       0.92      0.90      0.91     12746\n",
      "       cconj       0.99      0.99      0.99      6656\n",
      "         det       0.94      0.98      0.96     16228\n",
      "        intj       0.95      0.72      0.82       572\n",
      "        noun       0.90      0.93      0.92     34011\n",
      "         num       0.94      0.97      0.95      3753\n",
      "        part       0.71      0.99      0.83      5734\n",
      "        pron       0.97      0.92      0.95     18479\n",
      "       propn       0.93      0.83      0.88     11289\n",
      "       punct       0.99      0.99      0.99     22574\n",
      "       sconj       0.59      0.71      0.65      3836\n",
      "         sym       0.89      0.81      0.85       668\n",
      "        verb       0.89      0.88      0.89     22363\n",
      "           x       0.88      0.40      0.55       405\n",
      "\n",
      "    accuracy                           0.91    199895\n",
      "   macro avg       0.89      0.86      0.87    199895\n",
      "weighted avg       0.92      0.91      0.91    199895\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    classification_report(\n",
    "        decode(tag_tokenizer, y_train),\n",
    "        decode(tag_tokenizer, rnn_model.predict(train_data)),\n",
    "        zero_division=0\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "745/745 [==============================] - 27s 36ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         adj       0.91      0.85      0.88      1666\n",
      "         adp       0.89      0.86      0.88      2013\n",
      "         adv       0.92      0.79      0.85      1131\n",
      "         aux       0.93      0.89      0.91      1521\n",
      "       cconj       0.99      0.99      0.99       726\n",
      "         det       0.94      0.98      0.96      1870\n",
      "        intj       0.93      0.60      0.73        86\n",
      "        noun       0.69      0.92      0.79      3909\n",
      "         num       0.89      0.63      0.74       485\n",
      "        part       0.68      1.00      0.81       632\n",
      "        pron       0.97      0.93      0.95      2118\n",
      "       propn       0.91      0.47      0.62      1753\n",
      "       punct       0.99      0.99      0.99      2794\n",
      "       sconj       0.57      0.68      0.62       382\n",
      "         sym       0.83      0.85      0.84       101\n",
      "        verb       0.87      0.81      0.84      2544\n",
      "           x       0.00      0.00      0.00        80\n",
      "\n",
      "    accuracy                           0.86     23811\n",
      "   macro avg       0.82      0.78      0.79     23811\n",
      "weighted avg       0.87      0.86      0.86     23811\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    classification_report(\n",
    "        decode(tag_tokenizer, y_test),\n",
    "        decode(tag_tokenizer, rnn_model.predict(test_data)),\n",
    "        zero_division=0\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing word embeddings...\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = np.zeros(shape=(vocab_size, 300))\n",
    "\n",
    "print(\"Computing word embeddings...\")\n",
    "for w2idx, _word in enumerate(set(train_df.words)):\n",
    "    # Skip PAD and UNK tokens\n",
    "    if w2idx < 2:\n",
    "        continue\n",
    "    embedding_matrix[w2idx] = fasttext_model.get_word_vector(_word)\n",
    "\n",
    "# reclaim memory\n",
    "del fasttext_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n",
    "\n",
    "\n",
    "EMBEDDING_DIM = 300\n",
    "FILTERS = EMBEDDING_DIM\n",
    "\n",
    "# function adapted from https://stackoverflow.com/a/64973085\n",
    "def resblock(x, kernelsize, filters, dropout: bool = True):\n",
    "    fx = keras.layers.Conv1D(filters, kernelsize, activation='relu', padding='same')(x)\n",
    "\n",
    "    if dropout:\n",
    "        fx = keras.layers.Dropout(0.5)(fx)\n",
    "    else:\n",
    "        fx = keras.layers.BatchNormalization()(fx)\n",
    "\n",
    "    fx = keras.layers.Conv1D(filters, kernelsize, padding='same')(fx)\n",
    "    out = keras.layers.Add()([x,fx])\n",
    "    out = keras.layers.ReLU()(out)\n",
    "\n",
    "    if dropout:\n",
    "        out = keras.layers.Dropout(0.5)(out)\n",
    "    else:\n",
    "        out = keras.layers.BatchNormalization()(out)\n",
    "\n",
    "    return out\n",
    "\n",
    "\n",
    "def tune_cnn(hp: kt.HyperParameters):\n",
    "    kernel_size = hp.Choice(\"ngram-model\", [2, 3, 5])\n",
    "    inputs = keras.layers.Input(shape=(MAX_SEQUENCE_LENGTH,), dtype=\"int32\")\n",
    "\n",
    "    # Embeddings\n",
    "    x = tf.keras.layers.Embedding(\n",
    "        input_dim=vocab_size,\n",
    "        output_dim=EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        mask_zero=True,\n",
    "        trainable=False,\n",
    "    )(inputs)\n",
    "\n",
    "    if hp.Boolean(\"use-embeddings-dropout\"):\n",
    "        x = keras.layers.Dropout(0.25)(x)\n",
    "\n",
    "    for i in range(hp.Int(name=\"stacked-layers\", min_value=1, max_value=3)):\n",
    "        x = resblock(x, kernel_size, FILTERS, hp.Boolean(\"dropout-no-batchnorm\"))\n",
    "    \n",
    "    x = keras.layers.Dropout(0.5)(x)\n",
    "\n",
    "    outputs = keras.layers.TimeDistributed(\n",
    "        keras.layers.Dense(train_df.pos.unique().shape[0], activation=\"softmax\")\n",
    "    )(x)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "    # Model compilation\n",
    "    hp_learning_rate = hp.Choice(\"learning_rate\", values=[1e-3, 1e-4])\n",
    "    model.compile(\n",
    "        loss=\"categorical_crossentropy\",\n",
    "        optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),\n",
    "        metrics=[\"categorical_accuracy\"],\n",
    "    )\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Search space summary\n",
      "Default search space size: 5\n",
      "ngram-model (Choice)\n",
      "{'default': 2, 'conditions': [], 'values': [2, 3, 5], 'ordered': True}\n",
      "use-embeddings-dropout (Boolean)\n",
      "{'default': False, 'conditions': []}\n",
      "stacked-layers (Int)\n",
      "{'default': None, 'conditions': [], 'min_value': 1, 'max_value': 3, 'step': 1, 'sampling': 'linear'}\n",
      "dropout-no-batchnorm (Boolean)\n",
      "{'default': False, 'conditions': []}\n",
      "learning_rate (Choice)\n",
      "{'default': 0.001, 'conditions': [], 'values': [0.001, 0.0001], 'ordered': True}\n"
     ]
    }
   ],
   "source": [
    "tuner = kt.RandomSearch(\n",
    "    tune_cnn,\n",
    "    objective=kt.Objective(\"val_categorical_accuracy\", direction=\"max\"),\n",
    "    max_trials=8,\n",
    "    directory=\"KT_dir\",\n",
    "    project_name=\"KT_pos\",\n",
    ")\n",
    "\n",
    "tuner.search_space_summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 8 Complete [00h 45m 10s]\n",
      "val_categorical_accuracy: 0.8895503878593445\n",
      "\n",
      "Best val_categorical_accuracy So Far: 0.8972365260124207\n",
      "Total elapsed time: 07h 47m 45s\n"
     ]
    }
   ],
   "source": [
    "# do not save weights to save time, just pick the best model\n",
    "early_stopping = keras.callbacks.EarlyStopping(monitor=\"val_categorical_accuracy\", patience=2)\n",
    "\n",
    "tuner.search(\n",
    "    train_data,\n",
    "    y_train,\n",
    "    validation_data=(val_data, y_valid),\n",
    "    epochs=30,\n",
    "    batch_size=256,  # large batch size to fit more calculations\n",
    "    shuffle=True,\n",
    "    callbacks=[early_stopping],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 34)]                 0         []                            \n",
      "                                                                                                  \n",
      " embedding (Embedding)       (None, 34, 300)              4790100   ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " conv1d (Conv1D)             (None, 34, 300)              270300    ['embedding[0][0]']           \n",
      "                                                                                                  \n",
      " batch_normalization (Batch  (None, 34, 300)              1200      ['conv1d[0][0]']              \n",
      " Normalization)                                                                                   \n",
      "                                                                                                  \n",
      " conv1d_1 (Conv1D)           (None, 34, 300)              270300    ['batch_normalization[0][0]'] \n",
      "                                                                                                  \n",
      " add (Add)                   (None, 34, 300)              0         ['embedding[0][0]',           \n",
      "                                                                     'conv1d_1[0][0]']            \n",
      "                                                                                                  \n",
      " re_lu (ReLU)                (None, 34, 300)              0         ['add[0][0]']                 \n",
      "                                                                                                  \n",
      " batch_normalization_1 (Bat  (None, 34, 300)              1200      ['re_lu[0][0]']               \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " conv1d_2 (Conv1D)           (None, 34, 300)              270300    ['batch_normalization_1[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " batch_normalization_2 (Bat  (None, 34, 300)              1200      ['conv1d_2[0][0]']            \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " conv1d_3 (Conv1D)           (None, 34, 300)              270300    ['batch_normalization_2[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " add_1 (Add)                 (None, 34, 300)              0         ['batch_normalization_1[0][0]'\n",
      "                                                                    , 'conv1d_3[0][0]']           \n",
      "                                                                                                  \n",
      " re_lu_1 (ReLU)              (None, 34, 300)              0         ['add_1[0][0]']               \n",
      "                                                                                                  \n",
      " batch_normalization_3 (Bat  (None, 34, 300)              1200      ['re_lu_1[0][0]']             \n",
      " chNormalization)                                                                                 \n",
      "                                                                                                  \n",
      " dropout (Dropout)           (None, 34, 300)              0         ['batch_normalization_3[0][0]'\n",
      "                                                                    ]                             \n",
      "                                                                                                  \n",
      " time_distributed (TimeDist  (None, 34, 17)               5117      ['dropout[0][0]']             \n",
      " ributed)                                                                                         \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 5881217 (22.44 MB)\n",
      "Trainable params: 1088717 (4.15 MB)\n",
      "Non-trainable params: 4792500 (18.28 MB)\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "optimal_parameters = tuner.get_best_hyperparameters()[0].values\n",
    "\n",
    "best_model = tuner.get_best_models(num_models=1)[0]\n",
    "best_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Detecting that an object or model or tf.train.Checkpoint is being deleted with unrestored values. See the following logs for the specific values in question. To silence these warnings, use `status.expect_partial()`. See https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint#restorefor details about the status object returned by the restore function.\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.1\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.2\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.3\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.4\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.5\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.6\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.7\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.8\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.9\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.10\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.11\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.12\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.13\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.14\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.15\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.16\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.17\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.18\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.19\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.20\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.21\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.22\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.23\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.24\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.25\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.26\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.27\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.28\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.29\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.30\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.31\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.32\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.33\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.34\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.35\n",
      "WARNING:tensorflow:Value in checkpoint could not be found in the restored object: (root).optimizer._variables.36\n",
      "Epoch 1/80\n",
      "781/781 [==============================] - 118s 147ms/step - loss: 1.0238 - categorical_accuracy: 0.7046 - val_loss: 0.5163 - val_categorical_accuracy: 0.8340\n",
      "Epoch 2/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.5809 - categorical_accuracy: 0.8142 - val_loss: 0.4050 - val_categorical_accuracy: 0.8674\n",
      "Epoch 3/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.4843 - categorical_accuracy: 0.8419 - val_loss: 0.3662 - val_categorical_accuracy: 0.8765\n",
      "Epoch 4/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.4281 - categorical_accuracy: 0.8572 - val_loss: 0.3455 - val_categorical_accuracy: 0.8839\n",
      "Epoch 5/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.3882 - categorical_accuracy: 0.8695 - val_loss: 0.3300 - val_categorical_accuracy: 0.8898\n",
      "Epoch 6/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.3612 - categorical_accuracy: 0.8771 - val_loss: 0.3270 - val_categorical_accuracy: 0.8927\n",
      "Epoch 7/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.3410 - categorical_accuracy: 0.8828 - val_loss: 0.3205 - val_categorical_accuracy: 0.8941\n",
      "Epoch 8/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.3235 - categorical_accuracy: 0.8873 - val_loss: 0.3196 - val_categorical_accuracy: 0.8952\n",
      "Epoch 9/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.3103 - categorical_accuracy: 0.8912 - val_loss: 0.3153 - val_categorical_accuracy: 0.8958\n",
      "Epoch 10/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2993 - categorical_accuracy: 0.8941 - val_loss: 0.3150 - val_categorical_accuracy: 0.8958\n",
      "Epoch 11/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2895 - categorical_accuracy: 0.8972 - val_loss: 0.3164 - val_categorical_accuracy: 0.8931\n",
      "Epoch 12/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2850 - categorical_accuracy: 0.8991 - val_loss: 0.3164 - val_categorical_accuracy: 0.8979\n",
      "Epoch 13/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2772 - categorical_accuracy: 0.9003 - val_loss: 0.3172 - val_categorical_accuracy: 0.8948\n",
      "Epoch 14/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2723 - categorical_accuracy: 0.9013 - val_loss: 0.3147 - val_categorical_accuracy: 0.8971\n",
      "Epoch 15/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2694 - categorical_accuracy: 0.9019 - val_loss: 0.3155 - val_categorical_accuracy: 0.8967\n",
      "Epoch 16/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2657 - categorical_accuracy: 0.9030 - val_loss: 0.3155 - val_categorical_accuracy: 0.8968\n",
      "Epoch 17/80\n",
      "781/781 [==============================] - 115s 147ms/step - loss: 0.2621 - categorical_accuracy: 0.9035 - val_loss: 0.3181 - val_categorical_accuracy: 0.8968\n"
     ]
    }
   ],
   "source": [
    "# save pretrained weights\n",
    "embed_layer_weights = best_model.get_layer(\"embedding\").get_weights().copy()\n",
    "\n",
    "# reset all model weights\n",
    "best_model = tf.keras.models.clone_model(best_model)\n",
    "\n",
    "# restore pretrained word embedding and vectorization layers\n",
    "embed_layer_weights = best_model.get_layer(\"embedding\").set_weights(\n",
    "    embed_layer_weights\n",
    ")\n",
    "\n",
    "prod_early_stopping = keras.callbacks.EarlyStopping(\n",
    "    monitor=\"val_categorical_accuracy\", patience=5, restore_best_weights=True\n",
    ")\n",
    "\n",
    "# retrain re-initialized model now with history and more forgiving early stopping\n",
    "best_model.compile(\n",
    "    loss=\"categorical_crossentropy\",\n",
    "    optimizer=keras.optimizers.Adam(\n",
    "        learning_rate=optimal_parameters[\"learning_rate\"]\n",
    "    ),\n",
    "    metrics=[\"categorical_accuracy\"],\n",
    ")\n",
    "\n",
    "history = best_model.fit(\n",
    "    train_data,\n",
    "    y_train,\n",
    "    validation_data=(val_data, y_valid),\n",
    "    batch_size=256,\n",
    "    epochs=80,\n",
    "    shuffle=True,\n",
    "    callbacks=[prod_early_stopping],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model.save(os.path.join(OUTPUT_DIR, \"cnn_model.keras\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Figure saved to output/ex_3_fit.png\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0cAAAG+CAYAAABYuH97AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAC6eklEQVR4nOzdd3xT1fvA8U+a7r2gA1padtl7FBAQZIOoKEPZoMhXsIAoCIIgWkFA/IHFwVYUVMDFEmTLtOwlsxQ6aaGD7nF/f6QNhA6a0jYdz/v1yqvJzbn3PjeUJk/OOc9RKYqiIIQQQgghhBAVnJGhAxBCCCGEEEKI0kCSIyGEEEIIIYRAkiMhhBBCCCGEACQ5EkIIIYQQQghAkiMhhBBCCCGEACQ5EkIIIYQQQghAkiMhhBBCCCGEACQ5EkIIIYQQQghAkiMhhBBCCCGEACQ5EkIUoTVr1qBSqbQ3Y2Nj3NzcGDRoEFevXs3RvlOnTqhUKnr06JHjuaCgIFQqFQsXLtRu27dvn/bYR44cybHPiBEjsLa2fmKcH374oU6cpqameHt78/bbbxMTE5Oj/dGjR3n55Zdxc3PD1NQUV1dXBgwYkGsMAMeOHeOFF17A09MTMzMzXFxcaNu2LVOmTMkzpuzrLcgtKCjoiddYVhT0mvft2/dU58n+Ny+M7N+7p42hsC5dusTQoUOpXr065ubmODs706xZM9566y3i4uK07X744QeWLFnyVOcKCAhgzZo1ObZn/37m9lx+8otJpVLx4Ycf6h2jEEIUJ2NDByCEKH9Wr15N3bp1SU5O5p9//uHjjz9m7969XL58GQcHhxztd+7cyZ49e3j22WcLfI53332XgwcPPlWcO3bswM7Ojvj4eLZt28YXX3zB8ePHOXz4sPaD9NKlS/Hz86NVq1YsWLCAatWqERwczJdffkn79u354osveOutt7TH3Lp1K/369aNTp04sWLAANzc3wsLC+Pfff9mwYQOLFi3KNRY3N7ccydb48eOJjY1l/fr1OdqWF49f80cffcTevXvZs2ePzvZ69eo91XnGjBmTaxJeEM2aNePIkSNPHUNhnDp1inbt2uHj48OsWbPw8vIiKiqKM2fOsGHDBt555x1sbW0BTSJy/vx5/Pz8Cn2+gIAAnJ2dGTFihM727N/PGjVq6HW8/GI6cuQIVatWLXSsQghRHCQ5EkIUuQYNGtCiRQtA0zuUkZHB7Nmz+fXXXxk5cqRO29q1a5Oens67777LiRMnCvTtfo8ePdixYwd//PEHffv2LXSczZs3x9nZGYDnnnuO6OhovvvuOw4fPky7du34559/8PPzo1evXmzZsgVj44d/MgcNGsQLL7zA22+/TdOmTWnXrh0ACxYswNvbm507d+Zov2DBgjxjMTMzo02bNjrbbG1tSU1NzbG9LEpMTMTS0jLH9sevrVKlShgZGT3xmvM6Xl6qVq1a6A/itra2Bvs3WLJkCUZGRuzbtw8bGxvt9gEDBvDRRx+hKEqJxJHb7+fTKg+/10KI8keG1Qkhil12ohQREZHjORMTEz7++GMCAwPZuHFjgY43YsQI6tWrx/Tp08nIyCiyOLM/rN26dQsAf39/VCoVy5cv10l0AIyNjQkICEClUvHpp59qt0dHR+Ps7JyjPYCR0dP/yZ0zZw6tW7fG0dERW1tbmjVrxsqVK3N8SE5JSWHKlCm4urpiaWnJM888Q2BgIF5eXjl6BQ4dOkTbtm0xNzenSpUqfPDBB6xYsSLXIXwbN26kbdu2WFlZYW1tTffu3Tl16pROm+zhjefOnaNbt27Y2NjQpUuXQl9zp06daNCgAQcOHMDX1xdLS0tGjRqljadbt264ublhYWGBj48P06ZNIyEhQecYuQ2r8/Lyok+fPuzYsYNmzZphYWFB3bp1WbVqlU673IbVZV/jtWvX6NWrF9bW1nh4eDBlyhRSUlJ09r9z5w4DBgzAxsYGe3t7Xn31Ve0XAU8aphYdHY2trW2ew0Wzr6lTp05s3bqVW7du6QxHzFaQ3xsvLy8uXLjA/v37tft7eXkBuQ+ru3v3Lq+//joeHh6YmZlRqVIl2rVrx+7duwsUU27D6kJCQrTHNDU1xd3dnQEDBmj/dmRmZjJv3jzq1KmDhYUF9vb2NGrUiC+++CLf11EIIQpKeo6EEMXu5s2bgKaXKDcDBw5k4cKFzJw5k5deegkTE5N8j6dWq/H39+f5559n7dq12g/KT+vatWuApvciIyODvXv30qJFizx7HDw8PGjevDl79uwhIyMDtVpN27ZtWbFiBRMnTuTVV1+lWbNmT7wefQQFBfHGG2/g6ekJaOZDTZgwgZCQEGbNmqVtN3LkSDZu3Mi7777Ls88+y8WLF3nhhRd05qgAnD17lueee47atWuzdu1aLC0t+eqrr/j+++9znPuTTz5h5syZjBw5kpkzZ5Kamspnn31Ghw4dOH78uM6ws9TUVPr168cbb7zBtGnTSE9Pf6rrDgsL47XXXuPdd9/lk08+0SaaV69epVevXvj5+WFlZcXly5eZP38+x48fzzE0LzdnzpxhypQpTJs2DRcXF1asWMHo0aOpWbMmzzzzTL77pqWl0a9fP0aPHs2UKVM4cOAAH330EXZ2dtp/i4SEBDp37sy9e/eYP38+NWvWZMeOHQwcOLBA1922bVu2bt3Kq6++yhtvvEGrVq2wsLDI0S4gIIDXX3+d69evs2XLlhzPF+T3ZsuWLQwYMAA7OzsCAgIATY9RXoYOHcrJkyf5+OOPqV27NjExMZw8eZLo6OgCxfS4kJAQWrZsSVpaGu+//z6NGjUiOjqanTt3cv/+fVxcXFiwYAEffvghM2fO5JlnniEtLY3Lly/nOldQCCEKRRFCiCKyevVqBVCOHj2qpKWlKfHx8cqOHTsUV1dX5ZlnnlHS0tJ02nfs2FGpX7++oiiKsnv3bgVQli5dqiiKoty8eVMBlM8++0zbfu/evQqg/Pzzz4qiKEr79u2VqlWrKklJSYqiKMrw4cMVKyurJ8Y5e/ZsBVDCw8OVtLQ05f79+8r333+vWFhYKB4eHkpSUpISHh6uAMqgQYPyPdbAgQMVQImIiFAURVGioqKU9u3bK4ACKCYmJoqvr6/i7++vxMfHF/CVzPn65CYjI0NJS0tT5s6dqzg5OSmZmZmKoijKhQsXFEB57733dNr/+OOPCqAMHz5cu+3ll19WrKyslLt37+oct169egqg3Lx5U1EURQkODlaMjY2VCRMm6BwzPj5ecXV1VV555RXttuHDhyuAsmrVKr2uN3vfx/8NO3bsqADK33//ne++mZmZSlpamrJ//34FUM6cOaN9Lvvf/FHVqlVTzM3NlVu3bmm3JSUlKY6Ojsobb7yh3Zb9e7d3794c1/jTTz/pHLNXr15KnTp1tI+//PJLBVC2b9+u0+6NN95QAGX16tX5XlNycrLSv39/7e+TWq1WmjZtqsyYMUOJjIzUadu7d2+lWrVq+R5PUfL+vVEURalfv77SsWPHHPtk/398NF5ra2vFz88v33PlFxOgzJ49W/t41KhRiomJiXLx4sU8j9enTx+lSZMm+Z5TCCGehgyrE0IUuTZt2mBiYoKNjQ09evTAwcGB3377LdehZtm6dOlCt27dmDt3LvHx8QU6z/z587lz506hh9S4urpiYmKCg4MDr732Gs2aNWPHjh2Ym5sX+BhK1rCk7OFCTk5OHDx4kBMnTvDpp5/y/PPPc+XKFaZPn07Dhg2JiooqVKzZ9uzZQ9euXbGzs0OtVmNiYsKsWbOIjo4mMjISgP379wPwyiuv6Ow7YMCAHP8G+/fv59lnn9XOvQLN8L/H9925cyfp6ekMGzaM9PR07c3c3JyOHTvmWsntpZdeeqprfZSDg0OuBTtu3LjBkCFDcHV11b4eHTt2BDRV3p6kSZMm2t4UAHNzc2rXrq0dWpkflUqVY85bo0aNdPbdv3+/9v/BowYPHvzE44Om52bLli1cvHiRzz//nEGDBnH37l0+/vhjfHx8+O+//wp0nIL83uirVatWrFmzhnnz5nH06FHS0tIKdZxs27dvp3Pnzvj4+OR7zjNnzjB+/Hh27tyZoydUCCGeliRHQogit27dOk6cOMGePXt44403uHTpUoE+DM6fP5+oqCid8t358fX1pX///nz66afcv39f7zh3797NiRMnOH36NFFRURw6dEg7NMzZ2RlLS0vtkMC8BAUFYWlpiaOjo872Fi1a8N577/Hzzz8TGhrKpEmTCAoKyrcow5McP36cbt26AfDtt9/yzz//cOLECWbMmAFAUlISgHZYk4uLi87+xsbGODk56WyLjo7O0S63fbPnfLRs2RITExOd28aNG3MkfZaWltoqakUhtwp9Dx48oEOHDhw7dox58+axb98+Tpw4webNm4GHr0d+Hn89QJOQFGRfS0vLHIm0mZkZycnJ2scFfX2fxMfHBz8/P77//nuCg4NZvHgx0dHRfPDBB0/ct6C/N/rauHEjw4cPZ8WKFbRt2xZHR0eGDRtGeHh4oY539+7dJxbNmD59OgsXLuTo0aP07NkTJycnunTpwr///luocwohxONkzpEQosj5+PhoizB07tyZjIwMVqxYwS+//MKAAQPy3K9JkyYMHjyYxYsX06tXrwKdy9/fnwYNGvDJJ5/oHWfjxo11ekwepVar6dy5Mzt27ODOnTu5fmi7c+cOgYGB9OzZE7Vaned5TExMmD17Np9//jnnz5/XO85sGzZswMTEhD///FPnQ/mvv/6q0y77A39ERARVqlTRbk9PT9cmTo+2za1QxuMfcLNfp19++YVq1ao9MdbCrimkz/H27NlDaGgo+/bt0/YWAaVq/omTkxPHjx/Psb2wCQRoXotJkyYxd+7cAv0+FfT3Rl/Ozs4sWbKEJUuWEBwczO+//860adOIjIxkx44deh+vUqVK3LlzJ982xsbGTJ48mcmTJxMTE8Pu3bt5//336d69O7dv39argqEQQuRGeo6EEMVuwYIFODg4MGvWLDIzM/NtO2/ePFJTU5kzZ06Bjl23bl1GjRrF0qVLCQ4OLopwtaZPn46iKIwfPz5HVbyMjAzefPNNFEVh+vTp2u1hYWG5Hit7iJe7u3uh48leWPfRRCwpKYnvvvtOp112IYHHq//98ssvOQojdOzYkT179uj0/GRmZvLzzz/rtOvevTvGxsZcv36dFi1a5HoradkJ0+NFA77++usSjyUvHTt2JD4+nu3bt+ts37BhQ4H2z+v3KTQ0lLi4OJ3fp7x6vAr6e5PfMZ7E09OTt956i+eee46TJ08W6ng9e/Zk7969BR4qaG9vz4ABA/jf//7HvXv3ytXiyEIIw5GeIyFEsXNwcGD69Om8++67/PDDD7z22mt5tvX29ubNN9/Uax7Rhx9+yPr169m7dy9WVlZFETIA7dq1Y8mSJfj5+dG+fXveeustPD09tYvAHjt2jCVLluDr66vdp3v37lStWpW+fftSt25dMjMzOX36NIsWLcLa2pq333670PH07t2bxYsXM2TIEF5//XWio6NZuHBhjuSgfv36DB48mEWLFqFWq3n22We5cOECixYtws7OTqek+IwZM/jjjz/o0qULM2bMwMLCgq+++kpbCju7rZeXF3PnzmXGjBncuHFDO5csIiKC48ePY2VlVeCEtqj4+vri4ODAuHHjmD17NiYmJqxfv54zZ86UaBz5GT58OJ9//jmvvfYa8+bNo2bNmmzfvp2dO3cCTy7v/vrrrxMTE8NLL71EgwYNUKvVXL58mc8//xwjIyPee+89bduGDRuyefNmli9fTvPmzTEyMqJFixYF/r3JPsaGDRvYuHEj1atXx9zcnIYNG+ZoFxsbS+fOnRkyZAh169bFxsaGEydOsGPHDl588cUnxpSbuXPnsn37dp555hnef/99GjZsSExMDDt27GDy5MnUrVuXvn37atdRq1SpErdu3WLJkiVUq1aNWrVqFejfRAgh8mXgghBCiHIku1rdiRMncjyXlJSkeHp6KrVq1VLS09MVRcm7Gtvdu3cVW1vbJ1are9T777+vAHpVq3u0Qlt+jhw5ogwYMEBxcXFRjI2NlcqVKysvvviicvjw4RxtN27cqAwZMkSpVauWYm1trZiYmCienp7K0KFD863ClZvcXp9Vq1YpderUUczMzJTq1asr/v7+ysqVK3UqyymKpsrZ5MmTlcqVKyvm5uZKmzZtlCNHjih2dnbKpEmTdI558OBBpXXr1oqZmZni6uqqTJ06VZk/f74CKDExMTptf/31V6Vz586Kra2tYmZmplSrVk0ZMGCAsnv3bm2bglYNzE1e1eryqtp3+PBhpW3btoqlpaVSqVIlZcyYMcrJkydzVFbLq1pd7969cxyzY8eOOhXb8qpWl9s15nae4OBg5cUXX1Ssra0VGxsb5aWXXlK2bdumAMpvv/2W10uhKIqi7Ny5Uxk1apRSr149xc7OTjE2Nlbc3NyUF198UTly5IhO23v37ikDBgxQ7O3tFZVKpRNHQX9vgoKClG7duik2NjYKoK0093i1uuTkZGXcuHFKo0aNFFtbW8XCwkKpU6eOMnv2bCUhIaFAMfFYtTpFUZTbt28ro0aNUlxdXRUTExPF3d1deeWVV7TVIBctWqT4+voqzs7OiqmpqeLp6amMHj1aCQoKyvd1FEKIglIpSgktry2EEMKgDh8+TLt27Vi/fj1DhgzJt223bt0ICgriypUrJRRdxZK9ZlRwcPATixAIIYQoOTKsTgghyqFdu3Zx5MgRmjdvjoWFBWfOnOHTTz+lVq1aOsOeACZPnkzTpk3x8PDg3r17rF+/nl27drFy5UoDRV++LFu2DNDMj0tLS2PPnj383//9H6+99pokRkIIUcpIciSEEOWQra0tf/31F0uWLCE+Ph5nZ2d69uyJv79/jvLTGRkZzJo1i/DwcFQqFfXq1eO7777Ld26YKDhLS0s+//xzgoKCSElJwdPTk/fee4+ZM2caOjQhhBCPkWF1QgghhBBCCIGU8hZCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORJCCCGEEEIIQJIjIYQQQgghhAAkORIVxJo1a1CpVKhUKvbt25fjeUVRqFmzJiqVik6dOhXpuVUqFR9++KHe+wUFBaFSqVizZk2B9zl37hwqlQoTExPCwsL0PqcQQgjDKs/vV9ntFi5cWLgAhSgBkhyJCsXGxoaVK1fm2L5//36uX7+OjY2NAaIqOitWrAAgPT2ddevWGTgaIYQQhVXe36+EKK0kORIVysCBA9m0aRNxcXE621euXEnbtm3x9PQ0UGRPLyUlhfXr19O4cWOqVKnCqlWrDB1SnpKSklAUxdBhCCFEqVWe36+EKM0kORIVyuDBgwH48ccftdtiY2PZtGkTo0aNynWfe/fuMX78eKpUqYKpqSnVq1dnxowZpKSk6LSLi4tj7NixODk5YW1tTY8ePbhy5Uqux7x69SpDhgyhcuXKmJmZ4ePjw5dffvlU1/brr78SHR3NmDFjGD58OFeuXOHQoUM52qWkpDB37lx8fHwwNzfHycmJzp07c/jwYW2bzMxMli5dSpMmTbCwsMDe3p42bdrw+++/a9vkNfzCy8uLESNGaB9nDxH566+/GDVqFJUqVcLS0pKUlBSuXbvGyJEjqVWrFpaWllSpUoW+ffty7ty5HMeNiYlhypQpVK9eHTMzMypXrkyvXr24fPkyiqJQq1YtunfvnmO/Bw8eYGdnx//+9z89X1EhhDCc8vx+9STBwcG89tprOudctGgRmZmZOu2WL19O48aNsba2xsbGhrp16/L+++9rn09MTOSdd97B29sbc3NzHB0dadGihc5rKsTjjA0dgBAlydbWlgEDBrBq1SreeOMNQPPGY2RkxMCBA1myZIlO++TkZDp37sz169eZM2cOjRo14uDBg/j7+3P69Gm2bt0KaMaA9+/fn8OHDzNr1ixatmzJP//8Q8+ePXPEcPHiRXx9ffH09GTRokW4urqyc+dOJk6cSFRUFLNnzy7Uta1cuRIzMzNeffVV7t27h7+/PytXrqR9+/baNunp6fTs2ZODBw/i5+fHs88+S3p6OkePHiU4OBhfX18ARowYwffff8/o0aOZO3cupqamnDx5kqCgoELFBjBq1Ch69+7Nd999R0JCAiYmJoSGhuLk5MSnn35KpUqVuHfvHmvXrqV169acOnWKOnXqABAfH0/79u0JCgrivffeo3Xr1jx48IADBw4QFhZG3bp1mTBhAn5+fly9epVatWppz7tu3Tri4uIkORJClCnl+f0qP3fv3sXX15fU1FQ++ugjvLy8+PPPP3nnnXe4fv06AQEBAGzYsIHx48czYcIEFi5ciJGREdeuXePixYvaY02ePJnvvvuOefPm0bRpUxISEjh//jzR0dFFHrcoRxQhKoDVq1crgHLixAll7969CqCcP39eURRFadmypTJixAhFURSlfv36SseOHbX7ffXVVwqg/PTTTzrHmz9/vgIof/31l6IoirJ9+3YFUL744guddh9//LECKLNnz9Zu6969u1K1alUlNjZWp+1bb72lmJubK/fu3VMURVFu3rypAMrq1aufeH1BQUGKkZGRMmjQIO22jh07KlZWVkpcXJx227p16xRA+fbbb/M81oEDBxRAmTFjRr7nfPy6slWrVk0ZPny49nH2az9s2LAnXkd6erqSmpqq1KpVS5k0aZJ2+9y5cxVA2bVrV577xsXFKTY2Nsrbb7+ts71evXpK586dn3huIYQoDcrz+1V2u88++yzPNtOmTVMA5dixYzrb33zzTUWlUin//fefNgZ7e/t8z9egQQOlf//++bYR4nEyrE5UOB07dqRGjRqsWrWKc+fOceLEiTyHKOzZswcrKysGDBigsz172Njff/8NwN69ewF49dVXddoNGTJE53FycjJ///03L7zwApaWlqSnp2tvvXr1Ijk5maNHj+p9TatXryYzM1PnOkaNGkVCQgIbN27Ubtu+fTvm5uZ5Xm92G6DIe1peeumlHNvS09P55JNPqFevHqamphgbG2NqasrVq1e5dOmSTky1a9ema9eueR7fxsaGkSNHsmbNGhISEgDNv9/Fixd56623ivRahBCiJJTH96sn2bNnD/Xq1aNVq1Y5rkNRFPbs2QNAq1atiImJYfDgwfz2229ERUXlOFarVq3Yvn0706ZNY9++fSQlJRV5vKL8keRIVDgqlYqRI0fy/fff89VXX1G7dm06dOiQa9vo6GhcXV1RqVQ62ytXroyxsbG2az46OhpjY2OcnJx02rm6uuY4Xnp6OkuXLsXExETn1qtXL4Bc/8DnJzMzkzVr1uDu7k7z5s2JiYkhJiaGrl27YmVlpVPt6O7du7i7u2NklPd//bt376JWq3PE/rTc3NxybJs8eTIffPAB/fv3548//uDYsWOcOHGCxo0b67yJ3b17l6pVqz7xHBMmTCA+Pp7169cDsGzZMqpWrcrzzz9fdBcihBAlpLy9XxVEdHR0ru8X7u7u2ucBhg4dyqpVq7h16xYvvfQSlStXpnXr1uzatUu7z//93//x3nvv8euvv9K5c2ccHR3p378/V69eLfK4RfkhyZGokEaMGEFUVBRfffUVI0eOzLOdk5MTEREROSqrRUZGkp6ejrOzs7Zdenp6jnHM4eHhOo8dHBxQq9WMGDGCEydO5HrLftMpqN27d3Pr1i3t/B0HBwccHByoUqUKCQkJHD16VDsGu1KlSoSGhuaY1PqoSpUqkZGRkSP2x5mZmeWY5AvkOZb78TdsgO+//55hw4bxySef0L17d1q1akWLFi1yvOFWqlSJO3fu5BsPQM2aNenZsydffvklt2/f5vfff2fcuHGo1eon7iuEEKVReXq/KggnJ6dc1+kLDQ0F0F4HwMiRIzl8+DCxsbFs3boVRVHo06cPt27dAsDKyoo5c+Zw+fJlwsPDWb58OUePHqVv375FHrcoPyQ5EhVSlSpVmDp1Kn379mX48OF5tuvSpQsPHjzg119/1dmevYZQly5dAOjcuTOAtsci2w8//KDz2NLSks6dO3Pq1CkaNWpEixYtctwe/zbvSVauXImRkRG//vore/fu1bl99913ANqy3j179iQ5OTnfhfqyJ+UuX7483/N6eXlx9uxZnW179uzhwYMHBY5dpVJhZmams23r1q2EhITkiOnKlSva4RT5efvttzl79izDhw9HrVYzduzYAscjhBClTXl6vyqILl26cPHiRU6ePJnjOlQqlTb+R1lZWdGzZ09mzJhBamoqFy5cyNHGxcWFESNGMHjwYP777z8SExOLPHZRPki1OlFhffrpp09sM2zYML788kuGDx9OUFAQDRs25NChQ3zyySf06tVLOwemW7duPPPMM7z77rskJCTQokUL/vnnH21y8qgvvviC9u3b06FDB9588028vLyIj4/n2rVr/PHHHwVKALJFR0fz22+/0b179zyHjn3++eesW7cOf39/Bg8ezOrVqxk3bhz//fcfnTt3JjMzk2PHjuHj48OgQYPo0KEDQ4cOZd68eURERNCnTx/MzMw4deoUlpaWTJgwAdAMafjggw+YNWsWHTt25OLFiyxbtgw7O7sCx9+nTx/WrFlD3bp1adSoEYGBgXz22Wc5htD5+fmxceNGnn/+eaZNm0arVq1ISkpi//799OnTR+fN8rnnnqNevXrs3btXWwpWCCHKsvLwfvWoc+fO8csvv+TY3rJlSyZNmsS6devo3bs3c+fOpVq1amzdupWAgADefPNNateuDcDYsWOxsLCgXbt2uLm5ER4ejr+/P3Z2drRs2RKA1q1b06dPHxo1aoSDgwOXLl3iu+++o23btlhaWhYqdlEBGLYehBAl49HqP/l5vPqPoihKdHS0Mm7cOMXNzU0xNjZWqlWrpkyfPl1JTk7WaRcTE6OMGjVKsbe3VywtLZXnnntOuXz5cq5V3W7evKmMGjVKqVKlimJiYqJUqlRJ8fX1VebNm6fThidU/1myZIkCKL/++muebbIrGG3atElRFEVJSkpSZs2apdSqVUsxNTVVnJyclGeffVY5fPiwdp+MjAzl888/Vxo0aKCYmpoqdnZ2Stu2bZU//vhD2yYlJUV59913FQ8PD8XCwkLp2LGjcvr06Tyr1eX22t+/f18ZPXq0UrlyZcXS0lJp3769cvDgQaVjx445/h3u37+vvP3224qnp6diYmKiVK5cWendu7dy+fLlHMf98MMPFUA5evRonq+LEEKURuX1/erRdnndsve/deuWMmTIEMXJyUkxMTFR6tSpo3z22WdKRkaG9lhr165VOnfurLi4uCimpqaKu7u78sorryhnz57Vtpk2bZrSokULxcHBQTEzM1OqV6+uTJo0SYmKiso3TlGxqRRFlqkXQpQvLVq0QKVSceLECUOHIoQQQogyRIbVCSHKhbi4OM6fP8+ff/5JYGAgW7ZsMXRIQgghhChjJDkSQpQLJ0+epHPnzjg5OTF79mz69+9v6JCEEEIIUcbIsDohhBBCCCGEQEp5CyGEEEIIIQQgyZEQQgghhBBCAJIcCSGEEEIIIQQgBRkKLTMzk9DQUGxsbFCpVIYORwghyjxFUYiPj8fd3R0jo4r93Z28xwghRNEq6HuMJEeFFBoaioeHh6HDEEKIcuf27dtUrVrV0GEYlLzHCCFE8XjSe4wkR4VkY2MDaF5gW1tbA0cjhBBlX1xcHB4eHtq/rxWZvMcIIUTRKuh7jCRHhZQ9zMHW1lbeuIQQogjJMDJ5jxFCiOLypPeYij2oWwghhBBCCCGySHIkhBBCCCGEEEhyJIQQQgghhBCAzDkSQgghhBCiVMjMzCQ1NdXQYZRJJiYmqNXqpz6OJEdCCCGEEEIYWGpqKjdv3iQzM9PQoZRZ9vb2uLq6PlVhH0mOhBBCCCGEMCBFUQgLC0OtVuPh4VHhF8LWl6IoJCYmEhkZCYCbm1uhjyXJkRBCCCGEEAaUnp5OYmIi7u7uWFpaGjqcMsnCwgKAyMhIKleuXOghdpKWCiGEEEIIYUAZGRkAmJqaGjiSsi07sUxLSyv0MSQ5EkIIIYQQohSQRbCfTlG8fpIcCSGEEEIIIQSSHAkhhBBCCCEMzMvLiyVLlhg6DCnIIIQQFVVKegb3ElK5n5BGUloGKWkZJKdnkJyWSXJaBklpD+9rnsvanvrwvua5TJLTs7dn4GZnwU9vtDX05VVYU346w4Grd/n0xYZ08XExdDhCiHKsU6dONGnSpEiSmhMnTmBlZfX0QT0lSY6EEKKcyE52oh+kEp2Qyr2ElIf3H6QSnZCStV3T5kFKerHEIUt0GFZ8chp341MIiUkydChCiApOURQyMjIwNn5yylGpUqUSiOjJDJ4cBQQE8NlnnxEWFkb9+vVZsmQJHTp0yLP9l19+ybJlywgKCsLT05MZM2YwbNgwnTabNm3igw8+4Pr169SoUYOPP/6YF1544anOK4QQxUVRFFLSM0lKzSAhNZ3E1AwSUh77mZqueT4lg8TUdGIS04ok2TE2UmFvaYqlqRpzEyPMTdSYG6sxy7pvYfLIdhM15sZGmGXd133OKGs/NTbmBn9rqdDc7TXlbENjkg0ciRCiPBsxYgT79+9n//79fPHFFwCsXr2akSNHsmPHDmbMmMHZs2fZuXMnnp6eTJ48maNHj5KQkICPjw/+/v507dpVezwvLy/8/Pzw8/MDNMUVvv32W7Zu3crOnTupUqUKixYtol+/fsV6XQZ9B9u4cSN+fn4EBATQrl07vv76a3r27MnFixfx9PTM0X758uVMnz6db7/9lpYtW3L8+HHGjh2Lg4MDffv2BeDIkSMMHDiQjz76iBdeeIEtW7bwyiuvcOjQIVq3bl2o8wohRH4yMhXiktK4n5jK/cQ0YhJTiUnUPM7+GZuUliPZSUx5mAxlZCpFEouxkQpHK1OcrM1wsjLNum+add/skfuaNrbmxlIdqZxxszMHIDxWeo6EKKsURSEpLcMg57YwURfofeGLL77gypUrNGjQgLlz5wJw4cIFAN59910WLlxI9erVsbe3586dO/Tq1Yt58+Zhbm7O2rVr6du3L//991++n73nzJnDggUL+Oyzz1i6dCmvvvoqt27dwtHRsWguNhcqRVGK5h25EFq3bk2zZs1Yvny5dpuPjw/9+/fH398/R3tfX1/atWvHZ599pt3m5+fHv//+y6FDhwAYOHAgcXFxbN++XdumR48eODg48OOPPxbqvLmJi4vDzs6O2NhYbG1t9btwIUSplZmpcC8xlci4FKITUvJMdmKytt9PTCMuOY2i+ktqbmKElakxFqZqrEyNsTTL+mmqxspM89PSVI2dhUm5S3bk7+pDT/Na/H4mlIk/nqKVt6PM/RKijEhOTubmzZt4e3tjbm5OYmo69WbtNEgsF+d2x9K0YP0nj8852rdvH507d+bXX3/l+eefz3ff+vXr8+abb/LWW28BufcczZw5k48++giAhIQEbGxs2LZtGz169Mj1mI+/jo8q6N9Vg/UcpaamEhgYyLRp03S2d+vWjcOHD+e6T0pKSo4LtbCw4Pjx46SlpWFiYsKRI0eYNGmSTpvu3btr/9EKc97sc6ekpGgfx8XFPfEahRClR0amQnRCCpFxKUTGJxMZl0JE1v2IuBTuxicTGZ/C3fgU0gvZi2NjZoydpQkOlqbYZ/10sDTBztIUewsTrM2NtQmPpcnDZOdh0mOM2qhsJjai9HDP6jkKk54jIYSBtGjRQudxQkICc+bM4c8//yQ0NJT09HSSkpIIDg7O9ziNGjXS3reyssLGxobIyMhiiTmbwZKjqKgoMjIycHHRraTj4uJCeHh4rvt0796dFStW0L9/f5o1a0ZgYCCrVq0iLS2NqKgo3NzcCA8Pz/eYhTkvgL+/P3PmzCnMpQohillKega37yVyKzqR8LhknQQoMj6FiLhkoh6koE/O42RlirO1mTbJsbc0wT4r2Xn8sX3WYxO1rI5Q3hw4cIDPPvuMwMBAwsLC2LJlC/379893n/379zN58mQuXLiAu7s77777LuPGjSuZgAFX7bC6ZDIzFYwk4RaizLEwUXNxbneDnftpPV51burUqezcuZOFCxdSs2ZNLCwsGDBgAKmpqfkex8TEROexSqUis5ir/hh81uzjwz8URclzSMgHH3xAeHg4bdq0QVEUXFxcGDFiBAsWLECtfvgPWZBj6nNegOnTpzN58mTt47i4ODw8PPK/OCFEkUnPyOTO/SRuRicQFJXAzaxbUHQCIfeTCpT4qFTgbG1GZRvNzcXWnMo2ZlSyNcfFxozKWY+drc0wNZZER2i+7WzcuDEjR47kpZdeemL7mzdv0qtXL8aOHcv333/PP//8w/jx46lUqVKB9i8KLrbmqFSQlqEQlZBCZRvzJ+8khChVVCpVgYe2GZKpqSkZGU+eG3Xw4EFGjBihLZD24MEDgoKCijm6wjHYq+7s7Ixarc7RWxMZGZmjVyebhYUFq1at4uuvvyYiIgI3Nze++eYbbGxscHZ2BsDV1TXfYxbmvABmZmaYmZnpfZ1CiILLzFQIjU0iKCqRm1EPuBmVSFBWMhR8LzGf4W4KzU3v0MvqMmnWVbjn3AJLpypUtjF/mATZagoUGEvvjtBDz5496dmzZ4Hbf/XVV3h6emqHcvv4+PDvv/+ycOHCEkuOTNRGVLYxIyIuhbCYZEmOhBDFxsvLi2PHjhEUFIS1tXWevTo1a9Zk8+bN9O3bF5VKxQcffFDsPUCFZbDkyNTUlObNm7Nr1y6dMtu7du164gQuExMTqlatCsCGDRvo06cPRkaaDzxt27Zl165dOvOO/vrrL3x9fZ/6vEKIohMak8SJoHtcDI3T9gLdupdIanrefyzNjI3wcrLCy9kSb2dr6lnH0+T+btxv/Ypx9GVIQnO7Czh4gWdb8GwDlXzBppam66ikZaRB9DW4dxOM1GBsBsbmmp9qs0cemz+8ry793xaK3B05coRu3brpbOvevTsrV67Uzo3NTVHPa3Wzs9AkR7FJNPawf6pjCSFEXt555x2GDx9OvXr1SEpKYvXq1bm2+/zzzxk1ahS+vr44Ozvz3nvvldr5+wZ9B548eTJDhw6lRYsWtG3blm+++Ybg4GDt2Ozp06cTEhLCunXrALhy5QrHjx+ndevW3L9/n8WLF3P+/HnWrl2rPebbb7/NM888w/z583n++ef57bff2L17t7aaXUHOK4QoWpmZCtfuPuBE0D1O3LzHiaD7eS5QaaJW4eFoibeTFd7OVng5a356O1vhamuOUVoCXP4TzmyAY/uArN4ktSlU7wTxYRB+Hu4HaW5nNFUqsXB8mCxV8wXXRmBsWnQXqSgQEwyRlyDyguZnxEWIugKZafodS2WkmyxpkybTh49VKs1qq5npoGRofmZmgJL58L72uYzHHj+2n1LE3945eMPEk0V7zDIir3mv6enp2rmxuSnqea3u9uacvg1hsbLWkRCi+NSuXZsjR47obBsxYkSOdl5eXuzZs0dn2//+9z+dx48Ps8utoHZMTEyh4tSHQZOjgQMHEh0dzdy5cwkLC6NBgwZs27aNatWqARAWFqZTxSIjI4NFixbx33//YWJiQufOnTl8+DBeXl7aNr6+vmzYsIGZM2fywQcfUKNGDTZu3Khd46gg5xWiTMlIh7QESE2E1ITc76cX4QcklUrTK+PWBCzsc22Smp7J+dBYbSL07617xCTqJghqIxX13W1p4mFP9awkqLqzNe725jmHvmVmwM0DsGcDXPpDc13ZPNpA40FQvz9YOGi2JcfC7RMQfASCj0LIv5B0D/7bqrkBGFtA1RYPE6aqLcG8gCWTE6Ig8qIm+YnMvl2G1Pjc25vagFMNzWuXnqL590hPeeSWrJtAKZmQlqi5lUVFnWyVMbnNac1t+6OKel6rm51mIVhJjoQQQj8GH7sxfvx4xo8fn+tza9as0Xns4+PDqVOnnnjMAQMGMGDAgEKfV4gSk56i+aCdGJX1M/rhz5T4vJOdR+9npDz5PMXFsQa4NyXFpQmXjWpwIM6Nf24nc/p2DMlpuh+QzU2MaOrhQEtvR1p5OdLE0x5rsyf8CYq4CGc3wNmfND1C2Ry8NQlRo1fAsXrO/cztoFZXzQ00r3PYmYfJUvARSLoPQQc1N9D01rg00CRL1dpqfppaw93/HukJyvqZkEcZUSMTqFQHKvtA5Xqam0s9sPN48pC+zIyHiVJG6iMJ1OM/s+4rimaYnpEajIxBlfXTyOixx1ltdB4ba643+7HKCCjCIYdGT1/pqKzKa96rsbExTk5Oee5X1PNasxeCDc2jh1YIIUTuDJ4cCVFuKIomaUmMgoRoTYKjTXqytz32OK+ehsJQqcHUSnMzsQRTS82HexNLzVCsoppvk5lORsRF1LHBcO863LuO2flfaAw0UFR0U6pwjupctaiF4tYUl1rNaVbDjQZV7ApW6jo+As7/ohk2F3724XZze2jwIjQerOnl0ed6jM3Ao5Xm1u5tzbCyqCu6yVLMLc35ws/C8a+fcMCs3rPs5KeyD1Sur+kdUuc+p+SJjNRZ/2aWhdtflApt27bljz/+0Nn2119/0aJFizznGxUH6TkSQojCkeRIiIJIT4UH4RAXBvGhEJd1iw97uC0+AtIL8S2tSg2WTmDl/MhPZ80QL1MrMLHK+tCcfT/rsfZ+1k1tWiwFB9IzMvkvIp5TwTGcDL7PqTsx3IxKwJ54GhndoKHqJo2MbtBUfZPKqmjqqO5QhzugHIBQINwY/qsH7k2hSjPNz8r1dJOI1ET4b5smIbq+RzMXBjQ9MbW7Q6OBmp/GRfTNupERVK6rubUYqdkWF6qbLIWfBxSwqpyVAD16q6t5zUW59+DBA65du6Z9fPPmTU6fPo2joyOenp455saOGzeOZcuWMXnyZMaOHcuRI0dYuXIlP/74Y4nG7WaftRCs9BwJIYReJDkSFZuiQHLMI0lPWFbCE6r7M+FuwY+pNsuZ6OT62BksHTU9Ikalp7z03fgUTgXf59TtGE7eus/ZO7EkpemuYaBSgauLO9W8GlDby4GGXo5UtreA+HAIPQ2hJyH0FISc1PSSZffInMwqnqI2A9eGmkQpLQku/qbbi1alhWbYXIOXNK9RSbB115yvQVa55eQ4TaU5q7yHQony799//6Vz587ax9nzgoYPH86aNWtyzI319vZm27ZtTJo0iS+//BJ3d3f+7//+r8TKeGdzz+o5iohPISNTQS0LwQohRIFIciQqDkWBezfg9jG4fRzunNA8Luikd7Up2LiCjbvmg7StO9i4ga2bZpuNK1hV0vQoGKJkdCGkpmdyMSxOkwxl9QzduZ/zm2YbM2OaeNrT1MOepp4ONPN0wM4ylyFCNq5Qp4fmBprXPPaOJlHKTphCT2kKJoT8q7lls/eERoM0vUTONYvpivVQ0OIMolzr1KlTrhWTsj0+NxagY8eOnDxp2Gp9lWzMMDZSkZ6pEBmfrB1mJ4QQIn+SHInyKzVR84H89jFN5bI7xzXzgHJj4ZCV9LhlJTzuOX9aOpWZpCcvYbFJnLwVo+0ZOhcSm2NdIZUKale2oamnfdbNgZqVrDEqzDfPKhXYe2hu9fpptmUnqdmJUkaaptKcR5tS1YMmRFmmNlLhYmtOSEwSYbGSHAkhREFJciTKB0WB2NuaHqHbxzUJUcR5zRouj1KbaYZyebQCj9aaifS27mBSPj84hMQksedyJIevRXEqOIbwuJyTsx0sTWjq6aDtFWrkYYeteTFOHFepNIULnGpAw/yrSgohCs/NLis5ikkGT0NHI4QQZYMkR6JsSk+BsLNZvULHNEPkHi31nM3G/WEi5NGq6Bf+LGUyMhXO3Ilhz6VIdl+K4HK4bjU8tZGKuq6aXqFmng409XTAy8ky3/VXhBBlk5u9Bdy6T1isFGUQQoiCkuRIlB0RF+DMj5qeodBTmrVgHmVkrEl+PFqDR0vNT7uqhom1BMUnp3HoahR/X45k7+VIohMevi5GKmhezYFOdSrTvJoDjaraYWkq/+2FqAgernUk5byFECWrU6dONGnShCVLlhg6FL3JpyRRuikK3NgHh5fC9b91n7N0frh2jUdrcGtSYdaICY5O5O/LEfx9KZJjN6NJy3g4YdzG3JiOtSvRxacyHWtXxtGq/PaUCSHylp0cSc+REEIUnCRHonTKSIPzmzVJUcQ5zTaVEfj0hTq9NIuAOlYv8wUSCio9I5OTwTHahOha5AOd56s7W/Fs3co861OZll6OBVtsVQhRrmUXYQiVhWCFEKLAJDkSpUtynGYtnKPLIS5Es83EEpoOhTZvgqO3YeMrQbGJaey/epe/L0Ww77+7xCalaZ8zNlLR0suRLj6VebZuZapXsjZgpEKI0shdFoIVQpSAhIQE3nzzTTZv3oyNjQ3vvPOOzvOpqanMnDmT9evXExMTQ4MGDZg/fz6dOnUiNjYWV1dXtmzZQo8ePbT7bN68maFDhxIREYG1dcl+xpHkSJQOsXfg2FcQuBZS4jTbrCpD6zegxaiSWwjUwO4npLL9fDh/nAnleNA9MjIfDpeztzShcx1NMvRM7UrYWRRjRTkhRJmX3XN090EKaRmZ0qMsRFmiKAVfh7GomVjqNTJn6tSp7N27ly1btuDq6sr7779PYGAgTZo0AWDkyJEEBQWxYcMG3N3dtYnQuXPnqFWrFr1792b9+vU6ydEPP/zA888/X+KJEUhyJAwt7CwcWQbnNz0su+1cB3wnQKNXwNjMsPGVgISUdHZfiuD306Hsv3KX9EcSotou1jxb14WuPpVp6ukgq9wLIQrMycoUU7URqRmZRMQlU9WhYszJFKJcSEuET9wNc+73QzUL2hfAgwcPWLlyJevWreO5554DYO3atVStqimIdf36dX788Ufu3LmDu7vmet555x127NjB6tWr+eSTT3j11VcZNmwYiYmJWFpaEhcXx9atW9m0aVPxXN8TSHIkSp6iwPU9cPj/NMUWsnl1AN+JULNruV8MNDU9kwNX7vLbmVB2X4wgKS1D+1w9N1ueb+JOr4ZueDjKhxkhROEYGalwtTMn+F4iYbGSHAkhit7169dJTU2lbdu22m2Ojo7UqVMHgJMnT6IoCrVr19bZLyUlBScnJwB69+6NsbExv//+O4MGDWLTpk3Y2NjQrVu3kruQR0hyJEpOeqqmh+jwUoi8oNmmUkP9/pqeIvemBg2vuGVkKhy7Gc0fZ0LZdi5cZw6Rl5Ml/Rq706+JOzUr2xgwSiFEeZKdHIXKvCMhyhYTS00PjqHOXUCKouT7fGZmJmq1msDAQNRqtc5z2UPmTE1NGTBgAD/88AODBg3ihx9+YODAgRgbGyZNkeRIFL+kGAhco5lTlL1Qq4kVNB8OrceBQzVDRlesFEXhXEgsv50O5c+zoUTEpWifq2xjRt/G7vRr7E6jqnayEKsQosi5a8t5S8U6IcoUlarAQ9sMqWbNmpiYmHD06FE8PT0BuH//PleuXKFjx440bdqUjIwMIiMj6dChQ57HefXVV+nWrRsXLlxg7969fPTRRyV1CTlIciSKT1IMHFwE/66C1KzS09au0GYcNB8BFg6GjK5YXYt8wO9nQvn9dAhB0Q8nVNqaG9OroRv9mrjT2ttJ5hAJIYqVm72mKINUrBNCFAdra2tGjx7N1KlTcXJywsXFhRkzZmCUNT2idu3a2jlFixYtomnTpkRFRbFnzx4aNmxIr169AOjYsSMuLi68+uqreHl50aZNG4NdkyRHouhlpEPgatj7CSTd02yrXE8zdK7BADAun4uShsYk8ceZUH4/E8qF0DjtdnMTI7r6uPB8kyo8U9sZM2N1PkcRQoiik91zJGsdCSGKy2effcaDBw/o168fNjY2TJkyhdjYWO3zq1evZt68eUyZMoWQkBCcnJxo27atNjECUKlUDB48mM8++4xZs2YZ4jIexqI8abCgyFVcXBx2dnbExsZia2tr6HBKj6u7YOcMiPpP87hSXeg6B2p3L5cLtiqKwoGrUaw6dJP9V+5qtxsbqXimdiX6NXbnuXouWJnJ9xBCPIn8XX2oqF6L3RcjGLPuXxpWseOPCe2LMEIhRFFKTk7m5s2beHt7Y25ubuhwyqz8XseC/l2VT2yiaERe0iRF1//WPLZwhM7vQ/ORoC5/v2ZJqRlsPnWH1f8EcS3ygXZ7K29H+jXWVJpztCqfPWRCiLLDLXsh2FgZVieEEAVR/j61ipKVEAV7P9YUXFAywchEM6eowztgYW/o6IpcaEwS647c4sfjwdpqc1amal5p6cEIXy+qOZX+yZNCiIojeyHYqAeppKRnyLBeIYR4AkmOROGkp2iqzx1YCClZ82t8+sJzc8GxumFjKwYng++z6tBNtp8PJyNrkVYPRwtG+Hrzcouq2JqbGDhCIYTIycHSBDNjI1LSMwmPTZYvcIQQ4gkkORL6URS49DvsmgX3gzTbXBtBD3/wKl/j2dMyMtl+PpxVh25y+naMdnub6o6MbOdNVx8XqTYnhCjVVCoV7vYW3IxKIDRGkiMhhHgSSY5EwYWe0swruvWP5rG1K3SZBY0HQ1bJxvLgfkIqP54IZt3hW4THaSo8maqN6NfEnZHtvKjvbmfgCIUQouDc7My5GZUg846EEKIAJDkSTxYXBn/PhTM/AgoYW2jKcrd7G8ysDR1dkbkaEc+qf4LYcuoOyWmZADhbm/Jam2q82roalWzMDByhEELoL3vekSwEK0TpJ0Wkn05mZuZTH0OSI5G31EQ4vBT+WQJpWQuZNnwFus4Gu6oGDa2oZGYq7L96l1WHbnLwapR2ez03W0a396ZPYzeZwCyEKNPcpWKdEKWeiYkJKpWKu3fvUqlSJVTlcPmT4qQoCqmpqdy9excjIyNMTQtfMdjgyVFAQACfffYZYWFh1K9fnyVLltChQ4c8269fv54FCxZw9epV7Ozs6NGjBwsXLsTJyQmATp06sX///hz79erVi61btwLw4YcfMmfOHJ3nXVxcCA8PL8IrK8MyM+Hcz/D3HIgL0WzzaA3d/aFqc8PGVkTSMzL5OfAO3x68wY27CYBmGaZu9VwY1c6bVt6O8odJCFEuaHuOYqTnSIjSSq1WU7VqVe7cuUNQUJChwymzLC0t8fT0xOgppnsYNDnauHEjfn5+BAQE0K5dO77++mt69uzJxYsX8fT0zNH+0KFDDBs2jM8//5y+ffsSEhLCuHHjGDNmDFu2bAFg8+bNpKamaveJjo6mcePGvPzyyzrHql+/Prt379Y+VquldwCA8HPwx9sQEqh5bOcJz30I9V8sN4u4HrkezZw/LnA5PB4AGzNjXmnpwfC2Xng6WRo4OiGEKFpudpqeo1AZVidEqWZtbU2tWrVIS0szdChlklqtxtjY+Km/3DZocrR48WJGjx7NmDFjAFiyZAk7d+5k+fLl+Pv752h/9OhRvLy8mDhxIgDe3t688cYbLFiwQNvG0dFRZ58NGzZgaWmZIzkyNjbG1dW1qC+p7MrM0Ayh2zMPMtPA1AY6TIY248GkfKzUHBKTxCdbL7H1XBgAdhYmTHi2JoNaeWJtZvBOVCGEKBayEKwQZYdarZYv7A3MYCXGUlNTCQwMpFu3bjrbu3XrxuHDh3Pdx9fXlzt37rBt2zYURSEiIoJffvmF3r1753melStXMmjQIKysdMuXXr16FXd3d7y9vRk0aBA3btx4+osqq2KCYW1f2D1bkxjV6QUTAjXJUTlIjJLTMvhi91W6LNrH1nNhGKlgaJtq7HunE2M6VJfESAhRrmUPq4tJTCMpNcPA0QghROlmsE+FUVFRZGRk4OLiorM9v7k/vr6+rF+/noEDB5KcnEx6ejr9+vVj6dKlubY/fvw458+fZ+XKlTrbW7duzbp166hduzYRERHMmzcPX19fLly4oJ279LiUlBRSUlK0j+Pi4vS53NJJUeDsRtg2VbOQq4kV9PwUmg4tF0PoFEVhx/lw5m29REiM5hvTVt6OfNi3PvXcbQ0cnRBClAxbc2OsTNUkpGYQGptEjUrlp8qoEEIUNYMvTvP4uEBFUfIcK3jx4kUmTpzIrFmzCAwMZMeOHdy8eZNx48bl2n7lypU0aNCAVq1a6Wzv2bMnL730Eg0bNqRr167aQg1r167NM05/f3/s7Oy0Nw8PD30us/RJvAc/j4Atb2gSo6qt4M1D0GxYuUiMrkTE8+qKY7y5/iQhMUm42ZmzdHBTNr7eRhIjIUSFolKpcLOXogxCCFEQBkuOnJ2dUavVOXqJIiMjc/QmZfP396ddu3ZMnTqVRo0a0b17dwICAli1ahVhYWE6bRMTE9mwYYN2PlN+rKysaNiwIVevXs2zzfTp04mNjdXebt++XYCrLKWu74HlvnDxVzAyhs4zYeR2cKxu6MieWmxiGh/+foGeXxzk8PVoTI2NmPhsTf6e0pG+jd2lAp0QolACAgLw9vbG3Nyc5s2bc/DgwXzbf/nll/j4+GBhYUGdOnVYt25dCUWau+yiDDLvSAgh8mewYXWmpqY0b96cXbt28cILL2i379q1i+effz7XfRITEzE21g05e9La44tm/fTTT6SkpPDaa689MZaUlBQuXbqUbwlxMzMzzMzK+CKgaUmw+0M49pXmsVMtePEbqNLMoGEVhYxMhZ/+vc1nO//jXoKmWmH3+i7M7F0PD0epQCeEKDx9K6suX76c6dOn8+2339KyZUuOHz/O2LFjcXBwoG/fvga4AnCXhWCFEKJADDoTffLkyQwdOpQWLVrQtm1bvvnmG4KDg7XD5KZPn05ISIj2G7e+ffsyduxYli9fTvfu3QkLC8PPz49WrVrh7u6uc+yVK1fSv3//XOcQvfPOO/Tt2xdPT08iIyOZN28ecXFxDB8+vPgv2lDCzsDm1+HuZc3jlmPguY/AtOwnDv8G3WP27xe4EKqZB1arsjWz+9anfS1nA0cmhCgP9K2s+t133/HGG28wcOBAAKpXr87Ro0eZP3++wZIjV+k5EkKIAjFocjRw4ECio6OZO3cuYWFhNGjQgG3btlGtWjUAwsLCCA4O1rYfMWIE8fHxLFu2jClTpmBvb8+zzz7L/PnzdY575coVDh06xF9//ZXree/cucPgwYOJioqiUqVKtGnThqNHj2rPW65kZsA/X8DeTzSV6Kxd4PkvodZzho7sqYXHJvPp9kv8ejoUABtzYyZ1rc3QttUwURt8Op0QohzIrqw6bdo0ne35VVZNSUnB3Fy30qeFhQXHjx8nLS0NExOTXPcpzqI/7lnlvENlzpEQQuTL4DWMx48fz/jx43N9bs2aNTm2TZgwgQkTJuR7zNq1a+cYZveoDRs26BVjmXU/CLaMg+Ajmsd1+0Df/wOr3CvylRUp6RmsPHSTZXuukZiagUoFA1t48E73Ojhbl/Ghj0KIUqUwlVW7d+/OihUr6N+/P82aNSMwMJBVq1aRlpZGVFQUbm5uOfbx9/dnzpw5xXIN8LCct/QcCSFE/gyeHIlioChw5kfY9i6kxoOpNfScD01eLdOV6BRF4e9LkXy09SK3ohMBaOZpz5x+DWhY1c7A0QkhyjN9Kqt+8MEHhIeH06ZNGxRFwcXFhREjRrBgwYI8F3ecPn06kydP1j6Oi4sr0qqo2T1HUq1OCCHyJ8lReZN4D/54Gy79rnns0QZe+AocvQ0b11PKzFSY/fsFvjt6C4DKNmZM71WX/k2qSAU6IUSxKUxlVQsLC1atWsXXX39NREQEbm5ufPPNN9jY2ODsnPtcyOIu+pPdcxSfkk58cho25jmH9gkhhCgF6xyJInRtNwS01SRGRsbw7AcwcluZT4zSMzKZ8vMZvjt6C5UK3uhYnT3vdOKFplUlMRJCFKtHK6s+ateuXfj6+ua7r4mJCVWrVkWtVrNhwwb69OmDkZFh3natzIyxNdd8HxouFeuEECJP0nNUHqQmwu7ZcPwbzWPn2poS3e5NDRtXEUhJz2DCD6f462IExkYqFr3SmOebVDF0WEKICkTfyqpXrlzh+PHjtG7dmvv377N48WLOnz+f70LjJcHd3oK48HhCY5Op5WJj0FiEEKK0kuSoPNgwBG7s1dxv9Tp0nVMuSnQnpqbzxneBHLwahamxEQFDmtG1Xu7DWIQQorjoW1k1IyODRYsW8d9//2FiYkLnzp05fPgwXl5eBroCDVc7cy6HxxMWI0UZhBAiL5IclXXh5zSJkZEJDNkANbsaOqIiEZuUxqg1Jwi8dR9LUzUrhrXAt6asWySEMAx9Kqv6+Phw6tSpEohKP9nzjkJlWJ0QQuRJkqOy7uR3mp91e5ebxCjqQQrDVh7nYlgctubGrBnVimaeDoYOSwghyjT37IVgpedICCHyJMlRWZaWBGez1mxqNsywsRSRsNgkXl1xjBt3E3C2NuW70a3xcbM1dFhCCFHmudlnr3UkPUdCCJEXSY7Kskt/QnIs2HlC9c6Gjuap3YpOYMi3xwiJScLdzpzvx7SmeiVrQ4clhBDlQnbPUagsBCuEEHmS5KgsO5lV+ajpa2Cg8rBF5UpEPK+tOEZkfApeTpasH9uGKlnfcgohhHh62T1H4bHJ+S5iK4QQFVnZ/kRdkd27AUEHARU0GWLoaJ7K2TsxvPL1ESLjU6jrasNP49pKYiSEEEXMLavnKDE1g7ikdANHI4QQpZMkR2XVqe81P2t2AXsPw8byFI7diGbIt8eISUyjsYc9G15vQ2Ubc0OHJYQQ5Y65iRoHSxNAhtYJIUReJDkqizLS4dR6zf0yXIhh33+RDFt1nAcp6bSp7sj6Ma2xtzQ1dFhCCFFuZZfzDpPkSAghciXJUVl0bTc8CAdLZ6jd09DRFMq2c2GMXfcvKemZPFu3MmtGtsLaTKbACSFEcXK3zyrKECMV64QQIjfyabQsOrlO87PxIDAuez0tP/97m/c2nSVTgT6N3Ph8YBNM1JKnCyFEcZOeIyGEyJ8kR2VNfDhc2aG5XwaH1K355yYf/nERgEEtPfj4hYaojaRikhBClAQ3++yFYKXnSAghciPJUVlz+gdQMsCjNVSqY+hoCkxRFAL2Xeeznf8BMLq9NzN7+0gpWSGEKEHudrIQrBBC5EeSo7JEUeDUd5r7ZajXSFEUPt1xma/33wDAr2st3u5SSxIjIYQoYdnlvGVYnRBC5E6So7Lk1j+a9Y1MbaBef0NHUyCZmQof/Hae9ceCAZjZ24cxHaobOCohhKiY3B7pOZKFYIUQIidJjsqS7EIMDV8CM2vDxlIAmZkK7/xyhs0nQ1Cp4JMXGjK4laehwxJCiArLxc4MgJT0TO4lpOJkbWbgiIQQonSREmFlRVIMXPxNc79p2RhSt2jXf2w+GYKxkYovBjWVxEgIIQzMzFiNc1ZCJPOOhBAiJ0mOyopzP0N6MlSuD1WaGTqaJ/r539t8ufc6APNfakS/xu4GjkgIIQQ8utaRzDsSQojHSXJUVmQPqWs2DEr5GPEj16N5f8s5ACY8W5OXmlc1cERCCCGyPSzKID1HQgjxOEmOyoLQ0xB+FtSm0OgVQ0eTrxt3HzDu+0DSMhT6NHJjUtfahg5JCCHEI9yknLcQQuRJkqOyILt8t09fsHQ0bCz5uJeQyqg1J4hNSqOppz0LX26MkSzwKoQQpUr2sDop5y2EEDlJclTapSbC2Z8190vx2kYp6RmM+y6QoOhEqjpY8M3QFpibqA0dlhBCiMe4ZvccxUjPkRBCPM7gyVFAQADe3t6Ym5vTvHlzDh48mG/79evX07hxYywtLXFzc2PkyJFER0drn1+zZg0qlSrHLTlZ901A3/MazKXfISUW7KuB1zOGjiZXiqIwfdM5jgfdw8bMmFUjWlLJRsrDCiFEaeSeNecoVHqOhBAiB72To06dOrFu3TqSkp7+j+rGjRvx8/NjxowZnDp1ig4dOtCzZ0+Cg4NzbX/o0CGGDRvG6NGjuXDhAj///DMnTpxgzJgxOu1sbW0JCwvTuZmbmxf6vAZ1MmtIXdOhYGTwXDZXy/ZcY/OpENRGKgJea0ZtFxtDhySEECIPbvaanqOIuGQyMxUDRyOEEKWL3p+2mzdvzrvvvourqytjx47l6NGjhT754sWLGT16NGPGjMHHx4clS5bg4eHB8uXLc21/9OhRvLy8mDhxIt7e3rRv35433niDf//9V6edSqXC1dVV5/Y05zWYqGtw6xCojKDJEENHk6vfz4SyaNcVAD56vgEdalUycERCCCHy42JjhpEK0jIUoh6kGDocIYQoVfROjhYtWkRISAjr1q3j7t27PPPMM9SrV4+FCxcSERFR4OOkpqYSGBhIt27ddLZ369aNw4cP57qPr68vd+7cYdu2bSiKQkREBL/88gu9e/fWaffgwQOqVatG1apV6dOnD6dOnXqq8xpMdiGGms+BXRXDxpKLwFv3eOfnMwCM7eDNkNayyKsQQpR2xmojKttkD62TeUdCCPGoQo3TUqvVPP/88/z666+EhIQwZMgQPvjgAzw8POjfvz979ux54jGioqLIyMjAxcVFZ7uLiwvh4eG57uPr68v69esZOHAgpqamuLq6Ym9vz9KlS7Vt6taty5o1a/j999/58ccfMTc3p127dly9erXQ5wVISUkhLi5O51asMtLg9A+a+82GFu+5CiE4OpHX1wWSmp7Jc/VcmNbTx9AhCSGEKCC3rIp14TLvSAghdDzVJJbjx48za9YsFi5cSOXKlZk+fTqVK1emb9++vPPOOwU6huqxBU0VRcmxLdvFixeZOHEis2bNIjAwkB07dnDz5k3GjRunbdOmTRtee+01GjduTIcOHfjpp5+oXbu2TgKl73kB/P39sbOz0948PDwKdH2FdvUvSIgEq0pQu0fxnktPsUlpjFp7guiEVOq72/LFoCaopWS3EEKUGe5ZFetCpWKdEELoMNZ3h8jISL777jtWr17N1atX6du3Lxs2bKB79+7a5OKVV16hf//+LFy4MM/jODs7o1arc/TWREZG5ujVyebv70+7du2YOnUqAI0aNcLKyooOHTowb9483NzccuxjZGREy5YttT1HhTkvwPTp05k8ebL2cVxcXPEmSCfXaX42GQJqk+I7j57SMjL53/qTXIt8gKutOSuHt8TSVO9fIyGEEAbkaidrHQkhRG707jmqWrUqK1asYPjw4dy5c4dffvmFHj166PS6tGrVipYtW+Z7HFNTU5o3b86uXbt0tu/atQtfX99c90lMTMTosYptarVmLR1Fyb3ijqIonD59Wps4Fea8AGZmZtja2urcik1cqKbnCDRV6koJRVGY9dt5Dl2LwtJUzcoRLbRvsEIIIcoONzuZcySEELnROzn6+++/uXTpElOnTqVSpdwrk9na2rJ3794nHmvy5MmsWLGCVatWcenSJSZNmkRwcLB2mNz06dMZNuzhwqd9+/Zl8+bNLF++nBs3bvDPP/8wceJEWrVqhbu7OwBz5sxh586d3Lhxg9OnTzN69GhOnz6tM/TuSec1uNPrQckET19wrmXoaLS+PXiDH4/fxkgFSwc3pb67naFDEkKIElHUa/IZmrt99kKw0nMkhBCP0ns8VNWqVbl69Sq1aul+aL969SomJiZ4eXkV+FgDBw4kOjqauXPnEhYWRoMGDdi2bRvVqlUDICwsTGftoREjRhAfH8+yZcuYMmUK9vb2PPvss8yfP1/bJiYmhtdff53w8HDs7Oxo2rQpBw4coFWrVgU+r0FlZsKp7zX3mw3Lv20J2nE+HP/tlwGY2bseXXzyHoIohBDlSfbaeAEBAbRr146vv/6anj17cvHiRTw9c1bpzF6T7/PPP6dv376EhIQwbtw4xowZw5YtWwxwBTm5aYfVSc+REEI8SqXkNR4tDx07dmTUqFEMHz5cZ/v333/PihUr2LdvX1HGV2rFxcVhZ2dHbGxs0Q6xu7Ef1vUDM1uY8h+YWhbdsQvp3J1YXv76MMlpmQxtU425z9fPt3iFEEIURrH9XX1KrVu3plmzZjpr4fn4+NC/f3/8/f1ztF+4cCHLly/n+vXr2m1Lly5lwYIF3L59u0DnLO7XIiIumdaf/I2RCq7M64mxunQuMi6EEEWloH9X9f5reOrUKdq1a5dje5s2bTh9+rS+hxOPyy7E0HBAqUiMQmOSGL32BMlpmXSsXYnZfetJYiSEqDCKc02+R5X0chHO1mYYG6nIVOCuLAQrhBBaeidHKpWK+Pj4HNtjY2PJyMgokqAqrMR7cOkPzf1SMKTuQUo6o9acIDI+hTouNiwb0lS+XRRCVCjFtSbf40p6uQi1kQoX26yiDFLOWwghtPT+pNuhQwf8/f11EqGMjAz8/f1p3759kQZX4Zz7GTJSwLUhuDUxaCjpGZlM+OEkl8PjcbY2Y+WIFtiYl56S4kIIUZKKek2+x02fPp3Y2FjtraDD756Gm5TzFkKIHPQuyLBgwQKeeeYZ6tSpQ4cOHQA4ePAgcXFx7Nmzp8gDrDAUBQLXau43HQYGHro2b+sl9v53FzNjI1YMb0FVB8MP8RNCiJJWUmvymZmZYWZmVvQXkA83ewu4dZ8w6TkSQggtvXuO6tWrx9mzZ3nllVeIjIwkPj6eYcOGcfnyZRo0aFAcMVYMoSch8gKozaDRywYNZe3hINYcDgLg84FNaOJhb9B4hBDCUEpqTT5DcNeudSQ9R0IIkU3vniMAd3d3Pvnkk6KOpWI7+Z3mZ73nwcLBYGEcv3mPOX9cAODdHnXo1TDnN5xCCFGRTJ48maFDh9KiRQvatm3LN998k2NNvpCQENat0xTU6du3L2PHjmX58uV0796dsLAw/Pz8dNbkKw20w+qk50gIIbQKlRyB5pux4OBgUlNTdbY3atToqYOqcFIT4NwvmvsGLsSw6K//yFSgfxN33uxYw6CxCCFEaVAca/KVBm7ZC8FKz5EQQmjpnRzdvXuXkSNHsn379lyfl4p1hXDhV0iNBwdv8DJcUYvAW/c5dvMeJmoV7/WsKyW7hRAiy/jx4xk/fnyuz61ZsybHtgkTJjBhwoRijurpuNtlJ0fScySEENn0nnPk5+fH/fv3OXr0KBYWFuzYsYO1a9dSq1Ytfv/99+KIsfw7lTWkrtlQgxZiWL5Ps2DhC02r4Jb1pimEEKJ8cs0aVnf3QQqp6ZkGjkYIIUoHvXuO9uzZw2+//UbLli0xMjKiWrVqPPfcc9ja2uLv75/vInciF3evQPARUKmh8RCDhXElIp7dlyJQqeANGU4nhBDlnpOVKaZqI1IzMomIS8bDUaqSCiGE3j1HCQkJVK5cGQBHR0fu3r0LQMOGDTl58mTRRlcRnNJM4KVWN7A1XPGDr/Zreo161HelRiVrg8UhhBCiZBgZqbS9RzK0TgghNPROjurUqcN///0HQJMmTfj6668JCQnhq6++ynXtBpGP9FQ4/aPmvgELMdy5n8jvp0MBeLOT9BoJIURFIQvBCiGELr2H1fn5+REWFgbA7Nmz6d69O+vXr8fU1DTXSakiH1d2QGIUWLtqeo4MZMXBm6RnKrSv6UyjqvYGi0MIIUTJcs+qWBcq5byFEAIoRHL06quvau83bdqUoKAgLl++jKenJ87OzkUaXLl3MmtIXZMhoC50VfWnEv0ghQ0nNCVopddICCEqFuk5EkIIXXoNq0tLS6N69epcvHhRu83S0pJmzZpJYqSv2DtwbbfmftPXDBbGmsNBJKdl0qiqHb41nAwWhxBCiJL3cK0j6TkSQgjQMzkyMTEhJSVF1r8pCqd/ABTw6gBOhumxiU9OY+3hIADGd6oh/65CCFHBuNlKz5EQQjxK74IMEyZMYP78+aSnpxdHPBVDZiaczF7byHCFGH48HkxccjrVK1nRrZ6rweIQQghhGG72WcmRzDkSQgigEHOOjh07xt9//81ff/1Fw4YNsbKy0nl+8+bNRRZcuXVzH8QGg5kd+PQ1SAgp6RmsOHgTgHEda2BkJL1GQghR0bhnLfgdnZBKcloG5iZqA0ckhBCGpXdyZG9vz0svvVQcsVQcSTFg4wZ1+4CJhUFC2HwyhMj4FNzszOnfpIpBYhBCCGFY9pYmmJsYkZyWSXhsMl7OVk/eSQghyjG9k6PVq1cXRxwVS4MXwacfpCUY5PQZmQpfZy36OqZDdUyN9R5dKYQQohxQqVS421lwIyqB0NgkSY6EEBWefCo2FLUxmNsZ5NTbz4cRFJ2IvaUJg1p6GCQGIYQQpYPMOxJCiIf07jny9vbOt6rZjRs3niogUbwURWH5Pk2v0fC2XliZGWZ9JSGEEKWDW9a8o/A4SY6EEELvT8Z+fn46j9PS0jh16hQ7duxg6tSpRRWXKCYHrkZxITQOCxM1I3y9DB2OEEIIA8teCDY0Rsp5CyGE3snR22+/nev2L7/8kn///fepAxLFa/m+awAMbuWJg5WpgaMRQghhaNk9R7IQrBBCFOGco549e7Jp06aiOpwoBieD73P0xj1M1CrGPuNt6HCEEEKUAtlzjqTnSAghijA5+uWXX3B0dCyqw4likD3XqH+TKtpvCoUQQlRs7tJzJIQQWnoPq2vatKlOQQZFUQgPD+fu3bsEBAQUaXCi6FyNiGfXxQhUKnijYw1DhyOEEKKUyO45ik1KIzE1HUtTKdQjhKi49O456t+/P88//7z29uKLLzJ79mzOnz/P66+/rncAAQEBeHt7Y25uTvPmzTl48GC+7devX0/jxo2xtLTEzc2NkSNHEh0drX3+22+/pUOHDjg4OODg4EDXrl05fvy4zjE+/PBDVCqVzs3V1VXv2MuS5VnrGnWv50rNytYGjkYIIURpYWtugnVW5dJQKecthKjg9P56aPbs2UV28o0bN+Ln50dAQADt2rXj66+/pmfPnly8eBFPT88c7Q8dOsSwYcP4/PPP6du3LyEhIYwbN44xY8awZcsWAPbt28fgwYPx9fXF3NycBQsW0K1bNy5cuECVKlW0x6pfvz67d+/WPlar1UV2XaXNnfuJ/H46FIA3O0mvkRBCCF1uduZcjXxAeGyyfIEmhKjQ9O452rZtGzt37syxfefOnWzfvl2vYy1evJjRo0czZswYfHx8WLJkCR4eHixfvjzX9kePHsXLy4uJEyfi7e1N+/bteeONN3Sq5K1fv57x48fTpEkT6taty7fffktmZiZ///23zrGMjY1xdXXV3ipVqqRX7GXJioM3Sc9UaFfTicYe9oYORwghRCnjml3OO1aKMgghKja9k6Np06aRkZGRY7uiKEybNq3Ax0lNTSUwMJBu3brpbO/WrRuHDx/OdR9fX1/u3LnDtm3bUBSFiIgIfvnlF3r37p3neRITE0lLS8tRLOLq1au4u7vj7e3NoEGDnrh4bUpKCnFxcTq3siD6QQobTgQD8GbHmgaORgghRGmkLcogw+qEEBWc3snR1atXqVevXo7tdevW5dq1awU+TlRUFBkZGbi4uOhsd3FxITw8PNd9fH19Wb9+PQMHDsTU1BRXV1fs7e1ZunRpnueZNm0aVapUoWvXrtptrVu3Zt26dezcuZNvv/2W8PBwfH19deYuPc7f3x87OzvtzcPDo8DXakhrDweRnJZJo6p2tKvpZOhwhBBClELZRRnCpOdICFHB6Z0c2dnZ5drLcu3aNaysrPQO4NHKd6DpgXp8W7aLFy8yceJEZs2aRWBgIDt27ODmzZuMGzcu1/YLFizgxx9/ZPPmzZibm2u39+zZk5deeomGDRvStWtXtm7dCsDatWvzjHP69OnExsZqb7dv39b3Ukvcg5R01hwOAuDNjjXyfF2FEEJUbNk9R6FSzlsIUcHpXZChX79++Pn5sWXLFmrU0Ezuv3btGlOmTKFfv34FPo6zszNqtTpHL1FkZGSO3qRs/v7+tGvXjqlTpwLQqFEjrKys6NChA/PmzcPNzU3bduHChXzyySfs3r2bRo0a5RuLlZUVDRs25OrVq3m2MTMzw8zMrKCXVyr8eCyYuOR0qjtb0a1++a7GJ4QQovC0PUeyEKwQooLTu+fos88+w8rKirp16+Lt7Y23tzc+Pj44OTmxcOHCAh/H1NSU5s2bs2vXLp3tu3btwtfXN9d9EhMTMTLSDTm7ypyiKDoxfvTRR+zYsYMWLVo8MZaUlBQuXbqkk1yVdSnpGaw4pOnhG9exBmoj6TUSQgiROzdZCFYIIYBC9BzZ2dlx+PBhdu3axZkzZ7CwsKBRo0Y888wzep988uTJDB06lBYtWtC2bVu++eYbgoODtcPkpk+fTkhICOvWrQOgb9++jB07luXLl9O9e3fCwsLw8/OjVatWuLu7A5qhdB988AE//PADXl5e2p4pa2trrK015Unfeecd+vbti6enJ5GRkcybN4+4uDiGDx+u9zWUVltOhhARl4KrrTn9m1Z58g5CCCEqLLesanUPUtKJT07DxtzEwBEJIYRh6N1zBJp5Qt26dWPq1Km89dZbhUqMAAYOHMiSJUuYO3cuTZo04cCBA2zbto1q1aoBEBYWRnBwsLb9iBEjWLx4McuWLaNBgwa8/PLL1KlTh82bN2vbBAQEkJqayoABA3Bzc9PeHu3VunPnDoMHD6ZOnTq8+OKLmJqacvToUe15y7qMTIWvD2h6jcZ08MbUuFD/zEIIIbLos2D5iBEjciw0rlKpqF+/fglGrB8rM2NszTXfl0rvkRCiIlMpj45HK4CJEydSs2ZNJk6cqLN92bJlXLt2jSVLlhRlfKVWXFwcdnZ2xMbGYmtra+hwdGw9G8b/fjiJvaUJ/7z3LFZmencQCiFEiSutf1c3btzI0KFDdRYsX7FiRZ4LlsfGxpKU9HDuTnp6Oo0bN2bChAl8+OGHBTqnIV6LHksOcDk8njUjW9KpTuUSOacQQpSUgv5d1btLYdOmTbRr1y7Hdl9fX3755Rd9DyeKmKIoLN+vKak+vK2XJEZCCPGU9F2w3M7OTmeR8X///Zf79+8zcuTIEo5cP9lD66TnSAhRkemdHEVHR2NnZ5dju62tLVFRUUUSlCi8g1ejOB8Sh4WJmuG+XoYORwghyrTCLFj+uJUrV9K1a9d8h26XhoXG3eyzF4KVinVCiIpL7+SoZs2a7NixI8f27du3U7169SIJShTe8n3XARjUygNHK1MDRyOEEGVbYRYsf1RYWBjbt29nzJgx+bYrDQuNu2f1HMlaR0KIikzvMVeTJ0/mrbfe4u7duzz77LMA/P333yxatKjCzDcqrU4F3+fIjWiMjVSM7SCJqhBCFBV9Fix/1Jo1a7C3t6d///75tps+fTqTJ0/WPo6LiyvxBOlhOW/pORJCVFx6J0ejRo0iJSWFjz/+mI8++ggALy8vli9fzrBhw4o8QFFw2b1G/ZtWwT1reIQQQojCK8yC5dkURWHVqlUMHToUU9P8e/JLw0Lj2oVgpedICFGBFarG85tvvsmdO3eIiIggLi6OGzduMGzYMO7evVvU8YkCuhoRz18XI1CpYFxH6TUSQoiiUJgFy7Pt37+fa9euMXr06OIMschoe45iktGzkK0QQpQbT7UATqVKlbCysmLbtm28+OKLVK1atajiEnr6ar9mXaNu9VyoWdnGwNEIIUT5MXnyZFasWMGqVau4dOkSkyZNyrFgeW4jJ1auXEnr1q1p0KBBSYdcKNnV6pLSMohNSjNwNEIIYRiFrvN848YNVq1axdq1a3nw4AG9e/dmw4YNRRmbKKCQmCR+Ox0CwJudaho4GiGEKF8GDhxIdHQ0c+fOJSwsjAYNGuS7YDlo1jratGkTX3zxhSFCLhRzEzWOVqbcS0glNCYZe0sp6iOEqHj0So6Sk5P55ZdfWLFiBUePHuW5554jLCyM06dPl5lvxsqjbw/cID1TwbeGE0087A0djhBClDvjx49n/PjxuT63Zs2aHNvs7OxITEws5qiKnpudOfcSUgmLTaKee+lZiFcIIUpKgYfVjR8/Hnd3d7788ktefvllQkJC+OOPP1CpVBgZPdXoPPGUtp4LA2DsMzLXSAghROFlzzuSct5CiIqqwD1H33zzDe+99x7Tpk3DxkbmtJQWyWkZ3I1PAaBJVXvDBiOEEKJMc8+uWCcLwQohKqgCd/msW7eO48eP4+bmxsCBA/nzzz9JT08vzthEAYRmvYFZmqqxtzQxcDRCCCHKsuyeo3DpORJCVFAFTo6GDBnCrl27OH/+PHXr1uV///sfbm5uZGZmcvHixeKMUeQjNEbzBuZub1GgBQmFEEKIvGRXrAuVhWCFEBWU3pOFvLy8mDNnDkFBQXz33Xe89NJLvPbaa1StWpWJEycWR4wiHyExmgm/VWTRVyGEEE8pOzmShWCFEBVVoUt5q1QqevToQY8ePbh37x7r1q1j9erVRRmbKICQR3qOhBBCiKeR/V4SFqtZCFZGJAghKpoiKTPn6OiIn58fZ86cKYrDCT1kzzmqkjWJVgghhMbatWvZunWr9vG7776Lvb09vr6+3Lp1y4CRlV4utuaoVJCankl0QqqhwxFCiBInNbjLuJD7WcmRg/QcCSHEoz755BMsLDR/G48cOcKyZctYsGABzs7OTJo0ycDRlU6mxkY4W5sBEBYjQ+uEEBVPoYfVidIhe9Ksu50kR0II8ajbt29Ts2ZNAH799VcGDBjA66+/Trt27ejUqZNhgyvF3O3MuRufQmhsEg2r2hk6HCGEKFHSc1SGZWYq2m/2pOdICCF0WVtbEx0dDcBff/1F165dATA3NycpSaqx5UXKeQshKjLpOSrDoh6kkJqRiZFKM05cCCHEQ8899xxjxoyhadOmXLlyhd69ewNw4cIFvLy8DBtcKeYq5byFEBVYgZKjs2fPFviAjRo1KnQwQj93sooxuNqaY6KWTkAhhHjUl19+ycyZM7l9+zabNm3CyckJgMDAQAYPHmzg6Eov96wCPzLnSAhRERUoOWrSpAkqlQpFUXJ9Pvs5lUpFRkZGkQYo8pZdqU7KeAshRE729vYsW7Ysx/Y5c+YYIJqyI3tYXZj0HAkhKqACJUc3b94s7jhEIUilOiGEyNuOHTuwtramffv2gKYn6dtvv6VevXp8+eWXODg4GDjC0im75yhUeo6EEBVQgZKjatWqFXccohCk50gIIfI2depU5s+fD8C5c+eYMmUKkydPZs+ePUyePFkWLs9Dds9RRFwyGZkKaiNZCFYIUXEUuiDDxYsXCQ4OJjVVd5G4fv36PXVQomBCtAvASnIkhBCPu3nzJvXq1QNg06ZN9OnTh08++YSTJ0/Sq1cvA0dXelW2McNIBemZClEPUqTgjxCiQtE7Obpx4wYvvPAC586d05mHpFJpvlmSOUclJyS7jLckR0IIkYOpqSmJiYkA7N69m2HDhgHg6OhIXFycIUMr1YzVRrjYmhMWm0xYbLIkR0KICkXvEmdvv/023t7eREREYGlpyYULFzhw4AAtWrRg3759egcQEBCAt7c35ubmNG/enIMHD+bbfv369TRu3BhLS0vc3NwYOXKkdh2LbJs2baJevXqYmZlRr149tmzZ8tTnLY1kWJ0QQuStffv2TJ48mY8++ojjx49rS3lfuXKFqlWrGji60i27nHdYjBRlEEJULHonR0eOHGHu3LlUqlQJIyMjjIyMaN++Pf7+/kycOFGvY23cuBE/Pz9mzJjBqVOn6NChAz179iQ4ODjX9ocOHWLYsGGMHj2aCxcu8PPPP3PixAnGjBmjE9/AgQMZOnQoZ86cYejQobzyyiscO3as0OctjR6kpBOblAY8nDwrhBDioWXLlmFsbMwvv/zC8uXLqVKlCgDbt2+nR48eBo6udHPPmncUKgvBCiEqGJWSV33uPDg4OBAYGEj16tWpUaMGK1asoHPnzly/fp2GDRtqhzAUROvWrWnWrBnLly/XbvPx8aF///74+/vnaL9w4UKWL1/O9evXtduWLl3KggULuH37NgADBw4kLi6O7du3a9v06NEDBwcHfvzxx0KdNzdxcXHY2dkRGxuLra1tga+5qFyJiKfb5wewNTfm7IfdS/z8QghR1Az9d7U0MfRrMe/Pi6w4dJMx7b2Z2adeiZ9fCCGKWkH/rurdc9SgQQPtorCtW7dmwYIF/PPPP8ydO5fq1asX+DipqakEBgbSrVs3ne3dunXj8OHDue7j6+vLnTt32LZtG4qiEBERwS+//KIdKgGanqPHj9m9e3ftMQtz3tLoYRlvSwNHIoQQpVdGRgabNm1i3rx5fPzxx2zevFnmxhaAm332WkfScySEqFj0Lsgwc+ZMEhISAJg3bx59+vShQ4cOODk5sXHjxgIfJyoqioyMDFxcXHS2u7i4EB4enus+vr6+rF+/noEDB5KcnEx6ejr9+vVj6dKl2jbh4eH5HrMw5wVISUkhJSVF+9jQk3kfVqqTIXVCCJGba9eu0atXL0JCQqhTpw6KonDlyhU8PDzYunUrNWrUMHSIpZZ71pyjUFkIVghRwejdc9S9e3defPFFAKpXr87FixeJiooiMjKSZ599Vu8AsqvcZVMUJce2bBcvXmTixInMmjWLwMBAduzYwc2bNxk3bpzex9TnvAD+/v7Y2dlpbx4eHk+8tuIkZbyFECJ/EydOpEaNGty+fZuTJ09y6tQpgoOD8fb21nuObEWj7TmShWCFEBWM3j1HsbGxZGRk4OjoqN3m6OjIvXv3MDY2LvDYaGdnZ9RqdY7emsjIyBy9Otn8/f1p164dU6dOBaBRo0ZYWVnRoUMH5s2bh5ubG66urvkeszDnBZg+fTqTJ0/WPo6LizNogiSV6oQQIn/79+/n6NGjOu9XTk5OfPrpp7Rr186AkZV+blk9R5HxyaRnZGKs1vu7VCGEKJP0/ms3aNAgNmzYkGP7Tz/9xKBBgwp8HFNTU5o3b86uXbt0tu/atQtfX99c90lMTMTISDdktVoNoF1vqW3btjmO+ddff2mPWZjzApiZmWFra6tzM6SHc44kORJCiNyYmZkRHx+fY/uDBw8wNTU1QERlh7O1GcZGKjIViIxPefIOQghRTuidHB07dozOnTvn2N6pUyedctkFMXnyZFasWMGqVau4dOkSkyZNIjg4WDtMbvr06dpF+wD69u3L5s2bWb58OTdu3OCff/5h4sSJtGrVCnd3d0CzDtNff/3F/PnzuXz5MvPnz2f37t34+fkV+LxlgfQcCSFE/vr06cPrr7/OsWPHUBQFRVE4evQo48aNo1+/foYOr1RTG6m0i7+GybwjIUQFovewupSUFNLT03NsT0tLIylJvz+gAwcOJDo6mrlz5xIWFkaDBg3Ytm0b1apVAyAsLExn7aERI0YQHx/PsmXLmDJlCvb29jz77LPMnz9f28bX15cNGzYwc+ZMPvjgA2rUqMHGjRtp3bp1gc9b2qVlZBIepxkHLnOOhBAid//3f//H8OHDadu2LSYmJoDmver5559nyZIlhg2uDHC3NyckJonQmGSal423RyGEeGp6r3PUqVMnGjZsqFMhDuB///sfZ8+e5eDBg0UaYGllyDUo7txPpP38vZioVfz3UU+MjPIuJCGEEGVFcf1dvXbtGpcuXUJRFOrVq0fNmjWL7NjFxdDrHAFM/PEUv58J5f1edXn9GansJ4Qo2wr6d1XvnqOPP/6Yrl27cubMGbp06QLA33//zYkTJ/jrr78KH7EosNCs6kFudhaSGAkhxCMeLZyTm3379mnvL168uMDHDQgI4LPPPiMsLIz69euzZMkSOnTokGf7lJQU5s6dy/fff094eDhVq1ZlxowZjBo1qsDnNDS3rKUiQqVinRCiAtE7OWrXrh1Hjhzhs88+46effsLCwoJGjRqxcuVKatWqVRwxiseExCQCMqROCCEed+rUqQK1y2/phsdt3LgRPz8/AgICaNeuHV9//TU9e/bk4sWLeHp65rrPK6+8QkREBCtXrqRmzZpERkbmOiS9NHO3y14IVuYcCSEqDr2TI4AmTZqwfv36oo5FFFD2t3hSjEEIIXTt3bu3yI+5ePFiRo8ezZgxYwBYsmQJO3fuZPny5fj7++dov2PHDvbv38+NGze0ZcS9vLyKPK7i5ppVzjs8VnqOhBAVR4Gq1cXFxencz+8mit8dKeMthBAlIjU1lcDAQLp166azvVu3bhw+fDjXfX7//XdatGjBggULqFKlCrVr1+add97Jt2hRSkpKqXs/ze45CpXkSAhRgRSo58jBwYGwsDAqV66Mvb19rsMRFEVBpVKRkZFR5EEKXdllvKtkjQcXQghRPKKiosjIyMixSLiLi0uOxcSz3bhxg0OHDmFubs6WLVuIiopi/Pjx3Lt3j1WrVuW6j7+/P3PmzCny+J9G9pyjqAcppKZnYmosC8EKIcq/AiVHe/bs0Q4NKI4hC0I/IdrkyNLAkQghRMXw+JeC2V8I5iYzMxOVSsX69euxs7MDNEPzBgwYwJdffomFRc5e/+nTp+sUk4iLi8PDw6MIr0B/TlammBobkZqeSURcMh6O8p4jhCj/CpQcdezYEYD09HT27dvHqFGjDP5Hu6JSFOWRBWCl50gIIYqTs7MzarU6Ry9RZGRkjt6kbG5ublSpUkWbGAH4+PigKAp37tzJtXiRmZkZZmZmRRv8U1KpVLjZmXMrOpHQmCRJjoQQFYJefeTGxsYsXLhQhs4ZUExiGompmtdfCjIIIUTxMjU1pXnz5uzatUtn+65du/D19c11n3bt2hEaGsqDBw+0265cuYKRkRFVq1Yt1niLmltWUYYwmXckhKgg9B5A3KVLF511IkTJyh5S52xtirmJ2sDRCCFE+Td58mRWrFjBqlWruHTpEpMmTSI4OJhx48YBmiFxw4YN07YfMmQITk5OjBw5kosXL3LgwAGmTp3KqFGjch1SV5o9LMog5byFEBWD3qW8e/bsyfTp0zl//jzNmzfHyspK5/l+/foVWXAip4dD6srWG6wQQpRVAwcOJDo6mrlz5xIWFkaDBg3Ytm0b1apVAyAsLIzg4GBte2tra3bt2sWECRNo0aIFTk5OvPLKK8ybN89Ql1BoUs5bCFHR6J0cvfnmm0DuK4tLtbri97AYgyRHQghRUsaPH8/48eNzfW7NmjU5ttWtWzfHULyyyC3rvSZ7fT0hhCjv9E6OMjMziyMOUUDScySEEKKkZC8ZcTk8jvSMTIzVUs5bCFG+yV+5MkZ6joQQQpSUll6O2FuacOd+Er+dDjV0OEIIUewKlRzt37+fvn37UrNmTWrVqkW/fv04ePBgUccmchGSNbRBeo6EEEIUNxtzE954pgYAS/6+QlqGjB4RQpRveidH33//PV27dsXS0pKJEyfy1ltvYWFhQZcuXfjhhx+KI0bxiJD7mp6jqg6SHAkhhCh+w32r4Wxtyu17Sfz87x1DhyOEEMVK7+To448/ZsGCBWzcuJGJEyfy9ttvs3HjRj799FM++uij4ohRZElOyyDqQQogPUdCCCFKhqWpMeM71QRg6Z6rJKdJ4SUhRPmld3J048YN+vbtm2N7v379uHnzZpEEJXKXvQifhYkaB0sTA0cjhBCiohjS2hM3O3PCYpP54Vjwk3cQQogySu/kyMPDg7///jvH9r///hsPD48iCUrk7mGlOnNUKpWBoxFCCFFRmJuomfBsLQAC9l0jMTXdwBEJIUTx0LuU95QpU5g4cSKnT5/G19cXlUrFoUOHWLNmDV988UVxxCiyZM83kiF1QgghStrLLary1f7rBN9LZO3hW7zZqYahQxJCiCJXqEVgXV1dWbRoET/99BMAPj4+bNy4keeff77IAxQPZZfxlmIMQgghSpqJ2oi3u9Riys9n+Gr/dV5t44mtuQzxFkKUL3onRwAvvPACL7zwQlHHIp5AO6zOTpIjIYQQJa9/0yoE7LvG9bsJrDp0E7+utQ0dkhBCFClZBLYM0S4AKz1HQgghDEBtpGLyc3UAWHHwJvcTUg0ckRBCFC29kyMHBwccHR1z3JycnKhSpQodO3Zk9erVxRFrhfewIIMkR0IIIQyjZwNXfNxseZCSztcHbhg6HCGEKFJ6J0ezZs3CyMiI3r17M2fOHD788EN69+6NkZER//vf/6hduzZvvvkm3377bXHEW2FlZiqExmhKeVeR5EgIIYSBGBmpmPKcZjjdmsM3iYxPNnBEQghRdPSec3To0CHmzZvHuHHjdLZ//fXX/PXXX2zatIlGjRrxf//3f4wdO7bIAq3oohJSSM3IxEgFrnbmhg5HCCFEBdbFpzJNPOw5fTuG5fuuM7tvfUOHJIQQRULvnqOdO3fStWvXHNu7dOnCzp07AejVqxc3bkhXe1HKLuPtYmuOiVqmigkhhDAclUrFO900c4/WHw3WDvsWQoiyTu9P2Y6Ojvzxxx85tv/xxx84OjoCkJCQgI2NTYGOFxAQgLe3N+bm5jRv3pyDBw/m2XbEiBGoVKoct/r1H35j1alTp1zb9O7dW9vmww8/zPG8q6trQV8Cg8geUifzjYQQQpQG7Wo60drbkdSMTJbuuWbocIQQokjoPazugw8+4M0332Tv3r20atUKlUrF8ePH2bZtG1999RUAu3btomPHjk881saNG/Hz8yMgIIB27drx9ddf07NnTy5evIinp2eO9l988QWffvqp9nF6ejqNGzfm5Zdf1m7bvHkzqakPq+dER0fnaANQv359du/erX2sVqsL/iIYQEhMIiDzjYQQQpQOKpWKd7rX4eWvjvDzv7cZ17E61ZysDB2WEEI8Fb2To7Fjx1KvXj2WLVvG5s2bURSFunXrsn//fnx9fQGYMmVKgY61ePFiRo8ezZgxYwBYsmQJO3fuZPny5fj7++dob2dnh52dnfbxr7/+yv379xk5cqR2W3bvVbYNGzZgaWmZIzkyNjYu9b1Fj5KeIyGEEKVNSy9HOtauxP4rd/ni76ssfqWJoUMSQoinUqhFYNu1a0e7du2e6sSpqakEBgYybdo0ne3dunXj8OHDBTrGypUr6dq1K9WqVcu3zaBBg7Cy0v026+rVq7i7u2NmZkbr1q355JNPqF69uv4XUkK0axzZSzEGIYQQpceUbrXZf+Uuv54KYXynGtSsXLBh9UIIURoVamb/9evXmTlzJkOGDCEyMhKAHTt2cOHChQIfIyoqioyMDFxcXHS2u7i4EB4e/sT9w8LC2L59u7bXKTfHjx/n/PnzOdq0bt2adevWsXPnTr799lvCw8Px9fUlOjo6z2OlpKQQFxencytJ2QUZZAFYIYQQpUmjqvZ0q+dCpgKf775q6HCEEOKp6J0c7d+/n4YNG3Ls2DE2bdrEgwcPADh79iyzZ8/WOwCVSqXzWFGUHNtys2bNGuzt7enfv3+ebVauXEmDBg1o1aqVzvaePXvy0ksv0bBhQ7p27crWrVsBWLt2bZ7H8vf31w7rs7Ozw8PD44kxFqXQWFkAVgghROk0uVttVCrYejaMC6Gxhg5HCCEKTe/kaNq0acybN49du3Zhamqq3d65c2eOHDlS4OM4OzujVqtz9BJFRkbm6E16nKIorFq1iqFDh+rE8KjExEQ2bNiQb89SNisrKxo2bMjVq3l/4zV9+nRiY2O1t9u3bz/xuEUlISWdmMQ0QAoyCCGEKH3qutrSt5E7AJ/vumLgaIQQovD0To7OnTvHCy+8kGN7pUqV8h2W9jhTU1OaN2/Orl27dLbv2rVLW9ghL/v37+fatWuMHj06zzY//fQTKSkpvPbaa0+MJSUlhUuXLuHm5pZnGzMzM2xtbXVuJSV7/Qgbc2NszE1K7LxCCCFEQfl1rYWRCnZfiuRU8H1DhyOEEIWid3Jkb29PWFhYju2nTp2iSpUqeh1r8uTJrFixglWrVnHp0iUmTZpEcHAw48aNAzS9NcOGDcux38qVK2ndujUNGjTI89grV66kf//+ODk55XjunXfeYf/+/dy8eZNjx44xYMAA4uLiGD58uF7xl5Q72mIM0mskhBCidKpeyZqXmlUFYLH0Hgkhyii9k6MhQ4bw3nvvER4ejkqlIjMzk3/++Yd33nkn10QmPwMHDmTJkiXMnTuXJk2acODAAbZt26atPhcWFkZwcLDOPrGxsWzatCnfXqMrV65w6NChPNvcuXOHwYMHU6dOHV588UVMTU05evRovlXvDClUkiMhhBAFlZ4K5zdBZkaJn3pil1qYqFUcvBrF0RsFH00ihBClhUpRFEWfHdLS0hgxYgQbNmxAURSMjY3JyMhgyJAhrFmzptQvplpU4uLisLOzIzY2ttiH2C3YcZmAfdcZ1rYac5/Pu7dMCCHKspL8u1raFfq1UBRY3g4iL8DA78Gnb/EFmYeZv57j+6PBtPRy4Kc32haoyJIQQhS3gv5d1bvnyMTEhPXr13P16lV++uknvv/+ey5fvsx3331XYRKjkpbdcySV6oQQQuRLpYLa3TX3Dy8zSAgTnq2FmbERJ4Luc+BqlEFiEEKIwtI7OZo7dy6JiYlUr16dAQMG8Morr1CrVi2SkpKYO3duccRY4YXIsDohhBAF1ep1MDKB20fhzr8lfnoXW3OGttEMU1/013/oOUBFCCEMSu/kaM6cOdq1jR6VmJjInDlziiQooSs0JhmQniMhhDCUgIAAvL29MTc3p3nz5hw8eDDPtvv27UOlUuW4Xb58uWSCtXWDhi9r7h9eWjLnfMy4TjWwNFVz9k4suy5GGCQGIYQoDL2To7wWaT1z5gyOjo5FEpR4KD0jk/A4TXIkPUdCCFHyNm7ciJ+fHzNmzODUqVN06NCBnj175igY9Lj//vuPsLAw7a1WrVolFDHQ9n+an5d+h/tBJXfeLM7WZoxs5wVoKtdlZkrvkRCibChwcuTg4ICjoyMqlYratWvj6OiovdnZ2fHcc8/xyiuvFGesFVJEfAoZmQomahWVbcwMHY4QQlQ4ixcvZvTo0YwZMwYfHx+WLFmCh4cHy5cvz3e/ypUr4+rqqr2V6Lxc1wZQvTMomXDs65I77yNe71ADG3NjLofH8+e5nEuACCFEaWRc0IZLlixBURRGjRrFnDlzsLOz0z5namqKl5cXbdu2LZYgK7LsYgyuduYYGUnFHyGEKEmpqakEBgYybdo0ne3dunXj8OHD+e7btGlTkpOTqVevHjNnzqRz5855tk1JSSElJUX7OC4u7ukCB2j7FtzYCyfXQcf3wML+6Y+pBztLE17vUJ1Fu66wZNcVejVwxVit94AVIYQoUQVOjrIXSPX29sbX1xcTE5NiC0o8FHJfijEIIYShREVFkZGRgYuLi852FxcXwsPDc93Hzc2Nb775hubNm5OSksJ3331Hly5d2LdvH88880yu+/j7+xf9vN2aXaCSD9y9BCfXQru3i/b4BTCyvTer/rnJjagEtpwK4eUWHiUegxBC6EPvr3A6duyoTYySkpKIi4vTuYmiFSJlvIUQwuAen2ub1/xbgDp16jB27FiaNWtG27ZtCQgIoHfv3ixcuDDP40+fPp3Y2Fjt7fbt20UR9MO5R8e+hoy0pz+mnqzNjHmzUw0Avvj7KqnpmSUegxBC6EPv5CgxMZG33nqLypUrY21tjYODg85NFK3s5KiqJEdCCFHinJ2dUavVOXqJIiMjc/Qm5adNmzZcvXo1z+fNzMywtbXVuRWJRq+AVWWIC4ELW4rmmHoa2saLyjZm3LmfxE//FkHSJ4QQxUjv5Gjq1Kns2bOHgIAAzMzMWLFiBXPmzMHd3Z1169YVR4wVmiwAK4QQhmNqakrz5s3ZtWuXzvZdu3bh6+tb4OOcOnUKNze3og7vyYzNNOsegaastwHWHLIwVfPWszUBWLrnKslpGSUegxBCFJTeydEff/xBQEAAAwYMwNjYmA4dOjBz5kw++eQT1q9fXxwxVmjaOUcOkhwJIYQhTJ48mRUrVrBq1SouXbrEpEmTCA4OZty4cYBmSNywYcO07ZcsWcKvv/7K1atXuXDhAtOnT2fTpk289dZbhrmAFqPA2ALCz0LQIYOEMLClB1XsLYiIS+H7o7cMEoMQQhSE3snRvXv38Pb2BsDW1pZ79+4B0L59ew4cOFC00VVwiqJIz5EQQhjYwIEDWbJkCXPnzqVJkyYcOHCAbdu2Ua1aNQDCwsJ01jxKTU3lnXfeoVGjRnTo0IFDhw6xdetWXnzxRcNcgJUTNBmsuX9kmUFCMDNWM7GLpvdo+b7rxCWX/PwnIYQoCL2To+rVqxMUFARAvXr1+OmnnwBNj5K9vX1RxlbhxSWlk5CqGX7gbifJkRBCGMr48eMJCgoiJSWFwMBAnapza9asYd++fdrH7777LteuXSMpKYl79+5x8OBBevXqZYCoH9Hmf4AKruyAu1cMEsJLzari7WxFdEIqb34fKMUZhBClkt7J0ciRIzlz5gygGUqQPfdo0qRJTJ06tcgDrMjuxCQC4GRlioVpCS4eKIQQonxxrgl1emruH/3SICEYq41YOrgpVqZq/rkWzbu/nCEzs+TnQAkhRH4KvM5RtkmTJmnvd+7cmcuXL/Pvv/9So0YNGjduXKTBVXShMcmADKkTQghRBNq+Bf9tgzMb4NkPwMq5xENoUMWO5a81Z9SaE/x6OhQXO3Om9/Qp8TiEECIvT71UtaenJy+++KIkRsUg5L6m50gWgBVCCPHUqvmCe1NIT4YTKw0WxjO1KzH/pUYAfL3/Bmv+uWmwWIQQ4nEFTo727NlDvXr1cl3oNTY2lvr163Pw4MEiDa6iC42VniMhhBBFRKXS9B4BnPgW0pINFspLzasytXsdAOb8eZHt58IMFosQQjyqwMnRkiVLGDt2bK4L09nZ2fHGG2+wePHiIg2uopMy3kIIIYpUvefBtiok3IWzGw0ayvhONXitjSeKAm9vPM2JoHsGjUcIIUCP5OjMmTP06NEjz+e7detGYGBgkQQlNEKyynhXsTc3cCRCCCHKBbUJtNGsz8SRLyHTcBXjVCoVc/o1oFs9F1LTMxmz9l+uRsQbLB4hhAA9kqOIiAhMTEzyfN7Y2Ji7d+8WSVBC42FyZGngSIQQQpQbzYaBqQ1E/QfXdhs0FLWRiv8b3JRmnvbEJqUxYvUJIuIMN9xPCCEKnBxVqVKFc+fO5fn82bNncXNzK5KgBKSkZ3A3PgUAd+k5EkIIUVTM7aD5cM39I0sNGwtgbqJm5fCWVK9kRUhMEsNXHZdFYoUQBlPg5KhXr17MmjWL5OSc3+gkJSUxe/Zs+vTpU6TBVWRhWWW8zU2McLQyNXA0QgghypXW40ClhpsHIOysoaPBwcqUtSNb4WxtxuXweMZ9J4vECiEMo8DJ0cyZM7l37x61a9dmwYIF/Pbbb/z+++/Mnz+fOnXqcO/ePWbMmFGcsVYooVlD6tztLVCpVAaORgghRLli7wH1+2vuHzHMorCP83C0ZM3IlliZqjl8PZqpskisEMIACpwcubi4cPjwYRo0aMD06dN54YUX6N+/P++//z4NGjTgn3/+wcXFpThjrVAezjeSSnVCCCGKQXZZ7/O/QFyoYWPJkr1IrLGRit9OhzJ/52VDhySEqGD0WgS2WrVqbNu2jaioKI4dO8bRo0eJiopi27ZteHl5FVOIFZMkR0IIIYpVlWbg6QuZ6XDsa0NHoyWLxAohDEmv5Cibg4MDLVu2pFWrVjg4OBR1TALdYXVCCCFEsfDN6j0KXA0pDwwbyyNkkVghhKEUKjkSxU96joQQQhS72j3BsQYkx8Lp9YaORsfji8QevymLxAohip/Bk6OAgAC8vb0xNzenefPmHDx4MM+2I0aMQKVS5bjVr19f22bNmjW5tnm8yp4+5zWE0KxqddJzJIQQotgYGUHb8Zr7RwMgM8Ow8Twi5yKxJ2SRWCFEsTNocrRx40b8/PyYMWMGp06dokOHDvTs2ZPg4OBc23/xxReEhYVpb7dv38bR0ZGXX35Zp52tra1Ou7CwMMzNH64VpO95S1pmpqLtOarqIMmREEKIYtR4CFg4wv0guPynoaPR8egisXHJ6QxfdZzwWFkkVghRfAyaHC1evJjRo0czZswYfHx8WLJkCR4eHixfvjzX9nZ2dri6umpv//77L/fv32fkyJE67VQqlU47V1fXpzpvSYtOSCU1PROVClxsZQFYIYQQxcjUElqO1tw/vMywseTi0UViQ2OTGbFaFokVQhQfgyVHqampBAYG0q1bt/9v797joqrz/4G/Zoa5cB3uVwERE1Twhgp4ya0MY7M0d5Psu2hlm+5W31Xru+m2fTPrl3bd3Ezbysu6m+Z+81KtmuEmXsJLGZR5v5AgDCIoM1xkgJnP748DgyNXETgz+Ho+Hp8HZ858zjnvM0f5zJvPOZ+P3fqUlBRkZWW1ax8rV67E+PHjERkZabe+oqICkZGR6NWrFyZOnIjs7OybPq7ZbIbJZLIrXaWh1yjIUweNi+x3PhIRUU834reASgNcOATkH5I7miY4SSwRdRfZvnmXlJTAYrE0mRspKCgIRUVFbW5vMBiwfft2PP7443brY2NjsWbNGnz++edYv349dDodRo8ejdOnT9/UcRcvXgy9Xm8r4eHh7T3VG9Y4Uh17jYiIqBt4BgHxU6Xl/Y7XewRwklgi6h6yd0soFAq710KIJuuas2bNGnh7e2Py5Ml265OSkvCb3/wGgwcPxtixY/Gvf/0L/fr1w7vvvntTx12wYAGMRqOt5OfntxljRzUkR2E+bl12DCIiIjvJT0o/j38hPX/kgJpMEvslJ4klos4lW3Lk7+8PlUrVpLemuLi4Sa/O9YQQWLVqFdLT06HRaFqtq1QqMWLECFvPUUePq9Vq4eXlZVe6yoUr7DkiIqJuFjQAiL4LEFbggGM8g9scu0li95zDu/85DSHYg0REnUO25Eij0SAhIQEZGRl26zMyMjBq1KhWt929ezfOnDmDmTNntnkcIQRycnIQEhJy08ftLoWc44iIiOTQMCns9/8Arl6RN5ZWXDtJ7FsZp/D0+mxU1dTJHBUR9QSy3lY3b948fPTRR1i1ahWOHz+OuXPnIi8vD7NnzwYg3co2ffr0JtutXLkSiYmJiIuLa/LeSy+9hB07duDcuXPIycnBzJkzkZOTY9tne44rN04AS0REsuhzBxA4EKitBA6vkTuaVv3+F9F4edJAuCgV+PePBkxZnoW80iq5wyIiJ+ci58HT0tJQWlqKRYsWwWAwIC4uDtu2bbONPmcwGJrMPWQ0GrFx40YsXbq02X2WlZXhiSeeQFFREfR6PYYOHYo9e/Zg5MiR7T6u3BoHZGByRERE3UihkJ49+uz3wMG/AUlPAi6t374uF4VCgfTk3ogJ9sLvPz6ME0XluG/ZPrw7bShu7xcgd3hE5KQUgjfqdojJZIJer4fRaOzU54+qauow4H93AAB+XJgCL5260/ZNROTIuur3qjOS9bOoMwPvxAMVF4EHPgAGp3Xv8TvAYLyK2f/8Hj/kl0GpAP54Tyxm3d6nXQM8EdGtob2/V2UfrY7sNfQaeWpdmBgREVH3c9ECI5+Qlve/CzjB31BD9K7Y8EQSpg7vBasAlmw/gaf4HBIRdQCTIwfTMFJdmA9vqSMiIpkMfwxQuwFFR4DcPXJH0y46tQqv/WoQXp4cBxelAlvrn0M6X1opd2hE5ESYHDmYwrJqAHzeiIiIZOTmCwz5L2nZQSeFbY5CoUB6UiTWP5EEfw8tThSV4/5l32D3qUtyh0ZEToLJkYMpKJNG2uFIdUREjmP58uWIioqCTqdDQkIC9u7d267tvvnmG7i4uGDIkCFdG2BXSPodAAVw+ivg0km5o7khI3r74t9Pj8GQcG8Yr9bi0dWHsCLzLOdDIqI2MTlyMOw5IiJyLBs2bMCcOXPw/PPPIzs7G2PHjkVqamqT0VSvZzQaMX36dNx1113dFGkn84sGYu+Vlr9+GbDUyhvPDQrW67BhVhLShofDKoDXvjyBp9Zlo9LM55CIqGVMjhxMgW0Yb53MkRAREQC8/fbbmDlzJh5//HH0798f77zzDsLDw7FixYpWt5s1axYefvhhJCcnd1OkXWD0HAAK4PgXwN/vA8qL5I7ohmhdVFjyq3i8MjkOapUCW48Y8KsVfA6JiFrG5MjBFNQPyNCLAzIQEcmupqYGhw8fRkpKit36lJQUZGVltbjd6tWrcfbsWbz44ovtOo7ZbIbJZLIrDiF8BJD2T0DrBeTtB94fC/z8jdxR3RCFQoHfJEVi/W8bn0O67919yDxZLHdoROSAmBw5EItVoMjE2+qIiBxFSUkJLBYLgoKC7NYHBQWhqKj5XpTTp09j/vz5+Pjjj+Hi0r651hcvXgy9Xm8r4eHhNx17p+k/EXgiEwgcAFQWSz1IWc4xxPe1htc/hzQ0whum6jo8uuZbLM88w+eQiMgOkyMHctFUDYtVwEWpQKAnb6sjInIU108mKoRodoJRi8WChx9+GC+99BL69evX7v0vWLAARqPRVvLz82865k7lFw08vhOInwoIC/DVn4H/mwGYy+WO7IYE63X45IkkPDQiHEIAr395Ek+u+57PIRGRTfv+pEXdomEC2GC9DiolZ/UmIpKbv78/VCpVk16i4uLiJr1JAFBeXo7vvvsO2dnZeOqppwAAVqsVQgi4uLjgq6++wp133tlkO61WC61W2zUn0Vk07sCUD4DwkcCXC4BjnwHFx4Gp/wACY+WOrt2k55AGIb6XHgs/P4ptR4pwtrgSf0tPQG9/d7nDIyKZsefIgTQMxsBhvImIHINGo0FCQgIyMjLs1mdkZGDUqFFN6nt5eeHIkSPIycmxldmzZyMmJgY5OTlITEzsrtC7hkIBjPwt8Oh2wDMUKDkFfHgn8NNGuSO7Yf+VGIlPnkhCgKcWJy+W4/5l+7DrBJ9DIrrVMTlyIEyOiIgcz7x58/DRRx9h1apVOH78OObOnYu8vDzMnj0bgHRL3PTp0wEASqUScXFxdiUwMBA6nQ5xcXFwd+8hPRPhI4BZe4Co24HaSuDTx6TeJCcb7jshsulzSE+s/Q5niivkDo2IZMLkyIE0jFQXxpHqiIgcRlpaGt555x0sWrQIQ4YMwZ49e7Bt2zZERkYCAAwGQ5tzHvVIHgHAbzYDY+ZKrw8sB9ZMBEwGeeO6QUFe0nNIj4zqDaUC+OrYRaT8ZTcWbPoRF+sHSSKiW4dCcJiWDjGZTNDr9TAajfDy8uqUfT66+hB2nbyExVPiMW1kRKfsk4jIWXTF71Vn5XSfxYmtwObZgNkEuAcCD64Geo+RO6obdqa4HK99eRIZxy4CAHRqJWaOicKscdHw0qlljo6IbkZ7f6+y58iB8LY6IiJySrH31g/3PbB+uO/7nXK4776Bnvhw+nB8OjsZCZE+qK614r1dZzHu9V1YuS8X5jqL3CESURdjcuQghBC22+o4xxERETkdv2jg8QxgUFrjcN//mg5UO8iEtjdgeG9ffDo7GR+kJyA6wB1Xqmrx8r+P4a63dmNLdgGsVudK+oio/ZgcOQhTdR0qa6S/SIV6c44jIiJyQhp34IG/Ab98E1CqgeOfS6PZFR+XO7IbplAokDIwGDvm3I4lU+IR5KXFhStXMWdDDia+uw97Tl3iBLJEPRCTIwfR0Gvk666Bm4bTTxERkZO6drhvrzCg9DTw4V1OOdw3ALiolHhoZAQyn70D/zMhBp5aFxwzmDB91SH8ZuVBHLlglDtEIupETI4cRMMEsOw1IiKiHsE23Pe4xuG+t893uuG+G7hqVHjyjr7Y/cc7MHNMFDQqJb45U4r7lu3D0+uzkVdaJXeIRNQJmBw5CA7GQEREPY67P5C+GRgzT3p9cAWwOhU4vROwWuWNrYN83TV4YeIA/OeZcXhgaBgUCuCLHwpx19uZWPj5UZRUmOUOkYhuApMjB9HYc8TkiIiIehClChj/IvDQOkDrBVz4Fvj4V8C7w6QR7aouyx1hh4T7uuEvaUPw76fH4PZ+Aai1CKzJ+hnjXt+Fv/7nNCrNdXKHSEQdwOTIQVxgzxEREfVksfcCv/sGSPwdoNUDV3KlEe3e7g9seRIozJY7wg4ZGKrH2sdG4uPHExEfpkdljQVvZ5zCuDcy8d6uMygu50SyRM6EyZGDKGRyREREPZ13BJC6BHjmOHDfUiAoHqirBnL+CXzwC2lku5z1QK3zJRSj+/rjsydH491pQxHh64aSCjPe2HESoxZ/jd9/fBjfnCnhEOBEToDDojmIhtHqwnyYHBERUQ+ncQcSHgGGzQDyDwHffggc3QIUHJbKjj8Bw9KB4Y8BPr1lDrb9lEoF7hscigkDg/H5D4VYd/A8vs8rw7YjRdh2pAi9/dzwcGIEfp0QDl93jdzhElEzFIKD9HeIyWSCXq+H0WiEl5fXTe3LXGdBzJ+/BAB89+fx8PfQdkaIREROpTN/rzq7W/KzqLgEZK8FvlsNGPPrVyqA2+4GRvwW6DseUDrfDS/HDSasO5iHzdkFqKh/DkmjUiI1Phj/lRiJEb19oFAoZI6SqOdr7+9VJkcd1JkNV15pFW5/Yxe0LkqcePke/pIkolvSLZkQtOCW/iysFuDUDqk36ezXjeu9I4ERM4Gh6YCbr3zxdVCluQ5f/FCIjw/m4UhB49xIfQM98F+JEZgytBf0bmoZIyTq2dr7e1X2P8EsX74cUVFR0Ol0SEhIwN69e1us+8gjj0ChUDQpAwcOtNX58MMPMXbsWPj4+MDHxwfjx4/HoUOH7PazcOHCJvsIDg7usnNsy4UyaW6EMG9XJkZERHRrU6qA2F9KQ4A//T2Q9CSg0wNl54GM/wXeigU2/w64cFjuSG+Iu9YFD42MwBdPj8HnT43GQyPC4apW4UxxBV764hhGvroTz/zrB3yfdwX8uzWRfGRNjjZs2IA5c+bg+eefR3Z2NsaOHYvU1FTk5eU1W3/p0qUwGAy2kp+fD19fXzz44IO2OpmZmZg2bRp27dqF/fv3IyIiAikpKSgoKLDb18CBA+32deTIkS4919YUlkkPnnIYbyIiomv4RQP3vArMOwHc/y4QPAiwmIEf1gEf3SkN4nDgfaCs+e8NjmpQL28s+dUgHHz+Lrw8aSBigz1hrrNi4/cXMGV5FlKX7sU/DpxHebVzTphL5Mxkva0uMTERw4YNw4oVK2zr+vfvj8mTJ2Px4sVtbr9lyxZMmTIFubm5iIyMbLaOxWKBj48Pli1bhunTpwOQeo62bNmCnJycDsfembc8LN15Gn/ZeQppw8Px2q8H3dS+iIic1S19K9l1+Fm0QAjgwnfAtx8BRzcBlprG94LjgZh7pSHDg+MBJ7oTQwiB7/PKsO5gHv79YyHMddIEuW4aFSYNCcXDIyMR30svc5REzq29v1dlG62upqYGhw8fxvz58+3Wp6SkICsrq137WLlyJcaPH99iYgQAVVVVqK2tha+v/f3Jp0+fRmhoKLRaLRITE/Hqq6+iT58+Le7HbDbDbG6c9dpkMrUrxvbgBLBERETtoFAA4SOkMuH/AT/+CzixFcjLAoqOSGX3EkAfAcSkSolS5ChA5djP8igUCiRE+iAh0gcvTOyPTd8X4OOD53H2UiXWH8rH+kP5GBjqhclDwnDf4FAE63Vyh0zUY8mWHJWUlMBisSAoKMhufVBQEIqKitrc3mAwYPv27Vi3bl2r9ebPn4+wsDCMHz/eti4xMRFr165Fv379cPHiRbzyyisYNWoUjh49Cj8/v2b3s3jxYrz00kvtOLMbV1DGYbyJiIhuiLs/kPx7qVSWAqd3SInSmf8Axjzg0N+kotMDt02QEqW+dwFaT7kjb5W3mwaPjYnCo6N741DuZXx8MA9f/lSEo4UmHC004dXtx5EY5YtJQ8KQGhcMbzcOCU7UmWSf5+j6AQiEEO0alGDNmjXw9vbG5MmTW6zz+uuvY/369cjMzIRO1/hXltTUVNtyfHw8kpOTER0djb///e+YN29es/tasGCB3Xsmkwnh4eFtxtkejT1H/EsQERHRDXP3A4Y8LJWaKuBcJnByK3ByO1BVChz5l1RUGiBqnJQoxaQCnvINxtQWhUKBxD5+SOzjh8uVNdh6xIAvcgpx6OfLOHBOKv/72U8Y1y8Qk4aEYnz/ILhqVHKHTeT0ZEuO/P39oVKpmvQSFRcXN+lNup4QAqtWrUJ6ejo0mub/YvLmm2/i1Vdfxc6dOzFoUOvP8bi7uyM+Ph6nT59usY5Wq4VW2/nzDwkhbD1HvbzdOn3/REREtxSNmzTaXewvpWHB8w9JidKJrcDlc8CZDKn8ew7QawQQ80sgdiIQ0E/uyFvk665BelIk0pMiUVB2FV/8UIjPcgpx3GDCzuMXsfP4RbhpVJgwMBj3Dw7FmNv8oVbJPiAxkVOSLTnSaDRISEhARkYGHnjgAdv6jIwMTJo0qdVtd+/ejTNnzmDmzJnNvv/GG2/glVdewY4dOzB8+PA2YzGbzTh+/DjGjh17YyfRCUora2Cus0KhAO8hJiIi6kxKFRCZLJW7XwYunWxMlAoOAxe+lcp/XgL8+gKRowFXH+lWPJ0XoG346SX91OmlZY2HbBPShnm7Yva4aMweF41TF8vxeU4hPvuhAPmXr2JzdgE2ZxfAx02NeweFYNKQMCRE+ECpdJ7BKYjkJuttdfPmzUN6ejqGDx+O5ORkfPDBB8jLy8Ps2bMBSLeyFRQUYO3atXbbrVy5EomJiYiLi2uyz9dffx0vvPAC1q1bh969e9t6pjw8PODh4QEAePbZZ3HfffchIiICxcXFeOWVV2AymTBjxowuPuOmCq5IvUaBnlpoXPhXHiIioi6hUACBsVIZ+wxgMgAnt0nl3G6g9IxU2rezpgnT9UmUTg/oewHevQGf3tLEtZ08gl6/IE88OyEGz6T0Q3Z+GT7PKcS/fyxESUUN/nkgD/88kIcwb1fcNzgU9w8ORf8QT86nSNQGWZOjtLQ0lJaWYtGiRTAYDIiLi8O2bdtso88ZDIYmcx4ZjUZs3LgRS5cubXafy5cvR01NDX7961/brX/xxRexcOFCAMCFCxcwbdo0lJSUICAgAElJSThw4ECro951FY5UR0REJAOvEGDETKlUm4AzO6WeJbNJem02AtXG+uX6ddVGwFoLQEjvm42AMb99x9N4SEmSdyTgE3nNcm/AO0K6HbCDFAoFhkX4YFiED/58b3/sP1eKz3IK8eVPRSgou4r3d5/F+7vP4rZAD0waEor7B4chwo+38hM1R9Z5jpxZZ81B8dHec3hl63HcOygE7z08rBMjJCJyLpzbpxE/CwclBFBXbZ8wNZdEmU1A1WUpcbpyHigvbHvfHkGNyZJd8hQJeIVJtwjeoOpaC3adKMZnOYX4+mQxaurnTwKA6AB3jO7rj9F9/ZHUxw96V8ce7pzoZjn8PEckaRyMgT1HREREDk2hANSuUvFsffAoO7XV9YnSz1IpO1+/fF4qZiNQcVEqFw413V6plp6FUqkBpUv9TzWgcpFeK9XNvqdTqpGqUiPV0wU1CUrkldXiTKkZP5fVofSyJ0oPemHdQT2WwQuBweGIu60PkvqFYlikN7QuHPmOHJQQXTrJM5MjmfG2OiIioh5OrQP8b5NKc65eaUyW7BKnn6WkylIDVBbfVAgaAH3rS7Pf/i4DOAiYDrjBAC/U6Pyg9gqE3i8UPgGhUHgESnNLuQc0FlcfeQamsNQBFjNQV1+uXb7+Na6/Qeq6L9VNvmS38r4QgLVOurXSagEstfWvryktrrPUb9ew3gK46ACNuzT3lsajftkD0Hhes+xR/747oHbr0qSgS9WZAXN5Y6mpaGa5/mdN+XWvr6t755+B0X/oslCZHMnMNgEskyMiIqJbk6uPVEKHNn3PagHKDcDVsuu+bNc2fhG3va67Zv31r6+pV2eW5n+qvARUXoKl/BIUVSVQijp4KarghSrAXARcOgpcAnCihbgVSsDNX+pJUyilL+4KpVSguG6dopl1zdQToj65qan/WS0t11VLSWKdGRCWrroSjk2hbEyiNB6NyZPGA3DR1CdgdfYJmF3C1sL7zb22O24rCaTdey2srzPXP6vXScwVnbevZjA5kllhWTUA9hwRERFRM5QqadQ7fa8uO4QKkJKS6jKIiku4cCEPp87lorAgD8aSQnhZjfBTGOGvMMEPJgQoy6FHOSCsN92jddMUSqkHRqWRfrpoAJW2cVlxXc9Ws4/aN7OupUfyG25fvLao1NJ1Ul7znsqlhXouUl2FSkr4aiobe0RqKqUv/jXl1yxXSu9BSJ+3uf6ZNmelru8Ra+gt03pet1zfc9aw3Fw9N98uDZHJkYyqaupwubIGABDmw+SIiIiIZKJQAK4+ULj6IDygH8LrO7FqLVb8kF+GfWdKsOZMCbLzylBnFXBBHXxQjgClEf181YgJ8kC/QHf0C3RHmLcOSgjYvtCLa342WWe1XwdxTbKjvW5ZW5/4XLOsugW+ylqtQG1VY6JkLm+6bKlpmowpVW28bqmOCva3F16XKDZJHNt4X6VuTGw6MLBId7sF/kU5roZeIw+tC7x0vBRERI5q+fLleOONN2AwGDBw4EC88847LU4cvm/fPjz33HM4ceIEqqqqEBkZiVmzZmHu3LndHDXRzVOrlBje2xfDe/tizvh+qDDX4dvcy9h3pgTfnCnBsaJyHCsBUALgKACY4amzYEi4N4ZG+GBohDeGhnvD200j74k4M6WyvhfFA8ANDARCHcJv5DK69nkjTspGROSYNmzYgDlz5mD58uUYPXo0/va3vyE1NRXHjh1DREREk/ru7u546qmnMGjQILi7u2Pfvn2YNWsW3N3d8cQTT8hwBkSdx0PrgjtiA3FHbCAA4FK5Gd/nXUF2Xhmy867gxwtGlFfXYe/pEuw9XWLbro+/O4ZE1CdM4d6IDfaEi0qGwRyI2sB5jjqoM+agWH8oDws2HcEdMQFY/ejITo6QiMi5OOrcPomJiRg2bBhWrFhhW9e/f39MnjwZixcvbtc+pkyZAnd3d/zjH/9oV31H/SyI2lJnseLkxXJ8X58s5eSV4VxJZZN6rmoV4nvpMayhdynCG4GeOhkiplsF5zlyAg3DePN5IyIix1RTU4PDhw9j/vz5dutTUlKQlZXVrn1kZ2cjKysLr7zySot1zGYzzGaz7bXJ5MQPXNMtzUWlxMBQPQaG6pGeFAkAuFJZg5wLZbbepZy8MpSb63Ao9zIO5V62bRvm7YohEd4YFKZHXJgecaF66N04OS11LyZHMiq4wjmOiIgcWUlJCSwWC4KC7O/zDwoKQlFRUavb9urVC5cuXUJdXR0WLlyIxx9/vMW6ixcvxksvvdQpMRM5Gh93De6ICcQdMdKteFarwNlLFVKylC/dknfyYjkKyq6ioOwqtv5osG0b4euG+DA9BoZ5Ib4+YfJx5/NL1HWYHMmIcxwRETmH658LFUK0+azo3r17UVFRgQMHDmD+/Pno27cvpk2b1mzdBQsWYN68ebbXJpMJ4eHhNx84kQNSKhW4LcgTtwV5YuoI6d95eXUtfrxgRE5+GX4qMOJIgREXrlxF3uUq5F2uwtYjjQlTmLcr4sP0iO/V0MPkBT8PrVynQz0MkyMZMTkiInJs/v7+UKlUTXqJiouLm/QmXS8qKgoAEB8fj4sXL2LhwoUtJkdarRZaLb/c0a3LU6fG6L7+GN3X37aurKoGPxWYcKTAiJ8KjfipwIjzpVW2HqYvjzb+vwzV66REKUwv9TCF6RHgyf9TdOOYHMnEYhUoMnICWCIiR6bRaJCQkICMjAw88MADtvUZGRmYNGlSu/cjhLB7poiI2ubtpsGY2/wx5rbGhMl4tRZH6xOlIwUmHC0w4lxJJQqN1Sg0VuOrYxdtdYO9dIjwdYO/pwb+Hlr4uWtty/4eWgR4aOHnoYG7ll+HqRH/NcikuLwadVYBlVKBIC+OzkJE5KjmzZuH9PR0DB8+HMnJyfjggw+Ql5eH2bNnA5BuiSsoKMDatWsBAO+99x4iIiIQGxsLQJr36M0338TTTz8t2zkQ9RR6VzVGRftjVHRjwlReXYujhSb8VGC03ZJ3rqQSRaZqFJmq29ynq1pll0AFXJNA+Xk0Lgd5aeGp4wARPR2TI5k0jFQX7KWDSsk5joiIHFVaWhpKS0uxaNEiGAwGxMXFYdu2bYiMlEbiMhgMyMvLs9W3Wq1YsGABcnNz4eLigujoaCxZsgSzZs2S6xSIejRPnRpJffyQ1MfPtq7SXIcTRSYYjNUoKTejpKIGJRXm+tK4XF1rxdVaC/IvX0X+5attHivAU4voAHdEB3hIJdAD0QHuCNW7Qsnvcz0C5znqoJudg+KznAL84ZMcjIzyxb9mJXdBhEREzoVz+zTiZ0HU9YQQqKyxoLQ+UbpU3pg0lTaTTJVX17W4L51aiT7+jclSdIAH+gS4o4+/B1w1qm48K2oJ5zlycIVlUjcvB2MgIiIi6n4KhQIeWhd4aF0Q6efeZn1TdS3OXarE2eIKnL0klXOXKvFzaSWqa604ZjDhmKHpHGVh3q52SZNU3BHgqW1z1EvqfkyOZFJQVgWAyRERERGRM/DSqTEk3BtDwr3t1tdZrMi/ctUuaTp7qRJniitgvFprG11vz6lLdtupVQoEeGgR4KVDoKe2vugQ6CU93xToKa3389DyEYxuxORIJg09RxypjoiIiMh5uaiUiPJ3R5S/O8ajcYh/IQQuV9bg7KXK+l6mCtty/uUq1FqEbZS91igVgJ+HtkkCFeipRWB9YuXvoYWnTuoFc1Epu/qUezQmRzJpGJAh1Jsj1RERERH1NAqFAn4eUs/PyChfu/dq6qy4VGFGsakaxeVmqZiqUWwyo7i8cV1phRlWAVwqN+NSuRlH23FcN40KXjo1PHUu9UVt++nl6mL/nlZa9nJtrOOpdbmlB5dgciSTgitSctTLhz1HRERERLcSjYsSYd6ubT5eYbEKlFbUJ0/lDclT4/LFcjMumapxuaoG1bVWAEBVjQVVNRYUNX38qV0UCukWQm83NbzdNPBxU8PbtWFZAx93NfSuamnZTVNfTw0PrUuPeIaKyZEMTNW1KDdLI57wtjoiIiIiao5KqZBunfPSAdC3WremzooKcx1MV2tRXl2H8upamOp/Sq/rpO+g17xuWDbV162ps0IIabJd49VanC+tanesapUCelcpWfKpT6y8XdXwca9PoFw1jevd1LbESqd2rNH8mBzJoKHXyMdNDTcNLwERERER3RyNixK+Lhr4ums6vA9znQWmq3UwXq3BlapalFXV4kpVDcqqauqXa1FWVVO/rvF9c50VtRZhG/78RujUyvpEqSGZuiaxctNA79bQSyX1UAV66eDVhZPx8pu5DBqfN2KvERERERE5Bq2LCgGeKgR4am9ou6s1FpRdrcGVSil5Krtaa0ugrlRKr6WkqtaWaJVdrYXFKlBda4XBWA1DGwNTNHjqjr54dkJMR06vXZgcyaC61gp/Dw2fNyIiIiIip+eqUcFV44oQffu/2wohUG6uQ1llrZRYXZM4NfZMXbP+qpRo+dxEz1h7MDmSwb2DQnDvoBBYrELuUIiIiIiIup1CoYCXTg0vnRoRcJM7HBvZB0Jfvnw5oqKioNPpkJCQgL1797ZY95FHHoFCoWhSBg4caFdv48aNGDBgALRaLQYMGIDNmzff1HG7Cif0IiIiIiJyHLImRxs2bMCcOXPw/PPPIzs7G2PHjkVqairy8vKarb906VIYDAZbyc/Ph6+vLx588EFbnf379yMtLQ3p6en44YcfkJ6ejqlTp+LgwYMdPi4REREREfV8CiGEbPd2JSYmYtiwYVixYoVtXf/+/TF58mQsXry4ze23bNmCKVOmIDc3F5GRkQCAtLQ0mEwmbN++3VbvnnvugY+PD9avX98pxwUAk8kEvV4Po9EILy+vdm1DREQt4+/VRvwsiIg6V3t/r8rWc1RTU4PDhw8jJSXFbn1KSgqysrLatY+VK1di/PjxtsQIkHqOrt/nhAkTbPvsjOMSEREREVHPI9uADCUlJbBYLAgKCrJbHxQUhKKioja3NxgM2L59O9atW2e3vqioqNV9dvS4ZrMZZnPjuO0mUwenHSYiIiIiIock+4AMCoX9oARCiCbrmrNmzRp4e3tj8uTJHdrnjR538eLF0Ov1thIeHt5mjERERERE5DxkS478/f2hUqma9NYUFxc36dW5nhACq1atQnp6OjQa+7HOg4ODW91nR4+7YMECGI1GW8nPz2/zHImIiIiIyHnIlhxpNBokJCQgIyPDbn1GRgZGjRrV6ra7d+/GmTNnMHPmzCbvJScnN9nnV199ZdtnR4+r1Wrh5eVlV4iIiIiIqOeQdRLYefPmIT09HcOHD0dycjI++OAD5OXlYfbs2QCk3pqCggKsXbvWbruVK1ciMTERcXFxTfb5hz/8Abfffjtee+01TJo0CZ999hl27tyJffv2tfu4RERERER065E1OUpLS0NpaSkWLVoEg8GAuLg4bNu2zTb6nMFgaDL3kNFoxMaNG7F06dJm9zlq1Ch88skn+POf/4wXXngB0dHR2LBhAxITE9t9XCIiIiIiuvXIOs+RM+McFEREnYu/VxvxsyAi6lwOP88RERERERGRI5H1tjpn1tDhxvmOiIg6R8PvU97QwDaGiKiztbeNYXLUQeXl5QDA+Y6IiDpZeXk59Hq93GHIim0MEVHXaKuN4TNHHWS1WlFYWAhPT892TVp7PZPJhPDwcOTn5zvl/eTOHj/Ac3AEzh4/4Pzn4EjxCyFQXl6O0NBQKJW39l3fN9PGONI17Sieg/ycPX7A+c/B2eMHHOsc2tvGsOeog5RKJXr16nXT+3H2OZOcPX6A5+AInD1+wPnPwVHiv9V7jBp0RhvjKNf0ZvAc5Ofs8QPOfw7OHj/gOOfQnjbm1v7THBERERERUT0mR0RERERERGByJButVosXX3wRWq1W7lA6xNnjB3gOjsDZ4wec/xycPX5qqidcU56D/Jw9fsD5z8HZ4wec8xw4IAMRERERERHYc0RERERERASAyREREREREREAJkdEREREREQAmBx1qeXLlyMqKgo6nQ4JCQnYu3dvq/V3796NhIQE6HQ69OnTB++//343RWpv8eLFGDFiBDw9PREYGIjJkyfj5MmTrW6TmZkJhULRpJw4caKbora3cOHCJrEEBwe3uo2jfP4Nevfu3exn+uSTTzZbX+5rsGfPHtx3330IDQ2FQqHAli1b7N4XQmDhwoUIDQ2Fq6srfvGLX+Do0aNt7nfjxo0YMGAAtFotBgwYgM2bN3fRGbR+DrW1tXjuuecQHx8Pd3d3hIaGYvr06SgsLGx1n2vWrGn2ulRXV3dr/ADwyCOPNIkjKSmpzf125zWg9nHW9gVgG+MI18DZ2hfA+dsYZ29f2joHoOe0MUyOusiGDRswZ84cPP/888jOzsbYsWORmpqKvLy8Zuvn5ubil7/8JcaOHYvs7Gz86U9/wn//939j48aN3Ry59Av8ySefxIEDB5CRkYG6ujqkpKSgsrKyzW1PnjwJg8FgK7fddls3RNy8gQMH2sVy5MiRFus60uff4Ntvv7WLPyMjAwDw4IMPtrqdXNegsrISgwcPxrJly5p9//XXX8fbb7+NZcuW4dtvv0VwcDDuvvtulJeXt7jP/fv3Iy0tDenp6fjhhx+Qnp6OqVOn4uDBg91+DlVVVfj+++/xwgsv4Pvvv8emTZtw6tQp3H///W3u18vLy+6aGAwG6HS6bo2/wT333GMXx7Zt21rdZ3dfA2qbM7cvANsYR7gGzta+AM7fxjh7+wLcQm2MoC4xcuRIMXv2bLt1sbGxYv78+c3W/+Mf/yhiY2Pt1s2aNUskJSV1WYztVVxcLACI3bt3t1hn165dAoC4cuVK9wXWihdffFEMHjy43fUd+fNv8Ic//EFER0cLq9Xa7PuOdA0AiM2bN9teW61WERwcLJYsWWJbV11dLfR6vXj//fdb3M/UqVPFPffcY7duwoQJ4qGHHur0mK93/Tk059ChQwKAOH/+fIt1Vq9eLfR6fecG1w7NxT9jxgwxadKkG9qPnNeAmteT2hch2MY4AmdqX4Rw/jbG2dsXIXp2G8Oeoy5QU1ODw4cPIyUlxW59SkoKsrKymt1m//79TepPmDAB3333HWpra7ss1vYwGo0AAF9f3zbrDh06FCEhIbjrrruwa9eurg6tVadPn0ZoaCiioqLw0EMP4dy5cy3WdeTPH5D+Tf3zn//EY489BoVC0WpdR7oGDXJzc1FUVGT3GWu1WowbN67F/xNAy9eltW26k9FohEKhgLe3d6v1KioqEBkZiV69emHixInIzs7ungCbkZmZicDAQPTr1w+//e1vUVxc3Gp9R78Gt5qe1r4AbGPkvgbO3r4APbONccb2BegZbQyToy5QUlICi8WCoKAgu/VBQUEoKipqdpuioqJm69fV1aGkpKTLYm2LEALz5s3DmDFjEBcX12K9kJAQfPDBB9i4cSM2bdqEmJgY3HXXXdizZ083RtsoMTERa9euxY4dO/Dhhx+iqKgIo0aNQmlpabP1HfXzb7BlyxaUlZXhkUceabGOo12DazX8u7+R/xMN293oNt2luroa8+fPx8MPPwwvL68W68XGxmLNmjX4/PPPsX79euh0OowePRqnT5/uxmglqamp+Pjjj/H111/jrbfewrfffos777wTZrO5xW0c+RrcinpS+wKwjXGEa+Ds7QvQ89oYZ2xfgJ7TxrjIduRbwPV/gRFCtPpXmebqN7e+Oz311FP48ccfsW/fvlbrxcTEICYmxvY6OTkZ+fn5ePPNN3H77bd3dZhNpKam2pbj4+ORnJyM6Oho/P3vf8e8efOa3cYRP/8GK1euRGpqKkJDQ1us42jXoDk3+n+io9t0tdraWjz00EOwWq1Yvnx5q3WTkpLsHkgdPXo0hg0bhnfffRd//etfuzpUO2lpabbluLg4DB8+HJGRkdi6dSumTJnS4naOeA1udT2hfQHYxjS3vrv1lPYF6BltjLO2L0DPaWPYc9QF/P39oVKpmmS9xcXFTbLjBsHBwc3Wd3FxgZ+fX5fF2pqnn34an3/+OXbt2oVevXrd8PZJSUmy/fXieu7u7oiPj28xHkf8/BucP38eO3fuxOOPP37D2zrKNWgYxelG/k80bHej23S12tpaTJ06Fbm5ucjIyGj1r3rNUSqVGDFihENcl5CQEERGRrYaiyNeg1tZT2lfALYxjnANekL7AvScNqYntS+A87YxTI66gEajQUJCgm30lwYZGRkYNWpUs9skJyc3qf/VV19h+PDhUKvVXRZrc4QQeOqpp7Bp0yZ8/fXXiIqK6tB+srOzERIS0snRdYzZbMbx48dbjMeRPv/rrV69GoGBgbj33ntveFtHuQZRUVEIDg62+4xramqwe/fuFv9PAC1fl9a26UoNDdfp06exc+fODn2pEUIgJyfHIa5LaWkp8vPzW43F0a7Brc7Z2xeAbUwDR2hjekL7AvSMNqantS+AE7cx3T0CxK3ik08+EWq1WqxcuVIcO3ZMzJkzR7i7u4uff/5ZCCHE/PnzRXp6uq3+uXPnhJubm5g7d644duyYWLlypVCr1eLTTz/t9th/97vfCb1eLzIzM4XBYLCVqqoqW53r4//LX/4iNm/eLE6dOiV++uknMX/+fAFAbNy4sdvjF0KIZ555RmRmZopz586JAwcOiIkTJwpPT0+n+PyvZbFYREREhHjuueeavOdo16C8vFxkZ2eL7OxsAUC8/fbbIjs72zbSzpIlS4RerxebNm0SR44cEdOmTRMhISHCZDLZ9pGenm434tY333wjVCqVWLJkiTh+/LhYsmSJcHFxEQcOHOj2c6itrRX333+/6NWrl8jJybH7v2E2m1s8h4ULF4ovv/xSnD17VmRnZ4tHH31UuLi4iIMHD3Zr/OXl5eKZZ54RWVlZIjc3V+zatUskJyeLsLAwh7oG1DZnbl+EYBvjCNdACOdqX4Rw/jbG2duXts6hJ7UxTI660HvvvSciIyOFRqMRw4YNsxumdMaMGWLcuHF29TMzM8XQoUOFRqMRvXv3FitWrOjmiCUAmi2rV6+21bk+/tdee01ER0cLnU4nfHx8xJgxY8TWrVu7P/h6aWlpIiQkRKjVahEaGiqmTJkijh49anvfkT//a+3YsUMAECdPnmzynqNdg4ahXq8vM2bMEEJIQ62++OKLIjg4WGi1WnH77beLI0eO2O1j3LhxtvoN/u///k/ExMQItVotYmNju7Qxbu0ccnNzW/y/sWvXrhbPYc6cOSIiIkJoNBoREBAgUlJSRFZWVrfHX1VVJVJSUkRAQIBQq9UiIiJCzJgxQ+Tl5dntQ+5rQO3jrO2LEGxjHOEaCOFc7YsQzt/GOHv70tY59KQ2RiFE/ROBREREREREtzA+c0RERERERAQmR0RERERERACYHBEREREREQFgckRERERERASAyREREREREREAJkdEREREREQAmBwREREREREBYHJEREREREQEgMkREbUgMzMTCoUCZWVlcodCREQ9DNsYclRMjoiIiIiIiMDkiIiIiIiICACTIyKHJYTA66+/jj59+sDV1RWDBw/Gp59+CqDxdoStW7di8ODB0Ol0SExMxJEjR+z2sXHjRgwcOBBarRa9e/fGW2+9Zfe+2WzGH//4R4SHh0Or1eK2227DypUr7eocPnwYw4cPh5ubG0aNGoWTJ0927YkTEVGXYxtD1AJBRA7pT3/6k4iNjRVffvmlOHv2rFi9erXQarUiMzNT7Nq1SwAQ/fv3F1999ZX48ccfxcSJE0Xv3r1FTU2NEEKI7777TiiVSrFo0SJx8uRJsXr1auHq6ipWr15tO8bUqVNFeHi42LRpkzh79qzYuXOn+OSTT4QQwnaMxMREkZmZKY4ePSrGjh0rRo0aJcfHQUREnYhtDFHzmBwROaCKigqh0+lEVlaW3fqZM2eKadOm2RqVhkZGCCFKS0uFq6ur2LBhgxBCiIcffljcfffddtv/z//8jxgwYIAQQoiTJ08KACIjI6PZGBqOsXPnTtu6rVu3CgDi6tWrnXKeRETU/djGELWMt9UROaBjx46huroad999Nzw8PGxl7dq1OHv2rK1ecnKybdnX1xcxMTE4fvw4AOD48eMYPXq03X5Hjx6N06dPw2KxICcnByqVCuPGjWs1lkGDBtmWQ0JCAADFxcU3fY5ERCQPtjFELXOROwAiaspqtQIAtm7dirCwMLv3tFqtXeN1PYVCAUC6n7xhuYEQwrbs6urarljUanWTfTfER0REzodtDFHL2HNE5IAGDBgArVaLvLw89O3b166Eh4fb6h04cMC2fOXKFZw6dQqxsbG2fezbt89uv1lZWejXrx9UKhXi4+NhtVqxe/fu7jkpIiJyCGxjiFrGniMiB+Tp6Ylnn30Wc+fOhdVqxZgxY2AymZCVlQUPDw9ERkYCABYtWgQ/Pz8EBQXh+eefh7+/PyZPngwAeOaZZzBixAi8/PLLSEtLw/79+7Fs2TIsX74cANC7d2/MmDEDjz32GP76179i8ODBOH/+PIqLizF16lS5Tp2IiLoY2xiiVsj7yBMRtcRqtYqlS5eKmJgYoVarRUBAgJgwYYLYvXu37UHWL774QgwcOFBoNBoxYsQIkZOTY7ePTz/9VAwYMECo1WoREREh3njjDbv3r169KubOnStCQkKERqMRffv2FatWrRJCND4se+XKFVv97OxsAUDk5uZ29ekTEVEXYhtD1DyFENfcIEpETiEzMxN33HEHrly5Am9vb7nDISKiHoRtDN3K+MwRERERERERmBwREREREREBAHhbHREREREREdhzREREREREBIDJEREREREREQAmR0RERERERACYHBEREREREQFgckRERERERASAyREREREREREAJkdEREREREQAmBwREREREREBYHJEREREREQEAPj/6LQxcvmz2EYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 850x450 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(1, 2)\n",
    "ax1.plot(history.history['categorical_accuracy'])\n",
    "ax1.plot(history.history['val_categorical_accuracy'])\n",
    "ax1.set_title('Model Accuracy')\n",
    "ax1.set_ylabel('Categorical Accuracy')\n",
    "ax1.set_xlabel('epoch')\n",
    "\n",
    "# summarize history for loss\n",
    "ax2.plot(history.history['loss'])\n",
    "ax2.plot(history.history['val_loss'])\n",
    "ax2.set_title('Model Loss')\n",
    "ax2.set_ylabel('loss')\n",
    "ax2.set_xlabel('epoch')\n",
    "\n",
    "fig.suptitle(\"RNN POS Tagger Training Statistics\")\n",
    "fig.set_size_inches(8.5, 4.5)\n",
    "\n",
    "plt.legend(['train', 'dev'], loc='upper right')\n",
    "plt.tight_layout()\n",
    "\n",
    "tasks.util.save_plot(\"ex_3_fit.png\", OUTPUT_DIR)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "\n",
    "We will now calculate and export the results of our RNN, MLP and baseline classifiers on the training, validation and test sets. We will be using precision, recall and f1 scores as metrics. We **can not use PR-AUC scores**, since they are only defined for binary classification tasks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_res_df = tasks.util.get_statistics(\n",
    "    base_cls,\n",
    "    lambda x: np.array(x),\n",
    "    x_base_train,\n",
    "    x_base_valid,\n",
    "    x_base_test,\n",
    "    y_base_train,\n",
    "    y_base_valid,\n",
    "    y_base_test,\n",
    "    calculate_proba=False\n",
    ")\n",
    "base_res_df[\"model\"] = \"Baseline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "782/782 [==============================] - 1s 1ms/step\n",
      "751/751 [==============================] - 1s 1ms/step\n",
      "313/313 [==============================] - 0s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "mlp_res_df = tasks.util.get_statistics(\n",
    "    mlp,\n",
    "    lambda x: lb_mlp.inverse_transform(x),\n",
    "    x_train_mlp,\n",
    "    x_valid_mlp,\n",
    "    x_test_mlp,\n",
    "    y_train_mlp,\n",
    "    y_valid_mlp,\n",
    "    y_test_mlp,\n",
    "    calculate_proba=True\n",
    ")\n",
    "mlp_res_df[\"model\"] = \"MLP\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6247/6247 [==============================] - 192s 31ms/step\n",
      "751/751 [==============================] - 23s 30ms/step\n",
      "745/745 [==============================] - 23s 31ms/step\n"
     ]
    }
   ],
   "source": [
    "rnn_res_df = tasks.util.get_statistics(\n",
    "    rnn_model,\n",
    "    lambda output: np.array([x.upper() for x in decode(tag_tokenizer, output)]),\n",
    "    train_data,\n",
    "    val_data,\n",
    "    test_data,\n",
    "    y_train,\n",
    "    y_valid,\n",
    "    y_test,\n",
    "    calculate_proba=True,\n",
    "    time_distributed_func=lambda sequence: np.argmax(sequence, axis=2),\n",
    ")\n",
    "rnn_res_df[\"model\"] = \"RNN\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6247/6247 [==============================] - 44s 7ms/step\n",
      "751/751 [==============================] - 5s 7ms/step\n",
      "745/745 [==============================] - 5s 7ms/step\n"
     ]
    }
   ],
   "source": [
    "cnn_res_df = tasks.util.get_statistics(\n",
    "    best_model,\n",
    "    lambda output: np.array([x.upper() for x in decode(tag_tokenizer, output)]),\n",
    "    train_data,\n",
    "    val_data,\n",
    "    test_data,\n",
    "    y_train,\n",
    "    y_valid,\n",
    "    y_test,\n",
    "    calculate_proba=True,\n",
    "    time_distributed_func=lambda sequence: np.argmax(sequence, axis=2),\n",
    ")\n",
    "cnn_res_df[\"model\"] = \"CNN\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "      <th>auc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>tag</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Baseline</th>\n",
       "      <th>ADJ</th>\n",
       "      <td>0.824</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.824</td>\n",
       "      <td>0.903</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADP</th>\n",
       "      <td>0.667</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.800</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ADV</th>\n",
       "      <td>0.830</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.907</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUX</th>\n",
       "      <td>0.783</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.783</td>\n",
       "      <td>0.878</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCONJ</th>\n",
       "      <td>0.985</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.985</td>\n",
       "      <td>0.992</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">CNN</th>\n",
       "      <th>SCONJ</th>\n",
       "      <td>0.380</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.550</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SYM</th>\n",
       "      <td>0.832</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.832</td>\n",
       "      <td>0.908</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VERB</th>\n",
       "      <td>0.820</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.820</td>\n",
       "      <td>0.901</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X</th>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MACRO</th>\n",
       "      <td>0.841</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.841</td>\n",
       "      <td>0.841</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               accuracy precision recall     f1    auc\n",
       "model    tag                                          \n",
       "Baseline ADJ      0.824     1.000  0.824  0.903      -\n",
       "         ADP      0.667     1.000  0.667  0.800      -\n",
       "         ADV      0.830     1.000  0.830  0.907      -\n",
       "         AUX      0.783     1.000  0.783  0.878      -\n",
       "         CCONJ    0.985     1.000  0.985  0.992      -\n",
       "...                 ...       ...    ...    ...    ...\n",
       "CNN      SCONJ    0.380     1.000  0.380  0.550  1.000\n",
       "         SYM      0.832     1.000  0.832  0.908  1.000\n",
       "         VERB     0.820     1.000  0.820  0.901  1.000\n",
       "         X        0.000     0.000  0.000  0.000  1.000\n",
       "         MACRO    0.841     0.868  0.841  0.841  1.000\n",
       "\n",
       "[72 rows x 5 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_col(col: pd.Series):\n",
    "    return (\n",
    "        col.astype(str)\n",
    "        .str.replace(\"NaN\", \"0.0\")\n",
    "        .apply(lambda x: f\"{float(x):.3f}\" if x != \"-\" else x)\n",
    "    )\n",
    "\n",
    "\n",
    "def format_df(df):\n",
    "    df = df.copy()\n",
    "\n",
    "    df.accuracy = format_col(df.accuracy)\n",
    "    df.precision = format_col(df.precision)\n",
    "    df.recall = format_col(df.recall)\n",
    "    df.f1 = format_col(df.f1)\n",
    "    df.auc = format_col(df.auc)\n",
    "\n",
    "    df = df.drop(\"split\", axis=1)\n",
    "\n",
    "    df = df.set_index([\"model\", \"tag\"])\n",
    "\n",
    "    df = df.fillna(0.0)\n",
    "    return df\n",
    "\n",
    "\n",
    "final_df = pd.concat([base_res_df, mlp_res_df, rnn_res_df, cnn_res_df])\n",
    "\n",
    "final_train_df = format_df(final_df[final_df.split == \"training\"])\n",
    "final_valid_df = format_df(final_df[final_df.split == \"validation\"])\n",
    "final_test_df = format_df(final_df[final_df.split == \"test\"])\n",
    "final_test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_to_latex(\n",
    "    df: pd.DataFrame, name: str, col_format:str, caption: str = None, index: bool = False\n",
    "):\n",
    "    path = os.path.join(OUTPUT_DIR, name)\n",
    "\n",
    "    df.to_latex(\n",
    "        buf=path,\n",
    "        index=index,\n",
    "        formatters={\"name\": str.upper},\n",
    "        float_format=\"{:.3f}\".format,\n",
    "        label=\"tab::\" + name.split(\".\")[0],\n",
    "        caption=caption,\n",
    "        escape=True,\n",
    "        bold_rows=True,\n",
    "        encoding=\"utf-8\",\n",
    "        column_format=col_format,\n",
    "        longtable=True\n",
    "    )\n",
    "    print(f\"Dataframe exported to {path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe exported to output/ex_3_train.tex\n",
      "Dataframe exported to output/ex_3_valid.tex\n",
      "Dataframe exported to output/ex_3_test.tex\n"
     ]
    }
   ],
   "source": [
    "col_format = 13 * \"|l|\"\n",
    "export_to_latex(\n",
    "    final_train_df,\n",
    "    \"ex_3_train.tex\",\n",
    "    col_format,\n",
    "    \"Results on the training dataset.\",\n",
    "    index=True,\n",
    ")\n",
    "\n",
    "export_to_latex(\n",
    "    final_valid_df,\n",
    "    \"ex_3_valid.tex\",\n",
    "    col_format,\n",
    "    \"Results on the validation dataset.\",\n",
    "    index=True,\n",
    ")\n",
    "\n",
    "export_to_latex(\n",
    "    final_test_df,\n",
    "    \"ex_3_test.tex\",\n",
    "    col_format,\n",
    "    \"Results on the test dataset.\",\n",
    "    index=True,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}