notebooks/shallow_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finding the right model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report, f1_score\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost import plot_importance\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "module_path = os.path.abspath(os.path.join(os.pardir))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "from src.functions import load_OU_data, CourseScaler, plot_confusion, score_grid, \\\n",
    "smotecourses, process_courses, course_cross_validate, Course_GridSearchCV, \\\n",
    "model_evaluate_presentation, cross_val_presentation, GridSearchPresentationCV\n",
    "\n",
    "import pickle\n",
    "\n",
    "import warnings\n",
    "\n",
    "from IPython.display import display\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "sns.set_style('white')\n",
    "pd.set_option('display.max_columns',None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full = load_OU_data(prediction_window=.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_full.copy()\n",
    "df = df.drop(columns = ['id_student','region','highest_education', \\\n",
    "                   'imd_band','gender','age_band','disability','studied_credits',\n",
    "                   'module_presentation_length','date_registration'])\n",
    "df['final_result'] = np.array(['No Intervention' if w in ['Pass','Distinction'] \\\n",
    "              else 'Needs Intervention' for w in df['final_result']])\n",
    "test_df = df[df['code_presentation'] == '2014J']\n",
    "train_df = df[df['code_presentation'] != '2014J']\n",
    "\n",
    "X_train = train_df.drop(columns = ['final_result','code_presentation'])\n",
    "y_train = train_df['final_result']\n",
    "X_test = test_df.drop(columns = ['final_result','code_presentation'])\n",
    "y_test = test_df['final_result']\n",
    "\n",
    "X_train_transformed, y_train_transformed, X_test_transformed \\\n",
    "                                = process_courses(X_train, y_train, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FSM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7870333639883041"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAGACAYAAACN9jfJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABMVklEQVR4nO3deVhU9fs+8HtgANFRDFwy/ahI4ZohmUmGpoiKa5oKLrib5ha54S6hEqipueaSZmSKe2qahqUo7ihuiSiuuG8sA8owzPv3hz/OlxEQOgnnkPfrurqaOWeYeXhmjtzznDNnNEIIASIiIiL6RyyULoCIiIioKGKIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhiv7z4uPjUa9evVd2f3v37sX06dNfept9+/bhu+++y/fts4qPj0fNmjXRoUMH6T9PT0/4+vri5s2b/6r2grJ27VosW7bsld1f9erV0a5dO7MeTJw40ew2GzduxODBg3O9j/v378PPzw/t2rVDu3bt0KVLF4SHh7+yGvPr6NGjaNu2reyfz+n10L59e2zcuPEVVilPv3798Pjx4390u4EDB+Ly5csFXRpRodAqXQBRUePh4QEPD4+X3ubs2bNITEzM9+1fVKxYMfz666/SdSEEpk+fjrlz52LOnDn/vOgC1q1bt1d+n6tXr4a9vX225QkJCZgzZw62bduGDz/8MNefnzRpEj766CPMmzcPAHD58mV069YNjo6OcHJyeuX1FqQXXw/37t1D27ZtUadOHdSoUUOxuiIjI//x7ZYvX15Q5RAVOoYoeq0lJyfj66+/RkxMDDQaDdzd3TFy5EhotVrs378fs2fPhoWFBWrWrIlDhw7hl19+wbFjx7B7924sXboUe/bswZIlS6DRaGBpaYmxY8fC2toa69atQ0ZGBkqWLIkqVapIt3/w4AGmTp2KK1euwMLCAj4+PujVq1eedaalpeHBgwdwcHAAABgMBsyePRvHjx9HRkYGatWqhUmTJkGn0+HMmTMICAhAeno6KleujNu3b2PcuHEAgBkzZqB48eJITU3Fxo0bcfDgQSxZsgTp6ekoVqwY/P39Ua9ePcTFxWHixIkwGAwQQqBz587o0aNHrssXLFiAJ0+eYMqUKbh06RICAwORkJAAjUaDfv364dNPP8XRo0cxd+5c/O9//8OlS5dgMBgwZcoUNGzY8B89Z7t27UK5cuUwduxY7N+/P9fbPXjwAM+ePYPJZIKFhQXefvttLFmyBKVKlQIAnD59GtOnT8fTp09hZWWFsWPHws3NDSdOnMDMmTOl5X5+fmjcuDE2b96MjRs34unTp9DpdAgNDcWGDRuwdu1amEwmlC5dGpMnT84xoKWmpmLEiBG4fv06SpUqhcDAQLz55pto3Lgx1q9fD0dHRwBA37590aNHDzRv3vylPShfvjyqVKmCa9eu4e+//853XSdOnEBwcDBMJhMAYNCgQWjZsuVLX0/NmjVDx44dcfjwYdy5cwdeXl4YO3Ysxo8fDwDo3bs3li1bhpiYGCxduhQGgwGPHz/Gp59+Cj8/v2y369GjB7777ju8++67CAsLQ2hoKCwsLFCmTBlMnjwZjo6OGDduHHQ6HS5evIi7d++iWrVqmDNnDkqUKPGPXitEBU4Q/cfdvHlTuLi45Lhu7NixYtq0acJkMom0tDTRr18/sXTpUvH48WPRoEEDceHCBSGEEJs3bxbOzs7i5s2bYtOmTeLzzz8XQgjh4eEhTp06JYQQ4sCBA2LBggVCCCHmz58vvv76ayGEMLv90KFDRUhIiBBCiKSkJNGmTRtx7dq1bPXWqFFDtG/fXrRt21a4ubmJVq1aiTlz5gi9Xi+EEGLBggUiODhYmEwmIYQQ3377rZg6dapIT08XjRs3Fvv27RNCCHH48GFRvXp1ceTIEXHkyBFRo0YNER8fL4QQ4urVq6Jt27bi8ePHQgghYmNjRaNGjURKSooYP368WLp0qRBCiPv37ws/Pz+RkZGR6/LM3zc9PV14eHiI3bt3CyGEuHv3rnB3dxcnT54UR44cETVr1hR///23EEKIH374QfTo0SPH58XZ2Vm0bdtWtG/fXvrv4cOHZrfJ2tecHDp0SDRq1Eg0aNBADB48WCxfvlzcvXtXCCGEwWAQjRo1En/99ZcQQoizZ89KvXBzcxPR0dFSTxo0aCBu3LghNm3aJD744AORnJwshBDi6NGjonv37iI1NVV6/r28vLLVkdn3qKgoIYQQ69atE507dxZCCDF9+nTp9XD9+nXRpEkTYTQazX4+p9fvyZMnxQcffCBu3779j+rq1auX2LFjhxBCiAsXLoiAgAAhRO6vJyGEaNq0qQgODhZCPH8+3333XXHjxg3peXr06JEwmUyiZ8+e4urVq9LtatasKR49emR2u8z7O3PmjDh06JBo3ry5tHzTpk3Cy8tLmEwm4e/vL7y9vUVaWpowGAzi008/FRs3bsz1uSZSCidR9FqLiIjA2rVrodFoYG1tDR8fH6xevVra5ZO5q6Rjx445HtfUpk0bDBs2DE2aNEGjRo0wcODAlz7eoUOHMGbMGABAyZIlsWPHjhxvl3X3zYEDBzBmzBg0atRIeie+b98+JCcn49ChQwCA9PR0ODg4IDY2FgDQpEkTAEDDhg3xzjvvSPdboUIFVKxYEcDzXSz3799Hnz59pPUajQY3btyAp6cn/P39cebMGbi5uWHSpEmwsLDIdXmma9euIS0tDS1atADwfGLSokULHDhwAB9++CHeeust1KxZEwBQq1YtbNmyJdde5bY7L7/c3Nywb98+REdH48SJE/jrr7+waNEirF69GpaWlrCwsMAnn3wCAKhTpw62b9+O/fv3o3LlynjvvfcAAO+88w5cXV1x7NgxaDQaVK9eHTqdTnoOrl+/Dh8fH+kxExMTkZCQgNKlS5vVUr16dbi6ugJ4/loKCAhAcnIyunfvjp49e+Krr75CWFgYOnfuDEtLy2y/y7Nnz9ChQwcAQEZGBt544w3MmjULFSpUkO4/P3V5eXkhMDAQf/75Jz766COMHDlS+pmcXk+ZMndHly9fHg4ODkhMTMT//vc/ab1Go8H333+Pffv2YceOHYiLi4MQAk+fPs31+Tlw4ABat24tPcedOnXCjBkzEB8fDwBwd3eHtbU1AMDZ2VnaPU6kJgxR9FrL3K2R9brRaISlpSXEC18rmTUsZPrqq6/QuXNnHDx4EJs3b8ayZcuwefPmXB9Pq9VCo9FI12/evIk33nhD+gOYE3d3d/Tt2xcjR47Erl27ULJkSZhMJkyYMEEKSykpKdIuvxfrzvpHuXjx4ma/q5ubm3TMEADcuXMH5cqVQ40aNbB7924cOnQIhw8fxqJFi7Bu3To0bdo0x+VZ7/NFQggYjUYAz8NhJo1Gk63WV+XRo0dYsGABJk+ejPr166N+/foYPHgwJk6ciK1bt6Jr165mzwMAxMbGvrR+KyurbP3r0KGDFIpNJhPu378POzu7bPfx4mtHo9FAq9XC0dER1atXx969e7F9+3Zs2LAhx9/nxWOiXpTfunx8fNC0aVNERkbiwIEDWLhwIbZt25br6ymTjY2NWe0vPm+pqano2LEjmjdvjvr16+Ozzz5DeHj4S5/fnNYp8Voh+jf46Tx6rX388cdYs2YNhBAwGAxYv349PvroI7i6uuLatWuIiYkBAOzevRtJSUlmf3iNRiOaNWuG1NRUdOvWDVOnTkVcXJwUwjL/GGTl5uaGTZs2AXh+PFbv3r1x7dq1POvs168fSpUqhfnz55vVbTAYYDKZMHnyZMyZMwdOTk6wtrZGREQEAODMmTOIjY3NFhiA51OqyMhIxMXFAQD279+P9u3bIy0tDaNGjcLOnTvRpk0bTJ06FTqdDnfu3Ml1eSZHR0dYWVlhz549AJ4fAL1792589NFH+Xk6Xhk7OzscOnQIP/30k/TH9+nTp7hz5w5q1aqFatWqQaPRSAc8nz9/Hr1790bdunVx9epVnDlzBgBw6dIlHD9+HA0aNMj2GI0aNcJvv/2G+/fvA3j+CcXevXvnWM/Fixdx4cIFAEBYWBjef/992NraAgC6d++OmTNn4r333kP58uX/9e/+srp8fHxw4cIFdOrUCdOmTUNSUhISExNzfT3lJfN1fv36dej1evj5+aFZs2Y4duyYdF9Zb5fVxx9/jJ07d0qf2tu0aRNKly6NKlWq/OseEBUWTqLotZCamprtNAfr1q3DpEmTMH36dLRr1w7p6elwd3fH4MGDYW1tjTlz5sDf3x8WFhaoU6cOtFqt9IcPeD5VmjBhAkaPHi1NmIKCgmBtbQ03NzcMHz4cVlZWqF27tvQzU6ZMQUBAANq1awchBAYNGoQ6derkWb+VlRUmT56MAQMGoEuXLhgyZAhCQkLQsWNHZGRkoGbNmhg3bhy0Wi0WLFiAqVOnYs6cOahatSrKlCmDYsWKZdu18s477yAwMBAjR46EEAJarRZLlixB8eLFMWTIEEycOBFhYWGwtLRE8+bN0aBBA5QpUybH5ceOHZPqXLx4MaZPn44FCxYgIyMDQ4cORcOGDXH06NF/8xT+I1qtFj/88ANmzZqF0NBQFC9eHBqNBh07dkTnzp0BAAsWLEBQUBBmzpwJKysrLFiwAA4ODvjuu+8wbdo0PHv2DBqNBt988w0cHR1x6tQps8dwd3fHwIED0a9fP2g0Guh0OixcuDDHwFqtWjUsXLgQN2/ehIODA4KDg6V1TZs2xaRJk8x2v/0bL6tr9OjRCAoKwrx582BhYYFhw4ahUqVKub6e8uLp6Ynu3btj4cKF+OSTT+Dl5YVSpUqhcuXKePvtt3H9+nVUrlxZut3ixYuln23UqBH69OmD3r17w2Qywd7eHkuXLs1x4kukVhrBGSlRNnq9HosXL8bw4cNha2uL8+fPY9CgQThw4ECOfyTVJCQkBP3790eZMmVw584ddOjQAeHh4dKn0khdTp48icmTJ2PHjh2qf20RkTlOoohyoNPpYGVlhc6dO0Or1UKr1WLevHlF4o9cxYoV0adPH2i1Wun8UgxQ6uTv749jx44hJCSkSLy2iMgcJ1FEREREMnDnMxEREZEMDFFEREREMjBEEREREclQ6AeWf/jhh9IZk4mIiIjU7NatW7meoqXQQ1TFihVfekZnIiIiIrXo1KlTruu4O4+IiIhIBoYoIiIiIhkYooiIiIhk4BnLiYiIKJv09HTEx8fj2bNnSpdSKIoVK4ZKlSrBysoq3z/DEEVERETZxMfHo2TJkqhatep//muJhBB49OgR4uPj4ejomO+f4+48IiIiyubZs2dwcHD4zwcoANBoNHBwcPjHUzeGKCIiIsrR6xCgMsn5XRmiiIiIKE/P0jNUfX9K4DFRRERElKdiVpaoOu63V3Z/14LbvLL7UgpDFBEREanS1atXMX78eGi1WphMJnz77bdYvnw5zpw5g/T0dAwfPhzNmzdHcHAwoqKiAABt27ZF7969MW7cOCQkJCAhIQFLly7FihUrcOLECZhMJvTp0wdeXl7/uj6GKCJSvWfpGShmZama+yGiwnHo0CHUrVsXY8aMwYkTJ7B582Y8efIEGzduRGJiIlatWgVLS0vEx8dj/fr1MBqN6N69Oxo2bAgAaNiwIfr06YP9+/cjPj4ea9euRVpaGrp27YpGjRqhVKlS/6o+higiUr1XtRvhv7D7gOh10rlzZyxfvhwDBgxAyZIlUbduXbi4uAAA7Ozs4OfnhxUrVqB+/frQaDSwsrLCe++9h7i4OACQTlcQGxuL8+fPw9fXFwBgNBpx69atfx2ieGA5ERERqdLevXvx/vvvY/Xq1WjVqhXWrVuHs2fPAgCSk5PRv39/ODk5Sbvy0tPTcerUKVSpUgXA/33irlq1avjwww8RGhqK1atXw8vLC//73//+dX2cRBEREZEq1alTB/7+/liyZAlMJhPmz5+PLVu2oFu3bsjIyMDQoUPRpEkTHDt2DN7e3khPT0erVq1Qu3Zts/tp1qwZjh07hu7duyM1NRXNmzeHTqf71/UxRBER/Ue8ymO+ePwYvehZesYr3SWen9dY5cqVsXbtWrNlderUyXY7f3//bMuCg4OlyxqNBuPHj5dZae4Yov4BHtxKRGr2Kj+CzuPH6EWv+u/Wf+HvYJ4hymQyISAgABcvXoS1tTWmT58u7WsEgJUrV2LHjh3QaDQYPHgwPD09C7RgJfHg1vzhu2EiInod5BmiwsPDYTAYEBYWhujoaAQHB2PJkiUAgKSkJPz000/Ys2cPnj59ik8//fQ/HaIof/humIiIXgd5fjovKioK7u7uAAAXFxecO3dOWmdra4u33noLT58+xdOnT1+r79ghIiL6rxNCKF1CoZHzu+Y5idLr9WZHsFtaWsJoNEKrff6jFSpUQJs2bZCRkYFBgwb94wKIiIhIfYoVK4ZHjx7BwcHhPz8kEULg0aNHKFas2D/6uTxDlE6nQ0pKinTdZDJJASoiIgL379/H3r17AQD9+/eHq6sr6tat+4+KICIiInWpVKkS4uPj8eDBA6VLKRTFihVDpUqV/tHP5BmiXF1d8ddff6F169aIjo6Gs7OztM7Ozg7FihWDtbU1NBoNSpYsiaSkpH9eORGpBj+FSqQMtW17VlZW0hm/KWd5hihPT09ERkbCx8cHQggEBQVh1apVqFy5Mjw8PHDo0CF07doVFhYWcHV1RaNGjQqjbiIqIPwUav7wU6j0qnHbK3ryDFEWFhYIDAw0W+bk5CRdHjFiBEaMGPHqKyMiUjF+CpWI+N15Rdyz9AxV3Q/l36vsOZ8/Uiu1vs75bye9CjxjeRHH8W/RxUkGvQ7U+jrnv51Fl5p2pTNEERERUZGhpmDO3XlEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDNq8bmAymRAQEICLFy/C2toa06dPR5UqVQAAFy5cQFBQkHTb6OhoLFq0CI0bNy64iomIiIhUIM8QFR4eDoPBgLCwMERHRyM4OBhLliwBANSsWROhoaEAgF27dqFcuXIMUERERPRayDNERUVFwd3dHQDg4uKCc+fOZbtNamoqFixYgJ9//vnVV0hERESkQnkeE6XX66HT6aTrlpaWMBqNZrfZuHEjWrVqBXt7+1dfIRERESniWXqGKu9LLfKcROl0OqSkpEjXTSYTtFrzH9u+fTvmz5//6qsjIiIixRSzskTVcb+9kvu6FtzmldyPmuQ5iXJ1dUVERASA5weOOzs7m61PTk6GwWBAhQoVCqZCIiIiIhXKcxLl6emJyMhI+Pj4QAiBoKAgrFq1CpUrV4aHhweuXr2KihUrFkatRERERKqRZ4iysLBAYGCg2TInJyfpct26dbF48eJXXxkRERGRivFkm0REREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkgzavG5hMJgQEBODixYuwtrbG9OnTUaVKFWn9/v37sWjRIgghULt2bUydOhUajaZAiyYiIiJSWp6TqPDwcBgMBoSFhWHUqFEIDg6W1un1esyaNQvff/89NmzYgIoVK+LJkycFWjARERGRGuQZoqKiouDu7g4AcHFxwblz56R1p06dgrOzM0JCQtC9e3eUKVMG9vb2BVctERERkUrkuTtPr9dDp9NJ1y0tLWE0GqHVavHkyRMcPXoUW7duRfHixdGjRw+4uLjA0dGxQIsmIiIiUlqekyidToeUlBTpuslkglb7PHuVLl0a7777LsqWLYsSJUqgfv36uHDhQsFVS0RERKQSeYYoV1dXREREAACio6Ph7OwsratduzZiY2Px+PFjGI1GnD59Gm+//XbBVUtERESkEnnuzvP09ERkZCR8fHwghEBQUBBWrVqFypUrw8PDA6NGjcKAAQMAAK1atTILWURERET/VXmGKAsLCwQGBpotc3Jyki63adMGbdq0efWVEREREakYT7ZJREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJoM3rBiaTCQEBAbh48SKsra0xffp0VKlSRVo/ffp0nDx5EiVKlAAALF68GCVLliy4iomIiIhUIM8QFR4eDoPBgLCwMERHRyM4OBhLliyR1p8/fx4rVqyAvb19gRZKREREpCZ57s6LioqCu7s7AMDFxQXnzp2T1plMJly/fh1TpkyBj48PNm7cWHCVEhEREalInpMovV4PnU4nXbe0tITRaIRWq0Vqaip69uyJvn37IiMjA7169UKdOnVQo0aNAi2aiIiISGl5TqJ0Oh1SUlKk6yaTCVrt8+xla2uLXr16wdbWFjqdDg0bNkRMTEzBVUtERESkEnmGKFdXV0RERAAAoqOj4ezsLK27du0aunXrhoyMDKSnp+PkyZOoXbt2wVVLREREpBJ57s7z9PREZGQkfHx8IIRAUFAQVq1ahcqVK8PDwwMdOnRA165dYWVlhQ4dOuCdd94pjLqJiIiIFJVniLKwsEBgYKDZMicnJ+nygAEDMGDAgFdfGREREZGK8WSbRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCRDniHKZDJhypQp8Pb2hq+vL65fv57jbQYMGIC1a9cWSJFEREREapNniAoPD4fBYEBYWBhGjRqF4ODgbLeZN28ekpKSCqRAIiIiIjXKM0RFRUXB3d0dAODi4oJz586Zrf/999+h0Wik2xARERG9DvIMUXq9HjqdTrpuaWkJo9EIAIiNjcWOHTvw5ZdfFlyFRERERCqkzesGOp0OKSkp0nWTyQSt9vmPbd26Fffu3UPv3r1x69YtWFlZoWLFimjcuHHBVUxERESkAnmGKFdXV/z1119o3bo1oqOj4ezsLK0bO3asdHnBggUoU6YMAxQRERG9FvIMUZ6enoiMjISPjw+EEAgKCsKqVatQuXJleHh4FEaNRERERKqTZ4iysLBAYGCg2TInJ6dstxs+fPirq4qIiIhI5XiyTSIiIiIZGKKIiIiIZGCIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSgSGKiIiISAaGKCIiIiIZGKKIiIiIZGCIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSIc8QZTKZMGXKFHh7e8PX1xfXr183W79mzRp89tln6Ny5M3bu3FlghRIRERGpiTavG4SHh8NgMCAsLAzR0dEIDg7GkiVLAACPHz/G2rVrsWXLFqSlpaFNmzbw8vKCRqMp8MKJiIiIlJTnJCoqKgru7u4AABcXF5w7d05aZ29vj61bt8LKygoPHz6EjY0NAxQRERG9FvIMUXq9HjqdTrpuaWkJo9EoXddqtfj555/h7e2N9u3bF0yVRERERCqTZ4jS6XRISUmRrptMJmi15nsBe/bsiQMHDuD48eM4cuTIq6+SiIiISGXyDFGurq6IiIgAAERHR8PZ2Vlad+XKFQwbNgxCCFhZWcHa2hoWFvzAHxEREf335XlguaenJyIjI+Hj4wMhBIKCgrBq1SpUrlwZHh4eqFGjBry9vaHRaODu7o4GDRoURt1EREREisozRFlYWCAwMNBsmZOTk3R52LBhGDZs2KuvjIiIiEjFuO+NiIiISAaGKCIiIiIZGKKIiIiIZGCIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSgSGKiIiISAaGKCIiIiIZGKKIiIiIZGCIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSQZvXDUwmEwICAnDx4kVYW1tj+vTpqFKlirT+xx9/xG+//QYAaNKkCYYNG1Zw1RIRERGpRJ6TqPDwcBgMBoSFhWHUqFEIDg6W1t28eRPbtm3DunXrsH79ehw8eBAxMTEFWjARERGRGuQ5iYqKioK7uzsAwMXFBefOnZPWvfnmm1ixYgUsLS0BAEajETY2NgVUKhEREZF65DmJ0uv10Ol00nVLS0sYjUYAgJWVFezt7SGEQEhICGrVqgVHR8eCq5aIiIhIJfIMUTqdDikpKdJ1k8kErfb/BlhpaWkYPXo0UlJSMHXq1IKpkoiIiEhl8gxRrq6uiIiIAABER0fD2dlZWieEwJAhQ1C9enUEBgZKu/WIiIiI/uvyPCbK09MTkZGR8PHxgRACQUFBWLVqFSpXrgyTyYRjx47BYDDgwIEDAICRI0eiXr16BV44ERERkZLyDFEWFhYIDAw0W+bk5CRdPnv27KuvioiIiEjleLJNIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSgSGKiIiISAaGKCIiIiIZGKKIiIiIZGCIIiIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikoEhioiIiEgGhigiIiIiGRiiiIiIiGRgiCIiIiKSgSGKiIiISAaGKCIiIiIZGKKIiIiIZGCIIiIiIpIhzxBlMpkwZcoUeHt7w9fXF9evX892m8ePH6Nly5ZIS0srkCKJiIiI1CbPEBUeHg6DwYCwsDCMGjUKwcHBZusPHDiAfv364cGDBwVWJBEREZHa5BmioqKi4O7uDgBwcXHBuXPnzO/AwgKrVq1C6dKlC6RAIiIiIjXS5nUDvV4PnU4nXbe0tITRaIRW+/xHGzVqVHDVEREREalUnpMonU6HlJQU6brJZJICFBEREdHrKs8Q5erqioiICABAdHQ0nJ2dC7woIiIiIrXLc6Tk6emJyMhI+Pj4QAiBoKAgrFq1CpUrV4aHh0dh1EhERESkOnmGKAsLCwQGBpotc3Jyyna7P//889VVRURERKRyPNkmERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkA0MUERERkQwMUUREREQyMEQRERERycAQRURERCQDQxQRERGRDAxRRERERDIwRBERERHJwBBFREREJANDFBEREZEMDFFEREREMjBEEREREcnAEEVEREQkQ54hymQyYcqUKfD29oavry+uX79utn79+vXo1KkTunbtir/++qvACiUiIiJSE21eNwgPD4fBYEBYWBiio6MRHByMJUuWAAAePHiA0NBQbNq0CWlpaejevTsaNWoEa2vrAi+ciIiISEkaIYR42Q2++eYb1K1bF23atAEAuLu748CBAwCAvXv3Yv/+/QgMDAQADB06FIMGDULdunVzvb8PP/wQFStWfFX1ExERERWYW7du4ejRozmuy3MSpdfrodPppOuWlpYwGo3QarXQ6/UoWbKktK5EiRLQ6/Uvvb/cCiEiIiIqSvI8Jkqn0yElJUW6bjKZoNVqc1yXkpJiFqqIiIiI/qvyDFGurq6IiIgAAERHR8PZ2VlaV7duXURFRSEtLQ3JycmIi4szW09ERET0X5XnMVEmkwkBAQGIjY2FEAJBQUGIiIhA5cqV4eHhgfXr1yMsLAxCCAwaNAgtW7YsrNqJiIiIFJNniCIiIiKi7HiyTSIiIiIZGKKIiIiIZGCIIiIiIpIhz/NEqcWJEyewevVqREVFwcrKCpaWlqhXrx569OgBV1dXRWpKSUnB5s2bcfz4cTx58gQODg5wc3ND27ZtUaJECUVqAtirfyo2NhYJCQlwcHCAk5OTorVkUltNsbGxOHbsGBISEmBvbw83Nzc4OjoqXRYAdfWK294/o9frkZiYCHt7e9ja2ipaCwAkJibi5MmT0uu8fv36ivcoE3uVP4XdpyJxYPm0adOg0+nQpk0bvP3227CweD5Au3jxIrZt24aUlBQEBAQUak0bN27E7t270aRJE1SvXh1ly5ZFUlISTp8+jf3796Nly5bo0qVLodYEsFf5ZTAYsGzZMvz+++9wcHBAmTJlkJSUhPv378PLywt9+vRBsWLFXvua4uLiEBISgmLFisHZ2RnlypVDYmIizpw5A6PRiJEjR+Kdd94p1JoAdfaK217+bd26Fb/88ov0Bzg5ORmlSpVC9+7d0a5du0Kv5/Hjx5g9ezauXLkCR0dH6XV+4cIFODs748svv0SZMmUKvS6AvcovxfokioCHDx++dP2DBw8KqZL/s3///peu37dvXyFVYo69yh9/f39x8OBBkZGRYbbcZDKJffv2iTFjxrAmIcT8+fNFUlJSjusSEhLEvHnzCrmi59TYK257+ePv7y/CwsJEYmKi2fKkpCSxZs0aMXr06EKv6euvvxZXrlzJcd3ly5dFQEBAIVf0HHuVP0r2qUhMog4ePGh2XaPRwMHBATVq1FCoouzu3LkDo9GI//3vf4rWwV4RKYPbXv6kpaXBxsZG9vrXCXuVP0r2qUgcE/Xbb79lW3b//n1UqFAB06dPV6Ai4PDhwwgKCoKDgwPat2+POXPmwNbWFl27dsXAgQMVqQlgr/IrIyMDe/fuhb29PWrUqIHg4GAYjUYMGzYMlSpVYk3/n7e3NzQajdkyBweHPL9ovKCpsVfc9vLHxsYG58+fh52dHd58802sWLEC6enp6N27N0qVKqVIKBg5cmSOr/OePXuicuXKhV5PJvYqf5TsU5GYROXG29sbYWFhijx2586dMXv2bCQmJqJPnz4IDw9HyZIl4evrq1hNL8NemRs9ejSEEEhJScHdu3fRrFkzVKhQAdu3b8dPP/3Emv6/W7duZVt27949zJgxA5s2bVKgoufU2KvccNsz98033+Ds2bNIT09HqVKlUK5cOZQrVw4xMTFYunSpIjUdO3Ys27J79+5hzZo1WLdunQIVPcde5Y+SfSoSk6gXGQwG7NixI1saLky2traoWrUqAKBmzZpwcHAAgEI/mDUv7FXO4uPjsW7dOmRkZKB169YYMWIEgJynCa9zTRUrVsxxWeZB00pRY69exG0vZ6dOncL69euRlpaGVq1a4YcffgAA+Pr6KlZTgwYNcly+Zs2aQq7EHHuVP0r2qUieJ0oIgUuXLiE4OFixGrL+w6jV/l8WVdtgj73KmbW1NQDA0tIS5cuXl5abTCalSlJlTS9KSkqCh4cHPv74Y0XrKAq94raXs8xdKzY2Nma7XpUMmy/S6/Xw9fVV/DQe7FX+KNmnIjOJiomJQfHixfHmm29i+fLl0Ol0Zv94Frbz58/Dx8cHQghcvnxZuhwXF6dYTZnYq7wlJCTg4MGDEEIgMTHR7DJryp21tTV+/fVX6HQ6RetQa6+47eUtLS0N165dg8lkMrv87NkzxWp6kU6nw5QpUxQ5fUdW7FX+KNmnInFM1LfffovTp09Dr9ejbNmyqFmzJkqUKIGYmBh8++23itSU07EimXLaBVJY2Kv8GT9+fK7rvvnmm0Ks5P+osabLly9jzpw5sLOzQ7t27TBp0iRYWFhg4sSJaNq0qSI1AersFbe9/HnZLpbQ0NBCrOT/3L9/H8uXL4ednR2aN2+O4cOHw9LSEsHBwXBxcVGkJoC9yi9F+1RgJ094hby9vYUQQuj1etG0aVNpec+ePZUqSQghxIULF8T169dFWlqaWLBggVi4cKFITU1VtCa19iqrZcuWKV1CNnmd40cJT58+FWlpaYrW0L17d3H06FGxefNm8f7774uHDx+K5ORk6XWmFmp4/rjtFV19+/YVmzdvFgsXLhRubm4iLi5O3LlzR/To0UPp0lSHvTJXJHbnmUwm3L59G2+99Rbmzp0L4PmxGQaDQbGacnvXOWnSJMXedQLq7FXWj8QKIXD06FHExMQAgGK9unr1qtl1f39/zJw5E0IIxfbtq3HqYzKZpANJjx49Kh2YnPX4GiWo8fnjtpc/9+/fx4oVK1CqVCnVTDIMBgM6duwI4Pmnz6pVqwZA+WOPXnzt9O/fHytXroQQQjouUIma1NYrJftUJELU2LFjMXz4cGzYsAHvvfceAOCLL77AoEGDFKvp+PHjWLduHVJSUtCuXTvpY5RKfmoCUGevnJ2dsX//fowYMQIWFha4cuUKvL29FasHAPr27YtixYqhXLlyEELg6tWrmDx5MjQajWIfkZ86dSq+/PJL3Lp1CyNGjMDu3bthY2ODAQMGKBaiHB0dMXHiREybNk06QHrZsmWKfQVGJjU+f9z28mfcuHFo164dbt++jX79+uHnn39G8eLFMXr0aPz888+K1FSqVCksXrwYX3zxBVavXg0A+PXXXxU/keVHH30EGxsbFCtWDEIIPHz4EC1btoRGo8HevXsVqUmNvVK0T0qNwIq6Ll26iFu3bgkhhIiOjhZCCJGYmCi6du2qZFmqtW/fPjF06FDx6NEj4evrq3Q54uHDh2Lo0KHi4MGDQgh17HLx8fGRLvv7+0uXlRyTZ2RkiD/++MNs2datWxXfba3G50+t1LbtZX099+rVS7qs5HOYmpoqfvzxR7NlS5cuVXw38eXLl8Xnn38uYmJihBDqeJ2rsVdK9qlIHFiuRidOnMA333yDDRs2SOfM6dGjB/r3749mzZopXJ063bhxA19//TUePXqErVu3Kl0OjEYjQkJC4ODggMjISMUO1Mw0YcIEaDQaTJs2TXpNLVu2DH///TfmzZunaG1qpLbnT83UtO0NGTIEderUwRdffCHtAvr111+xfft2rFixQtHa1Eiv12PKlCn45JNPsHHjRtWdTFYtlOpTkQhRo0aNynWdkscfqZHae5WamorIyEh4enoqXYpk8+bN2Lx5s2K7EjKZTCb8+eefaN68ubTs119/RYsWLWBra6tITS8ee5SV0ufQyaSW54/bXv48ffoU69evR+/evaVly5Ytw2effSYdc1fYXnbcmlLHHr1o4cKF2LZtG/bs2aNoHWrvVWH3qUiEqD/++ANz585FQEBAtnW5nT31dcVe0avUrVs33Lx5E9WqVTM7QaOSxx6pFbe9oqtly5Z49OgR7OzsIISARqOR/q/UsUdqxV6ZKxIhCgBmzJgBV1dXeHl5KV0KAHW/62Sv8qbGCYsaa3r69Cl69uyJxYsXK3rSyBepsVcAt738UOMk4/Hjx+jfvz9+/PFH2NnZKVJDTtir/FGyT0UmRKkN33Xmnxp7pcYJixprAoBz584hPT0d9erVU6yGF6m1V2qjxm1PrZOMgwcPwtLSEm5uborV8CL2Kn+U7FORCVGPHz/G8ePHkZycjFKlSsHFxQXlypVTtCa1vevMxF7lTY0TFjXWpFZq7RW3vbypcZKhVuxV/ijZpyIRojZs2ICwsDC8//77KFGiBFJSUnD8+HF06dIF3bp1U7o8VWGv8k+NExa11fTkyRMsXrwYhw8fhl6vR8mSJVG/fn0MGzZMsYOAM6mtV9z28k9tk4y0tDSsXbsWR44cQXJysvQ679mzJ4oVK6ZobexV/ijVpyIRonx8fBAaGgorKytpmcFgQLdu3bBp0ybF6lLju072il6lQYMGoUOHDmjcuLEUDPbv348NGzbgxx9/VLo8VeG2V3SNHDkSNWrUMHudR0RE4PTp01i0aJHS5akKe2WuSJyx3Gg0Ii0tzewfp2fPnil6mvkX33VeunQJ33//veLvOtmr/FHjhEWNNen1erRu3Vq6rtPp0KZNG6xZs0aRejKpsVfc9vJHjZOM+/fvY86cOWbLatSoge7duytSTyb2Kn+U7FORCFFDhgxBp06dUKVKFZQsWRJ6vR7Xr1/HuHHjFKtp06ZNWLt2bY7vOpUMUexV/owbNw4dOnTAl19+aTZhGTVqlGITFjXW5ODggIULF6Jx48bQ6XRSTWXLllWknkxq7BW3vfwZP348atSoAT8/P7NJxqhRoxSbZNjY2GDr1q1wd3eXnruIiAgUL15ckXoysVf5o2SfikSIatasGRo3boy4uDjo9XrodDo4OTkp+iWoanzXCbBX+aXGCYsaa5o1axbWrl2L5cuXS68nV1dXhISEKFYToM5ecdvLHzVOMmbPno1Fixbhp59+UtXrnL3KHyX7VCRC1LVr1zBnzhxYW1tj2LBhqFq1KoDnX9j69ddfK1KTGt91AuxVfqlxwqLGmmxsbPDRRx+hadOmqFKlirT89OnT0pfsKkGNveK2lz9qnGS88cYb+PLLL6HVas2+HeDWrVuK1QSwV/mlZJ+KxIHlvr6+GDRoEIxGI2bNmoVZs2ahVq1a8PX1VfT7soxGo6redQLsVX5l7kOPiooyezfVrVs3xY41UGNNixYtwsGDB5GRkYFatWph6tSp0Gg06NWrl6LnY1Jjr7jt5c+TJ0+waNEinDx50uy5++KLLxQ7nm3Dhg1Yvnw5TCYTvL29MXDgQABQ/HXOXuWPon0q+O84/veyfiNzXFycaN26tbh9+7ai30h+9epVMXz4cDFq1Chx9epVafmUKVMUq0kI9uqfuHjxorh27ZrZsujoaIWqeU5tNXXt2lW6HBwcLKZOnSqEUMe3yautV9z28i8pKUmkpqaaLYuPj1eoGiE6d+4s0tLSRFpamhg5cqRYsmSJEEIdr3P2Kn+U6pNFwUa0V0Or1eLPP/9ERkYGqlWrhsmTJ2PQoEF4+PChYjVNnjwZXbt2Rdu2bTF06FD8/fffAIArV64oVhPAXuXXokWLMHXqVIwZMwYBAQHSWa+V/MoeNdYksgyq/f39kZycjBUrVih+7J8ae8VtL382bNiAzz77DO3atcPy5cul5ePHj1esJktLS1hbW8Pa2hohISE4cuQIduzYofjrnL3KHyX7VCRCVFBQEPbs2YPk5GQAQMOGDTFhwgSzgyWV8PHHH+OTTz7BggULMGbMGNy5c0fxjY69yp+IiAisXbsW69evh62trXTMilBw77Yaa2rdujU6d+6MhIQEAMA333yDw4cP4/Tp04rVBKizV9z28mf9+vXYsWMHdu7ciZiYGHz//fcAlH3uXF1dMXz4cCQnJ0Or1WL+/PlYuXIlYmJiFKsJYK/yS9E+Ffis6z+qT58+Yu/evcJoNAohhDh8+LBo166d8PLyUrgy9VFjr7p06WJ2feTIkWL58uWK7npRY01CCHHjxg3pucv0xx9/KFTNc2rtldqocdvz9vaWLqenp4vevXuL7du3K/7cHTlyRKSlpUnXnz17JlatWqVcQYK9yi8l+8QQJdPt27eFv7+/ePLkibTs8OHDon379soVpVJq7NWqVavEZ599JtWUlpYm+vXrJ+rWrcuaigD2Kn/UuO2FhISIYcOGiaSkJCGEEImJiaJjx47igw8+UKwmtWKv8kfJPhWJT+cRFYSbN2/irbfegqWlpbQsPDwczZs3Z01FAHtVdB09ehT16tWDtbU1gP/7tGWfPn2ULUyF2Kv8UapPRTpEPXr0CCVKlFD8CyKLAvaKSBnc9oj+u4rEgeW5mTFjBpYsWYK7d+8qXYrqsVf0Ki1btgzLli2D0WhUuhTV47ZXdG3ZsgWbN29Wuowi4XXtVZGeRN27dw/ly5dXugwzfNeZf+xV0XXgwAHUrl0bAGBvb69wNfRPcdszd/bsWbz77rvZlm/ZsgW1atVC6dKlVfe3Rm1e114VyUnUkSNHMHz4cHTq1EnpUrJR+l1nRkYG9uzZgxMnTkCv12PSpEkYN24c4uPjFannZZTuVU7UOGFRQ003b97EmTNncO/ePQCAu7s77O3tVReglOzVpk2bpMuXLl2SLi9cuLDQa8mLGrc9JScZs2bNki5Pnz5dutyxY0dUr15ddaFAjVMfNfaqMPpUJL47DwBSU1OxZcsWrF27Fg8ePMDkyZMVPbFebl78EsTC5u/vDyEEUlJScPfuXTRr1gwVKlTAhAkTFP36gpwo3auc1KxZE7Vr10ZSUpJqAoKSNcXHx8PPzw9WVlZwcHDA7du3YWtri7lz56JcuXKFWkt+KNmrX3/9FZ999hkAYNq0adL2duzYsUKtIz+U3PZym/oAQO3atRXZw5B1h0xsbGyhPvbLqLFX9+7dw+zZszFr1ix4enri6dOnSE1NxZIlS/Dhhx8Wai35UeB9KvDP/70CgYGBonXr1mLOnDni6tWron///kqXJIxGo9i9e7c4fvy4SE5OFhMnThT+/v7i5s2bitaVeb4Mo9EoWrRoIS3v3bu3QhWpt1dZP/b94MED8ejRI+WKyeLGjRvi9OnT4u7du0qXIoYOHSqOHz9utuzgwYNi6NChClVkTk29yvq1F7ldLmwbN26ULsfGxkqXFyxYoEQ5Qghhdu6eadOmKVZHVllrUvocTFmpsVfDhw+XzhOX+do+e/as6Nevn5JlKaZITKKioqJQu3ZtvPfee6hcubLiZwUH1Dvxyfx4p6WlpVnyNplMSpWkyl4dO3YM/v7+2Lp1K+zs7BATE4PJkydj1qxZqF+/viI1qXHq8/jx42z9aNSokdlXKyhBjb3K+u9SbpcLmxqnY0KFU5979+4hLCwMQgjpciZvb2/F6lJjrxITE7OdRqROnTrQ6/UKVaTsdKxIhKitW7fi5MmT2LBhA4KDgyGEQFxcHJycnBSrKT4+HuvWrUNGRgZat26NESNGAAB+++03xWoCgISEBBw8eBBCCCQmJppdVooaezVv3jyEhobCzs4OwPOvxli5ciUmTpyIX375RZGagoODMW7cOLPQEhkZicDAQMWOq9Fqc/4nQslQDqizVwkJCYiMjITJZFLNtpf1j3BulwubGt4Ev6hdu3Z48OBBtstKU2OvDAaDdHnFihXSZRsbGyXKAfD8GL/27dsDAN58802Ehobi3LlzmDt3LkNUJldXV7i6ukKv12Pbtm0YM2YMACh2cJ0aJz7A8/2/meGkVq1aZpeVosZeWVpaolKlSmbLHB0dYWGh3Gct1Dj1yQzlWSkdDAB19qp27drYsWMHAPVse2qcjqlx6tO5c2e8+eabijz2y6ixVyVLlsS1a9dQtWpVKThdv34dxYsXV6QeQNnpWJEJUZl0Oh26d++OVq1aYcOGDYrVocaJD/D8C2KzynzXkBlklKDGXgkhYDKZzEJTRkYG0tPTFatJjVOfrKE8KyWDAaDOXr247WXK+s69sKlxOqbGqc/YsWOlXZ1Lly7FoEGDFK7oOTX2ys/PD0OHDkWXLl1QpUoV3Lx5Exs2bMDs2bMVq0nJ6ViRC1FnzpzBmjVrEBkZiRYtWihWhxonPgAQExODefPmwcHBAW3atMFXX30FABg/fjw+/fRTRWpSY6/at2+PkSNHYvDgwahUqRLu3r2LxYsXw8vLS7Ga1Dj1yQwGBoMBycnJcHBwUKyWrNTYKz8/P8ybNw8AsHLlSvTr1w8AMGDAAMWO/VPjdEyNU5+suzcjIyNVE6LU2KtatWph1apV+PXXX7Fv3z689dZbWL58uaJ1KjkdKxIhymAw4LfffsOaNWtgbW0NvV6P8PBwRU8Up8aJDwAEBARg+PDhSExMxNChQ7FlyxbY29tjwIABioUoNfaqa9euKFGiBIKCgnD//n1UrFgRn332GVq3bq1YTWqc+iQmJmLy5Mk4f/487Ozs8ODBA7i5uWHKlCnQ6XSK1aXGXj169Ei6vG/fPilEKXn8kRqnY2qc+qjx2CNAnb0CAAcHBwwcOBCnT5+GwWBAfHy8oiFKyelYkQhRzZo1Q9u2bTF79mxUrVoVAwYMUPxMu2qc+ACAlZUVGjVqBAD46aefULVqVQBQdH+1WnvVqlUrtGnTBgCg1+sVf02pceozY8YMeHp6Yv78+dKyDRs2IDAwEDNnzlSsLjX2KquswUnJP9BqnI6pceqT027PTB9//LFidamxVydOnEBQUBA2b96MiRMnwtnZGVeuXEGPHj3QpUsXRWpScjpWJM5Y3rt3bxw6dAjffvst9u/fr+g7u0wBAQHw9fVFo0aNMHToUISFheGPP/5Q7JNdmbL+g5110qPksSJq7FVsbCxatWol7f45cuQIWrVqhcuXLytWU2JiIkaMGAEvLy8MHDgQ7u7uGDt2rKIfHb558ybatWtntqxLly6Kn+lajb1Sy4HbWb04HcvET+eZy9ztuXPnTmm3Z+Z/SlJjr5YsWSK9qXrjjTcwZ84cLF++3OyM/UrInI516tQJrq6uhfYtHUViEjVw4EAMHDgQx44dw4YNG3Du3DnMmjULHTp0gLOzsyI1qXHiAwCXL1/GqFGjIIQwuxwXF6dYTWrs1YwZMzBnzhzpFAfNmzeHvb09pk+fjh9//FGxmtQ29bGysspxudL/uKuxV2rc9rJSy3RMjVOfadOm5fphBSWpsVfp6enSJ5sdHR0BAGXLloWlpaUi9QDKTsfU96p5iQYNGqBBgwZISkrCtm3bMHbsWGzdulWRWtQ48QEgje4BwMfHJ8fLhU2NvTKZTNm+TsHV1VXRT+flNvXZvn27QhUBz549w7Vr17JNLp4+fapQRc+psVdq3/aUDr6ZcjvYHVAuGPTr10/avblp0ybpBKVKU2Ov0tLSpMuBgYHSZSVfXzlNxx48eIDhw4czROXEwsICQgizJ7OwqfVdZ4MGDfD48WPY2trC1tYW27dvR3p6erY/OIVJjb3KLcAp+SW/apz62NjYYPLkyTkuV5Iae6XT6XI8sD08PFyBap5T47anxqlP1jcJWc/yrjQ19qpatWrYu3cvPDw8pGV//fWXNJVSgpLTMXU9O3m4fPkyfv75Z/z+++9o0aIFQkJCFKtFje86gefnyAgLC4OVlRVcXFxw584dODg44NChQ4qdx0ONvWrcuDFCQkIwZMgQlCxZEikpKVi4cCEaNmyoWE1qnPqEhoYq9tgvo8ZeBQcHS9OMvn37YtWqVQCe78J+8USAhUWN254apz5qmdK9SI29GjNmDIYMGYJNmzZJn4R7+PAhvv/+e8VqUnI6ViRC1O7du7FmzRqkp6ejU6dOuHr1qlmjlKDGiQ8A/P7779i1axdSU1PRunVr7Nu3D1qtFj169FCsJjX26vPPP8fy5cvRsWNHPHv2DHZ2dujQoQMGDBigWE1qnPoYDAbMnTsXe/bsQVpaGkqUKIE2bdpgyJAhir5DVmOvsga6rBNNJQ/iVuN0TI1Tn6dPn+LatWswmUzZArqSExY19sre3h7r1q3DmTNnEB8fD09PT9SrV0/RIKrkdKxIhCh/f3/06tULffv2xRtvvIE9e/YoXZIqJz4AYGtrC61Wi1KlSsHR0VH6Q6fkHzw19kqj0eDzzz/H559/DuD5Hz2lx+ZqnPqEhISgbNmy2LlzJ2xsbKDX67FixQqEhIRg4sSJitWlxl6p8StW1DgdU+PUp1ixYlIozxrQNRqNol8or8ZeAcDVq1dRt25dREdH48qVKzh8+DD69OmDEiVKKFKPktOxIhGi9uzZg82bN6NHjx5wdnbGkydPlC5JlROfTOnp6dI7mMzLSh7ErcZe3b17F35+fli6dCns7Oywa9cuhIaGYsGCBWbf71eY1Dj1OX/+PNatWydd1+l08PPzg6+vryL1ZFJjr4QQ0vb24mWlqHE6psapjxpDOaDOXq1fvx7bt29HaGgotm7diq5du+L06dNYtmyZdA7AwqbkdKxIhKhy5cph8ODBGDx4MA4fPoz169ejWbNmaNmyJfz9/RWpSY0THwC4desWWrVqJW1omZeVfEejxl5NnToVAwYMkE5x0K5dO2i1WkydOlWxfftqnPqo8QBuQJ29ytz2gOchRQ3bnhqnY2qc+kydOhU9e/bEO++8k23dhQsXsHbtWkUOIVFjr7Zv345ly5YBAEqUKAEfHx907NgR3bp1UyxEAcpNx4pEiMrKzc0Nbm5uePz4MbZt26ZoLWqb+ADAn3/+qejj50ZtvUpJScm2O8PLy0vR0b1apz5Zn7tMSp/wVo29UuO2p8bpmBqnPl999RXmzZuHc+fOwdHREWXKlEFSUhIuXLiAunXrws/PT5G61Ngr4PkbYwDS12TZ2Ngo+jVQSk7HikSImjJlCnr27Gl2Yk17e3v06dNHsXcJapz4AOp8R6XGXuX2R0TJPy5qnPpkna5kUvq5A9TZKzVve4B6pmNq7FPp0qUREBAAvV6P06dP48mTJ3BwcMDEiRMVPSmwGnuVlpYmvYa6desG4PlrS8nTwyg5HSsSIWrkyJE5vkuIiYnBu+++q8i7BDW+6wTU+Y5Kjb2qW7cufvrpJ/Tq1UtaFhoaiurVqytYlfqmPmp87jKprVfc9vJHjX3KpNPppG9XUAM19qpx48aYPXs2Ro0aJZ2zcd68eWjSpEmh15KVUtMxjVB6Lv8PvPgu4b333lPsXYIa3yFkxV69nMFgwIwZM/Dnn3+ibNmySEpKwscff4xx48Yp9kXEzZo1yzYhyHzHt3fvXkVqUuNzB6izV5m47eWPmvqkdmrqVUZGBubOnYudO3eidOnSSEhIQMuWLTFmzBhYWCjzdbxdu3ZFWFiY2b8JQgj06NGjwL+jtUiFKDVJSEh46TuEESNGwN7eXukyVUHNvUpPT0dCQgLeeOMNxT8UoEZqfu4ob3z+qKBkZGTgyZMnKF26tOL/di5cuBBPnz7NNh0rXrw4Bg0aVKCPzRD1L6npHYLaqalXOR1nl0mpd+icGuSfmnulRmp7/qjoUuO/nUpOxxii6LWU2zv0zOPslHiHzqlB/rFXRMpQ47+dmZSYjjFE0WtNje/Q1ViTWrFXRMpQ07an5HSMIYqIiIiKLCWnYwxRREREVOQpMR1jiCIiIiKSQZmTOhAREREVcQxRRERERDIwRBH9R61duxYLFixQ5LG/+uorHD16NNf1vr6+iIuLK7DHj4iIwLhx4wrs/gvCzz///NL1Fy9exPHjxwE876/BYCiMsojoJRiiiIhUYMmSJS9dv2fPHly+fBkAMHfuXFhbWxdGWUT0EvyeC6Ii4NmzZxg/fjxu376N9PR0TJgwAevWrUN8fDwyMjLQt29ftG7dGidOnEBQUBBKlSoFS0tLuLi4AHj+5co7duyARqNB69atzb54+UWenp6oV68erl27Bjc3NyQnJ+PMmTNwdHTErFmzEB8fjwkTJiAjIwMajQaTJk1CjRo1sGbNGmzYsAFly5bFo0ePAACbN2/GlStXMHr0aKSlpcHLy8vsS3GTk5MxceJEPHnyBAAwadIksy+BFkJg2rRpOHPmDNLT0zF8+HA0b94cwcHBiIqKAgC0bdsWvXv3RlxcHCZMmABbW1vY2trCzs4OALBr1y78+OOPsLCwwPvvv4/Ro0fn+ru3bt0a9evXx6VLl2BnZ4c5c+bg999/x6ZNm2AymTBixAgkJCRku7+oqCiEhIRAq9XC1tYW3333HWxsbDB16lRcv34dJpMJfn5++PDDD9GuXTs0aNAAFy9ehEajweLFi/Hzzz8jMTERAQEBGD16NCZOnIjk5GTcv38f3bt3h4eHB7Zs2QIrKyvUrl0bfn5+2LVrFx48eJDjc9GiRQu4urri6tWrcHBwwIIFC2BpaSnvxUdEuRNEpHqrVq0Ss2bNEkIIcfXqVbFo0SIxY8YMIYQQycnJwtPTUzx69Ei0bdtWXLlyRQghxJQpU8T8+fPFpUuXhI+PjzAajcJoNApfX18RFxeX62PVrFlT3Lp1SxgMBuHi4iIuXbokTCaTaNq0qUhMTBTDhw8Xf/zxhxBCiL///lt07NhRPHjwQLRo0UKkpaUJg8Eg2rZtK44cOSI2bdok1f3s2TPRtGlTIYQQPXv2FJcvXxYzZ84Ua9askX4vHx8fs1r27Nkj/Pz8hBBCJCQkiLlz54o///xTDB06VJhMJmEwGETnzp1FTEyM+Pzzz8XBgweFEEIsXbpU+Pv7iydPnggvLy+RmpoqhBBi9OjR0m1y0rRpU3Hs2DEhhBAhISFi5cqVYtOmTWLw4MFCCJHr/QUHB4uVK1eKjIwM8ccff4hbt26JNWvWiJkzZwohhHj8+LFo3bq19BhRUVFCCCFGjhwpduzYIYQQ4qOPPhJCCHHu3Dmxe/duIYQQd+/eFZ6enkIIIebPny9++eUX6T6ePXuW43MhhBA1atQQt2/fFkII4e3tLU6dOpXr70xE8nESRVQEXLlyBY0bNwYAVK1aFQ8ePMBHH30EANDpdHBycsLNmzfx8OFDODo6AgBcXV1x48YNxMbG4vbt2+jTpw8AIDExEdevX0e1atVyfKzSpUvjrbfeAgAUL14cb7/9NgCgZMmSSEtLQ1xcHD744AMAQM2aNXH37l3cuHEDb7/9trSLqW7dutnuV+RwNpXY2FgcOXIEu3btkmrL6urVq9I0zc7ODn5+flixYgXq168PjUYDKysrvPfee4iLi8O1a9ekx3V1dcWVK1dw48YNPH78GJ9//jkAICUlBTdu3ECjRo1y/N21Wq30u7m6uiIiIgIuLi5ST3O7v8GDB+P7779H7969Ub58edStWxexsbGIiorCmTNnAABGoxGPHz8GANSqVQsAUKFCBaSlpZnVUKZMGaxevRp79uyBTqeD0WjMsVYAOT4XAPDGG2+gQoUKuT4GEb0aPCaKqAhwcnLC2bNnAQA3b97Eb7/9hhMnTgB4foK52NhYVKpUCeXLl5cO2M68fbVq1fD222/jp59+QmhoKDp16mS2y+xFGo0mz1oyH/vChQsoU6YMqlatisuXL+PZs2fIyMjAhQsXAAA2NjZ48OABAOD8+fPZ7qtatWro06cPQkNDMW/ePLRv3z7b+szfIzk5Gf3794eTk5O0Ky89PR2nTp1ClSpV4OTkhFOnTgEAzp07BwCoVKkSKlSogJUrVyI0NBQ9e/aUQllOjEYjYmJiAABRUVFSgMz8EtPc7m/btm3o2LEjQkND8c4772D9+vWoVq0a2rRpg9DQUCxfvhytWrVC6dKlc+1xZshcuXIlXFxcMHv2bLRq1UpartFoYDKZ8nwucrt/Inr1OIkiKgJ8fHwwYcIE9OzZExkZGVixYgXWrFmDbt26IS0tDcOGDYODgwMCAwMxduxY6HQ6lChRAnZ2dqhRowbc3NzQrVs3GAwG1K1bF+XLl5ddy9ixYzF58mSsXLkSRqMRM2bMgL29PQYOHAgfHx/Y29vD1tYWAODu7o61a9eiW7duqF27NkqUKGF2X4MHD8bEiROxfv166PV6DBs2DAAwY8YMdOrUCR4eHjh8+DC6deuGjIwMDB06FE2aNMGxY8fg7e2N9PR0tGrVCrVr18a4cePg7++PH374Afb29rCxsYG9vT369OkDX19fZGRkoGLFivDy8nrp77d8+XLcvn0bb731Fr766ivs2LFDWpfb/RkMBkyaNAm2trawsLBAYGAgypcvj0mTJqFnz57Q6/Xo3r37S79R3snJCaNHj0bnzp0xffp07Ny5EyVLloSlpSUMBgPq1KmDmTNnwsnJ6aXPBREVHp6xnIhUJzQ0FI0bN0aVKlUK9XGbNWuGXbt2wcbGplAfl4iKJk6iiF5De/fuxY8//phtea9eveDp6Vn4Bb3Aw8NDOi7rVTtz5gxmzZqVbXleEyoiohdxEkVEREQkAw8sJyIiIpKBIYqIiIhIBoYoIiIiIhkYooiIiIhkYIgiIiIikuH/AReYU6z2H6UQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code_module</th>\n",
       "      <th>code_presentation</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAA</th>\n",
       "      <th>2013J</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">BBB</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.809978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.780488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.808637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCC</th>\n",
       "      <th>2014B</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DDD</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.782471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.790948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.825059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">EEE</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.789731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.812155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FFF</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.780666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.781568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.792948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">GGG</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.699008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.777778</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  score\n",
       "code_module code_presentation          \n",
       "AAA         2013J                   NaN\n",
       "BBB         2013B              0.809978\n",
       "            2013J              0.780488\n",
       "            2014B              0.808637\n",
       "CCC         2014B                   NaN\n",
       "DDD         2013B              0.782471\n",
       "            2013J              0.790948\n",
       "            2014B              0.825059\n",
       "EEE         2013J              0.789731\n",
       "            2014B              0.812155\n",
       "FFF         2013B              0.780666\n",
       "            2013J              0.781568\n",
       "            2014B              0.792948\n",
       "GGG         2013J              0.699008\n",
       "            2014B              0.777778"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7404791929382093\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "Needs Intervention       0.61      0.68      0.64      2735\n",
      "   No Intervention       0.82      0.77      0.80      5195\n",
      "\n",
      "          accuracy                           0.74      7930\n",
      "         macro avg       0.72      0.73      0.72      7930\n",
      "      weighted avg       0.75      0.74      0.74      7930\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression()\n",
    "scores = cross_val_presentation(model, train_df, scoring = 'accuracy', verbose = 1)\n",
    "scores.plot(kind = 'bar', figsize = (10,5), title = 'Logistic Regression F1 Score by Presentation')\n",
    "plt.show()\n",
    "display(scores)\n",
    "y_pred = model.predict(X_test_transformed)\n",
    "print(accuracy_score(y_test, y_pred))\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridsearchCV for best logistic regression hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trying:\n",
      "{'C': 0.001, 'penalty': 'none', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7870886248194271\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'none', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'none', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'none', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'none', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7870886248194271\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'none', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'none', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'none', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'none', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7870886248194271\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'none', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'none', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'none', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7871537586694889\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l2', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7816749268239186\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l2', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7816749268239186\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l2', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7816749268239186\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l2', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7816749268239186\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l2', 'random_state': 111, 'solver': 'liblinear'}\n",
      "average score:  0.7810281706005875\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l2', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7863866792513687\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l2', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7863866792513687\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l2', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7863866792513687\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l2', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7863866792513687\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l2', 'random_state': 111, 'solver': 'liblinear'}\n",
      "average score:  0.7867529062672689\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l2', 'random_state': 111, 'solver': 'lbfgs'}\n",
      "average score:  0.7874915007960491\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l2', 'random_state': 111, 'solver': 'sag'}\n",
      "average score:  0.7874915007960491\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l2', 'random_state': 111, 'solver': 'saga'}\n",
      "average score:  0.7874915007960491\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l2', 'random_state': 111, 'solver': 'newton-cg'}\n",
      "average score:  0.7874915007960491\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l2', 'random_state': 111, 'solver': 'liblinear'}\n",
      "average score:  0.7874457404885398\n",
      "trying:\n",
      "{'l1_ratio': 0.4, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
      "average score:  0.7876331085133412\n",
      "trying:\n",
      "{'l1_ratio': 0.7, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
      "average score:  0.7879784966518429\n",
      "trying:\n",
      "{'l1_ratio': 0.9, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
      "average score:  0.7879677899281448\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l1', 'solver': 'saga'}\n",
      "average score:  0.7914764962798563\n",
      "trying:\n",
      "{'C': 0.001, 'penalty': 'l1', 'solver': 'liblinear'}\n",
      "average score:  0.7909719175617516\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l1', 'solver': 'saga'}\n",
      "average score:  0.7871533778468656\n",
      "trying:\n",
      "{'C': 0.01, 'penalty': 'l1', 'solver': 'liblinear'}\n",
      "average score:  0.7885791763617291\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l1', 'solver': 'saga'}\n",
      "average score:  0.7879969872930251\n",
      "trying:\n",
      "{'C': 0.1, 'penalty': 'l1', 'solver': 'liblinear'}\n",
      "average score:  0.7882197549724771\n",
      "best model\n",
      "LogisticRegression(C=0.1, l1_ratio=0.9, penalty='l1', random_state=111,\n",
      "                   solver='liblinear')\n",
      "best cv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code_module</th>\n",
       "      <th>code_presentation</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAA</th>\n",
       "      <th>2013J</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">BBB</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.809978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.791196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.812024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCC</th>\n",
       "      <th>2014B</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DDD</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.768743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.788075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.815603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">EEE</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.782396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.848987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FFF</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.781478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.781568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.796564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">GGG</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.701213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.81137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  score\n",
       "code_module code_presentation          \n",
       "AAA         2013J                   NaN\n",
       "BBB         2013B              0.809978\n",
       "            2013J              0.791196\n",
       "            2014B              0.812024\n",
       "CCC         2014B                   NaN\n",
       "DDD         2013B              0.768743\n",
       "            2013J              0.788075\n",
       "            2014B              0.815603\n",
       "EEE         2013J              0.782396\n",
       "            2014B              0.848987\n",
       "FFF         2013B              0.781478\n",
       "            2013J              0.781568\n",
       "            2014B              0.796564\n",
       "GGG         2013J              0.701213\n",
       "            2014B               0.81137"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross validated accuracy score:\n",
      "0.7914764962798563\n",
      "validation accuracy: \n",
      "0.7406052963430012\n",
      "validation set confusion matrix\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAD3CAYAAABcpJzyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmW0lEQVR4nO3de1wU9f4/8NcsC4is4iVFE0FRAe+GKFLhNzTS8JiWCHhCzVKrk2nq8ZB3RCEMScvj3cQ0UzLRTDOTNDUMUhTMUlG8JKKkgCkgt93P7w9/blLg7go7zHJezx77eDg7w8x73O3lh8985jOSEEKAiIjMRlXbBRAR1XUMWiIiM2PQEhGZGYOWiMjMGLRERGamNufONe/2MufuyUKlzdhQ2yWQArVv2LHa+5D8nYzeVuzLqvbxjGXWoCUikpUk1XYFlWLQElHdodDOUAYtEdUdbNESEZmZMnOWQUtEdYiVMpOWQUtEdQe7DoiIzEyZOcugJaI6RKXMpGXQElHdocycZdASUR3CPloiIjPjqAMiIjNTZs4yaImoDmHXARGRmXHUARGRmSkzZxm0RFSHsEVLRGRmDFoiIjNTZs4yaImoDuGoAyIiM+MTFoiIzIwtWiIiM+PFMCIiM2PXARGRmbHrgIjIzJSZswxaIqpDaqiPVqfTITw8HGfPnoWNjQ0WLFgAFxcXAMDp06cRFRWl3zYtLQ3Lli1D3759q9wfg5aI6o4a6jpITExEaWkp4uPjkZaWhujoaKxYsQIA0LFjR2zcuBEAsGfPHjRv3vyhIQswaImoDpFqqEWbmpoKX19fAECPHj1w6tSpv21TVFSEpUuX4tNPPzW4PwYtEdUZkgkt2vj4eMTHx+uXg4ODERwcDAAoKCiARqPRr7OyskJ5eTnU6j8j84svvsDAgQPRpEkTg8di0BJRnWFKz8GDwfpXGo0GhYWF+mWdTlchZAHgq6++wkcffWTUsRQ66oyIyHQqSTL69TCenp44dOgQgHsXu9zc3Cqsv3PnDkpLS9GyZUuj6mKLlojqDFO6Dh7G398fSUlJCAkJgRACUVFRiIuLg7OzM/r374+LFy+iVatWxtclhBA1UlklNO/2MteuyYKlzdhQ2yWQArVv2LHa+7Cb1tPobe/GpFb7eMZii5aI6gyF3hjGoCWiuqOmug5qGoOWiOoMBi0RkZlJCp3sgEFLRHUGW7RERGZmxYm/iYjMiy1aIiIzY9ASEZmZQnOWQUtEdQdbtEREZsagJSIyM5VKmRMSMmiJqM5QaIOWQUtEdQe7DoiIzIxBS0RkZoaenFBbGLREVGeoeAtu3SVJEpYMCUOXlh1Qqi3DW9sW4EJuln69v9uTmP7sWEiQkHb1DCZ/uRANbe2x/p9R0NjYoaS8DK/Fz8HvBbm1eBZU03Q6HZYvXIWL5y7B2lqNibMm4PHWfz5jatfnXyNx135IkoSXQofA1/9pFBYUYtHsxSgqvIvy8nKMfWcMOnbzqMWzsCxKnb1LmWMhLMzgTs/A1toW/Ve8hjl7/ouoQe/o12ls6iMyYCKGr58Mv+VjcDk/G4/ZN0Ko12D8cv08nls1HttO7sM7/zey9k6AzOLH71NQWlKK2HUL8cqEUVi7JE6/7o9bt/H1tm+waF00IpdHYO2SOAghsH3TTnTv1Q0LV0di8pyJWPH+6lo8A8sjSZLRLzkZbNFeu3YNu3btQklJif69CRMmmLUoS+PTpjsSzx4BABy9cgqerf589pG3Szf8cv08oga9g7ZNWmH90S9xs/AWfrl+Hm7NXAAADerZo0xbXiu1k/n8mn4aPZ/0BAB4dHXH+dPn9escGjXE0k2LYaW2Qk7277CxtYEkSRj6z8GwtrYGAGi1WljbWNdK7ZbKYi+GTZo0CT4+PkY/Vvd/UYN69vij+M9nwGuFDlYqK2h1WjS1b4S+7bzg8+HLKCwtwrdvrMFPv/2M3KI/0K9DHxybHI/G9R3w3MpxtXgGZA5FhUWwt6+vX1apVNCWa2GltgIAWKmt8NXnu7Fp9Ra8EDwIAKBpoAEA5N3Mx6I5izF+ymvyF27BFJqzhoPW3t4ekydPlqMWi3WnuBANbB/4H0qSoNVpAQB5RX8gNetXff9r0sUT6NbSDYHdn8OSgxuw7qft6NyiPTaFLkSfD/9ZK/WTedS3r4+7RXf1yzoh9CF73+CgQRj44nOYOzEC6cd+Rnevrrh0/hIWzojFa5NeQdeeXeQu26IptUVrsI+2Q4cO2L17Ny5cuICLFy/i4sWLctRlUZIvp+M5j6cAAL1ad8Ev1zP169KvnkEnR1c0re8AK5UVerXugjO/X0T+3du4XVwAALhRkI8G9exrpXYyn07dPXA06d4jrc/8fBZt2rno12VduooF06IhhIBarYa1jTVUkoTfLlzBe+/GYNqCKfB6yvhHZ9M9KpXK6JecDLZoT58+jdOnT+uXJUnChg0bzFqUpdn5y/fo194biW9+DAnAm19EYMLT/8SF3Cx8ffoQwr9Zhh2vLgUAJPyciF9zMjH/25VYNmwWxvkEQq1SY8K2yNo9CapxPs/0wYmUdEx9NQwA8M6ct7F905do6dQSff6vN1zd2mDqq2GQJAk9fTzRtWcXREyNQmlpKVbHrgUA1NfYY07sjNo8DYui0AYtJCGEMLRRfn4+rly5AicnJzRp0sTonWve7VWt4qhuSpvBf6jp79o37Gh4IwM8ljxv9LZn3tlT7eMZy2D7ec+ePQgJCcHKlSsRHByML7/8Uo66iIhMZrHDu9avX4+EhATY29ujoKAAo0ePxpAhQ+SojYjIJEq9GGYwaCVJgr39vQs1Go0Gtra2Zi+KiOhRKDRnDQdt69atER0dDS8vLxw7dgzOzs5y1EVEZDKlTvxtsKr33nsPrVu3xpEjR9C6dWvMnz9fjrqIiEym1D7aKoP2559/BgAkJyfDxcUF/fr1g4uLC1JSUmQrjojIFJJk/EtOVXYd/Pjjj+jatSt27979t3VPP/20WYsiInoUFncxbPz48QAAT09PDB8+XP8+b1YgIsWqoaDV6XQIDw/H2bNnYWNjgwULFsDF5c87+w4ePIhly5ZBCIHOnTtj7ty5Dw35KoN2165d2L9/P1JSUpCcnKw/eEZGBkaNGlUjJ0NEVJNqauLvxMRElJaWIj4+HmlpaYiOjsaKFSsAAAUFBYiJicGGDRvQpEkTrFmzBvn5+Q+9mavKoPX19UWzZs1w69YtBAcH//+TUKF169Y1ciJERDWtproOUlNT4evrCwDo0aMHTp06pV934sQJuLm5YeHChbhy5QqGDx9u8I7ZKoPWwcEB3t7e8Pb2Rm5urn4+Wq1WWxPnQURU40wJ2vj4eMTHx+uXg4OD9Y3KgoICaDQa/TorKyuUl5dDrVYjPz8fKSkp2LFjB+rXr4+XX34ZPXr0QNu2bas8lsFxtPPmzcPBgwfRvHlzCCEgSRK2bNli9MkQEcnFlKB9MFj/SqPRoLDwzzmmdTod1Op7cdmoUSN07doVzZo1AwB4eXnh9OnT1Qva9PR0JCYmKnYgMBHRfTU16MDT0xMHDhxAQEAA0tLS4Obmpl/XuXNnZGRkIC8vDw0bNkR6ejqCgoIeuj+DQevi4oKSkhLY2dlVv3oiIjOqqT5af39/JCUlISQkBEIIREVFIS4uDs7Ozujfvz+mTp2KsWPHAgAGDhxYIYgrY9Qzw/z8/PRDG9h1QERKVVO/eatUKkRERFR4r127dvo/Dxo0CIMGDTJ6fwaDNjY21oTyiIhqj8XdsKDfQK1GTEwM8vLyMHDgQLi7u6NVq1Zy1EZEZBKF5qzhSWVmz56NYcOGoaysDF5eXoiM5CNXiEiZLG5SmfuKi4vh4+MDSZLg6urK+WiJSLGUGrQGuw5sbW1x+PBh6HQ6pKWlwcbGRo66iIhMptQ+WoMt2vnz5yMhIQH5+flYt24dwsPDZSiLiMh0KpVk9EtOBlu0e/fuRXh4OBwcHOSoh4jo0Sm0RWswaLVaLcaMGYO2bdsiKCgI3t7ectRFRGQyi+06ePXVV5GQkIDRo0fjs88+w4ABA+Soi4jIZCrJ+JecDLZoi4uLsXfvXuzYsQNCCLz99tty1EVEZDKltmgNBu0LL7yAAQMGIDw8vMIM40RESmOl0MmvDAatn58fpk6dKkctRETVosyYNaKuzMxM3L59W45aiIiqRSVJRr/kZLBFm5mZCW9vbzRp0kTf//HDDz+YvTAiIlNZbB/tgQMH5KiDiKja5G6pGstg0J47dw5z587F7du38cILL6BDhw7w8/OTozYiIpMotUVrsI92wYIFeO+999C4cWMEBgZi6dKlctRFRGQytSQZ/ZK1LmM2cnFxgSRJaNKkCezt7c1dExHRI1Fqi9Zg0Do4OGDLli24e/cudu/ejYYNG8pRFxGRyZTaR2uw6yAqKgpZWVlo3LgxTp06haioKDnqIiIymWTCS04GW7QbNmzAv//9b/1ybGwsb2AgIkVSaou2yqDdunUrvvjiC2RmZuLQoUMAAJ1Oh7KyMgYtESmSxd2CO2TIEPj4+GDVqlV44403ANx7BG/Tpk1lK46IyBRKbdFWGf82NjZwcnLCvHnzkJubi+zsbGRlZSE9PV3O+oiIjGaxfbQTJ05Ebm4uWrZsCeDe8IlevXqZvTAiIlMptUVrMGhv3ryJLVu2yFELEVG1KDVoDfYct23bFjk5OXLUQkRULRb7uPHU1FT4+fmhSZMm+vc4excRKZGVQlu0BoP222+/laMOIqJqU2rXQZVBGxsbW2XzesqUKWYriIjoUVlc0Lq6uspZBxFRtVncpDIvvviinHUQEVWbMu8LM3KaRCIiS1BTLVqdTofw8HCcPXsWNjY2WLBgQYWngC9YsADHjx/XTxu7fPlyNGjQoMr9MWiJqM5Q19BcB4mJiSgtLUV8fDzS0tIQHR2NFStW6Nf/8ssvWLt2bYXRWA9jsKqioiJcv34dN2/exLJly3D16tVHr56IyIxqahxtamoqfH19AQA9evTAqVOn9Ot0Oh0uX76MOXPmICQkBF988YXBuoy6BTckJATffvst2rdvjzlz5uDjjz82uGMAuBl50Kjt6H+L3UC32i6BFEjsy6r2PlQmzGIQHx+P+Ph4/XJwcDCCg4MBAAUFBdBoNPp1VlZWKC8vh1qtRlFREUJDQzFmzBhotVqMGjUKXbp0gYeHR5XHMhi0xcXF6N+/PzZs2ID3338fR44cMfpEiIjkZEof7YPB+lcajQaFhYX6ZZ1OB7X6Xlza2dlh1KhRsLOzAwD06dMHZ86ceWjQGuw6KCsrwyeffILOnTvj/PnzuHv3rtEnQkQkJ5UkGf16GE9PT/083GlpaXBz+/O3sEuXLmHEiBHQarUoKyvD8ePH0blz54fuz2CLNiwsDImJiXjzzTexc+dOzJw505jzJSKSnUqqmYth/v7+SEpKQkhICIQQiIqKQlxcHJydndG/f38MGTIEQUFBsLa2xpAhQ9ChQ4eH7k8SQojKVmRnZ1f5Q48//rhRxRZri4zajv63sI+WKlMTfbSzU2Ybve187/nVPp6xqmzRTp48GQBw69YtFBYWokOHDjh//jwee+wxbN++XbYCiYiMJSn0loUqg/b+1bi33noLCxcuhEajQVFREec5ICLFsri5Du67fv26fphD/fr1cePGDbMXRUT0KCxuroP7nn76aYSGhqJLly44efIknn32WTnqIiIymST708CMYzBoJ0+ejFOnTuHy5csYOnToQ8eKERHVJot73Ph9OTk5WL9+PfLy8jBw4ECUlJSge/fuctRGRGQSlUIvhhmsavbs2Rg2bBjKysrg5eWFyMhIOeoiIjKZUp8ZZjBoi4uL4ePjA0mS4OrqCltbWznqIiIymVKD1mDXga2tLQ4fPgydToe0tDTY2NjIURcRkclMmVRGTgZbtPPnz0dCQgLy8/Oxbt06hIeHy1AWEZHpLLZF26JFC0ycOBGXL1+Gh4cHHB0d5aiLiMhkVjU010FNMxi0n376Kfbt24c//vgDL774on7CWyIipampSWVqmsGqdu/ejbi4ODRo0ACjR49Genq6HHUREZnMYrsOhBAVCuPFMCJSKou9M+wf//gHXn75ZWRnZ2PcuHG8BZeIFMtiJ5UJDQ2Fj48PMjIy0LZtW96CS0SKZXEXw3bs2PG3986cOYMzZ85g6NChZiyJiOjRSJYWtJmZmRWWhRBISEhAvXr1GLREpEgW10c7depU/Z9/++03hIWF4ZlnnsGMGTNkKYyIyFQW20e7adMmfPLJJ5g+fTr8/PzkqImI6JFY3MTfOTk5mD59OhwcHLB161Y4ODjIWRcRkcmUOtdBlUE7aNAg2NjYoE+fPoiIiKiwLjY21uyFERGZSqWyqu0SKlVl0C5fvlzOOoiIqs3iWrS9e/eWsw4iomqzuD5aIiJLY3HDu4iILA1btEREZmZxfbRERJZGJVnYqAMiIkvDrgMiIjPjxTAiIjNji5aIyMyUejFMmZM3EhE9ApVkZfTrYXQ6HebMmYPg4GCMHDkSly9frnSbsWPHYvPmzYbreuQzIiJSmJp6OGNiYiJKS0sRHx+PqVOnIjo6+m/bLFmyBLdv3zaqLnYdEFGdUVMXw1JTU+Hr6wsA6NGjB06dOlVh/TfffANJkvTbGMIWLRHVGSpJMvoVHx+Pl156Sf+Kj4/X76egoAAajUa/bGVlhfLycgBARkYGdu3ahUmTJhldF1u0RFRnmNKiDQ4ORnBwcKXrNBoNCgsL9cs6nQ5q9b243LFjB3JycjB69GhcvXoV1tbWaNWqFfr27VvlsRi0RFRn1NTwLk9PTxw4cAABAQFIS0uDm5ubft1//vMf/Z+XLl2Kxx577KEhCzBoiagOqalbcP39/ZGUlISQkBAIIRAVFYW4uDg4Ozujf//+Ju9PEkKIGqmsEsXaInPtmiyY3UA3wxvR/xyxL6va+9iXtcvobf2d/lHt4xmLLVoiqjN4ZxgRkZlxrgMiIjNji5aIyMysOB8tEZF5seuAiMjM2HVQh+l0OkRGRCHjbAZsbGwwN2IOnF2c9es3fvIpvvl6LwDAt+/TeOOt11FUdBfTp03H7du3YW1tjfnvzYejY/PaOgUyA0mSsHxiFLq7dkJJWSnGfjANmdmXAADd23XCkjfn6bft0/EJDJ07FgN7PYMe7ToDAFo0boZbhbfhM/GF2ijfIrFFW4ft/+4ASktLsXHzBpxMP4nY9z/Ah8uWAACyrmTh611f49MtG6FSqfBK6Bj0e9YPP6UcRcfOHfHGv17Hl9t3Yv3H6xE24z8PPxBZlKFPDUQ9G1s8OWkIvDt6Ivb12Rg69zUAQHrmr/D793AAQGDfQbh68zr2Hvsee499DwBQW6nxw+LtGPcBvxOmsNgWbUFBAQ4dOoTS0lL9e0OHDjVnTRbnxPETePLpJwEA3bp3wy+//Kpf59jCEctXL4OV1b1O+rLyctjY2iJ01MvQarUAgGvXrqFBgwbyF05m9XTnXvjm6PcAgJTTx+Hl1v1v29SvZ4d5o6ai75RhFd5/e+gYfJt6EKcunZGj1DpDpdB5sgwG7b/+9S80b94cLVu2BKDcfzFqU2FBIRo8ONOP6t5MP2q1GtbW1mjcuDGEEPggZjE8PDzQpo3Lve2srDB2zHiczziHlWtX1lb5ZCYN7Rvgj8I7+mWtTgsrlRW0Oq3+vdcGjsDWQ7uReztf/5612hqvDwpF7wny3blUV6gkCw1aIQQWLVokRy0Wy15jj8LCP2831ok/Z/oBgJKSEsydFY769e0xc870Cj+7Nm41Ll64iAlvTsTuvV/JVjOZ3+3CO2hgZ69fVkmqCiELAC/3exGB88dXeO9ZT18c+jkFt4vugEyj1Iagwfh3d3dHeno6SktL9S+q6IkneuCHwz8AAE6mn0SHDu3164QQmDRhMtzc3TBn3ix9F8LHqz/GVzvv3Zddv359qFTK/JeYHl3SL8cQ4N0PAODd0RM/X6zYDdCwfgPY2tgg68a1Cu8/+8TT2HP0gGx11iWSCf/JyWCL9qeffsL+/fv1y5Ik4bvvvjNrUZam37P98OORZIz652gIIRAROQ8b1m+Es3NraHU6pB5NRVlpKZIOJwEAJk5+G0NfGopZM+Zgx7Yd0Gp1iIicZ+AoZGm2J+2Bf09fJC3ZAUmSMGbRFEweNg7nsy/hqx/3wc3JFZeuX/nbz7m3bocNiV/UQsWWT6mjDoyevSs3NxeNGjXSt8iMwdm7qDKcvYsqUxOzdx27ecTobb0ee7LaxzOWwd9XU1JS0L9/f7z22mv6ORqJiJTIYrsOlixZgs8++wyOjo7IycnBhAkT8NRTT8lRGxGRSSx21IGVlRUcHR0BAI6OjrC1tTV7UUREj0KpfbQGg1aj0WDjxo3o1asXjh49CgcHBznqIiIymcUO74qJiUF2djYWL16Ma9euISoqSo66iIhMZnF9tNevX0eLFi1w8+ZNBAUF6d/Py8tjq5aIFMniug7i4uIwffp0zJkzB5Ik4f4oMEmSsGHDBtkKJCIylsVdDJs+/d6tomPGjEG/fv3073/99dfmr4qI6BEotY+2yqA9cOAAjh8/jt27dyMtLQ3AvXlXv/vuOwQEBMhVHxGR0Syu68DDwwO3bt2Cra0t2rZtC+DevxaDBg2SrTgiIlMoNWgN3oKr0+keecIT3oJLleEtuFSZmrgF9+wfPxu9rbtD12ofz1gGx9GuWbMGa9asQb169fTv/fDDD2YtiojoUSi1RWswaHfv3o3Dhw/Dzs5OjnqIiB6ZxY06uM/JyalCa5aISLkstEVbVlaGwYMHw83NTT90IjY21uyFERGZyuKGd903btw4OeogIqo2pfbRGuzQ6NSpE5KSkrB9+3bcunVLP5MXEZHSKHWuA4NBO2PGDLRu3RqXL1/GY489hpkzZ8pRFxGRySRJMvolJ4NBe+vWLQQGBkKtVsPT0xM6nU6OuoiITKYy4b+H0el0mDNnDoKDgzFy5Ehcvny5wvpNmzZh2LBhCAwMNGpaAoN9tACQmZkJ4N6MXqY8M4yISE411VJNTExEaWkp4uPjkZaWhujoaKxYsQLAvRkMN2/ejO3bt6OkpASDBg3C888//9BjGwzaWbNmYcaMGcjMzMTEiRMxd+7cGjkRIqKaVlN9r6mpqfD19QUA9OjRA6dOndKva9KkCXbs2AG1Wo2rV6/C1tbWYMAbDNrffvsNmzdvfuTbcImI5GJKizY+Ph7x8fH65eDgYAQHBwMACgoKoNFo9OusrKxQXl4OtfpeZKrVanz66adYunQpRo4cafBYBoP2xx9/xIcffoh+/fohMDAQrVu3NvpEiIjkZEqL9sFg/SuNRoPCwkL9sk6n04fsfaGhoQgKCsK4ceOQnJyMPn36VHksg83U2bNnY9u2bfDw8EBERAReeeUVI0+DiEheNTW8y9PTE4cOHQIApKWlwc3tz4mQLly4gAkTJkAIAWtra9jY2Bj8jd+oi2EnT57EDz/8gNzcXAwYMMCYHyEikl1NXQzz9/dHUlISQkJCIIRAVFQU4uLi4OzsjP79+8PDwwPBwcGQJAm+vr7o3bv3w+syNE1iQEAAPDw8MHz4cPj4+JhULKdJpMpwmkSqTE1Mk3j9rvH7aGHnVO3jGctgi/all17C2LFj5aiFiKhalHkDrhF9tIcOHYJWq5WjFiKiapJMeMnHYIs2Pz8fvr6+cHJy0t+6tmXLFjlqIyIyicXO3rVy5Uo56iAiqjalzt5lMGjVajViYmKQl5eHgQMHwt3dHa1atZKjNiIikyg1aI0aRzts2DCUlZXBy8sLkZGRctRFRFRnGAza4uJi+Pj4QJIkuLq6wtbWVo66iIhMZrHTJNra2uLw4cPQ6XRIS0uDjY2NHHUREdUZBoN2/vz5SEhIQH5+PtatW4d58+bJURcRkcmU+oQFgxfDDh8+jMWLF+uXN2zYgFGjRpm1KCKiR6HUi2FVBu2uXbuwf/9+pKSkIDk5GcC9GWwyMjIYtESkSBY3jtbX1xfNmjXDrVu39FOJqVQqTpNIRApmYUHr4OAAb29veHt7Izc3FyUlJQDA23GJSLGUGbNG9NHOmzcPBw8eRPPmzSGE4C24RKRgyoxag0Gbnp6OxMREPsqGiBRPqX20BtPTxcVF321ARKRkFju869q1a/Dz84OLiwsAsOuAiBRMmS1ag0EbGxsrRx1ERNWmzJh9SNA++Bjev6rqyZFERLVJqX20VQbtjRs35KyDiKgGKDNoDT6csTr4cEaqDB/OSJWpiYczFpUXGL1tfbWm2sczllGPGycisgRK7Trg4FgiIjMzqkV78OBBnDt3Dm3atMGzzz5r7pqIiB6JUmfvMthHGxsbi0uXLqFnz544duwYnJyc8O6778pVHxGRxTPYoj169Kj+BoXRo0cjKCjI7EUREdUlBvtoy8vLodPpAEA/qQwRERnPYIs2ICAAI0aMQPfu3XHy5EkEBATIURcRUZ1h1DjajIwMXLhwAa6urnBz4xhIIiJTVBm0O3bsqPKHhg4daqZyiIjqniq7DjIzMyssCyGQkJCAevXqMWiJiExgVNfBb7/9hrCwMLRt2xYzZsyARiPfrWtERJbO4KiDTZs2YezYsRg/fjyioqIUF7IpKSno2bMnrl27pn9v0aJFSEhIqNZ+g4KCkJVl+N7rhIQELFq0qMr1JSUl2Lp1a7VqMcWtW7fw1VdfAQBWr16NkydPynZsS1Ld783SpUuxefPmKtc/+DnIITs7G/v37wcAREZGIjs7W7Zjk2FVBm1OTg5effVVHDt2DFu3boWfn5+cdZnExsYG06dPhxnnx3lkN27ckDVoz549q/8fbvz48ejWrZtsx7Y05vzePPg5yCE5ORnHjx8HAMycOROPP/64bMcmw6rsox00aBBsbGzQp08fREREVFintMnA+/TpA51Oh02bNiE0NLTCuo0bN2LXrl2QJAkBAQEYNWoUrl27htmzZ6OkpAS2traYP38+WrZsicWLF+Pw4cNo0aIF8vPzAQCpqalYuHAh1Go17Ozs8OGHH1bZqn/uuefg6emJixcvomnTpli6dClWrlyJ8+fP47///S9Gjx6NmTNn6vc9a9YsuLu7w8/PD66urmjXrh0OHDiAL7/8EvXr18fHH38MKysrDBgw4G/1arVaTJ06FS1atMCVK1fQtWtXzJs3DytXrsSZM2cQHx+PEydOICAgAD4+Ppg+fTqysrKg1WoxZswYBAQEYOTIkfDw8MC5c+dQUFCADz/8EK1atTLvh6UgD/verFu3Drt374ZarYaXlxemTZtW6T6ysrIMfg59+/at9PN788030ahRI/Tt2xcJCQn4+uuvIUkSIiIi4OPjA2dnZyxYsAAA0KhRI0RFReHXX3/FmjVrYG1tjaysLAQEBGD8+PFYvXo1iouL8cQTT2D9+vUIDw9Hs2bNMG3aNBQUFECr1WLSpEnw8fHB4MGD0bt3b5w9exaSJGH58uVo0KCB2f++/6eJKqSkpFT5UpLk5GTxzjvviLy8PNG/f39x6dIlERMTI7Zt2ybOnTsnQkJCRHl5uSgvLxcjR44UmZmZYtKkSeL7778XQghx5MgRMWXKFHHy5EkxYsQIodVqxZ07d4SPj4+4cuWKiI6OFuvWrRNarVbs27dPXL16tcLxt23bJmJiYoQQQnh4eIjs7GwhhBDBwcHixIkT4sqVK2L48OFCCCHef/99sWnTJiGEEBcvXhQhISFCCCHc3d1FXl6efpvt27cLIYQYOnSoyMvLq7TeK1euiN69e4s7d+6I8vJy8cwzz4jff/9d//chhBBhYWHi4MGDYuPGjSIyMlIIIcSdO3eEv7+/yM3NFaGhoWLnzp1CCCE++OADsWrVqpr/gBTqYd+bM2fOiMDAQFFaWip0Op146623xP79+yv8/EcffSQ+++wzoz6Hqj4/b29vUVJSot/mp59+EiUlJSIgIECUlZWJ4cOHi3PnzgkhhPj888/FBx98IJKTk8Xzzz8vysrKRGFhofD09BRCVPwehoaGivPnz4vo6Gixfv16IYQQ169fF35+fkKn0wk/Pz+RmpoqhBBiypQpYteuXWb+26YqW7S9e/eWM++rrXHjxpgxYwbCwsLg6ekJ4N743+zsbLzyyisAgD/++AOXL19GRkYGVq1ahbVr10IIAbVajUuXLqFLly5QqVTQaDT68cJvvPEGVq5cidGjR8PR0fGhv4o3btwYLVu2BAC0bNnybw+1zMjIQHJyMvbs2aOv5/7PNW7cGAAwfPhwhIeHw9XVFW3btkXjxo0rrRcAnJ2d9a3rZs2aVfkQzczMTDz55JMAAI1Gg3bt2uHKlSsAgE6dOgEAWrRogZs3bxr7111nVPa9uXDhArp37w5ra2sAgJeXF86dO1dl95mhz6Gqz8/JyQk2NjYA7l0T2L59O27cuIF+/fpBrVYjMzMT8+bNAwCUlZWhTZs2AAA3Nzeo1Wqo1WrUq1evynPLzMzE4MGDAQCOjo7QaDTIzc0F8OfnXtn3lGpenZqPtl+/fti3bx+2b9+OadOmwdXVFe3bt8fatWshSRLWr18Pd3d3uLq64tVXX4WnpycyMzNx9OhRtG/fHps2bYJOp0NxcTHOnz8PANi5cydefPFFhIWFYdWqVfj8888xYcKESo9f2e3JKpVKfwuzq6srXnjhBQwePBi5ubn6vtsHH+Xepk0bCCGwdu1ajBgxQv9zf63XmOPd165dOxw7dgz+/v4oKChARkYGnJycTP3rrbMq+97ExcWhvLwcVlZWOHr06EOHNBrzuVf2+T34ufv4+CAmJgY5OTmYO3cuAKBt27ZYuHAhHn/8caSmpuqfemLq596pUyfk5OTg9u3baNSoUZX7IPOpU0EL3LsQkJycDADw8PCAj48PRowYgdLSUnTr1g2Ojo4ICwtDeHg4SkpKUFxcjJkzZ6Jjx47o27cvAgMD0bx5czRt2hQA0K1bN8yaNQt2dnZQqVR/6682pGnTpigrK0NMTAzeeOMNzJw5E59//jkKCgqqDOzAwEB89NFH6NOnDwBUWm9VnJ2dkZGRgfXr1+vfCwoKwuzZszFixAiUlJRgwoQJ+vOjex783ri7u+P555/HiBEjoNPp0LNnT5OnB33wczDm85MkCQMGDMCRI0fg7OwMAAgPD0dYWBjKy8shSRIiIyPx+++/V3o8Nzc3rFixAp07d9a/9/rrr2PGjBnYu3cviouLERERoW9Nk7zM+igbIiLiExaIiMyOQUtEZGYMWiIiM2PQEhGZGYOWiMjMGLRERGbGoCUiMrP/B95HAIkLtwR7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "LRparams = [{'solver': ['lbfgs','sag','saga','newton-cg'],\n",
    "         'penalty': ['none'],\n",
    "         'random_state':[111],\n",
    "          'C':[.001,.01,.1]},\n",
    "          {'solver': ['lbfgs','sag','saga','newton-cg','liblinear'],\n",
    "         'penalty': ['l2'],\n",
    "         'random_state':[111],\n",
    "          'C':[.001,.01,.1]},\n",
    "          {'solver':['saga'],\n",
    "          'penalty':['elasticnet'], 'l1_ratio':[.4,.7,.9]},\n",
    "          {'solver':['saga','liblinear'],\n",
    "          'penalty':['l1'],\n",
    "          'C':[.001,.01,.1]}]\n",
    "\n",
    "\n",
    "LRgrid = GridSearchPresentationCV(LogisticRegression(), LRparams,\n",
    "                                scoring='accuracy', verbose = 1)\n",
    "\n",
    "LRgrid.fit(train_df)\n",
    "LRmodel = score_grid(LRgrid, X_test_transformed, y_test)\n",
    "pickle.dump(LRmodel,open('../models/LRmodel3.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression Evaluation\n",
    "By optimizing the hyperparameters of the logistic regression model, which are all regularization parameters, we've barely moved the needle at all.  We gained .0002 accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More model types\n",
    "We used the logistic regression classifier to tune our features, but now it's time to try some other models.  We will use Course_GridsearchCV to optimize the hyperparameters for these as well."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree\n",
    "A decision tree is a promising candidate for this problem because it does not assume the independence of the features. Instead it seeks to find the best way to divide and subdivide the data in a tree structure based on the values of different variables.  We can tune how many features each split is allowed to consider.  Once the tree is build predictions are made by sending an observations down the tree sending it on a path to the predicted class as it reaches each split in the tree is sent in one or the other direction.  You can think of it like a deterministic Pachinko machine!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_features': 4, 'min_samples_split': 0.5, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7642471171317629\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_features': 4, 'min_samples_split': 0.6, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7642471171317629\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_features': 5, 'min_samples_split': 0.5, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7757951561758097\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_features': 5, 'min_samples_split': 0.6, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7757951561758097\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 20, 'max_features': 4, 'min_samples_split': 0.5, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7642471171317629\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 20, 'max_features': 4, 'min_samples_split': 0.6, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7642471171317629\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 20, 'max_features': 5, 'min_samples_split': 0.5, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7757951561758097\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 20, 'max_features': 5, 'min_samples_split': 0.6, 'random_state': 111, 'splitter': 'best'}\n",
      "average score:  0.7757951561758097\n",
      "best model\n",
      "DecisionTreeClassifier(criterion='entropy', max_depth=20, max_features=5,\n",
      "                       min_samples_split=0.6, random_state=111)\n",
      "best cv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code_module</th>\n",
       "      <th>code_presentation</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAA</th>\n",
       "      <th>2013J</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">BBB</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.7865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.766211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.745131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCC</th>\n",
       "      <th>2014B</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DDD</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.80359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.791667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.794326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">EEE</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.823961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.771639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FFF</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.800975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.77678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.792043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">GGG</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.693495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.739018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  score\n",
       "code_module code_presentation          \n",
       "AAA         2013J                   NaN\n",
       "BBB         2013B                0.7865\n",
       "            2013J              0.766211\n",
       "            2014B              0.745131\n",
       "CCC         2014B                   NaN\n",
       "DDD         2013B               0.80359\n",
       "            2013J              0.791667\n",
       "            2014B              0.794326\n",
       "EEE         2013J              0.823961\n",
       "            2014B              0.771639\n",
       "FFF         2013B              0.800975\n",
       "            2013J               0.77678\n",
       "            2014B              0.792043\n",
       "GGG         2013J              0.693495\n",
       "            2014B              0.739018"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross validated accuracy score:\n",
      "0.7757951561758097\n",
      "validation accuracy: \n",
      "0.7253467843631778\n",
      "validation set confusion matrix\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAD3CAYAAABcpJzyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAl1ElEQVR4nO3de1wU9f4/8NcsC3hZxTuaCIIJlCVGJFDiLyRPRse0vIC/vBxLPV08mno65B1vhBFqefJSJl7TzUJPeckkPYoopCiYnRRFJQg1BbyAcd3P9w8fTVKssyvsMEuvZ495PJqd2Zn3utu7t+/5zGckIYQAERHZjK6+AyAiauiYaImIbIyJlojIxphoiYhsjImWiMjG9LY8uPSCpy0PT3bquvFYfYdAGtTcsWWtjyH1dbN4X7Enr9bns5RNEy0Rkaokqb4jqBETLRE1HBpthjLRElHDwYqWiMjGtJlnmWiJqAFx0GamZaIlooaDrQMiIhvTZp5loiWiBkSnzUzLREtEDYc28ywTLRE1IOzREhHZGEcdEBHZmDbzLBMtETUgddQ6MJlMiI6OxunTp+Hk5IT58+fDw8MDAPDDDz8gJiZG3jcjIwMffPABevfubfZ4TLRE1HDU0aiDpKQklJeXw2g0IiMjA7GxsVi+fDkA4IEHHsD69esBALt27UK7du3ummQBJloiakisyLNGoxFGo1Fej4iIQEREBAAgPT0dISEhAIAePXrg5MmTf3j/rVu3sHTpUmzYsEHxXEy0RNRwWFHR3plYf6+4uBgGg0Fed3BwQGVlJfT631LmZ599hn79+qFVq1bKYVkcFRGR1ukky5e7MBgMKCkpkddNJlO1JAsAX375JYYMGWJZWNZ/EiIijZKsWO7C398fBw4cAHD7Ype3t3e17Tdv3kR5eTk6dOhgUVhsHRBRw1FHow769u2LlJQUREZGQgiBmJgYJCQkwN3dHWFhYTh//jw6duxoeVhCCFEnkdV0cD4zjGrAZ4ZRTerkmWEv+1q8r/j4VK3PZylWtETUcPAWXCIiG+PsXURENqbRy/tMtETUcLB1QERkY9rMs0y0RNSAsEdLRGRjbB0QEdmWxIqWiMi2JFa0RES2pdE8y0RLRA2HTqOZlomWiBoMtg6IiGxMp9PmrWFMtETUYGi0oGWiJaKGg60DIiIbY6IlIrIxSaOTHTDRElGDwYqWiMjGHHgLLhGRbbGiJSKyMSZaIiIb02ieZaIlooaDFS0RkY0x0RIR2RjnOiAisjGNFrRMtETUcLB1QERkY0y0REQ2xicsEBHZmI634DZckiRh2bh58Ov8AMoqyjFm2VvIvpQDAPDr/ACWvDRL3jfI+xEMXDgOu48fAABM/OtotG/RFlM3vFMvsZPtmEwmLJwXhzNZZ+Do6IgZc6ehk3snefsn6zbh6117AABPhDyOsa+NwfXr1zHrrWiUFJfApYULpkdPRavWrerrI9gdzt7VgA3s+Rc0cnTG41MHIdC7B+L/Nh0DY8cBADIv/IDQWcMAAIODw/FT4SXsPn4AjZycseq1WPS83w+fp35Vn+GTjfz3m/0oKy/D6o2r8F3mSSyJex/xS+MAAHm5P+Gr7buRsOlj6HQ6jBkxDk+G/T/s+GIXejzih9Hj/oa0w99i2XvLMWPu9Hr+JPbDbnu0Fy9exPbt21FWVia/Nn78eJsGZW96PRCAr47vBwCkZWUgoMvDf9iniXNjzIl8A71nRAAAGjk6Y+2+ROzJPAjfjl1UjZfUkXk8E48/EQwAeNjvIfzw/Sl5W/v2rnh/5RI4ODgAACorq+Dk7Izz587jtQmvAAD8HumOuJh49QO3Y1pNtIqjeydOnIji4mK0adNGXqi65k2a4fqtm/J6lakKDjqHavu8HBaBLYd2ouBmEQDgWskN7MlMVjVOUldJcQmaNmsqr+t0OlRWVgIA9I56tGjZAkIILIl7Hz4PeMOjszu8fbxxYN/t38WBfcko/aW0XmK3V5Jk+aImxYq2adOmmDRpkhqx2K0bt26iWWODvK7T6VBlqqq2z4u9B2Bw3Gtqh0b1qKmhKW6V3JLXhTBBr//tP7mysjLMm7kATZo2QdSMNwEAfxs7Eu++vQjjRr2CJ3o/Dtf27VSP257ZbUXbtWtX7NixA+fOncP58+dx/vx5NeKyKymn0hHu/yQAINC7B77LOV1te/MmzeDs6IS8gov1EB3VF79HuiMl+RAA4LvMk+jS9bcWkRAC//zHv9DV535Mm/2W3EI4fjQDAwcNwIdrV6CTuxv8HuleL7HbK51OZ/GiJsWK9ocffsAPP/wgr0uShHXr1tk0KHuzNW03+vr1QkrMZ5AkCaP//SYm9X8ZZy/l4MsjSfC+zxMXfs6r7zBJZU+GPYm0Q0fw0otjAQjMmjcDG9d+Ajd3N5iqTDh29DjKy8txKPkwAOD1N16Dh6c7Zk+dCwBo59qWF8KspNGCFpIQQijtVFRUhNzcXLi5uaFVK8uHmkgveNYqOGqYrhuP1XcIpEHNHVvW+hi+S56xeN9Tb+yq9fkspVg/79q1C5GRkVixYgUiIiLwn//8R424iIisJkmSxYuaFFsHa9asQWJiIpo2bYri4mKMGjUKAwYMUCM2IiKr1FUCNZlMiI6OxunTp+Hk5IT58+fDw8ND3r5//3588MEHEEKgW7dumD179l3PrVjRSpKEpk1vD1ExGAxwdnaug49BRFT36mp4V1JSEsrLy2E0GjFlyhTExsbK24qLixEXF4cVK1Zgy5Yt6NixI4qKiu56PMWKtlOnToiNjUVAQACOHj0Kd3d3yz4xEZHKrBlNYDQaYTQa5fWIiAhERNy+oSg9PR0hISEAgB49euDkyZPyfsePH4e3tzcWLlyI3NxcDBkyRPHalWKiffvtt2E0GnHo0CF06dIFU6ZMsfiDEBGpyZrWwZ2J9feKi4thMPw2Nt7BwQGVlZXQ6/UoKipCWloatm3bhiZNmuDFF19Ejx494Olp/uK/2fT/3XffAQBSU1Ph4eGBPn36wMPDA2lpaRZ/ECIiNdVV68BgMKCkpEReN5l+u9mkRYsWePjhh9G2bVs0bdoUAQEB1YbA1sRsRXv48GE8/PDD2LFjxx+29erV6+5REhHVg7q6GObv7499+/YhPDwcGRkZ8Pb2lrd169YNWVlZKCwsRPPmzZGZmYmhQ4fe9XhmE+24cePkEw4ZMkR+nTcrEJFm1VGi7du3L1JSUhAZGQkhBGJiYpCQkAB3d3eEhYVhypQpGDNmDACgX79+1RJxjWGZu2Fh+/bt2Lt3L9LS0hAUFATgdvmclZVVY5Vb48F5wwLVgDcsUE3q4oYF/4+et3jfY2O31vp8ljJb0YaEhKBt27a4du2a3DDW6XTo1KmTubcQEdUrrU4qYzbRuri4IDAwEIGBgSgoKJDno62qqjL3FiKiemV3ifZXc+bMwf79+9GuXTsIISBJEjZv3qxGbEREVrHbRJuZmYmkpCTVpxUjIrKWRvOscqL18PBAWVkZGjdurEY8RET3zG4r2osXLyI0NFSeUIGtAyLSKq3+zVsx0cbH8+FwRGQf7Lai1ev1iIuLQ2FhIfr16wcfHx907NhRjdiIiKyi0TyrPE3izJkzMWjQIFRUVCAgIAALFixQIy4iIqtpdeJvxURbWlqK4OBgSJIELy8vzkdLRJql1USr2DpwdnZGcnIyTCYTMjIy4OTkpEZcRERW02qPVrGinTdvHhITE1FUVITVq1cjOjpahbCIiKyn00kWL2pSrGh3796N6OhouLi4qBEPEdG902hFq5hoq6qqMHr0aHh6emLo0KEIDAxUIy4iIqvZbevgpZdeQmJiIkaNGoVPPvkETz/9tBpxERFZTSdZvqhJsaItLS3F7t27sW3bNggh8I9//EONuIiIrKbVilYx0T733HN4+umnER0dXe255kREWuNgr7fghoaG8sm3RGQXtJlmLYgrOzsbN27cUCMWIqJa0UmSxYuaFCva7OxsBAYGolWrVnL/4+DBgzYPjIjIWnbbo923b58acRAR1ZralaqlFBPtmTNnMHv2bNy4cQPPPfccunbtitDQUDViIyKyilYrWsUe7fz58/H222+jZcuWGDx4MJYuXapGXEREVtNLksWLqnFZspOHhwckSUKrVq3QtGlTW8dERHRPtFrRKiZaFxcXbN68Gb/88gt27NiB5s2bqxEXEZHVtNqjVWwdxMTEIC8vDy1btsTJkycRExOjRlxERFaTrFjUpFjRrlu3Dv/85z/l9fj4eN7AQESapNWK1myi3bJlCz777DNkZ2fjwIEDAACTyYSKigomWiLSJLu7BXfAgAEIDg7GypUr8corrwC4/Sjf1q1bqxYcEZE1tFrRmk3/Tk5OcHNzw5w5c1BQUID8/Hzk5eUhMzNTzfiIiCxmtz3aCRMmoKCgAB06dABwe/jEY489ZvPAiIispdWKVjHRXr16FZs3b1YjFiKiWtFqolXsHHt6euLy5ctqxEJEVCt2+7jx9PR0hIaGolWrVvJrnL2LiLTIQaMVrWKi/frrr9WIg4io1rTaOjCbaOPj482W15MnT7ZZQERE98ruEq2Xl5eacRAR1ZrdTSrz/PPPqxkHEVGtafO+MAunSSQisgd1VdGaTCZER0fj9OnTcHJywvz586s9BXz+/Pk4duyYPG3ssmXL0KxZM7PHY6IlogZDX0dzHSQlJaG8vBxGoxEZGRmIjY3F8uXL5e3ff/89Vq1aVW001t0oRnXr1i1cunQJV69exQcffICffvrp3qMnIrKhuhpHm56ejpCQEABAjx49cPLkSXmbyWRCTk4OZs2ahcjISHz22WeKcVl0C25kZCS+/vpr3H///Zg1axY+/vhjxQMDwC9bvrdoP/pzadzPu75DIA0Se/JqfQydFbMYGI1GGI1GeT0iIgIREREAgOLiYhgMBnmbg4MDKisrodfrcevWLQwfPhyjR49GVVUVRo4ciYceegi+vr5mz6WYaEtLSxEWFoZ169bhnXfewaFDhyz+IEREarKmR3tnYv09g8GAkpISed1kMkGvv50uGzdujJEjR6Jx48YAgKCgIJw6dequiVaxdVBRUYG1a9eiW7duOHv2LH755ReLPwgRkZp0kmTxcjf+/v7yPNwZGRnw9v7tb2EXLlzAsGHDUFVVhYqKChw7dgzdunW76/EUK9qoqCgkJSXh1VdfxRdffIHp06db8nmJiFSnk+rmYljfvn2RkpKCyMhICCEQExODhIQEuLu7IywsDAMGDMDQoUPh6OiIAQMGoGvXrnc9niSEEDVtyM/PN/um++67z6JgS6tuWbQf/bmwR0s1qYse7cy0mRbvOy9wXq3PZymzFe2kSZMAANeuXUNJSQm6du2Ks2fPok2bNti6datqARIRWUrS6C0LZhPtr1fjXn/9dSxcuBAGgwG3bt3iPAdEpFl2N9fBry5duiQPc2jSpAmuXLli86CIiO6F3c118KtevXph+PDheOihh3DixAk89dRTasRFRGQ1SfWngVlGMdFOmjQJJ0+eRE5ODgYOHHjXsWJERPXJ7h43/qvLly9jzZo1KCwsRL9+/VBWVgY/Pz81YiMisopOoxfDFKOaOXMmBg0ahIqKCgQEBGDBggVqxEVEZDWtPjNMMdGWlpYiODgYkiTBy8sLzs7OasRFRGQ1rSZaxdaBs7MzkpOTYTKZkJGRAScnJzXiIiKymjWTyqhJsaKdN28eEhMTUVRUhNWrVyM6OlqFsIiIrGe3FW379u0xYcIE5OTkwNfXF66urmrERURkNYc6muugrikm2g0bNmDPnj24fv06nn/+eXnCWyIiramrSWXqmmJUO3bsQEJCApo1a4ZRo0YhMzNTjbiIiKxmt60DIUS1wHgxjIi0ym7vDPvrX/+KF198Efn5+Rg7dixvwSUizbLbSWWGDx+O4OBgZGVlwdPTk7fgEpFm2d3FsG3btv3htVOnTuHUqVMYOHCgDUMiIro3kr0l2uzs7GrrQggkJiaiUaNGTLREpEl216OdMmWK/O8//vgjoqKi8OSTT2LatGmqBEZEZC277dFu3LgRa9euxdSpUxEaGqpGTERE98TuJv6+fPkypk6dChcXF2zZsgUuLi5qxkVEZDWtznVgNtE+++yzcHJyQlBQEObOnVttW3x8vM0DIyKylk7nUN8h1Mhsol22bJmacRAR1ZrdVbQ9e/ZUMw4iolqzux4tEZG9sbvhXURE9oYVLRGRjdldj5aIyN7oJDsbdUBEZG/YOiAisjFeDCMisjFWtERENsaLYURENsaLYURENsbWARGRjfFiGBGRjdntxN9ERPaCFS0RkY2xR0tEZGNaHXWgzWfzEhHdAx0ki5e7MZlMmDVrFiIiIjBixAjk5OTUuM+YMWOwadMmC+IiImogJEmyeLmbpKQklJeXw2g0YsqUKYiNjf3DPkuWLMGNGzcsioutAyJqMOrqYlh6ejpCQkIAAD169MDJkyerbf/qq68gSZK8jxJWtETUYFhT0RqNRrzwwgvyYjQa5eMUFxfDYDDI6w4ODqisrAQAZGVlYfv27Zg4caLFcbGiJaIGw8GKi2ERERGIiIiocZvBYEBJSYm8bjKZoNffTpfbtm3D5cuXMWrUKPz0009wdHREx44d0bt3b7PnYqIlogajrloH/v7+2LdvH8LDw5GRkQFvb29527/+9S/535cuXYo2bdrcNckCTLRE1IDU1Tjavn37IiUlBZGRkRBCICYmBgkJCXB3d0dYWJj1cQkhRJ1EVoPSqlu2OrSmmEwmLJgbg6zTWXBycsLsubPg7uEub1+/dgO+2rkbABDSuxdeef3v+Pij1Th08BAA4OaNm7h6tQB7k5PqJX61Ne7nrbxTAyBJEpZNiIGf14MoqyjHmEVvIjv/AgDAr8uDWPLqHHnfoAcewcDZY5B57n/Y8Nb7cNI7ovDmNQyPnYDiX0rMnKFhEXvyan2Mg5e+sXjfXu2tT5j3ihfD6sDeb/ahvLwc6zetw8TJExD/ziJ5W15uHnZu34l1n6zBhs3rcPjQYWSdzsLLY1/Cx2tX4eO1q9CuvSvmx86rx09AtjDwiX5o5OSMxycOwFsfv434v8+Ut2Vm/w+h/xyC0H8OwQdfrMHnybuw++h/ERXxGtZ+vQW9Jw/C8bPfY8wzw+rxE9ifuhreVdcUWwfFxcU4cOAAysvL5dcGDhxoy5jszvFjx/F4r8cBAN39uuP77/8nb3Nt74plH34AB4fbTfqKyko4OTvL25P2fIPmzZvj8SeC1Q2abK5Xt8fw1ZH/AgDSfjiGAG+/P+zTpFFjzBk5Bb0nDwIATFoeLSeCTm3vQ87Pta/y/kx0Gq0dFRPta6+9hnbt2qFDhw4AtHsvcX0qKS5BszuHguhuDwXR6/VwdHREy5YtIYTAorjF8PX1RefOHvK+qz9ajdi4t+sjbLKx5k2b4XrJTXm9ylQFB50DqkxV8msv9xuGLQd2oOBGkfyag84BmSu/RiMnZ8zdsFjVmO2dTrLTRCuEwLvvvqtGLHarqaEpSkp+60ebxG9DQQCgrKwMs2dEo0mTppg+a6r8evbZbDRr1qxaP5cajhslN9GscVN5XSfpqiVZAHixz/MYPG9ctdcqqyrRbUwfhD3SC+ui3sOTUwarEm9DoNVCUDH9+/j4IDMzE+Xl5fJC1T3ySA8cTD4IADiReQJdu94vbxNCYOL4SfD28casOTPkFgIApB5OwxMhT6geL6kj5fujCA/sAwAIfMAf350/VW178ybN4OzkhLwrF+XXPvjHAjzpd7sNdfOXEphMJvUCbgAkK/5Rk2JF++2332Lv3r3yuiRJ+OYby6/s/Rn0eaoPDh9Kxcj/PwpCCMxdMAfr1qyHu3snVJlMSD+SjorycqQkpwAAJkz6B/x6+CHnQg6CgoPqOXqyla0pu9D30RCkLNkGSZIw+t3JmDRoLM7mX8CXh/fA280LFy7lVnvP+9tWY8XEWMwa/gZMwoTXlk6rp+jtk1bno7V4eFdBQQFatGhRrSJT8mcZ3kXW+bMM7yLr1MXwrqNXD1m8b0Cbx2t9Pksptg7S0tIQFhaGl19+WR7ES0SkRXbbOliyZAk++eQTuLq64vLlyxg/fjyeeIJ9RSLSHrsddeDg4ABXV1cAgKurK5zvGANKRKQlWu3RKiZag8GA9evX47HHHsORI0fg4uKiRlxERFaz2+FdcXFxyM/Px+LFi3Hx4kXExMSoERcRkdXsrkd76dIltG/fHlevXsXQoUPl1wsLC1nVEpEm2V3rICEhAVOnTsWsWbMgSRJ+HQUmSRLWrVunWoBERJayu4thU6fevlV09OjR6NOnj/z6zp07bR8VEdE90GqP1myi3bdvH44dO4YdO3YgIyMDwO15V7/55huEh4erFR8RkcXsrnXg6+uLa9euwdnZGZ6engBu/9/i2WefVS04IiJraDXRKt6CazKZoNPdW9+Dt+BSTXgLLtWkLm7BPX39O4v39XF5uNbns5TiONqPPvoIH330ERo1aiS/dvDgQZsGRUR0L7Ra0Som2h07diA5ORmNGzdWIx4iontmd6MOfuXm5latmiUi0i47rWgrKirQv39/eHt7y0Mn4uPjbR4YEZG17G5416/Gjh2rRhxERLWm1R6tYkPjwQcfREpKCrZu3Ypr167JM3kREWmNVuc6UEy006ZNQ6dOnZCTk4M2bdpg+vTpasRFRGS1Xx/VbsmiJsVEe+3aNQwePBh6vR7+/v58WBwRaZbOin/UpNijBYDs7GwAt2f0suaZYUREarLbi2EzZszAtGnTkJ2djQkTJmD27NlqxEVEZDWtXgxTTLQ//vgjNm3adM+34RIRqUWrFa1i9jx8+DAGDBiAxYsXIzc3V2l3IqJ6o9VRB4qTygBAeXk5vvnmGyQmJqKiogJr1qyx6OCcVIZqwkllqCZ1ManMxVs/WrxvhybutT6fpSzqB5w4cQIHDx5EQUEBgoODbR0TEdE90erwLsUebXh4OHx9fTFkyBAsWLBAjZiIiO6RNnu0ion2hRdewJgxY9SIhYioVrSZZi1oHRw4cABVVVVqxEJEVEuSFYt6FCvaoqIihISEwM3NTe5tbN68WY3YiIisotXhXYqJdsWKFWrEQURUa3Z7w4Jer0dcXBwKCwvRr18/+Pj4oGPHjmrERkRkFa0mWsUe7cyZMzFo0CBUVFQgICCAIw+IiKykmGhLS0sRHBwMSZLg5eUFZ2dnNeIiIrJaXY2jNZlMmDVrFiIiIjBixAjk5ORU275x40YMGjQIgwcPxs6dOxXjUmwdODs7Izk5GSaTCRkZGXByclI8KBGRPUtKSkJ5eTmMRiMyMjIQGxuL5cuXAwAKCwuxadMmbN26FWVlZXj22WfxzDPP3DV5K1a08+bNQ2JiIoqKirB69WrMmTOn7j4NEVEdqqu5DtLT0xESEgIA6NGjB06ePClva9WqFbZt2wZHR0dcvXoVzs7OihWyYkWbnJyMxYsXy+vr1q3DyJEjld5GRKQ6ay6GGY1GGI1GeT0iIgIREREAgOLiYhgMBnmbg4MDKisrodffTpl6vR4bNmzA0qVLMWLECMVzmU2027dvx969e5GWlobU1FQAt/sWWVlZTLREpEnWjKO9M7H+nsFgQElJibxuMpnkJPur4cOHY+jQoRg7dixSU1MRFBRk9lxmE21ISAjatm2La9euycHodDp06tTJ4g9CRKSuuhne5e/vj3379iE8PBwZGRnw9v5txrlz585h0aJFWLp0KRwdHeHk5KQ4X7fZROvi4oLAwEAEBgaioKAAZWVlAMDbcYlIs+pqFG3fvn2RkpKCyMhICCEQExODhIQEuLu7IywsDL6+voiIiIAkSQgJCUHPnj3vHpfSfLRz5szB/v370a5dOwghrLoFl/PRUk04Hy3VpC7mo71eXmjxvi5OrWp9PkspXgzLzMxEUlISH2VDRJqn1bkOFLOnh4eH3DYgItIyrT7KRrGivXjxIkJDQ+Hh4QEAnL2LiDRMmxWtYqKNj49XIw4iolrTZpq9S6K9cyDv75kbe0ZEVJ+02qM1m2ivXLmiZhxERHVAm4nWoseN3ysO76KacHgX1aQuhnfdqiy2eN8meoPyTnVEsUdLRGQvtNo64OBYIiIbs6ii3b9/P86cOYPOnTvjqaeesnVMRET3RKuPslHs0cbHx+PChQt49NFHcfToUbi5ueGtt95SKz4iIrunWNEeOXJEvkFh1KhRGDp0qM2DIiJqSBR7tJWVlTCZTAAgTypDRESWU6xow8PDMWzYMPj5+eHEiRMIDw9XIy4iogbDonG0WVlZOHfuHLy8vKpNgEtERMrMJtpt27aZfdPAgQNtFA4RUcNjtnWQnZ1dbV0IgcTERDRq1IiJlojICha1Dn788UdERUXB09MT06ZNq/Z0SCIiujvFUQcbN27EmDFjMG7cOMTExGguyaalpeHRRx/FxYsX5dfeffddJCYm1uq4Q4cORV6e8r3XiYmJePfdd81uLysrw5YtW2oVizWuXbuGL7/8EgDw4Ycf4sSJE6qd257U9nezdOlSbNq0yez2O78HNeTn52Pv3r0AgAULFiA/P1+1c5Mys4n28uXLeOmll3D06FFs2bIFoaGhasZlFScnJ0ydOhU2nB/nnl25ckXVRHv69Gn5P7hx48ahe/fuqp3b3tjyd3Pn96CG1NRUHDt2DAAwffp03Hfffaqdm5SZ7dE+++yzcHJyQlBQEObOnVttm9YmAw8KCoLJZMLGjRsxfPjwatvWr1+P7du3Q5IkhIeHY+TIkbh48SJmzpyJsrIyODs7Y968eejQoQMWL16M5ORktG/fHkVFRQCA9PR0LFy4EHq9Ho0bN8Z7771ntqr/y1/+An9/f5w/fx6tW7fG0qVLsWLFCpw9exb//ve/MWrUKEyfPl0+9owZM+Dj44PQ0FB4eXmhS5cu2LdvH/7zn/+gSZMm+Pjjj+Hg4ICnn376D/FWVVVhypQpaN++PXJzc/Hwww9jzpw5WLFiBU6dOgWj0Yjjx48jPDwcwcHBmDp1KvLy8lBVVYXRo0cjPDwcI0aMgK+vL86cOYPi4mK899576Nixo22/LA252+9m9erV2LFjB/R6PQICAvDmm2/WeIy8vDzF76F37941fn+vvvoqWrRogd69eyMxMRE7d+6EJEmYO3cugoOD4e7ujvnz5wMAWrRogZiYGPzvf//DRx99BEdHR+Tl5SE8PBzjxo3Dhx9+iNLSUjzyyCNYs2YNoqOj0bZtW7z55psoLi5GVVUVJk6ciODgYPTv3x89e/bE6dOnIUkSli1bhmbNmtn8z/tPTZiRlpZmdtGS1NRU8cYbb4jCwkIRFhYmLly4IOLi4sTnn38uzpw5IyIjI0VlZaWorKwUI0aMENnZ2WLixIniv//9rxBCiEOHDonJkyeLEydOiGHDhomqqipx8+ZNERwcLHJzc0VsbKxYvXq1qKqqEnv27BE//fRTtfN//vnnIi4uTgghhK+vr8jPzxdCCBERESGOHz8ucnNzxZAhQ4QQQrzzzjti48aNQgghzp8/LyIjI4UQQvj4+IjCwkJ5n61btwohhBg4cKAoLCysMd7c3FzRs2dPcfPmTVFZWSmefPJJ8fPPP8t/HkIIERUVJfbv3y/Wr18vFixYIIQQ4ubNm6Jv376ioKBADB8+XHzxxRdCCCEWLVokVq5cWfdfkEbd7Xdz6tQpMXjwYFFeXi5MJpN4/fXXxd69e6u9//333xeffPKJRd+Due8vMDBQlJWVyft8++23oqysTISHh4uKigoxZMgQcebMGSGEEJ9++qlYtGiRSE1NFc8884yoqKgQJSUlwt/fXwhR/Xc4fPhwcfbsWREbGyvWrFkjhBDi0qVLIjQ0VJhMJhEaGirS09OFEEJMnjxZbN++3cZ/2mS2olV6TrnWtGzZEtOmTUNUVBT8/f0B3B7/m5+fj7/97W8AgOvXryMnJwdZWVlYuXIlVq1aBSEE9Ho9Lly4gIceegg6nQ4Gg0EeL/zKK69gxYoVGDVqFFxdXe/6V/GWLVuiQ4cOAIAOHTr84aGWWVlZSE1Nxa5du+R4fn1fy5YtAQBDhgxBdHQ0vLy84OnpiZYtW9YYLwC4u7vL1XXbtm3NPkQzOzsbjz/+OADAYDCgS5cuyM3NBQA8+OCDAID27dvj6tWrlv5xNxg1/W7OnTsHPz8/ODo6AgACAgJw5swZs+0zpe/B3Pfn5uYGJycnALevCWzduhVXrlxBnz59oNfrkZ2djTlz5gAAKioq0LlzZwCAt7c39Ho99Ho9GjVqZPazZWdno3///gAAV1dXGAwGFBQUAPjte6/pd0p1r0HNR9unTx/s2bMHW7duxZtvvgkvLy/cf//9WLVqFSRJwpo1a+Dj4wMvLy+89NJL8Pf3R3Z2No4cOYL7778fGzduhMlkQmlpKc6ePQsA+OKLL/D8888jKioKK1euxKefforx48fXeP6abk/W6XTyLcxeXl547rnn0L9/fxQUFMi92zsf5d65c2cIIbBq1SoMGzZMft/v47XkfL/q0qULjh49ir59+6K4uBhZWVlwc3Oz9o+3warpd5OQkIDKyko4ODjgyJEjdx3SaMn3XtP3d+f3HhwcjLi4OFy+fBmzZ88GAHh6emLhwoW47777kJ6eLj/1xNrv/cEHH8Tly5dx48YNtGjRwuwxyHYaVKIFbl8ISE1NBQD4+voiODgYw4YNQ3l5Obp37w5XV1dERUUhOjoaZWVlKC0txfTp0/HAAw+gd+/eGDx4MNq1a4fWrVsDALp3744ZM2agcePG0Ol0f+hXK2ndujUqKioQFxeHV155BdOnT8enn36K4uJiswl78ODBeP/99xEUFAQANcZrjru7O7KysrBmzRr5taFDh2LmzJkYNmwYysrKMH78ePnz0W13/m58fHzwzDPPYNiwYTCZTHj00Uetnh70zu/Bku9PkiQ8/fTTOHToENzd3QEA0dHRiIqKQmVlJSRJwoIFC/Dzzz/XeD5vb28sX74c3bp1k1/7+9//jmnTpmH37t0oLS3F3Llz5Wqa1GXTR9kQERGfsEBEZHNMtERENsZES0RkY0y0REQ2xkRLRGRjTLRERDbGREtEZGP/Bwi76iFOwTpNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "DTparams = {'criterion':['entropy'],\n",
    "           'splitter':['best'],\n",
    "           'max_depth':[4,20],'min_samples_split':[.5,.6],\n",
    "           'max_features':[4,5],'random_state':[111]}\n",
    "\n",
    "\n",
    "DTgrid = GridSearchPresentationCV(DecisionTreeClassifier(), DTparams,\n",
    "                                scoring='accuracy', verbose = 1)\n",
    "\n",
    "DTgrid.fit(train_df)\n",
    "DTmodel = score_grid(DTgrid, X_test_transformed, y_test)\n",
    "pickle.dump(DTmodel,open('../models/DTmodel3.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree Evaluation \n",
    "As expected, this model is better at predicting how a student will do.  It doesn't rely on establishing a straight line through the data to model it, which is probably not the best approach in this problem space.  We see a nice increase in accuracy here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest Classifier\n",
    "This is an interesting extension to the decision tree model.  It creates a whole forest of decision trees and trains each one on a subset of the data and a subset of the features.  This is a technique called bagging, or [Boostrap AGGregation](https://machinelearningmastery.com/bagging-and-random-forest-ensemble-algorithms-for-machine-learning/) (check the link for more on this).  It works on the principle that a bunch of bad predictors, on average will be more accurate than one good predictor.  This worked for Francis Galton in [guessing the weight of an ox](https://crowdsourcingweek.com/blog/using-the-crowd-to-predict/), maybe it will work here!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': 0.7, 'min_samples_split': 0.1, 'n_estimators': 150}\n",
      "average score:  0.7814574060399397\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': 0.7, 'min_samples_split': 0.1, 'n_estimators': 200}\n",
      "average score:  0.7825448807114502\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': 0.7, 'min_samples_split': 0.3, 'n_estimators': 150}\n",
      "average score:  0.773973851962928\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': 0.7, 'min_samples_split': 0.3, 'n_estimators': 200}\n",
      "average score:  0.7705220431228281\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': None, 'min_samples_split': 0.1, 'n_estimators': 150}\n",
      "average score:  0.7869489235651188\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': None, 'min_samples_split': 0.1, 'n_estimators': 200}\n",
      "average score:  0.786279947947604\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': None, 'min_samples_split': 0.3, 'n_estimators': 150}\n",
      "average score:  0.7709464295769174\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 4, 'max_samples': None, 'min_samples_split': 0.3, 'n_estimators': 200}\n",
      "average score:  0.7701999594978512\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': 0.7, 'min_samples_split': 0.1, 'n_estimators': 150}\n",
      "average score:  0.7802523495406452\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': 0.7, 'min_samples_split': 0.1, 'n_estimators': 200}\n",
      "average score:  0.7796281333262987\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': 0.7, 'min_samples_split': 0.3, 'n_estimators': 150}\n",
      "average score:  0.773973851962928\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': 0.7, 'min_samples_split': 0.3, 'n_estimators': 200}\n",
      "average score:  0.7705220431228281\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': None, 'min_samples_split': 0.1, 'n_estimators': 150}\n",
      "average score:  0.7848615373411094\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': None, 'min_samples_split': 0.1, 'n_estimators': 200}\n",
      "average score:  0.783708452283899\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': None, 'min_samples_split': 0.3, 'n_estimators': 150}\n",
      "average score:  0.7709464295769174\n",
      "trying:\n",
      "{'criterion': 'entropy', 'max_depth': 7, 'max_samples': None, 'min_samples_split': 0.3, 'n_estimators': 200}\n",
      "average score:  0.7701999594978512\n",
      "best model\n",
      "RandomForestClassifier(criterion='entropy', max_depth=7, min_samples_split=0.3,\n",
      "                       n_estimators=200, n_jobs=-1, random_state=111)\n",
      "best cv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code_module</th>\n",
       "      <th>code_presentation</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AAA</th>\n",
       "      <th>2013J</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">BBB</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.804842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.760857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.776461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CCC</th>\n",
       "      <th>2014B</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">DDD</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.790919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.782328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.813239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">EEE</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.810513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.821363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">FFF</th>\n",
       "      <th>2013B</th>\n",
       "      <td>0.793664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013J</th>\n",
       "      <td>0.795332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.790235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">GGG</th>\n",
       "      <th>2013J</th>\n",
       "      <td>0.708931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014B</th>\n",
       "      <td>0.781654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  score\n",
       "code_module code_presentation          \n",
       "AAA         2013J                   NaN\n",
       "BBB         2013B              0.804842\n",
       "            2013J              0.760857\n",
       "            2014B              0.776461\n",
       "CCC         2014B                   NaN\n",
       "DDD         2013B              0.790919\n",
       "            2013J              0.782328\n",
       "            2014B              0.813239\n",
       "EEE         2013J              0.810513\n",
       "            2014B              0.821363\n",
       "FFF         2013B              0.793664\n",
       "            2013J              0.795332\n",
       "            2014B              0.790235\n",
       "GGG         2013J              0.708931\n",
       "            2014B              0.781654"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross validated accuracy score:\n",
      "0.7869489235651188\n",
      "validation accuracy: \n",
      "0.7269861286254728\n",
      "validation set confusion matrix\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAD3CAYAAABcpJzyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmOElEQVR4nO3de1wU9f4/8Ncs6+JlTfGGFwRBRdS8hKTSEX8hcTQ8qR0v4PHC8ZLHyqNHzUjxAl4IQ7L0eCsV85KShVaamqZHCYMUA7MUBC+hKCpoCsp1P98/+DlJgbsr7DBLr6ePedTszM68h13efnzP5/MZSQghQEREFqOp7gCIiGo6JloiIgtjoiUisjAmWiIiC2OiJSKyMK0lDy75Oljy8GSlHuxPre4QSIVq29St9DHMyTni4JVKn89UFk20RESKkqTqjqBcTLREVHOotBjKREtENQdbtEREFqbOPMtES0Q1iI06My0TLRHVHCwdEBFZmDrzLBMtEdUgGnVmWiZaIqo51JlnmWiJqAZhjZaIyMLY64CIyMLUmWeZaImoBmHpgIjIwtjrgIjIwtSZZ5loiagGYYuWiMjCmGiJiCysivKswWBASEgIUlJSoNPpsHjxYjg5OQEAzp49i7CwMHnfpKQkrFq1Cn379q3weEy0RFRzVFGvg0OHDqGwsBDR0dFISkpCeHg41qxZAwDo2LEjtmzZAgDYt28fmjVr9tgkCzDRElFNUkVPWEhMTISXlxcAoHv37jhz5swf9rl//z5WrlyJrVu3Gj0eEy0R1RxmtGijo6MRHR0tr/v7+8Pf3x8AkJubC71eL2+zsbFBcXExtNrfUuann36KAQMGoFGjRkbPxURLRDWHGTfDHk2sv6fX65GXlyevGwyGMkkWAL788kusWLHCtLBMjoqISO00ZiyP4e7ujmPHjgEovdnl6upaZvu9e/dQWFiIFi1amBQWW7REVHNU0c0wX19fxMXFISAgAEIIhIWFISoqCo6OjvDx8cHFixfRqlUr08MSQogqiay8g/s6WOrQZMUe7E+t7hBIhWrb1K30MaTXOpu8r1j9U6XPZyq2aImo5uCABSIiC+PsXUREliWxRUtEZFkSW7RERJal0jzLREtENYdGpZmWiZaIagyWDoiILEyjUedgVyZaIqoxVNqgZaIlopqDpQMiIgtjoiUisjBJpY/BZaIlohqDLVoiIguz4RBcIiLLYouWiMjCmGiJiCxMpXmWiZaIag62aImILIyJlojIwjjXARGRham0QctES0Q1B0sHREQWxkRLRGRhfMICEZGFaTgEt+aSJAmrp4ahm0snFBQVYuK7s5CeeQkA0K1tJ7z3aqi8b++Oz2DIgon4PiUJqVHHcOZSCgBgV9x+rNi1oTrCJwsxGAxYsjAMqSmp0Ol0WLBwPhydHOXtWz7aiv1fHQAAePXtg8mv/wtCCPh694fT/9+va7eumDZjarXEb404e1cNNuQvA1BbZ4vnpg1Gr47uiPzXPAxZMAEAkJz+M7zfGA4AGNZ3IK7euo4DJ/8Hn2f6YPuRzzF11bzqDJ0s6PA3R1BYWIgt2zfjdPJpRL7zLt5f9R4A4ErGFXy15yts3bEFGo0G/xw9Dv1e8Ebt2rXRsVNHrFz9fvUGb6WstkZ77do17NmzBwUFBfJrU6ZMsWhQ1qZP52ex/8T/AAAJZ0/Bw7XbH/apW7sOQsfORN8ZQwEAPVy7okf7Lvhf5Ke4cecWpq6aj+s5N5QMmyzsh1M/4Lk+zwEobZn+9NPP8jb75vZY/cEq2NjYAACKiouhs7XFzz+fxY2sG5jwz1dQ29YWs956A22c21RH+FZJrYnWaO/eadOmITc3F02aNJEXKuupevXxa949eb3EUAIbjU2ZfSYMGImdx/Yi++5tAMC5X9Iw/6NleH7mMOyOO4CVry9SNGayvLzcPNTX6+V1G40NiouLAQC1atWCnZ0dhBCIfOdduLm5oU0bJzRt2gQTXhmPDZs+xIRJEzAnKLi6wrdKkmT6oiSjLdp69eph+vTpSsRite7m3UP9OvXkdY2kQYmhpMw+o/q9jGGLJsnrh5PicL/gAQBgV9w+LAycqUywpJh6+nrIy7svrxuEAVrtb79yBQUFWDA3BHXr1kPw/NkAgE6dO0FrU7qPe49ncOPGTQghVNtSUxu1/pyMtmjbt2+PvXv34sKFC7h48SIuXryoRFxWJe6nk/Dr1Q8A0KujO368eK7M9qfq1oetTocrN6/Jr62fEYGhffwAAD7P9EHi+R+VC5gU8cwz3fFt7LcAgNPJp9G+fTt5mxAC06ZMh2sHV8wPnSuXENau/gBbt2wDAKScS0Hz5s1VmzzUSKPRmLwoyWiL9uzZszh79qy8LkkSNm/ebNGgrM2uuH3w7eGFuPd2Q5IkjFs2A9OHvoK0zEv48ruDcHVwwaXrGWXe89aGt7FxZiReGxSIvPz7mPjurGqKniyl3wv98N3xeIz9RyCEEFi4JBSbN22Bo2NrlBgMSDyRiKLCQsTFxgEApk7/NyZMHIc5QcGIPRoLGxstFoWFGjkLPUqtfydJQghhbKfbt28jIyMDDg4OaNSokekH93WoVHBUMz3Yn1rdIZAK1bapW+ljuL33osn7nvvPvkqfz1RG28/79u1DQEAA1q5dC39/f3z++edKxEVEZDZJkkxelGS0dLBp0ybExMSgXr16yM3NRWBgIAYPHqxEbEREZlFrPdtoopUkCfXqld5R1+v1sLW1tXhQRERPQqV51niibd26NcLDw+Hh4YGTJ0/C0dHR2FuIiKpFVfUmMBgMCAkJQUpKCnQ6HRYvXgwnJyd5+9GjR7Fq1SoIIdC5c2csWLDgsa1po1G9/fbbaN26NY4fP47WrVtj0SJ2rCcidaqqGu2hQ4dQWFiI6OhozJw5E+Hh4fK23NxcREREYO3atdi5cydatWqF27dvP/Z4FSbaH38s7dcZHx8PJycn9OvXD05OTkhISDDnuomIFFNVI8MSExPh5eUFAOjevTvOnDkjb/vhhx/g6uqKpUuX4h//+AeaNGlitDdWhaWD7777Dl26dMHevXv/sK1Pnz6Pj5KIqBqYczMsOjoa0dHR8rq/vz/8/f0BlLZa9Y8On7YpHT6t1Wpx+/ZtJCQkYPfu3ahbty5GjRqF7t27w9nZucJzVZhoJ00qHS7q7u6O4cOHy69zsAIRqZYZifbRxPp7er0eeXl58rrB8Nvw6YYNG6JLly5o2rQpAMDDwwNnz559skS7Z88eHD58GAkJCYiPj5dPlpqairFjx5p8MURESqmqib/d3d1x5MgR+Pn5ISkpCa6urvK2zp07IzU1FTk5OXjqqaeQnJyMESNGPPZ4FSZaLy8vNG3aFHfu3JGzvkajQevWravkQoiIqlpV9aP19fVFXFwcAgICIIRAWFgYoqKi4OjoCB8fH8ycORMTJ04EAAwYMKBMIi43LlOG4GZnZ5eZj7Zly5YmBcshuFQeDsGl8lTFEFyPDUNN3vfkhM8qfT5TGe1HGxoaiqNHj6JZs2bydG07duxQIjYiIrNY7ciw5ORkHDp0SPFpxYiIzKXSPGs80To5OaGgoAB16tRRIh4ioidmtS3aa9euwdvbWx5+xtIBEamVWv/lbTTRRkZGKhEHEVGlWW2LVqvVIiIiAjk5ORgwYAA6dOiAVq1aKREbEZFZVJpnjU8qM2/ePAwdOhRFRUXw8PDAkiVLlIiLiMhsap3422iizc/Ph6enJyRJgouLC+ejJSLVUmuiNVo6sLW1RWxsLAwGA5KSkqDT6ZSIi4jIbGqt0Rpt0S5atAgxMTG4ffs2Nm7ciJCQEAXCIiIyn0YjmbwoyWiL9sCBAwgJCUGDBg2UiIeI6MmptEVrNNGWlJRg3LhxcHZ2xogRI9CrVy8l4iIiMpvVlg7Gjx+PmJgYBAYG4uOPP0b//v2ViIuIyGwayfRFSUZbtPn5+Thw4AB2794NIQT+/e9/KxEXEZHZ1NqiNZpoBw0ahP79+yMkJKTMUyCJiNTGxlqH4Hp7e2PmzJlKxEJEVCnqTLMmxJWeno67d+8qEQsRUaVoJMnkRUlGW7Tp6eno1asXGjVqJNc/vv32W4sHRkRkLqut0R45ckSJOIiIKk3plqqpjCba8+fPY8GCBbh79y4GDRqE9u3bw9vbW4nYiIjMotYWrdEa7eLFi/H222/Dzs4Ow4YNw8qVK5WIi4jIbFpJMnlRNC5TdnJycoIkSWjUqBHq1atn6ZiIiJ6IWlu0RhNtgwYNsGPHDjx48AB79+7FU089pURcRERmU2uN1mjpICwsDFeuXIGdnR3OnDmDsLAwJeIiIjKbZMaiJKMt2s2bN+ONN96Q1yMjIzmAgYhUSa0t2goT7c6dO/Hpp58iPT0dx44dAwAYDAYUFRUx0RKRKlndENzBgwfD09MT69atw+TJkwGUPsq3cePGigVHRGQOtbZoK0z/Op0ODg4OCA0NRXZ2NjIzM3HlyhUkJycrGR8RkcmstkY7depUZGdno0WLFgBKu088++yzFg+MiMhcam3RGk20t27dwo4dO5SIhYioUtSaaI1Wjp2dnZGVlaVELERElWK1jxtPTEyEt7c3GjVqJL/G2buISI1sVNqiNZpov/76ayXiICKqNLWWDipMtJGRkRU2r2fMmGGxgIiInpTVJVoXFxcl4yAiqjSrm1Tm5ZdfVjIOIqJKU+e4MBOnSSQisgZV1aI1GAwICQlBSkoKdDodFi9eXOYp4IsXL8apU6fkaWNXr16N+vXrV3g8JloiqjG0VTTXwaFDh1BYWIjo6GgkJSUhPDwca9askbf/9NNPWL9+fZneWI9jNKr79+/j+vXruHXrFlatWoWrV68+efRERBZUVf1oExMT4eXlBQDo3r07zpw5I28zGAy4fPky5s+fj4CAAHz66adG4zJpCG5AQAC+/vprtGvXDvPnz8eGDRuMHhgAbu9NMmk/+nOpM6prdYdAKiR2pFX6GBozZjGIjo5GdHS0vO7v7w9/f38AQG5uLvR6vbzNxsYGxcXF0Gq1uH//PkaPHo1x48ahpKQEY8eOxdNPPw03N7cKz2U00ebn58PHxwebN2/GO++8g+PHj5t8IURESjKnRvtoYv09vV6PvLw8ed1gMECrLU2XderUwdixY1GnTh0AQO/evXHu3LnHJlqjpYOioiJ89NFH6Ny5M9LS0vDgwQOTL4SISEkaSTJ5eRx3d3d5Hu6kpCS4urrK2y5duoSRI0eipKQERUVFOHXqFDp37vzY4xlt0QYFBeHQoUN49dVX8cUXXyA4ONiU6yUiUpxGqpqbYb6+voiLi0NAQACEEAgLC0NUVBQcHR3h4+ODwYMHY8SIEahVqxYGDx6M9u3bP/Z4khBClLchMzOzwje1bNnSpGDvFN4yaT/6c7Eb27u6QyAVqooa7byEeSbvu6jXokqfz1QVtminT58OALhz5w7y8vLQvn17pKWloUmTJti1a5diARIRmUpS6ZCFChPtw7txr7/+OpYuXQq9Xo/79+9zngMiUi2rm+vgoevXr8vdHOrWrYubN29aPCgioidhdXMdPNSnTx+MHj0aTz/9NE6fPo0XXnhBibiIiMwmKf40MNMYTbTTp0/HmTNncPnyZQwZMuSxfcWIiKqT1T1u/KGsrCxs2rQJOTk5GDBgAAoKCtCtWzclYiMiMotGpTfDjEY1b948DB06FEVFRfDw8MCSJUuUiIuIyGxqfWaY0USbn58PT09PSJIEFxcX2NraKhEXEZHZ1JpojZYObG1tERsbC4PBgKSkJOh0OiXiIiIymzmTyijJaIt20aJFiImJwe3bt7Fx40aEhIQoEBYRkfmstkXbvHlzTJ06FZcvX4abmxvs7e2ViIuIyGw2VTTXQVUzmmi3bt2KgwcP4tdff8XLL78sT3hLRKQ2VTWpTFUzGtXevXsRFRWF+vXrIzAwEMnJyUrERURkNqstHQghygTGm2FEpFZWOzLsb3/7G0aNGoXMzEy88sorHIJLRKpltZPKjB49Gp6enkhNTYWzszOH4BKRalndzbDdu3f/4bVz587h3LlzGDJkiAVDIiJ6MpK1Jdr09PQy60IIxMTEoHbt2ky0RKRKVlejnTlzpvz/v/zyC4KCgvD8889jzpw5igRGRGQuq63Rbtu2DR999BFmz54Nb29vJWIiInoiVjfxd1ZWFmbPno0GDRpg586daNCggZJxERGZTa1zHVSYaAcOHAidTofevXtj4cKFZbZFRkZaPDAiInNpNDbVHUK5Kky0q1evVjIOIqJKs7oWbc+ePZWMg4io0qyuRktEZG2srnsXEZG1YYuWiMjCrK5GS0RkbTSSlfU6ICKyNiwdEBFZGG+GERFZGFu0REQWxpthREQWxpthREQWxtIBEZGF8WYYEZGFWe3E30RE1kKtLVp1PsmMiOgJSJJk8vI4BoMB8+fPh7+/P8aMGYPLly+Xu8/EiROxfft2o3Ex0RJRjaGRbExeHufQoUMoLCxEdHQ0Zs6cifDw8D/s89577+Hu3bsmxcXSARHVGOb0o42OjkZ0dLS87u/vD39/fwBAYmIivLy8AADdu3fHmTNnyrx3//79kCRJ3scYJloiqjHM6d71aGL9vdzcXOj1enndxsYGxcXF0Gq1SE1NxZ49e7BixQqsWrXKpHMx0RJRjVFVN8P0ej3y8vLkdYPBAK22NF3u3r0bWVlZCAwMxNWrV1GrVi20atUKffv2rfB4TLREVGNU1YAFd3d3HDlyBH5+fkhKSoKrq6u87c0335T/f+XKlWjSpMljkyzAREtENYhNFQ3B9fX1RVxcHAICAiCEQFhYGKKiouDo6AgfHx+zjycJIUSVRFaOO4W3LHVosmJ2Y3tXdwikQmJHWqWPcezaQZP37dvCt9LnMxVbtERUY3CugxrMYDDgncXLcD4lDTqdDnNC30JrRwd5+/bNO3Bw/zcAgOe8PDHx1fHIvZeLubMW4P6D+9DV0iE0fD4aN2lcXZdAFiBJElaPD0U3p44oKC7ExHVzkJ5V2vG9m1NHvBc4V963d7vuGBL5Kn6+ch4bJ4dDa6OFBAmTPgxG6rWL1XUJVocjw2qwo4ePobCgEBu2fYDX/jMZ70eslLddzbiK/Xu/xodb1mLDtg+QcPx7nE9Jw57Pv0JbVxd88NEavDDAB1ujPq7GKyBLGOLhi9o6Wzw3fzje+jgCkWNmy9uSL5+F98JR8F44CqsObMVn3+/HgeRjWDRiOv57YAu8F45C2O41eHvkG9V4BdanqkaGVTWjLdrc3FwcO3YMhYWF8mtDhgyxZExWJ/nUafTuU1p37NLtaZz7+Zy8zb65Pd5f+y5sbEqL9MXFxbC11aFd+7a4fLG0dZOXlyd3HaGao4+bB/YnHQMAJKQlwcPl6T/sU9e2DkKHT0Pf0JEAgJlb38av9+8BALQ2NsgvKlAu4BpAo9K2o9Hf7tdeew3NmjVDixYtAKi3BlKd8vLyoNfXk9c1mt86N2tradHQriGEEFgRuQqubq5wbOOI/IICJBz/Hv6DR+Hur3ex7qPV1XgFZAlP1dHj1wf35PUSgwE2GhuUGErk1yZ4D8fO+H3IvncbAOT/urZwxrLRb2HIsleVDdrKaSQrTbRCCCxbtkyJWKxWvXr1cD/vvrz+aOdmACgoKMDieW+jbr26eHPuTADAhjUbMXrcKPx9xBCcT0nD7OnB2BazWfHYyXLuPshF/dq/jS7SSJoySRYARvUZhGHL/13mtec79cbqCSEYs+oN1mfNpNaGoNH036FDByQnJ6OwsFBeqKyuz3TB8djvAAA/Jp9Bu/Zt5W1CCMya+hbad2iH2QvelEsI9Z96Cvr6pb+EjRrblRmFQjVDXEoi/J75fwCAXu2648eMlDLbn6qjh61WhyvZ1+TXnu/UG+8HzsWAt8cj8ULZ8fVknGTGH0XjMtaPdtCgQcjNzf3tDZKEb775xqSD/1n60T7sdZCWmg4hBOYtCsbx2O/g4NgKBoMB894MwdNdO8v7vzZtMpq3tMeSBeF4cP8BiouLMen1iej1XM9qvArl/Fn60T7sddDVyQ0SJIxbGwS/7s8jLesyvkz8Bh4uXRD88mt4OfK38kDS0i9hq9Xh+q+lvzspmRcwef286roERVVFP9qTN+NM3tej6V8qfT5TmTxgITs7Gw0bNpRbZKb4syRaMs+fJdGSeaok0d46bvK+Hk2eq/T5TGW0dJCQkAAfHx9MmDBBHpZGRKRGai0dGL0Z9t577+Hjjz+Gvb09srKyMGXKFPzlL8o1uYmITGW1vQ5sbGxgb28PALC3t4etra3FgyIiehJqHRlmNNHq9Xps2bIFzz77LE6cOIEGDRooERcRkdmstntXREQEMjMzsXz5cly7dg1hYWFKxEVEZDarq9Fev34dzZs3x61btzBixAj59ZycHLZqiUiVrK50EBUVhdmzZ2P+/PmQJAkPe4FJkoTNmzmCiYjUx+puhs2eXTrT0Lhx49CvXz/59a+++sryURERPQG11mgrTLRHjhzBqVOnsHfvXiQlJQEoHQH1zTffwM/PT6n4iIhMZnWlAzc3N9y5cwe2trZwdnYGUPq3xcCBAxULjojIHGpNtEaH4BoMBmg0T1b34BBcKg+H4FJ5qmIIbsqvP5q8b4cGXSp9PlMZ7Uf74Ycf4sMPP0Tt2rXl17799luLBkVE9CTU2qI1mmj37t2L2NhY1KlTR4l4iIiemNX1OnjIwcGhTGuWiEi9rLRFW1RUhJdeegmurq5y14nIyEiLB0ZEZC6r69710CuvvKJEHERElabWGq3RgkanTp0QFxeHXbt24c6dO/JMXkREaqPWuQ6MJto5c+agdevWuHz5Mpo0aYLg4GAl4iIiMpskSSYvSjKaaO/cuYNhw4ZBq9XC3d0dBoNBibiIiMymMeOPkozWaAEgPT0dQOmMXuY8M4yISElWezNs7ty5mDNnDtLT0zF16lQsWLBAibiIiMym1pthRhPtL7/8gu3btz/xMFwiIqWotUVrNHt+9913GDx4MJYvX46MjAwlYiIieiJq7XVgdFIZACgsLMQ333yDmJgYFBUVYdOmTSYdnJPKUHk4qQyVpyomlbl2/xeT921R17HS5zOVSfWA06dP49tvv0V2djY8PT0tHRMR0RNRa/cuozVaPz8/uLm5Yfjw4ViyZIkSMRERPSF11miNJtq///3vmDhxohKxEBFVijrTrAmlg2PHjqGkpESJWIiIKkkyY6mYwWDA/Pnz4e/vjzFjxuDy5ctltm/btg1Dhw7FsGHDTHqOotEW7e3bt+Hl5QUHBwe5trFjxw6jByYiUlpV1V4PHTqEwsJCREdHIykpCeHh4VizZg0AICcnB9u3b8euXbtQUFCAgQMH4sUXX3zsuY0m2rVr11ZJ4EREllZV3bYSExPh5eUFAOjevTvOnDkjb2vUqBF2794NrVaLq1evwtbW1miCN5potVotIiIikJOTgwEDBqBDhw5o1apVJS+DiKjqmZNoo6OjER0dLa/7+/vD398fAJCbmwu9Xi9vs7GxQXFxMbTa0pSp1WqxdetWrFy5EmPGjDF6LqM12nnz5mHo0KEoKiqCh4cHex4QUY3g7++PmJgYeXmYZAFAr9cjLy9PXjcYDHKSfWj06NGIjY3FiRMnEB8f/9hzGU20+fn58PT0hCRJcHFxga2trbnXQ0SkiKrqR+vu7o5jx44BAJKSkuDq6ipvu3DhAqZMmQIhBGrVqgWdTmd0igKjpQNbW1vExsbCYDAgKSkJOp3OlOslIrJavr6+iIuLQ0BAAIQQCAsLQ1RUFBwdHeHj4wM3Nzf4+/tDkiR4eXmhZ8+ejz2e0SG4169fx9KlS5Gamoq2bdvizTffhIODg0nBcggulYdDcKk8VTEEN6fghsn7NrJtVunzmcpoizY2NhbLly+X1zdv3oyxY8daNCgioidhddMk7tmzB4cPH0ZCQoJc6DUYDEhNTWWiJSJVUus0iRUmWi8vLzRt2hR37tyR78ZpNBq0bt1aseCIiMxjZYm2QYMG6NWrF3r16oXs7GwUFBQAAIfjEpFqqTPNmlCjDQ0NxdGjR9GsWTMIITgEl4hUTJ2p1miiTU5OxqFDh/goGyJSPbXWaI1mTycnJ7lsQESkZmp9lI3RFu21a9fg7e0NJycnAGDpgIhUTJ0tWqOJNjIyUok4iIgqTZ1p9jGJ9tFZbX7v0ckXiIjUQq012goT7c2bN5WMg4ioClhZop0yZYqScRARVZrVDcElIrI2ai0dsHMsEZGFmdSiPXr0KM6fP482bdrghRdesHRMRERPRK2lA6Pz0UZGRuLSpUvo0aMHTp48CQcHB7z11ltKxUdEZPWMtmhPnDghD1AIDAzEiBEjLB4UEVFNYrRGW1xcDIPBAADypDJERGQ6oy1aPz8/jBw5Et26dcPp06fh5+enRFxERDWG0RotAKSmpuLChQtwcXEp8zRIIiIyrsJEu3v37grfNGTIEAuFQ0RU81RYOkhPTy+zLoRATEwMateuzURLRGQGk0oHv/zyC4KCguDs7Iw5c+ZAr9crERsRUY1gtNfBtm3bMHHiREyaNAlhYWGqS7IJCQno0aMHrl27Jr+2bNkyxMTEVOq4I0aMwJUrV4zuFxMTg2XLllW4vaCgADt37qxULOa4c+cOvvzySwDABx98gNOnTyt2bmtS2e/NypUrsX379gq3P/o5KCEzMxOHDx8GACxZsgSZmZmKnZuMqzDRZmVlYfz48Th58iR27twJb29vJeMyi06nw+zZs2FC41xxN2/eVDTRpqSkyL9wkyZNQteuXRU7t7Wx5Pfm0c9BCfHx8Th16hQAIDg4GC1btlTs3GRchTXagQMHQqfToXfv3li4cGGZbWqbDLx3794wGAzYtm0bRo8eXWbbli1bsGfPHkiSBD8/P4wdOxbXrl3DvHnzUFBQAFtbWyxatAgtWrTA8uXLERsbi+bNm+P27dsAgMTERCxduhRarRZ16tTB+++/X2Gr/q9//Svc3d1x8eJFNG7cGCtXrsTatWuRlpaG//73vwgMDERwcLB87Llz56JDhw7w9vaGi4sL2rZtiyNHjuDzzz9H3bp1sWHDBtjY2KB///5/iLekpAQzZ85E8+bNkZGRgS5duiA0NBRr167FuXPnEB0djR9++AF+fn7w9PTE7NmzceXKFZSUlGDcuHHw8/PDmDFj4ObmhvPnzyM3Nxfvv/8+WrVqZdkPS0Ue973ZuHEj9u7dC61WCw8PD8yaNavcY1y5csXo59C3b99yP79XX30VDRs2RN++fRETE4OvvvoKkiRh4cKF8PT0hKOjIxYvXgwAaNiwIcLCwvDzzz/jww8/RK1atXDlyhX4+flh0qRJ+OCDD5Cfn49nnnkGmzZtQkhICJo2bYpZs2YhNzcXJSUlmDZtGjw9PfHSSy+hZ8+eSElJgSRJWL16NerXr2/xn/efmqhAQkJChYuaxMfHi//85z8iJydH+Pj4iEuXLomIiAjx2WefifPnz4uAgABRXFwsiouLxZgxY0R6erqYNm2a+N///ieEEOL48eNixowZ4vTp02LkyJGipKRE3Lt3T3h6eoqMjAwRHh4uNm7cKEpKSsTBgwfF1atXy5z/s88+ExEREUIIIdzc3ERmZqYQQgh/f3/xww8/iIyMDDF8+HAhhBDvvPOO2LZtmxBCiIsXL4qAgAAhhBAdOnQQOTk58j67du0SQggxZMgQkZOTU268GRkZomfPnuLevXuiuLhYPP/88+LGjRvyz0MIIYKCgsTRo0fFli1bxJIlS4QQQty7d0/4+vqK7OxsMXr0aPHFF18IIYR49913xbp166r+A1Kpx31vzp07J4YNGyYKCwuFwWAQr7/+ujh8+HCZ969YsUJ8/PHHJn0OFX1+vXr1EgUFBfI+33//vSgoKBB+fn6iqKhIDB8+XJw/f14IIcQnn3wi3n33XREfHy9efPFFUVRUJPLy8oS7u7sQouz3cPTo0SItLU2Eh4eLTZs2CSGEuH79uvD29hYGg0F4e3uLxMREIYQQM2bMEHv27LHwT5sqbNH27NlTyXxfaXZ2dpgzZw6CgoLg7u4OoLT/b2ZmJv75z38CAH799VdcvnwZqampWLduHdavXw8hBLRaLS5duoSnn34aGo0Ger1e7i88efJkrF27FoGBgbC3t3/sP8Xt7OzQokULAECLFi3+8FDL1NRUxMfHY9++fXI8D99nZ2cHABg+fDhCQkLg4uICZ2dn2NnZlRsvADg6Osqt66ZNm1b4EM309HQ899xzAAC9Xo+2bdsiIyMDANCpUycAQPPmzXHr1i1Tf9w1RnnfmwsXLqBbt26oVasWAMDDwwPnz5+vsHxm7HOo6PNzcHCATqcDUHpPYNeuXbh58yb69esHrVaL9PR0hIaGAgCKiorQpk0bAICrqyu0Wi20Wi1q165d4bWlp6fjpZdeAgDY29tDr9cjOzsbwG+fe3nfU6p6NWo+2n79+uHgwYPYtWsXZs2aBRcXF7Rr1w7r16+HJEnYtGkTOnToABcXF4wfPx7u7u5IT0/HiRMn0K5dO2zbtg0GgwH5+flIS0sDAHzxxRd4+eWXERQUhHXr1uGTTz6pcFL08oYnazQaeQizi4sLBg0ahJdeegnZ2dly7fbRR7m3adMGQgisX78eI0eOlN/3+3hNOd9Dbdu2xcmTJ+Hr64vc3FykpqbCwcHB3B9vjVXe9yYqKgrFxcWwsbHBiRMnHtul0ZTPvbzP79HP3dPTExEREcjKysKCBQsAAM7Ozli6dClatmyJxMRE+akn5n7unTp1QlZWFu7evYuGDRtWeAyynBqVaIHSGwHx8fEAADc3N3h6emLkyJEoLCxE165dYW9vj6CgIISEhKCgoAD5+fkIDg5Gx44d0bdvXwwbNgzNmjVD48aNAQBdu3bF3LlzUadOHWg0mj/Uq41p3LgxioqKEBERgcmTJyM4OBiffPIJcnNzK0zYw4YNw4oVK9C7d28AKDfeijg6OiI1NRWbNm2SXxsxYgTmzZuHkSNHoqCgAFOmTJGvj0o9+r3p0KEDXnzxRYwcORIGgwE9evQwe3rQRz8HUz4/SZLQv39/HD9+HI6OjgCAkJAQBAUFobi4GJIkYcmSJbhx40a553N1dcWaNWvQuXNn+bV//etfmDNnDg4cOID8/HwsXLhQbk2TskzqR0tERE+OT1ggIrIwJloiIgtjoiUisjAmWiIiC2OiJSKyMCZaIiILY6IlIrKw/wP2xRzE/nNXtgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "RFparams = {'n_estimators':[150,200],\n",
    "           'criterion':['entropy'],\n",
    "           'max_depth':[4,7],\n",
    "           'min_samples_split':[.1,.3],\n",
    "           'max_samples':[.7,None]}\n",
    "           \n",
    "RFgrid = GridSearchPresentationCV(RandomForestClassifier(n_jobs = -1,\n",
    "                                                        random_state=111), \n",
    "                                  RFparams, scoring = 'accuracy', verbose = 1)\n",
    "\n",
    "RFgrid.fit(train_df)\n",
    "\n",
    "RFmodel = score_grid(RFgrid, X_test_transformed, y_test)\n",
    "pickle.dump(RFmodel, open('../models/RFmodel3.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Classifier Evaluation:\n",
    "Apparently bagging is not the right approach to this problem.  Our single well tuned decision tree outperformed the random forest classifier.  I guess the wisdom of the crowd is not always superior to the wisdom of the expert!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# eXtreme Gradient Boost model:\n",
    "XGBoost models have gained a lot of popularity recently and won a lot of Kaggle competitions.  It uses another popular idea called [boosting](https://en.wikipedia.org/wiki/Gradient_boosting).  That's a pretty involved wikipedia article, but the TLDR is that it's a similar ensemble method like random forest, but whereas random forest trains a bunch of trees in parallel and takes the aggregate of their predictions, boosting stacks the trees on top of each other and each one tries to improve on the one below it by predicting where the previous one made mistakes.  I think of it as like a line of morons each grading the next one's paper, which is an analysis of the previous one's paper.  Each one gets a lot wrong, but something right so the right answers percolate through and some of the wrong answers get corrected at each step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 400, 'num_parallel_tree': 3, 'subsample': 0.5}\n",
      "average score:  0.7911727375735815\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 400, 'num_parallel_tree': 3, 'subsample': 0.7}\n",
      "average score:  0.7927651004421473\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 400, 'num_parallel_tree': 4, 'subsample': 0.5}\n",
      "average score:  0.7911094907309091\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 400, 'num_parallel_tree': 4, 'subsample': 0.7}\n",
      "average score:  0.7925209492707292\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 750, 'num_parallel_tree': 3, 'subsample': 0.5}\n",
      "average score:  0.7891140348230992\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 750, 'num_parallel_tree': 3, 'subsample': 0.7}\n",
      "average score:  0.789243245700921\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 750, 'num_parallel_tree': 4, 'subsample': 0.5}\n",
      "average score:  0.7893984321553836\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 750, 'num_parallel_tree': 4, 'subsample': 0.7}\n",
      "average score:  0.7890650486253061\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 5, 'n_estimators': 400, 'num_parallel_tree': 3, 'subsample': 0.5}\n",
      "average score:  0.7906254805674717\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 5, 'n_estimators': 400, 'num_parallel_tree': 3, 'subsample': 0.7}\n",
      "average score:  0.7924356478268647\n",
      "trying:\n",
      "{'base_score': 0.4, 'colsample_bytree': 0.7, 'eval_metric': 'logloss', 'learning_rate': 0.01, 'max_depth': 4, 'min_child_weight': 5, 'n_estimators': 400, 'num_parallel_tree': 4, 'subsample': 0.5}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-e57177a05e48>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     13\u001b[0m                                                  n_jobs=-1, random_state=111),\n\u001b[0;32m     14\u001b[0m                                  XGBparams, scoring = 'accuracy', verbose = 1)\n\u001b[1;32m---> 15\u001b[1;33m \u001b[0mXBGgrid\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[0mXGBmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscore_grid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXBGgrid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\data_science\\data_science_projects\\ou_students\\ou_student_predictions\\src\\functions.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, df, show_progress)\u001b[0m\n\u001b[0;32m    457\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    458\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 459\u001b[1;33m             \u001b[0mscores\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcross_val_presentation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscoring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    460\u001b[0m             \u001b[0mnew_score\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    461\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\data_science\\data_science_projects\\ou_students\\ou_student_predictions\\src\\functions.py\u001b[0m in \u001b[0;36mcross_val_presentation\u001b[1;34m(model, df, scoring, verbose)\u001b[0m\n\u001b[0;32m    370\u001b[0m             \u001b[0mtest\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'code_module'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m&\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'code_presentation'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mpresentation\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    371\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'code_module'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 372\u001b[1;33m                 \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_evaluate_presentation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscoring\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    373\u001b[0m                 \u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpresentation\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'score'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    374\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mverbose\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\data_science\\data_science_projects\\ou_students\\ou_student_predictions\\src\\functions.py\u001b[0m in \u001b[0;36mmodel_evaluate_presentation\u001b[1;34m(model, train, test, scoring)\u001b[0m\n\u001b[0;32m    390\u001b[0m     \u001b[0mtransformed_big_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprocess_courses\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    391\u001b[0m     \u001b[0mX_train_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_test_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransformed_big_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 392\u001b[1;33m     \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_transformed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    393\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test_transformed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    394\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mscoring\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\student_predictor_env\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    420\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    421\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marg\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 422\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    423\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    424\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\student_predictor_env\\lib\\site-packages\\xgboost\\sklearn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight, base_margin, eval_set, eval_metric, early_stopping_rounds, verbose, xgb_model, sample_weight_eval_set, feature_weights, callbacks)\u001b[0m\n\u001b[0;32m    907\u001b[0m             eval_group=None, label_transform=label_transform)\n\u001b[0;32m    908\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 909\u001b[1;33m         self._Booster = train(xgb_options, train_dmatrix,\n\u001b[0m\u001b[0;32m    910\u001b[0m                               \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_num_boosting_rounds\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    911\u001b[0m                               \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\student_predictor_env\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m    225\u001b[0m     \u001b[0mBooster\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0ma\u001b[0m \u001b[0mtrained\u001b[0m \u001b[0mbooster\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    226\u001b[0m     \"\"\"\n\u001b[1;32m--> 227\u001b[1;33m     bst = _train_internal(params, dtrain,\n\u001b[0m\u001b[0;32m    228\u001b[0m                           \u001b[0mnum_boost_round\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnum_boost_round\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m                           \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\student_predictor_env\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks, evals_result, maximize, verbose_eval, early_stopping_rounds)\u001b[0m\n\u001b[0;32m    100\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\student_predictor_env\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1278\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1279\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1280\u001b[1;33m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle,\n\u001b[0m\u001b[0;32m   1281\u001b[0m                                                     \u001b[0mctypes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mc_int\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1282\u001b[0m                                                     dtrain.handle))\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "XGBparams = {'n_estimators':[400, 750],\n",
    "             'max_depth':[4, 7],\n",
    "             'learning_rate':[.01, .1], \n",
    "             'subsample':[.5,.7],\n",
    "             'min_child_weight':[4,5],\n",
    "             'num_parallel_tree':[3,4],\n",
    "             'eval_metric':['logloss'],\n",
    "            'colsample_bytree':[.7,.8],\n",
    "            'base_score':[.4,.6]}\n",
    "           \n",
    "XBGgrid = GridSearchPresentationCV(XGBClassifier(objective='binary:logistic', \n",
    "                                                 eval_metric='logloss',\n",
    "                                                 n_jobs=-1, random_state=111),\n",
    "                                 XGBparams, scoring = 'accuracy', verbose = 1)\n",
    "XBGgrid.fit(train_df)\n",
    "\n",
    "XGBmodel = score_grid(XBGgrid, X_test_transformed, y_test)\n",
    "\n",
    "pickle.dump(XGBmodel, open('../models/XGBmodel5.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_importance(XGBmodel, importance_type = 'gain')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost evaluation:\n",
    "XGBoost gave us the best cross-validated score of these four model types.  While it performs slightly worse on the validation set than the random forest, looking at the scores across folds, it seems likely that this represents the validation set being a sort of bad split.  XGBoost also performs equally well on both classes.  It correctly identifies 74% of students who need intervention to succeed, while misclassifying 27% of passing students as needing intervention.  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-Nearest Neighbors\n",
    "This model positions each observation (student registration in this case) from the training set in n_feature dimensional space.  New observations from the test set are voted on by the closest K observations from the training set to determine which class the new observation should belong to. There are two big benefits to this modeling technique for our dataset.  The first is that it is fast with few features.  We have only 6 features, so the dimensionality of the space is low, observations are more tightly packed, and the model makes determinations quickly.  The second benefit is that it is non-linear.  We've had some success with more linear models, and decision tree models.  Our error analysis and EDA show that there are strong linear relationships between our variables and the success of students.  However there is a solid 20% of students that cannot seem to be classified with these linear models.  The decision tree models similar seem to fail to classify these students.  Perhaps this model can help find where they belong.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KNNparams = {'n_neighbors':[30, 50, 100, 150, 200, 300],\n",
    "            'weights':['distance', 'uniform'],\n",
    "             'p':[1,2,5,10,15,20]}\n",
    "             \n",
    "KNNgrid = GridSearchPresentationCV(KNeighborsClassifier(n_jobs = -1), KNNparams, \n",
    "                                scoring='accuracy', verbose = 1)\n",
    "\n",
    "KNNgrid.fit(train_df)\n",
    "KNNmodel = score_grid(KNNgrid, X_test_transformed, y_test)\n",
    "pickle.dump(KNNmodel, open('../models/KNNmodel2.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN Evaluation\n",
    "This model, like the others, seems to top out around 77%.  We still haven't found a way to capture the pattern for how those last 20-23% of students fail or withdraw.  Looking at our error analysis from the final report notebook we see that those misclassfied students really look a lot like successful ones according to all of our variables.  It may just be that we don't have the right variable to find the connection.  We may be running up against the irreducible error for this feature set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LRmodel.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = LRmodel.predict(X_test_transformed)\n",
    "plot_confusion(y_test, y_pred)\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DTmodel.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = DTmodel.predict(X_test_transformed)\n",
    "plot_confusion(y_test, y_pred)\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RFmodel.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = RFmodel.predict(X_test_transformed)\n",
    "plot_confusion(y_test, y_pred)\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XGBmodel.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = XGBmodel.predict(X_test_transformed)\n",
    "plot_confusion(y_test, y_pred)\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KNNmodel.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = KNNmodel.predict(X_test_transformed)\n",
    "plot_confusion(y_test, y_pred)\n",
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3,2, figsize=(15,10))\n",
    "axes = axes.ravel()\n",
    "models = [KNNmodel, XGBmodel, RFmodel, DTmodel, LRmodel]\n",
    "for i, model in enumerate(models):\n",
    "    y_pred = model.predict(X_test_transformed)\n",
    "    plot_confusion(y_test, y_pred, ax=axes[i])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = pickle.load(open('../models/KNNmodel2.pkl','rb'))\n",
    "best_model.fit(X_train_transformed,y_train_transformed)\n",
    "y_pred = best_model.predict(X_test_transformed)\n",
    "eval_df = test_df\n",
    "eval_df['prediction'] = y_pred\n",
    "modules = eval_df['code_module'].unique()\n",
    "course_acc = pd.DataFrame(columns = ['score'],index = modules)\n",
    "fig, axes = plt.subplots(4,2, figsize = (10,10))\n",
    "axes = axes.ravel()\n",
    "for i, module in enumerate(modules):\n",
    "    preds = eval_df[eval_df['code_module'] == module]['prediction']\n",
    "    trues = eval_df[eval_df['code_module'] == module]['final_result']\n",
    "    accuracy = accuracy_score(trues,preds)\n",
    "    course_acc.loc[module,'score'] = accuracy\n",
    "    plot_confusion(trues,preds, ax = axes[i])\n",
    "    axes[i].set_title(f'module: {module}: accuracy score: {round(accuracy,2)}')\n",
    "plt.tight_layout()\n",
    "plt.savefig('../figures/KNNmodel2accbymodule.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "student_predictor_env",
   "language": "python",
   "name": "student_predictor_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}