sample-project/introduction-to-scikit-learn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Scikit-Learn (sklearn)\n",
    "\n",
    "This notebook demonstrates some of the most useful functions of the beautiful Scikit-Learn library.\n",
    "\n",
    "What we're going to cover:\n",
    "0. An end-to-end Scikit-Learn workflow\n",
    "1. Getting the data ready\n",
    "2. Choose the right estimator/algorithm for our problems\n",
    "3. Fit the model/algorithm and use it to make predictions on our data\n",
    "4. Evaluating a model\n",
    "5. Improve a model\n",
    "6. Save and load a trained model\n",
    "7. Putting it all together!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's listify the contents\n",
    "what_were_covering = [\n",
    "    \"0. An end-to-end Scikit-Learn workflow\",\n",
    "    \"1. Getting the data ready\",\n",
    "    \"2. Choose the right estimator/algorithm for our problems\",\n",
    "    \"3. Fit the model/algorithm and use it to make predictions on our data\",\n",
    "    \"4. Evaluating a model\",\n",
    "    \"5. Improve a model\",\n",
    "    \"6. Save and load a trained model\",\n",
    "    \"7. Putting it all together!\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. An end-to-end Scikit-Learn workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>264</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>131</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>303 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0     63    1   3       145   233    1        0      150      0      2.3   \n",
       "1     37    1   2       130   250    0        1      187      0      3.5   \n",
       "2     41    0   1       130   204    0        0      172      0      1.4   \n",
       "3     56    1   1       120   236    0        1      178      0      0.8   \n",
       "4     57    0   0       120   354    0        1      163      1      0.6   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "299   45    1   3       110   264    0        1      132      0      1.2   \n",
       "300   68    1   0       144   193    1        1      141      0      3.4   \n",
       "301   57    1   0       130   131    0        1      115      1      1.2   \n",
       "302   57    0   1       130   236    0        0      174      0      0.0   \n",
       "\n",
       "     slope  ca  thal  target  \n",
       "0        0   0     1       1  \n",
       "1        0   0     2       1  \n",
       "2        2   0     2       1  \n",
       "3        2   0     2       1  \n",
       "4        2   0     2       1  \n",
       "..     ...  ..   ...     ...  \n",
       "298      1   0     3       0  \n",
       "299      1   0     3       0  \n",
       "300      1   2     3       0  \n",
       "301      1   1     3       0  \n",
       "302      1   1     2       0  \n",
       "\n",
       "[303 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. Get the data ready\n",
    "import pandas as pd\n",
    "heart_disease = pd.read_csv(\"data/heart-disease.csv\")\n",
    "heart_disease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create X (features matrix) choose from age to thal\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "\n",
    "# Create y (labels)\n",
    "y = heart_disease[\"target\"] # 0: no heart disease, 1: got heart disease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>264</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>131</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>303 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0     63    1   3       145   233    1        0      150      0      2.3   \n",
       "1     37    1   2       130   250    0        1      187      0      3.5   \n",
       "2     41    0   1       130   204    0        0      172      0      1.4   \n",
       "3     56    1   1       120   236    0        1      178      0      0.8   \n",
       "4     57    0   0       120   354    0        1      163      1      0.6   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "299   45    1   3       110   264    0        1      132      0      1.2   \n",
       "300   68    1   0       144   193    1        1      141      0      3.4   \n",
       "301   57    1   0       130   131    0        1      115      1      1.2   \n",
       "302   57    0   1       130   236    0        0      174      0      0.0   \n",
       "\n",
       "     slope  ca  thal  \n",
       "0        0   0     1  \n",
       "1        0   0     2  \n",
       "2        2   0     2  \n",
       "3        2   0     2  \n",
       "4        2   0     2  \n",
       "..     ...  ..   ...  \n",
       "298      1   0     3  \n",
       "299      1   0     3  \n",
       "300      1   2     3  \n",
       "301      1   1     3  \n",
       "302      1   1     2  \n",
       "\n",
       "[303 rows x 13 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1\n",
       "1      1\n",
       "2      1\n",
       "3      1\n",
       "4      1\n",
       "      ..\n",
       "298    0\n",
       "299    0\n",
       "300    0\n",
       "301    0\n",
       "302    0\n",
       "Name: target, Length: 303, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "System:\n",
      "    python: 3.7.6 (default, Dec 30 2019, 19:38:36)  [Clang 10.0.0 (clang-1000.11.45.5)]\n",
      "executable: /usr/local/opt/python/bin/python3.7\n",
      "   machine: Darwin-17.7.0-x86_64-i386-64bit\n",
      "\n",
      "Python dependencies:\n",
      "       pip: 19.3.1\n",
      "setuptools: 42.0.2\n",
      "   sklearn: 0.23.0\n",
      "     numpy: 1.17.4\n",
      "     scipy: 1.3.3\n",
      "    Cython: None\n",
      "    pandas: 0.25.3\n",
      "matplotlib: 3.1.2\n",
      "    joblib: 0.14.0\n",
      "\n",
      "Built with OpenMP: True\n"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "sklearn.show_versions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. Choose the right model and hyperparameters\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "# We'll keep the default hyperparameters\n",
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Fit the model to the training data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# test_size=0.2, 80% of data for training and 20% for testing\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a forest of trees from the training set (X, y)\n",
    "clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1,\n",
       "       0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0,\n",
       "       1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make a prediction\n",
    "y_preds = clf.predict(np.array(X_test))\n",
    "y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19     1\n",
       "187    0\n",
       "163    1\n",
       "247    0\n",
       "85     1\n",
       "      ..\n",
       "86     1\n",
       "273    0\n",
       "15     1\n",
       "269    0\n",
       "201    0\n",
       "Name: target, Length: 61, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4. Evaluate the model on the training data and test data\n",
    "\n",
    "# Returns the mean accuracy on the given test data and labels\n",
    "clf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7049180327868853"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.61      0.68        31\n",
      "           1       0.67      0.80      0.73        30\n",
      "\n",
      "    accuracy                           0.70        61\n",
      "   macro avg       0.71      0.71      0.70        61\n",
      "weighted avg       0.71      0.70      0.70        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The precision will be \"how many are correctly classified among that class\"\n",
    "- The recall means \"how many of this class you find over the whole number of element of this class\"\n",
    "- The f1-score is the harmonic mean between precision & recall\n",
    "- The support is the number of occurence of the given class in your dataset (so you have 37.5K of class 0 and 37.5K of class 1, which is a really well balanced dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[19, 12],\n",
       "       [ 6, 24]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute confusion matrix to evaluate the accuracy of a classification\n",
    "confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7049180327868853"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy classification score\n",
    "accuracy_score(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying model with 10 estimators...\n",
      "Model accuracy on test set: 77.05%\n",
      "\n",
      "Trying model with 20 estimators...\n",
      "Model accuracy on test set: 75.41%\n",
      "\n",
      "Trying model with 30 estimators...\n",
      "Model accuracy on test set: 78.69%\n",
      "\n",
      "Trying model with 40 estimators...\n",
      "Model accuracy on test set: 68.85%\n",
      "\n",
      "Trying model with 50 estimators...\n",
      "Model accuracy on test set: 77.05%\n",
      "\n",
      "Trying model with 60 estimators...\n",
      "Model accuracy on test set: 75.41%\n",
      "\n",
      "Trying model with 70 estimators...\n",
      "Model accuracy on test set: 81.97%\n",
      "\n",
      "Trying model with 80 estimators...\n",
      "Model accuracy on test set: 78.69%\n",
      "\n",
      "Trying model with 90 estimators...\n",
      "Model accuracy on test set: 72.13%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 5. Improve a model\n",
    "# Try different amount of n_estimators\n",
    "np.random.seed(42)\n",
    "for i in range(10, 100, 10):\n",
    "    print(f\"Trying model with {i} estimators...\")\n",
    "    clf = RandomForestClassifier(n_estimators=i).fit(X_train, y_train)\n",
    "    print(f\"Model accuracy on test set: {clf.score(X_test, y_test) * 100:.2f}%\")\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Save a model and load it\n",
    "import pickle # Python object serialization\n",
    "\n",
    "pickle.dump(clf, open(\"random_forst_model_1.pkl\", \"wb\")) # write binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7213114754098361"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_model = pickle.load(open(\"random_forst_model_1.pkl\", \"rb\")) # read binary\n",
    "loaded_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Getting our data ready to be used with machine learning\n",
    "\n",
    "Three main things we have to do:\n",
    "1. Split the data into features and labels (usually `X` & `y`)\n",
    "2. Filling (also called imputing) or disregarding missing values\n",
    "3. Converting non-numerical values to numerical values (also called feature encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  \n",
       "0   0     1  \n",
       "1   0     2  \n",
       "2   0     2  \n",
       "3   0     2  \n",
       "4   0     2  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split the data into features and labels (usually X & y)\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    1\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split the data into features and labels (usually X & y)\n",
    "y = heart_disease[\"target\"]\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into training and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((242, 13), (61, 13), (242,), (61,))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "242.4"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape[0] * 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "242 + 61"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(heart_disease)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Make sure it's all numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "      <td>15323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "      <td>19943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "      <td>28343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "      <td>13434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "      <td>14043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors  Price\n",
       "0   Honda  White          35431      4  15323\n",
       "1     BMW   Blue         192714      5  19943\n",
       "2   Honda  White          84714      4  28343\n",
       "3  Toyota  White         154365      4  13434\n",
       "4  Nissan   Blue         181577      3  14043"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales = pd.read_csv(\"data/car-sales-extended.csv\")\n",
    "car_sales.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    856\n",
       "5     79\n",
       "3     65\n",
       "Name: Doors, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# treat doors as categorical\n",
    "car_sales[\"Doors\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car_sales)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             object\n",
       "Colour           object\n",
       "Odometer (KM)     int64\n",
       "Doors             int64\n",
       "Price             int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X/y\n",
    "X = car_sales.drop(\"Price\", axis=1)\n",
    "y = car_sales[\"Price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors\n",
       "0   Honda  White          35431      4\n",
       "1     BMW   Blue         192714      5\n",
       "2   Honda  White          84714      4\n",
       "3  Toyota  White         154365      4\n",
       "4  Nissan   Blue         181577      3"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/one-hot-encoding.jpg\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Doors</th>\n",
       "      <th>Make_BMW</th>\n",
       "      <th>Make_Honda</th>\n",
       "      <th>Make_Nissan</th>\n",
       "      <th>Make_Toyota</th>\n",
       "      <th>Colour_Black</th>\n",
       "      <th>Colour_Blue</th>\n",
       "      <th>Colour_Green</th>\n",
       "      <th>Colour_Red</th>\n",
       "      <th>Colour_White</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Doors  Make_BMW  Make_Honda  Make_Nissan  Make_Toyota  Colour_Black  \\\n",
       "0        4         0           1            0            0             0   \n",
       "1        5         1           0            0            0             0   \n",
       "2        4         0           1            0            0             0   \n",
       "3        4         0           0            0            1             0   \n",
       "4        3         0           0            1            0             0   \n",
       "..     ...       ...         ...          ...          ...           ...   \n",
       "995      4         0           0            0            1             1   \n",
       "996      3         0           0            1            0             0   \n",
       "997      4         0           0            1            0             0   \n",
       "998      4         0           1            0            0             0   \n",
       "999      4         0           0            0            1             0   \n",
       "\n",
       "     Colour_Blue  Colour_Green  Colour_Red  Colour_White  \n",
       "0              0             0           0             1  \n",
       "1              1             0           0             0  \n",
       "2              0             0           0             1  \n",
       "3              0             0           0             1  \n",
       "4              1             0           0             0  \n",
       "..           ...           ...         ...           ...  \n",
       "995            0             0           0             0  \n",
       "996            0             0           0             1  \n",
       "997            1             0           0             0  \n",
       "998            0             0           0             1  \n",
       "999            1             0           0             0  \n",
       "\n",
       "[1000 rows x 10 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(car_sales[[\"Make\", \"Colour\", \"Doors\"]])\n",
    "dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 3.54310e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        1.00000e+00, 1.92714e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 8.47140e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 6.66040e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.15883e+05],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 2.48360e+05]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn the categories into numbers\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "# Encode categorical integer features as a one-hot numeric array\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "\n",
    "# Applies transformers to columns of an array or pandas DataFrame\n",
    "transformer = ColumnTransformer([(\"one_hot\", one_hot, categorical_features)], remainder=\"passthrough\")\n",
    "\n",
    "transformed_X = transformer.fit_transform(X)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35431.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>192714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84714.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154365.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>181577.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35820.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>155144.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66604.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>215883.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>248360.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3    4    5    6    7    8    9   10   11        12\n",
       "0    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   35431.0\n",
       "1    1.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  192714.0\n",
       "2    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0   84714.0\n",
       "3    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  154365.0\n",
       "4    0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  1.0  0.0  0.0  181577.0\n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...       ...\n",
       "995  0.0  0.0  0.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   35820.0\n",
       "996  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  1.0  0.0  0.0  155144.0\n",
       "997  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0   66604.0\n",
       "998  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  215883.0\n",
       "999  0.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  248360.0\n",
       "\n",
       "[1000 rows x 13 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(transformed_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor()"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try to refit the model\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into training and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(transformed_X, y, test_size=0.2)\n",
    "\n",
    "\n",
    "# Build machine learning model\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3235867221569877"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 What if there were missing values?\n",
    "\n",
    "1. Fill them with some value (also known as imputation).\n",
    "2. Remove the samples with missing data altogether."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import car sales missing data\n",
    "car_sales_missing = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_missing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Black</td>\n",
       "      <td>35820.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32042.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>NaN</td>\n",
       "      <td>White</td>\n",
       "      <td>155144.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>66604.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>31570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>215883.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Blue</td>\n",
       "      <td>248360.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12732.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Make Colour  Odometer (KM)  Doors    Price\n",
       "0     Honda  White        35431.0    4.0  15323.0\n",
       "1       BMW   Blue       192714.0    5.0  19943.0\n",
       "2     Honda  White        84714.0    4.0  28343.0\n",
       "3    Toyota  White       154365.0    4.0  13434.0\n",
       "4    Nissan   Blue       181577.0    3.0  14043.0\n",
       "..      ...    ...            ...    ...      ...\n",
       "995  Toyota  Black        35820.0    4.0  32042.0\n",
       "996     NaN  White       155144.0    3.0   5716.0\n",
       "997  Nissan   Blue        66604.0    4.0  31570.0\n",
       "998   Honda  White       215883.0    4.0   4001.0\n",
       "999  Toyota   Blue       248360.0    4.0  12732.0\n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show number of column with missing value\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0    811\n",
       "5.0     75\n",
       "3.0     64\n",
       "Name: Doors, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing[\"Doors\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1: Fill missing data with Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fill the \"Make\" column\n",
    "car_sales_missing[\"Make\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "# Fill the \"Colour\" column\n",
    "car_sales_missing[\"Colour\"].fillna(\"missing\", inplace=True)\n",
    "\n",
    "# Fill the \"Odometer (KM)\" column. Filled with mean values\n",
    "car_sales_missing[\"Odometer (KM)\"].fillna(car_sales_missing[\"Odometer (KM)\"].mean(), inplace=True)\n",
    "\n",
    "# Fill the \"Doors\" column. Most cars have 4 doors\n",
    "car_sales_missing[\"Doors\"].fillna(4, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              0\n",
       "Colour            0\n",
       "Odometer (KM)     0\n",
       "Doors             0\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show number of column with missing value\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove rows with missing Price value\n",
    "car_sales_missing.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "Doors            0\n",
       "Price            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show number of column with missing value\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "950"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car_sales_missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Doors</th>\n",
       "      <th>Make_BMW</th>\n",
       "      <th>Make_Honda</th>\n",
       "      <th>Make_Nissan</th>\n",
       "      <th>Make_Toyota</th>\n",
       "      <th>Colour_Black</th>\n",
       "      <th>Colour_Blue</th>\n",
       "      <th>Colour_Green</th>\n",
       "      <th>Colour_Red</th>\n",
       "      <th>Colour_White</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Doors  Make_BMW  Make_Honda  Make_Nissan  Make_Toyota  Colour_Black  \\\n",
       "0        4         0           1            0            0             0   \n",
       "1        5         1           0            0            0             0   \n",
       "2        4         0           1            0            0             0   \n",
       "3        4         0           0            0            1             0   \n",
       "4        3         0           0            1            0             0   \n",
       "..     ...       ...         ...          ...          ...           ...   \n",
       "995      4         0           0            0            1             1   \n",
       "996      3         0           0            1            0             0   \n",
       "997      4         0           0            1            0             0   \n",
       "998      4         0           1            0            0             0   \n",
       "999      4         0           0            0            1             0   \n",
       "\n",
       "     Colour_Blue  Colour_Green  Colour_Red  Colour_White  \n",
       "0              0             0           0             1  \n",
       "1              1             0           0             0  \n",
       "2              0             0           0             1  \n",
       "3              0             0           0             1  \n",
       "4              1             0           0             0  \n",
       "..           ...           ...         ...           ...  \n",
       "995            0             0           0             0  \n",
       "996            0             0           0             1  \n",
       "997            1             0           0             0  \n",
       "998            0             0           0             1  \n",
       "999            1             0           0             0  \n",
       "\n",
       "[1000 rows x 10 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(car_sales[[\"Make\", \"Colour\", \"Doors\"]])\n",
    "dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create X & y\n",
    "X = car_sales_missing.drop(\"Price\", axis=1)\n",
    "y = car_sales_missing[\"Price\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        3.54310e+04, 1.53230e+04],\n",
       "       [1.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        1.92714e+05, 1.99430e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        8.47140e+04, 2.83430e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 1.00000e+00, ..., 0.00000e+00,\n",
       "        6.66040e+04, 3.15700e+04],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.15883e+05, 4.00100e+03],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 0.00000e+00,\n",
       "        2.48360e+05, 1.27320e+04]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn categorical into numberical data\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\",\n",
    "                                   one_hot,\n",
    "                                   categorical_features)],\n",
    "                                   remainder=\"passthrough\")\n",
    "\n",
    "transformed_X = transformer.fit_transform(car_sales_missing)\n",
    "transformed_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35820.0</td>\n",
       "      <td>32042.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>155144.0</td>\n",
       "      <td>5716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66604.0</td>\n",
       "      <td>31570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>948</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>215883.0</td>\n",
       "      <td>4001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>949</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>248360.0</td>\n",
       "      <td>12732.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>950 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3    4    5    6    7    8    9   10   11   12   13  \\\n",
       "0    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0   \n",
       "1    1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0   \n",
       "2    0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0   \n",
       "3    0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0   \n",
       "4    0.0  0.0  1.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0   \n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "945  0.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   \n",
       "946  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  1.0  0.0  1.0  0.0  0.0   \n",
       "947  0.0  0.0  1.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   \n",
       "948  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0   \n",
       "949  0.0  0.0  0.0  1.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0   \n",
       "\n",
       "           14       15  \n",
       "0     35431.0  15323.0  \n",
       "1    192714.0  19943.0  \n",
       "2     84714.0  28343.0  \n",
       "3    154365.0  13434.0  \n",
       "4    181577.0  14043.0  \n",
       "..        ...      ...  \n",
       "945   35820.0  32042.0  \n",
       "946  155144.0   5716.0  \n",
       "947   66604.0  31570.0  \n",
       "948  215883.0   4001.0  \n",
       "949  248360.0  12732.0  \n",
       "\n",
       "[950 rows x 16 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(transformed_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3281222657899089"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build machine learning model\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2: Filling missing values with Scikit-Learn\n",
    "\n",
    "The main takeaways:\n",
    "- Split your data first (into train/test)\n",
    "- Fill/transform the training set and test sets separately"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "car_sales_missing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             47\n",
       "Colour           46\n",
       "Odometer (KM)    48\n",
       "Doors            47\n",
       "Price             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the rows with no labels\n",
    "car_sales_missing.dropna(subset=[\"Price\"], inplace=True)\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into X & y\n",
    "X = car_sales_missing.drop(\"Price\", axis=1)\n",
    "y = car_sales_missing[\"Price\"]\n",
    "\n",
    "# Split data into train and test\n",
    "np.random.seed(42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             47\n",
       "Colour           46\n",
       "Odometer (KM)    48\n",
       "Doors            47\n",
       "dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check missing values\n",
    "X.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Honda', 'White', 4.0, 71934.0],\n",
       "       ['Toyota', 'Red', 4.0, 162665.0],\n",
       "       ['Honda', 'White', 4.0, 42844.0],\n",
       "       ...,\n",
       "       ['Toyota', 'White', 4.0, 196225.0],\n",
       "       ['Honda', 'Blue', 4.0, 133117.0],\n",
       "       ['Honda', 'missing', 4.0, 150582.0]], dtype=object)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fill missing values with Scikit-Learn\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "# Fill categorical values with 'missing' & numerical values with mean\n",
    "cat_imputer = SimpleImputer(strategy=\"constant\", fill_value=\"missing\")\n",
    "door_imputer = SimpleImputer(strategy=\"constant\", fill_value=4)\n",
    "num_imputer = SimpleImputer(strategy=\"mean\")\n",
    "\n",
    "# Define columns\n",
    "cat_features = [\"Make\", \"Colour\"]\n",
    "door_feature = [\"Doors\"]\n",
    "num_features = [\"Odometer (KM)\"]\n",
    "\n",
    "# Create an imputer (something that fills missing data)\n",
    "imputer = ColumnTransformer([\n",
    "    (\"cat_imputer\", cat_imputer, cat_features),\n",
    "    (\"door_imputer\", door_imputer, door_feature),\n",
    "    (\"num_imputer\", num_imputer, num_features)\n",
    "])\n",
    "\n",
    "# Fill train and test values separately\n",
    "filled_X_train = imputer.fit_transform(X_train)\n",
    "filled_X_test = imputer.transform(X_test)\n",
    "\n",
    "# Check filled X_train\n",
    "filled_X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Doors            0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get our transformed data array's back into DataFrame's\n",
    "car_sales_filled_train = pd.DataFrame(filled_X_train, \n",
    "                                      columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "\n",
    "car_sales_filled_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Doors            0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_filled_test = pd.DataFrame(filled_X_test, \n",
    "                                     columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "car_sales_filled_test.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 7.19340e+04],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.62665e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 4.28440e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.96225e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.33117e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.50582e+05]])"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let's one hot encode the features with the same code as before \n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\", \n",
    "                                 one_hot, \n",
    "                                 categorical_features)],\n",
    "                                 remainder=\"passthrough\")\n",
    "\n",
    "# Fill train and test values separately\n",
    "transformed_X_train = transformer.fit_transform(car_sales_filled_train)\n",
    "transformed_X_test = transformer.transform(car_sales_filled_test)\n",
    "\n",
    "# Check transformed and filled X_train\n",
    "transformed_X_train.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21229043336119102"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we've transformed X, let's see if we can fit a model\n",
    "np.random.seed(42)\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "\n",
    "# Make sure to use transformed (filled and one-hot encoded X data)\n",
    "model.fit(transformed_X_train, y_train) # train\n",
    "model.score(transformed_X_test, y_test) # test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(950, 1000)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check length of transformed data (filled and one-hot encoded)\n",
    "# vs. length of original data\n",
    "len(transformed_X_train.toarray())+len(transformed_X_test.toarray()), len(car_sales)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Choosing the right estimator/algorithm for our problem\n",
    "\n",
    "Scikit-Learn uses estimator as another term for machine learning model or algorithm.\n",
    "\n",
    "- Classification - predicting whether a sample is one thing or another\n",
    "- Regression - predicting a number"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Picking a machine learning model for a regression problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Boston housing dataset\n",
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "boston;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert dataset into pandas dataframe\n",
    "boston_df = pd.DataFrame(boston[\"data\"], columns=boston[\"feature_names\"])\n",
    "boston_df[\"target\"] = pd.Series(boston[\"target\"])\n",
    "boston_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "506"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many samples?\n",
    "len(boston_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6662221670168523"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try the Ridge Regression model\n",
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Ridge model\n",
    "model = Ridge()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Check the score of the Ridge model on test data\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How do we improve this score?\n",
    "\n",
    "What if Ridge wasn't working?\n",
    "\n",
    "Let's refer back to the map... https://scikitlearn.org/stable/tutorial/machine_learning_map/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8922527442109116"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's try the Random Forst Regressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instatiate Random Forest Regressor\n",
    "rf = RandomForestRegressor(n_estimators=100)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Regressor\n",
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6662221670168523"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the Ridge model again\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Choosing an estimator for a classification problem\n",
    "Let's go to the map... https://scikit-learn.org/stable/tutorial/machine_learning_map/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease = pd.read_csv(\"data/heart-disease.csv\")\n",
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "303"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(heart_disease)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Consulting the map and it says to try LinearSVC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/svm/_base.py:975: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8688524590163934"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the LinearSVC estimator class\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate LinearSVC\n",
    "clf = LinearSVC(max_iter=10000)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the LinearSVC\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    165\n",
       "0    138\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestClassifier estimator class\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Random Forest Classifier\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Classifier\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tidbit:\n",
    "\n",
    "1. If you have structured data, used ensemble methods\n",
    "2. If you have unstructured data, use deep learning or transfer learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>241</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>264</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>132</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>193</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>141</td>\n",
       "      <td>0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>131</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>115</td>\n",
       "      <td>1</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>303 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0     63    1   3       145   233    1        0      150      0      2.3   \n",
       "1     37    1   2       130   250    0        1      187      0      3.5   \n",
       "2     41    0   1       130   204    0        0      172      0      1.4   \n",
       "3     56    1   1       120   236    0        1      178      0      0.8   \n",
       "4     57    0   0       120   354    0        1      163      1      0.6   \n",
       "..   ...  ...  ..       ...   ...  ...      ...      ...    ...      ...   \n",
       "298   57    0   0       140   241    0        1      123      1      0.2   \n",
       "299   45    1   3       110   264    0        1      132      0      1.2   \n",
       "300   68    1   0       144   193    1        1      141      0      3.4   \n",
       "301   57    1   0       130   131    0        1      115      1      1.2   \n",
       "302   57    0   1       130   236    0        0      174      0      0.0   \n",
       "\n",
       "     slope  ca  thal  target  \n",
       "0        0   0     1       1  \n",
       "1        0   0     2       1  \n",
       "2        2   0     2       1  \n",
       "3        2   0     2       1  \n",
       "4        2   0     2       1  \n",
       "..     ...  ..   ...     ...  \n",
       "298      1   0     3       0  \n",
       "299      1   0     3       0  \n",
       "300      1   2     3       0  \n",
       "301      1   1     3       0  \n",
       "302      1   1     2       0  \n",
       "\n",
       "[303 rows x 14 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Fit the model/algorithm on our data and use it to make predictions\n",
    "\n",
    "### 3.1 Fitting the model to the data\n",
    "\n",
    "Different names for:\n",
    "\n",
    "- X = features, features variables, data\n",
    "- y = labels, targets, target variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the RandomForestClassifier estimator class\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Setup random seed\n",
    "np.random.seed(42)\n",
    "\n",
    "# Make the data\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate Random Forest Classifier\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "# Fit the model to the data (training the machine learning model)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Evaluate the Random Forest Classifier (use the patterns the model has learned)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  \n",
       "0   0     1  \n",
       "1   0     2  \n",
       "2   0     2  \n",
       "3   0     2  \n",
       "4   0     2  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "298    0\n",
       "299    0\n",
       "300    0\n",
       "301    0\n",
       "302    0\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 Make predictions using a machine learning model\n",
    "\n",
    "2 ways to make predictions:\n",
    "1. predict()\n",
    "2. predict_proba()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use a trained model to make predictions\n",
    "clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0,\n",
       "       0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare predictions to truth labels to evaluate the model\n",
    "y_preds = clf.predict(X_test)\n",
    "np.mean(y_preds == y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make predictions with predict_proba() - use this if someone asks you \"what's the probability your model is assigning to each prediction?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.89, 0.11],\n",
       "       [0.49, 0.51],\n",
       "       [0.43, 0.57],\n",
       "       [0.84, 0.16],\n",
       "       [0.18, 0.82]])"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict_proba() returns probabilities of a classification label \n",
    "clf.predict_proba(X_test[:5])\n",
    "# [% for 0, % for 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's predict() on the same data...\n",
    "clf.predict(X_test[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>276</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>59</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>170</td>\n",
       "      <td>288</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>150</td>\n",
       "      <td>126</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>173</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>134</td>\n",
       "      <td>409</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>1</td>\n",
       "      <td>1.9</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>110</td>\n",
       "      <td>265</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "179   57    1   0       150   276    0        0      112      1      0.6   \n",
       "228   59    1   3       170   288    0        0      159      0      0.2   \n",
       "111   57    1   2       150   126    1        1      173      0      0.2   \n",
       "246   56    0   0       134   409    0        0      150      1      1.9   \n",
       "60    71    0   2       110   265    1        0      130      0      0.0   \n",
       "\n",
       "     slope  ca  thal  \n",
       "179      1   1     1  \n",
       "228      1   0     3  \n",
       "111      2   1     3  \n",
       "246      1   2     3  \n",
       "60       2   1     2  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    165\n",
       "0    138\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "predict() can also be used for regression models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  target  \n",
       "0     15.3  396.90   4.98    24.0  \n",
       "1     17.8  396.90   9.14    21.6  \n",
       "2     17.8  392.83   4.03    34.7  \n",
       "3     18.7  394.63   2.94    33.4  \n",
       "4     18.7  396.90   5.33    36.2  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit model\n",
    "model = RandomForestRegressor(n_estimators=100).fit(X_train, y_train)\n",
    "\n",
    "# Make predictions\n",
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([23.081, 30.574, 16.759, 23.46 , 16.893, 21.644, 19.113, 15.334,\n",
       "       21.14 , 20.639])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([23.6, 32.4, 13.6, 22.8, 16.1, 20. , 17.8, 14. , 19.6, 16.8])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y_test[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compare the predictions to the truth\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mean_absolute_error(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0. An end-to-end Scikit-Learn workflow',\n",
       " '1. Getting the data ready',\n",
       " '2. Choose the right estimator/algorithm for our problems',\n",
       " '3. Fit the model/algorithm and use it to make predictions on our data',\n",
       " '4. Evaluating a model',\n",
       " '5. Improve a model',\n",
       " '6. Save and load a trained model',\n",
       " '7. Putting it all together!']"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "what_were_covering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Evaluating a machine learning model\n",
    "\n",
    "Three ways to evaluate Scikit-Learn models/esitmators:\n",
    "- Estimator score method\n",
    "- The scoring parameter\n",
    "- Problem-specific metric functions.\n",
    "\n",
    "### 4.1 Evaluating a model with the score method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do the same but for regression..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Create the data\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "# Split into training and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate and fit model\n",
    "model = RandomForestRegressor(n_estimators=100).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9763520974033731"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Evaluating a model using the scoring parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "\n",
    "clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8524590163934426"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/cross-validation.jpg\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.86885246, 0.81967213, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate a score by cross-validation\n",
    "cross_val_score(clf, X, y, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90322581, 0.80645161, 0.87096774, 0.9       , 0.86666667,\n",
       "       0.8       , 0.73333333, 0.86666667, 0.73333333, 0.8       ])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(clf, X, y, cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8524590163934426, 0.8248087431693989)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Single training and test split score\n",
    "clf_single_score = clf.score(X_test, y_test)\n",
    "\n",
    "# Take the mean of 5-fold cross-validation score\n",
    "clf_cross_val_score = np.mean(cross_val_score(clf, X, y, cv=5))\n",
    "\n",
    "# Compare the two\n",
    "clf_single_score, clf_cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78688525, 0.86885246, 0.80327869, 0.78333333, 0.76666667])"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scoring parameter set to None by default\n",
    "cross_val_score(clf, X, y, cv=5, scoring=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.1 Classification model evaluation metrics\n",
    "\n",
    "1. Accuracy\n",
    "2. Area under ROC curve\n",
    "3. Confusion matrix\n",
    "4. Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>1</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  slope  \\\n",
       "0   63    1   3       145   233    1        0      150      0      2.3      0   \n",
       "1   37    1   2       130   250    0        1      187      0      3.5      0   \n",
       "2   41    0   1       130   204    0        0      172      0      1.4      2   \n",
       "3   56    1   1       120   236    0        1      178      0      0.8      2   \n",
       "4   57    0   0       120   354    0        1      163      1      0.6      2   \n",
       "\n",
       "   ca  thal  target  \n",
       "0   0     1       1  \n",
       "1   0     2       1  \n",
       "2   0     2       1  \n",
       "3   0     2       1  \n",
       "4   0     2       1  "
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "heart_disease.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.90163934, 0.83606557, 0.78333333, 0.78333333])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "cross_val_score = cross_val_score(clf, X, y, cv=5)\n",
    "cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8248087431693989"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(cross_val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Heart Disease Classifier Cross-Validated Accuracy: 82.48%\n"
     ]
    }
   ],
   "source": [
    "print(f\"Heart Disease Classifier Cross-Validated Accuracy: {np.mean(cross_val_score) *100:.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Area under the receiver operating characteristic curve (AUC/ROC)**\n",
    "\n",
    "- Area under curve (AUC)\n",
    "- ROC curve\n",
    "\n",
    "ROC curves are a comparison of a model's true postive rate (tpr) versus a models false positive rate (fpr).\n",
    "- True positive = model predicts 1 when truth is 1\n",
    "- False positive = model predicts 1 when truth is 0\n",
    "- True negative = model predicts 0 when truth is 0\n",
    "- False negative = model predicts 0 when truth is 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create X_test... etc\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.51, 0.49],\n",
       "        [0.17, 0.83],\n",
       "        [0.51, 0.49],\n",
       "        [0.72, 0.28],\n",
       "        [0.43, 0.57],\n",
       "        [0.12, 0.88],\n",
       "        [0.3 , 0.7 ],\n",
       "        [0.97, 0.03],\n",
       "        [0.15, 0.85],\n",
       "        [0.4 , 0.6 ]]), 61)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "# Fit the classifier\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions with probabilities\n",
    "y_probs = clf.predict_proba(X_test)\n",
    "\n",
    "y_probs[:10], len(y_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.49, 0.83, 0.49, 0.28, 0.57, 0.88, 0.7 , 0.03, 0.85, 0.6 ]), 61)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probs_positive = y_probs[:, 1]\n",
    "y_probs_positive[:10], len(y_probs_positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.        , 0.03448276, 0.03448276, 0.03448276, 0.03448276,\n",
       "        0.03448276, 0.03448276, 0.06896552, 0.06896552, 0.06896552,\n",
       "        0.10344828, 0.10344828, 0.13793103, 0.13793103, 0.13793103,\n",
       "        0.20689655, 0.20689655, 0.20689655, 0.27586207, 0.37931034,\n",
       "        0.37931034, 0.48275862, 0.48275862, 0.55172414, 0.55172414,\n",
       "        1.        ]),\n",
       " 26,\n",
       " array([0.     , 0.     , 0.125  , 0.1875 , 0.25   , 0.3125 , 0.375  ,\n",
       "        0.375  , 0.5    , 0.53125, 0.5625 , 0.59375, 0.59375, 0.625  ,\n",
       "        0.71875, 0.71875, 0.78125, 0.8125 , 0.90625, 0.90625, 0.9375 ,\n",
       "        0.9375 , 0.96875, 0.96875, 1.     , 1.     ]),\n",
       " 26,\n",
       " array([2.  , 1.  , 0.98, 0.95, 0.92, 0.88, 0.87, 0.86, 0.85, 0.83, 0.81,\n",
       "        0.8 , 0.77, 0.76, 0.71, 0.66, 0.6 , 0.57, 0.49, 0.4 , 0.38, 0.31,\n",
       "        0.28, 0.23, 0.2 , 0.  ]))"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate fpr, tpr and thresholds\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_probs_positive)\n",
    "\n",
    "# Check the false positive rates\n",
    "fpr, len(fpr), tpr, len(tpr), thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3gUVffA8e8hlNBBQKSH3nukSO+CKHZBRNAoUsSuL4oVy09QEREQkSpKE18UFQUEkVdEEaRIlSZNpEkLIZTk/P6YCa4hZYHsbnb3fJ4nT6btzJnd2T1z7525I6qKMcaY8JUl0AEYY4wJLEsExhgT5iwRGGNMmLNEYIwxYc4SgTHGhDlLBMYYE+YsEWQyItJdROYHOo7MRERiRaRcALYbJSIqIln9vW1fEJH1ItLyEl53ycekiHQQkc8u5bWXSkRyiMgmESniz+0GM0sEaRCRP0TklPtD9JeITBKRPL7cpqp+rKrtfbkNTyJyjYgsEpETInJMRL4QkWr+2n4K8SwWkfs8p6lqHlXd7qPtVRKRT0TkkLv/a0XkMRGJ8MX2LpWbkCpczjpUtbqqLk5nOxckv8s8Jl8FXvdYv4rISfc7tVdEhiV/r0Wks4gsd5c7LCIfi0jJZMsUE5HxIrLPPXY3ichLIpJbVU8DE4CB6exrUHz2/mCJIH3Xq2oeoA5QF3g6wPFckpTOakWkMTAf+BwoDpQF1gBLfXEGntnOrEWkPPAzsBuoqar5gduAaCBvBm8rYPseqG2LyNVAflX9Kdms2u53qgVwB3Cvx2tuBaYCw4HCQHXgNPCDiBR0l7kCWAbkBBqral6gHVAAKO+uairQU0RypBJbhn72me3Yvmiqan+p/AF/AG09xocCX3mM5wDeBHYB+4ExQE6P+V2A1cBxYBtwrTs9PzAe2AfsBV4BItx5vYAf3OH3gDeTxfQ58Jg7XBz4FDgI7AAe8ljuRWAW8JG7/ftS2L//AaNTmP418KE73BLYAzwDHHLfk+7evAcer/0P8BcwBSgIfOnGfMQdLuku/yqQAMQDscBId7oCFdzhScAo4CvgBM6XubxHPO2BzcAxYDTwfUr77i77kefnmcL8KHfbPd39OwQM8pjfAOcH6aj7WY4EsnvMV6A/sAXY4U57B+fH5ziwEmjmsXyE+z5vc/dtJVAKWOKu66T7vtzhLt8Z5/g6CvwI1Ep27P4HWIvzQ5oVj+PZjX2FG8d+YJg7fZe7rVj3rzEex6S7THVgAfC3+9pnUnn/ngfGJZt2/rN0x2cCo9xhAXYCTyV7TRZgHTDYHX8F+A3Iks73dwvQ4hI/+5bAntR+D7jw+/U8cAq4wmP5uu4xk80dvxfYiHPczwPK+Ps3LdX9DXQAmfkv2Qdf0j343vGY/zYwB7gC5yziC+D/3HkNcH6M2rkHcgmgijtvNvA+kBu4ElgOPODOO/+lA5rj/GiIO17QPdiKu+tc6R6A2YFywHagg8eBeha40V02Z7J9y4Xzo9sqhf2+B9jnDrcEzgHDcH70W+D8IFX24j1Ieu0Q97U5gULALe728wKfAJ95bHsxyX64uTARHHbf36zAx8B0d15h90t5szvvYfc9SC0R/AXck8bnH+Vu+wM39to4P6pV3fn1gUbutqJwvuSPJIt7gfveJCXHu9z3ICvwuBtDpDvvSZxjrDLOj2JtoFDy98AdrwscABriJJCeOMdrDo9jdzVOIsnpMS3peF4G9HCH8wCNku1zVo9t9eKfYzIvTtJ7HIh0xxum8v59AjyZxmdZxV3Xox7jCpRNYV0vAcvc4Z+Al7z4/s7B4+ToIj/7lqSfCP71/QIWAfd7LP8GMMYd7gJsBaq6n/2zwI+B/o07H2ugA8jMf+4HH4tzdqbAQqCAO09wfhA9z0Yb88+Z3/vA2ymssyjOj4lnyaEb8J077PmlE5wztObu+P3AIne4IbAr2bqfBibqPwfqkjT2raS7T1VSmHctcNYdbonzY57bY/5M4Dkv3oOWwBncH7pU4qgDHPEYX0z6iWCcx7xOwCZ3+O6kHwuP92938vV5zD+LW0pLZX6Uu+2SHtOWA11TWf4RYHayuFunc4wdwakqAack0yWV5ZIngveAl5Mtsxn3DNg9du9N4XhO+iFbgvPjWjiVfU4tEXQDVnn5/VkA9ElhP467x40C0/gneTV1p11wvAB9gC3u8Jbk601l+x8Dz1/iZ9+S9BPBkmTz7+Of72fSsZf03f0aiPFYNgsQRyYpFVgbQfpuVKcOsiXOGUthd3oRnLPalSJyVESOAt+408E5E9uWwvrKANmAfR6vex+nZPAv6hwx03G+fAB34hzcSespnrQOdz3P4CSaJLvT2K8jQCJQLIV5xXCKtOeXVdWTHuM7cUol6b0HAAdVNT5pRERyicj7IrJTRI7j/CAVuMgGur88huNwzmhxYzq/z+77tyeN9Rwm5f33antuY+OX7oUEx4HX+Of4SPKvz0BEnhCRjW7j5FGcasKk16R2zKSkDPB4ss+/FM57kOK2k4kBKgGbROQXEens5XYvJsYjpFzfXg/nPbwD54Qmtzs96ZhL75j09nPLi1NtlhJv15GW5O/vp0BjESmGU5pPxKl+Befzesfjs/obJ1mUuMwYMoQlAi+p6vc4Z6NvupMO4VTTVFfVAu5ffnUawcA5SMpfuCZ245QICnu8Lp+qVk9l09OAW0WkDM6X5lOP9ezwWEcBVc2rqp08w05jf07iVA/clsLs23FKP0kKikhuj/HSwJ9evAcpxfA4TtVHQ1XNh/OFAedLkWbMXtiHU9JxViginuMp+BanmupSvQdsAiq6+/IM/+xHkvP7IyLNgKdw3t+CqloAp/ow6TWpHTMp2Q28muzzz6Wq01LadnKqukVVu+GcgAwBZrmfcXrv/26cakhvrMVJNiltX1V1Js4x+Lw7eTNO4v7XMSkiWXA+p6Rj8lvgJnd6WqriXPyQkvQ++5M4JzlJMUTw7xMcSPZeqeoRnIsv7sA5aZvunoyA8749kOzzyqmqP6azD35hieDiDAfaiUhtVU3EqTt+W0SuBBCREiLSwV12PHCPiLQRkSzuvCqqug/nYHlLRPK588qLSIuUNqiqq3B+cMcB81Q16QxnOXBCRP4jIjlFJEJEarhXanhrIM6VFQ+JSF4RKSgir+BU77yUbNmXRCS7+2PWGfjEi/cgJXlxksdR9+qPF5LN34/3PzTJfQXUFJEb3as4+gNXpbH8C8A1IvKGiFzlxl9BRD4SkQJebC8vTjVHrIhUAfp6sfw5nIbyrCLyPJDPY/444GURqSiOWiJSyJ2X/H35AOgjIg3dZXOLyHUi4tUVLyJyl4gUcT/DpGMq0Y0tkdQ/gy+BYiLyiDjX6+cVkYapLDsXp00pLa8D94vIVe6P5hPAsyJyp4hEup/LOJz36W33NcPc8cnuCVLScTdMRGoljeO0zSS/YilJep/970Ck+55mw6nTT/EKpGSm4lRRJl39lGQM8LSIVHe3lV9EUjoJCwhLBBdBVQ8CH/LPGcx/cBqAfnKrBr7FOdtFVZfjNLq+jXPW9z1O8RCcAyU7sAGn+DyLtIupU4G2eBxYqpqA84NcB+eKoaRkkf8i9ucHoANO4+o+nCqfukBTVd3isehfbpx/4lRN9VHVTem9B6kYjtOwdgjnS/pNsvnv4JSAjojICG/3xd2fQzhnk0Nxiv7VcK6MOZ3K8ttwkl4UsF5EjuGUuFbgtAul5wmcM78TOD/MM9JZfh7O/v6O817H8+/qhWE47S/zcRLMeJz3Cpw66clu1cLtqroCp81oJM5nsxWnLt9b1+LscyzOe95VVU+pahzO1VtL3W018nyRqp7AuQDiepzjYgvQKqUNqOqvwLE0EgWq+htO9eCT7vgMoAfwKM5nuMF9D5qo6mF3mb+Ba3Dq+X8WkRM4pYVj7vsAzucyWZ17ClLabpqfvaoeA/rhfKf24pQQ0qpmTDIHqAj8parnSyOqOhun5DXd/Z6sAzp6sT6/SLoaxZgUiXMn6keqmlYVS6bkVh3swbnc9btAxxOORKQ90E9Vb/TjNnPgVAk1V9UD/tpuMAvumyCMScatlvoZp/rpSZz699SqB4yPqep8nBKOP7d5GufCDuMlqxoyoaYxzlUth3CqL25U1VOBDcmYzM2qhowxJsxZicAYY8Jc0LURFC5cWKOiogIdhjHGBJWVK1ceUtUUu+YOukQQFRXFihUrAh2GMcYEFRHZmdo8qxoyxpgwZ4nAGGPCnCUCY4wJc0HXRpCSs2fPsmfPHuLj49Nf2KQoMjKSkiVLki1btkCHYozxs5BIBHv27CFv3rxERUXhdDhpLoaqcvjwYfbs2UPZsmUDHY4xxs98VjUkIhNE5ICIrEtlvojICBHZKs5Do+td6rbi4+MpVKiQJYFLJCIUKlTISlTGhClfthFMwunhMDUdcXrpqwj0xunb/ZJZErg89v4ZE758VjWkqktEJCqNRbrgPCBdcbowLiAixdz++o0JL7E7YPtknEcBGPNvJ+Pg4BEhqkFnKHQxjxzxTiDbCErw777Y97jTLkgEItIbp9RA6dKl/RLcxYqIiKBmzZqcO3eOsmXLMmXKFAoUcJ5tsn79egYMGMDevXtJTEzk7rvv5tlnnz1/Fv7111/z3HPPERcXR44cOWjdujVvvfVWIHfH+Nu2cbD+NS58wJkJd4vWl+f+cbeSP1c8K75ZQZYQSwReU9WxwFiA6OjoTNlLXs6cOVm9ejUAPXv2ZNSoUQwaNIhTp05xww038N5779G+fXvi4uK45ZZbGD16NP3792fdunU8+OCDfPXVV1SpUoWEhATGjh0b4L0xfqcJkCUHdLV2GuM4ejSeJ5/8nnHjfqNChQK8Pa4DWSqX8sm2AnkfwV6cB2EnKelOC3qNGzdm715nV6ZOnUqTJk1o3749ALly5WLkyJG8/vrrAAwdOpRBgwZRpYrTfXpERAR9+6b3xENjTChLSEjkmmumMmHCOp566mrWru1Jixa+SQIQ2BLBHOBBEZmO81D2YxnSPrDyETiy+rJX8y8F60D94V4tmpCQwMKFC4mJiQGcaqH69ev/a5ny5csTGxvL8ePHWbduHY8//njGxmuMCUqHD5/iiisiiYjIwquvNqNUqbxER6f12O2M4cvLR6cBy4DKIrJHRGJEpI+I9HEXmQtsx3nG6Ac4zwcNWqdOnaJOnTpcddVV7N+/n3bt2gU6JGNMkFBVPvpoA5UqjWfcuN8AuOmmin5JAuDbq4a6pTNfgf4ZvmEvz9wzWlIbQVxcHB06dGDUqFE89NBDVKtWjSVLlvxr2e3bt5MnTx7y5ctH9erVWblyJbVr1w5I3OYSnDkG+xeCZuAVPsc3Zdy6TFDZvfs4ffosYO7cHTRqVIwmTYr7PYagaCwOJrly5WLEiBHceOON9OvXj+7du/Paa6/x7bff0rZtW06dOsVDDz3EU089BcCTTz7JzTffTNOmTalUqRKJiYmMHTuWPn36pLMlEzCbR8Bvz2f8enMWy/h1mkxt2rSNPPDAAhISEhk+vBUPPliXiAj/N91aIvCBunXrUqtWLaZNm0aPHj34/PPPGTBgAP379ychIYEePXrw4IMPAlCrVi2GDx9Ot27diIuLQ0To3LlzgPfApCnhFEgEdMzgtihLBGGnYMFIGjYsxtix7ShbtkDA4gi6ZxZHR0dr8gfTbNy4kapVqwYootBh76OXVj8Dm96ErmcCHYkJMufOJfL22ys4cyaRQYMaAU77gD/u7BeRlaoandI8KxEYY4wfrFlzgJiYeaxcuZ/bb698PgFkhu5d7HkExhjjQ6dPn+O5534gOvojdu8+wSefXM/06Z0zRQJIYiUCE5xO/w2nDwZm22cOB2a7Jiht2XKEIUOWc+edVRg2rBWFCuUMdEgXsERggo8qfFEBzhwJXAxZ8wZu2ybTi409w+efb6V792rUqFGETZvupVy5wDUGp8cSgQlC6iSBUrc4f4GQp3xgtmsyvQUL/qB37/ns3HmcevWKUrVqoUydBMASgQlmBWpBVJr3LRrjN0eOxPPEE4uZMGEdlSoV5Pvvu1K1aqFAh+UVayzOQPv37+fOO++kXLly1K9fn8aNGzN79myfbnPFihU89NBDPt2GMSZtCQmJNGkylcmT1/P00w1Zs6YnzZqVDHRYXrMSQQZRVW688UZ69uzJ1KlTAdi5cydz5szx6Xajo6OJjk7x0mBjjI8dOhTHFVfkJCIiC6+91ozSpfNRr17RQId10axEkEEWLVpE9uzZ/9U1RJkyZRgwYACTJk06fycxQOfOnVm8eDEA8+fPp3HjxtSrV4/bbruN2NhYAAYOHEi1atWoVasWTzzxBACffPIJNWrUoHbt2jRv3hyAxYsXn78T+cUXX+Tee++lZcuWlCtXjhEjRpzf5ssvv0zlypVp2rQp3bp148033/Tp+2FMKFNVPvxwPZUqTWDcuLUA3HhjxaBMAhCiJYKWLadfMO322yvTr19d4uLO0qnTpxfM79WrBr161eDQoThuvfXfZ/GLF3dNd5vr16+nXr16FxXnoUOHeOWVV/j222/JnTs3Q4YMYdiwYfTv35/Zs2ezadMmRISjR48CMHjwYObNm0eJEiXOT0tu06ZNfPfdd5w4cYLKlSvTt29fVq9ezaeffsqaNWs4e/Ys9erVu6Br7IuiibDqKdgy2nmgit+5d8OLnccY/9u58xgPPLCAefP+4JpritO8efBUAaUmJBNBZtC/f39++OEHsmfPTv/+KXey+tNPP7FhwwaaNGkCwJkzZ2jcuDH58+cnMjKSmJgYOnfufP6Mv0mTJvTq1Yvbb7+dm2++OcV1XnfddeTIkYMcOXJw5ZVXsn//fpYuXUqXLl2IjIwkMjKS66+//tJ37NwpWNYDdn8Kpe+APGUvfV2XQyKgbI/AbNuErY8+2kDfvgtQhXffbU2/fnXJkiXz3Bh2qUIyEaR1Bp8rV7Y05xcunMurEkBy1atX59NP/ylpjBo1ikOHDhEdHU3WrFlJTPyny+L4eOdxhKpKu3btmDZt2gXrW758OQsXLmTWrFmMHDmSRYsWMWbMGH7++We++uor6tevz8qVKy94XY4cOc4PR0REcO7cuYvel1SdPgxLusDBH6HeMKjyaMat25ggUKRITpo0KcH777ejTJn8gQ4nw1jZOoO0bt2a+Ph43nvvvfPT4uLiAIiKimL16tUkJiaye/duli9fDkCjRo1YunQpW7duBeDkyZP8/vvvxMbGcuzYMTp16sTbb7/NmjVrANi2bRsNGzZk8ODBFClShN27d3sVW5MmTfjiiy+Ij48nNjaWL7/88uJ3MHYHLGgCh1dA0xmWBExYOHs2gddf/5mXX14GQIcOZfn661tCKglAiJYIAkFE+Oyzz3j00UcZOnQoRYoUOV/v36RJE8qWLUu1atWoWrXq+baEIkWKMGnSJLp168bp06cBeOWVV8ibNy9dunQhPj4eVWXYsGGA8+yCLVu2oKq0adOG2rVr8/3336cb29VXX80NN9xArVq1KFq0KDVr1iR//os4kP9eCYuvg8Qz0HoBXNns4t8gY4LMqlX7iYmZx6pVB+jatUqm6iQuo1k31GEiNjaWPHnyEBcXR/PmzRk7duwFjdspvo9/fg0/3AY5CkPLryG/vc8mtMXHn2Pw4GUMHbqcwoVzMnp0W26+uVKgw7ps1g21oXfv3mzYsIH4+Hh69uzp3RVO28bD8gecO3hbfmUPTjFhYevWI7z55i/cfXd13nqrJQULRgY6JJ+zRBAmkm5y84oq/PYSrHsJinWApp9ANutkzYSu2NgzzJ69hR49qlOjRhE2b743oE8M87eQSQT+espPqDpfRZh41ikFbJ8I5e6BBu9DlmyBDc4YH5o3bwe9e89n9+4TREdfRdWqhcIqCUCIXDUUGRnJ4cOHCbb2jsxCVTl8+DCR2bPC99c7SaDGC9BwvCUBE7IOHz5Fz55zufbaT8mVKxv/+1+3oOkkLqOFRImgZMmS7Nmzh4MHA/SgkhAQmQ1Kbu0DR5ZCw3FQPibQIRnjM04ncdPYuvUIgwY14tlnGxEZGRI/h5ckJPY8W7ZslC0boDtcQ8GxjbC4I5w+BC2+gOIdAx2RMT5x8GAchQo5ncQNGdKcMmXyUafOlYEOK+BComrIXIYDPzg3iiXEQ9vvLQmYkKSqTJz4G5UqjeeDD5xO4rp0qWBJwGWJIJzt+hQWtYXIK6H9MrjiMjqiMyaT+uOPY3ToMIt7751HzZpFaNWqVKBDynRComrIXIJN78Cvj0LhxtBiDuQIz0YyE9qmTFlP377fIgKjR7flgQdqh0QncRnNEkG40URY9SRsGgalbobGH0HWnIGOyhifKFo0N82bl2TMmHaULp0v0OFkWpYIwklCPCzrCbtmQqWHnB5Es0QEOipjMszZswkMHfoLCQmJPP/8NbRvH0X79lGBDivTs0QQLk7/DUtuhIP/g7pvOb2H2g14JoT8+ut+7r33G9asOcidd1a1m0wvgiWCcHBqHyxsA7HboMl0KHNHoCMyJsOcOnWWl15axptv/kKRIrmYPbsLN95YMdBhBRWfXjUkIteKyGYR2SoiA1OYX1pEvhORVSKyVkQ6+TKesLXjQzi+EVrNsyRgQs727ccYNmwFvXrVYMOGeywJXAKfJQIRiQBGAR2BakA3EamWbLFngZmqWhfoCoz2VTxhLfGs879I08DGYUwGOX78NJMmrQOgevXCbNkSw7hxHcKip1Bf8GWJoAGwVVW3q+oZYDrQJdkyCiQ15ecH/vRhPMaYEDB37nZq1JhETMw8Nm48DBByTwzzN18mghKA57MU97jTPL0I3CUie4C5wICUViQivUVkhYissP6EjAlPhw7F0aPHXK677r/kzZudpUvDt5O4jBboO4u7AZNUtSTQCZgiIhfEpKpjVTVaVaOLFCni9yCNMYGV1Enc9OmbeP75xvz6aw8aNSoe6LBChi+vGtoLeN7LXdKd5ikGuBZAVZeJSCRQGDjgw7iMMUFi//6TFCmSi4iILLz5ZkvKlMlHrVp2MpjRfFki+AWoKCJlRSQ7TmPwnGTL7ALaAIhIVSASsLofY8KcqjJ+/G9UrjyBsWPXAHD99eUtCfiIz0oEqnpORB4E5gERwARVXS8ig4EVqjoHeBz4QEQexWk47qX2dBljwtr27Ue5//75LFq0ixYtStK2bZlAhxTy0k0EIlIHaAYUB04B64CFqnosvdeq6lycRmDPac97DG8AmlxkzMaYEDV58jr69fuWiIgsjBnTjvvvr2WdxPlBqlVDItJDRFYALwEFgZ3AcaAtsFhExotISf+EaYwJB8WL56F169Js2HCP9RTqR2mVCK4AWqjqyZRmikg0UBXnslBjjLloZ84k8PrrP5OYqLz4YhPatYuiXbuoQIcVdlItEajqO0C8iDyUyvwVqrrAZ5EZY0LaL7/so379Kbzwwo9s334Max4MnDSvGlLVBKCHn2IxxoSBuLizPPHEYho1msqRI/HMmXMTH37YyXoKDSBvrhr6n4gMB2YA56uJVHWtz6IyxoSsHTuO8e67q7j//loMGdKc/PlzBDqksOdNIrja/e/5QFsFmmd8OMaYUHTs2Gn++9/fueeemlSvXpitW2MoVcqeGJZZpJsIVLWZPwIxxoSmr77axgMPLGDfvpM0blycKlUKWRLIZNK9s1hECorIMBFZLiI/i8hbIlLQH8EZY4LXwYNxdO/+FZ07z6ZgwUiWLbuTKlWsk7jMyJuqoenAT0B3d/xOnPaC9r4KyhgT3BISEmnadBo7dhzjpZeuYeDAhmTPbs/Hzqy8SQQlVPUFj/GXRGSdrwIKe3vnwvL7QBMybp1nYzNuXcak4a+/TnLllU4ncW+91ZKoqHzUqGH9A2V23iSChSJyq6rOAhCRmwG7f8BXjq52njFc/n6QDDyDylcJstgjqo1vJCYqH3ywlief/J4hQ5rTt28dOncuH+iwjJe8+WW4GxggIu7zDskGHBORXoCq6hW+Ci6sRY+EiOyBjsKYdG3deoT775/P4sW7ad26NB06RAU6JHORvEkEhX0ehTEmKE2c+Bv9+i0ke/YsfPBBe2JiatqNYUHIm+cRfK2qCZ5/ntN8HaAxJvMqXTofHTpEsWHDPdx3Xy1LAkEq1RKB+zCZSKCoiOQFkj7hfEBpP8RmjMlkTp8+x//9n9NJ3ODBTWnTpgxt2tjzAoJdWlVD/YHHgCuB9fyTCI4DY3wclzEmk/n5533ExHzD+vWH6dmzOqpqJYAQkWoiUNW3gbdF5BFVHe7HmIwxmcjJk2d47rmlDB++khIl8vLllzdx3XV2RVAoSevBNI0AUksCIpJHRKr5KjBjTOawc+dxRo9eTZ8+tVm/vpclgRCUVtXQnSLyBvA1sBLnofKRQAWglfv/CZ9HaIzxu6NH45k163fuu68W1aoVZuvW+yhZMm+gwzI+klbV0EMiUhi4DeeZBMVwnlm8EZisqov9EqExxq8+/3wrffsu4MCBOJo2LUGVKoUsCYS4NO8jUNVDwHvunzEmhB04cJKHHlrEjBmbqVWrCHPm3GSdxIUJ63PAGENCQiJNmkxj164TvPJKU5566mqyZbNO4sKFJQJjwtiff8Zy1VW5iYjIwjvvtCYqKh/VqllnAuHGmzuLjTEhJjFRee+91VSpMoExY1YD0KlTOUsCYcqbB9PkFJGnRWSMO15BRDr6PjRjjC/8/vvftGo1g379vqVhw2J07Fg20CGZAPOmRDAB567ipu74n8BrPovIGOMz48f/Ru3aH7J27UEmTOjA/Pm3UrZsgUCHZQLMmzaCiqraTURuA1DVOLH7yo0JSlFR+ejYsSyjRrWhWLE8gQ7HZBLeJIIzIhIJKICIlAXO+DQqY0yGOH36HC+//BMAr7xincSZlHlTNfQy8A1QUkQmA98Bz/g0KmPMZfvxx73UqfMhr776E/v2xaKqgQ7JZFLplghU9WsRWQFcg9NW8KSqHvB5ZMaYSxIbe4ZBg37g3Xd/pVSpvHzzzS106GANwiZ13lw1NF9VD6rq56r6maoeEJH53qxcRK4Vkc0islVEBqayzO0iskFE1ovI1IvdAWPMv+3adZz3319D//51WbfuHrAq1gEAAB6YSURBVEsCJl0+ezCNiEQAo4B2wB7gFxGZo6obPJapCDwNNFHVIyJy5SXviTFh7MiReD75ZDO9e9emWrXCbN9+P8WLW2Ow8Y4vH0zTANiqqtsBRGQ60AXY4LHM/cAoVT0CYFVOxly82bO30K/ftxw8GEeLFqWoXPkKSwLmovjywTQlgN0e43uAhsmWqQQgIkuBCOBFVf0m+YpEpDfQG6B06SB9SuapfRC3N/3l4vb4PhYTEv766yQDBixk1qzfqVPnSr766mYqV74i0GGZIORNY/FwEakCVMOpKkqanhH1+VmBikBLoCSwRERqqurRZDGMBcYCREdHB+elD1/XgXgvCzxZcoBY7x8mdQkJiTRrNo3du0/w2mvNeOKJaOskzlyydBOBiDwLtAeqAPOADsAPQHqJYC9QymO8pDvN0x7gZ1U9C+wQkd9xEsMvXkUfTM4chVK3Qrme6S+bqxRksf4AzYX27DlB8eJ5iIjIwogRrSlbNr91FW0umze/NncAdYBfVbWHiBQDJnnxul+Aiu4NaHuBrsCdyZb5DOgGTHQfglMJ2O5l7MEnbwUo0TnQUZgglJiojBq1iqef/h9DhjSnf/+6dOxYLtBhmRDhTSI4paoJInLOvXroLyDdWxNV9ZyIPIhTiogAJqjqehEZDKxQ1TnuvPYisgFIwLlH4fAl740xIWjTpsPcd998li7dS4cOUXTubAnAZCxvEsEqESmA0/ncCpyrhpZ7s3JVnQvMTTbteY9hxbky6TFvAzYmnIwbt5YHH1xIrlzZmDy5Iz16VMO6+jIZLc1E4HYu96LbeDtKROYB+VT1V79EZ0yYK1++ANdfX56RI9tQtGjuQIdjQlR6zyxWEVkA1HDHt/olqlCSmADbJ0DiWciSLdDRmEwuPv4cgwcvA+C115rRqlVpWrUK0kumTdDw5hrF1SJS1+eRhKKDy2B+Q1jeG4o0hQq9Ax2RycSWLnU6ifu///uZgwfjrJM44zfetBHUxekeYhtwEucOY1XVej6NLJid2gerB8KODyFnCbhmKpTpCla3a1Jw4sQZnnnmf4watYoyZfIxb96ttG8fFeiwTBjxJhHc4PMoQkXCGfh9BPw2GBJPQ7WnofozkM1u9zep27PnBOPG/caAAfV49dWm5MmTPdAhmTDjzZ3F2/wRSND7cx78+jAc3wzFr4P6w537BoxJweHDp5g5czN9+9ahatVCbN9+nz0xzASM3b56uWK3w6+PwZ7PIU8FaPEllLgu0FGZTEpV+fTT3+nffyF//x1P69alqVz5CksCJqAsEVyOg0thYRunO4ja/wdVHoWIHIGOymRS+/bF0r//QmbP3kL9+kWZP/9W6yTOZApeJQIRKYnzEPvvRCQHkFVVT/o2tCBw8AenLaDzRshjD/8wqXM6iZvO3r2xDB3anEcfjSZrVutY0GQO3nQ6dy/wIJAfKI/TvcRooK1vQwsikVcFOgKTSe3efZwSJfISEZGFUaPaULZsfipVslKAyVy8OSV5CGiE07UEqvo7zsNqjDGpSEhIZMSIX6lSZQLvvbcagA4dyloSMJmSN1VD8ap6Jql/E/cRlHZBvDGp2LjxMDEx81i27E86dizL9deXD3RIxqTJm0SwVESeAiJFpBXOIyy/9G1YxgSnsWPXMGDAIvLmzc6UKZ3o3r2qdRJnMj1vqoaeAk4Am4CHgYXAIF8GZUywqlixIDfdVIENG3px113WU6gJDt6UCK4Dxqnqe74Oxphgc+rUWV588UdEhNdfb26dxJmg5E2J4DZgq4hMFJFr3TYCY8LekiW7qV37Q4YO/YVjx05bJ3EmaKWbCFS1B84jJL8A7gG2i8gYXwdmTGZ1/Php+vVbQIsWM0hISGThwtt57712Vg1kgpZXN5Sp6mkR+Rw4hfPYyduBPr4MzJjM6s8/Y5k0aT2PPVafwYObkDu3dRJngps3N5S1w3mAfVvgB+BDLnwIvTEh7dChOGbO3Ey/fnWpUqUQO3bcb08MMyHDmxJBb2AGMEBVT/k4HmMyFVVl5szNDBiwkKNHT9O2bRkqVbrCkoAJKd50Q32bPwIxJrP5889Y+vZdwJw524iOLsrChdfancEmJKWaCETke1VtISJHAM/LIZKeUGbfCBOyEhISad7c6STuzTdb8PDD9a2TOBOy0ioRtHL/F/ZHIMZkBjt3HqNkSaeTuNGj21KuXH4qVCgY6LCM8alUT3FUNdEdHK+qCZ5/wHj/hGeMfyQkJDJs2AqqVp14vpO49u2jLAmYsOBNY3EtzxH3hrKrfROOMf63bt1BYmLmsXz5X3TuXI4bb6wY6JCM8au02gj+AwwE8orI30mTcdoLrERgQsKYMat56KFF5M+fg6lTr6Nr1yp2Y5gJO2mVCIYCbwH/h5MQAHCrhowJaqqKiFC1aiFuu60yw4e3okiRXIEOy5iASCsRVFDVLSIyBaieNDHpbElV1/o4NmMyXFzcWZ5/fikREcKQIS1o0aIULVqUCnRYxgRUWolgIBADjEphngLNfRKRMT6yePEu7rtvPtu2HaVfvzrnSwXGhLtUE4Gqxrj/m/kvHGMy3rFjp3nqqe8ZO3Yt5csXYNGi262raGM8pHuHjIjcLCJ53eGBIjJTRGr7PjRjMsa+fbF89NEGnngimrVre1oSMCYZb26VfFFVT4jINUAn4GPgfW9W7j6/YLOIbBWRgWksd4uIqIhEexe2MWk7eDCOd9/9FYAqVQrxxx+9eeONluTKlS3AkRmT+XiTCJKuEuoMvK+qnwM50nuRe7/BKKAjUA3oJiLVUlguL84jMH/2NmhjUqOqTJ26kapVJ/L444v5/Xfnyme7IsiY1HmTCPaJyCigKzBXRLJ7+boGwFZV3a6qZ4DpQJcUlnsZGALEexmzMSnavfs4118/m+7dv6JChQKsWnW3dRJnjBe8+UG/Hfge6KSqR3D6Hkq1msdDCWC3x/ged9p5IlIPKKWqX6W1IhHpLSIrRGTFwYMHvdi0CTfnziXSsuUMvvtuF2+/3YqlS7tRvbp1k2WMN7zphjpWRNYDLUWkJfA/Vf36cjcsIlmAYUAvL2IYC4wFiI6OtgfDmvP++OMYpUrlJWvWLLz/fnvKlctPuXIFAh2WMUHFm6uGHgQ+AUq7fzNFpJ8X694LeN6pU9KdliQvUANYLCJ/AI2AOdZgbLxx7lwib775C1WrTmT0aKeTuLZty1gSMOYSePuEsgaqGgsgIq8BPwKj03ndL0BFESmLkwC64vGIS1U9hkcX1yKyGHhCVVdczA6Y8LN27UFiYr5hxYr9dOlSgVtuqRTokIwJat4kAgHOeIyfdaelSVXPuaWJeTgPvJ+gqutFZDCwQlXnXErAfrGgGfy9Mv3lEs86/+3uVL8ZPXoVDz/8HQUL5mDGjM7cdltluzvYmMvkTSKYAvwsIp/iJIAbgcnerFxV5wJzk017PpVlW3qzTr/4eyXkrwFFW6a/bJ7yEBHp85DCXVJ3EDVqFKZr1yq8/XZLChe2S0KNyQjeNBYPdattmuL0MdRHVX/xdWABV7Ql1B0a6CjC3smTZ3j22aVkzSq88UZLmjcvRfPm1kmcMRnJ24ewxgOnPf4b43MLF+6kZs3JDB++ktOnE1C1C8aM8QVvrhoaBEwDiuFc+TNVRJ72dWAmfB09Gs99982jbdtPyJo1C0uWdGXEiDbWFmCMj3jTRnA3UFdV4wBE5FVgFc4Da4zJcPv3xzF9+ib+858GvPBCY3LmtP6BjPElbxLBvmTLZXWnGZNh9u8/yfTpm3j44fpUrnwFf/xxvzUGG+Mn3iSCv4H1IjIPp7G4PfCLiAwDUNXHfBifCXGqyscfb+ThhxcRG3uWTp3KUbFiQUsCxviRN4ngK/cvyU8+isWEmV27jtOnzwK+/noHjRsXZ/z4DlSsWDDQYRkTdry5fHS8PwIx4SWpk7gDB+IYMaI1/frVISLC24vYjDEZyZsSgTEZZvv2o5Qpk4+sWbPwwQftKV++AFFR+QMdljFhzU7BjF+cO5fIkCE/U63aREaNcjqJa9OmjCUBYzIBr0sEIpJDVe1mMnPRVq8+QEzMPH79dT833VSR226zTuKMyUy8uaGsgYj8Bmxxx2uLyLs+j8yEhJEjf+Xqqz9i794TzJp1A//9bxeKFcsT6LCMMR68qRoagfO84sMAqroGaOXLoEzwS+oOolatInTvXpUNG+6x7qKNyaS8qRrKoqo7k93en5Dawia8xcaeYdCgH8iWLQtvvmmdxBkTDLwpEewWkQaAikiEiDwC/O7juEwQmj//D2rUmMS77/7K2bOJ1kmcMUHCmxJBX5zqodLAfuBbd5oxABw5Es9jj33HpEnrqVz5CpYs6UrTpiUDHZYxxkve3FB2AOcxk8ak6MCBOGbN+p2nn27I8883JjLSbk8xJpik+40VkQ9w+hj6F1Xt7ZOITFD466+TTJu2kUcfjXY7ietNoUI5Ax2WMeYSeHPq9q3HcCRwE7DbN+GYzE5V+fDD9Tz66GLi4s7SuXN5KlYsaEnAmCDmTdXQDM9xEZkC/OCziEym9ccfx3jggQXMn/8HTZqUYNy49tZJnDEh4FIqc8sCRTM6EJO5nTuXSKtWMzh06BSjRrWhT586ZMliTwwzJhR400ZwhH/aCLLgPJ9goC+DMpnH1q1HKFs2P1mzZmHChGspVy4/ZcpY/0DGhJI07yMQ5y6y2kAR96+gqpZT1Zn+CM4EztmzCbz22k9Urz7pfCdxrVqVtiRgTAhKs0Sgqioic1W1hr8CMoH366/7iYmZx+rVB7jttkrccUflQIdkjPEhb+4sXi0idX0eickURoz4lQYNPuKvv07y3/92YebMGyhaNHegwzLG+FCqJQIRyaqq54C6OM8o3gacBASnsFDPTzEaP1BVRIS6da/k7rur89ZbLSlYMDLQYRlj/CCtqqHlQD3gBj/FYgLgxIkzPP30EnLkiOCtt1rRrFlJmjWz7iGMCSdpJQIBUNVtforF+Nk33+zggQfms3v3CR55pP75UoExJryklQiKiMhjqc1U1WE+iMf4weHDp3jsse/48MMNVK16BUuX3knjxsUDHZYxJkDSSgQRQB7ckoEJHYcPn2L27K0891wjBg1qRI4c1kmcMeEsrV+Afao6+HJWLiLXAu/gJJVxqvp6svmPAfcB54CDwL2quvNytmlStm9fLB9/vJHHH4+mUqUr2LmztzUGG2OAtC8fvaySgIhEAKOAjkA1oJuIVEu22CogWlVrAbOAoZezTXMhVWXChN+oWnUizz23lK1bjwJYEjDGnJdWImhzmetuAGxV1e2qegaYDnTxXEBVv1PVOHf0J8AuV8lAO3YcpX37WcTEzKN27SKsWXO3dRJnjLlAqlVDqvr3Za67BP/urnoP0DCN5WOAr1OaISK9gd4ApUuXvsywwsO5c4m0bj2Tw4fjee+9tvTuXds6iTPGpChTtBKKyF1ANNAipfmqOhYYCxAdHW0Pwk3Dli1HKFfO6SRu4sRrKV++AKVK5Qt0WMaYTMybLiYu1V6glMd4SXfav4hIW2AQcIOqnvZhPCHt7NkEXnllGTVqTGLkyFUAtGxZ2pKAMSZdviwR/AJUFJGyOAmgK3Cn5wJuH0bvA9e6z0Y2l2DFir+IiZnH2rUH6dq1Ct26VQl0SMaYIOKzRKCq50TkQWAezuWjE1R1vYgMBlao6hzgDZx7FT5x72jdparWpcVFeOedlTz22GKuuio3n39+IzfcUCHQIRljgoxP2whUdS4wN9m05z2G2/py+6EsqTuI6OiriImpydChzSlQwC4JNcZcvEzRWGy8d/z4af7znyVERmbl7bdb0aRJCZo0KRHosIwxQcyXjcUmg82du53q1ScxduxasmYVVO0CKmPM5bMSQRA4dCiORx75jo8/3kj16oWYNetOGjYsFuiwjDEhwhJBEDhy5DRffLGNF15ozDPPNCJ79ohAh2SMCSGWCDKpvXtP8PHHG3nyyaupWLEgO3f2tsZgY4xPWBtBJqOqfPDBWqpVm8iLL/7Itm1OJ3GWBIwxvmKJIBPZtu0obdrMpHfv+dSrV5S1a3tSoYJ1EmeM8S2rGsokzp1LpE2bmfz9dzzvv9+O++6rZZ3EGWP8whJBgG3e/Dflyxcga9YsTJ7ckfLlC1CyZN5Ah2WMCSNWNRQgZ84k8NJLP1Kz5iRGjXI6iWvRopQlAWOM31mJIACWL99HTMw81q07xJ13VqV796qBDskYE8YsEfjZ8OErefzxxRQrlpsvvriJzp3LBzokY0yYs0TgJ0mdxDVocBX331+LIUOakz9/jkCHZYwxlgh87dix0zz11PfkzJmV4cNbc801JbjmGuskzhiTeVhjsQ998cU2qlWbyLhxv5EjR4R1EmeMyZSsROADBw/G8fDDi5g2bRM1axbms8+6cPXV1kmcMSZzskTgA8eOnWbu3B289NI1DBzY0DqJM8ZkapYIMsju3cf56KONDBzYgAoVnE7irDHYGBMMrI3gMiUmKmPGrKZ69Um88sqy853EWRIwxgQLSwSXYcuWI7RuPYO+fb+lQYOr+O23XtZJnDEm6FjV0CU6dy6Rdu0+4ejR04wf34F77qmBiHUSZ4wJPpYILtLGjYepWLEgWbNmYcqUTpQvX4DixfMEOixjjLlkVjXkpdOnz/HCC0upVWsyI0c6ncQ1a1bSkoAxJuhZicALP/30JzEx89iw4TA9elSjR49qgQ7JGGMyjCWCdLz11i88+eT3lCyZl7lzb6Zjx3KBDskYYzKUJYJUJCY69WaNGxenT5/avP56c/Lls0tCjTGhxxJBMkePxvP4mC7kKpqHd6djncQZY0KeNRZ7+OyzLVSrNpHJS+qQN5daJ3HGmLBgiQA4cOAkt98+h5tu+pyiRXOz/NWxvDbgpN0XYIwJC5YIgOPHz7BgwU5efbUpy5d3p17ZfYEOyRhj/CZs2wh27TrOlCkbeOaZhlSoUJBdux4gb97sgQ7LGGP8zqclAhG5VkQ2i8hWERmYwvwcIjLDnf+ziET5Mh5wOokbPXoV1atP5LXXfjrfSZwlAWNMuPJZIhCRCGAU0BGoBnQTkeR3YsUAR1S1AvA2MMRX8QBs3vw3LVvOoH//hTRuXJz16++xTuKMMWHPl1VDDYCtqrodQESmA12ADR7LdAFedIdnASNFRNQHl+uc2zyeDs13cexkdiY+spyebSYi61+E9SksnHAqozdvjDGZli8TQQlgt8f4HqBhasuo6jkROQYUAg55LiQivYHeAKVLl76kYLLmLsxHz/9A+eKnKVY4O04hJRX5a0Dp2y5pO8YYE2yCorFYVccCYwGio6MvrbRQsgtN+3fJyLCMMSYk+LKxeC9QymO8pDstxWVEJCuQHzjsw5iMMcYk48tE8AtQUUTKikh2oCswJ9kyc4Ce7vCtwCJftA8YY4xJnc+qhtw6/weBeUAEMEFV14vIYGCFqs4BxgNTRGQr8DdOsjDGGONHPm0jUNW5wNxk0573GI4HrFXWGGMCyLqYMMaYMGeJwBhjwpwlAmOMCXOWCIwxJsxJsF2tKSIHgZ2X+PLCJLtrOQzYPocH2+fwcDn7XEZVi6Q0I+gSweUQkRWqGh3oOPzJ9jk82D6HB1/ts1UNGWNMmLNEYIwxYS7cEsHYQAcQALbP4cH2OTz4ZJ/Dqo3AGGPMhcKtRGCMMSYZSwTGGBPmQjIRiMi1IrJZRLaKyMAU5ucQkRnu/J9FJMr/UWYsL/b5MRHZICJrRWShiJQJRJwZKb199ljuFhFREQn6Sw292WcRud39rNeLyFR/x5jRvDi2S4vIdyKyyj2+OwUizowiIhNE5ICIrEtlvojICPf9WCsi9S57o6oaUn84XV5vA8oB2YE1QLVky/QDxrjDXYEZgY7bD/vcCsjlDvcNh312l8sLLAF+AqIDHbcfPueKwCqgoDt+ZaDj9sM+jwX6usPVgD8CHfdl7nNzoB6wLpX5nYCvAQEaAT9f7jZDsUTQANiqqttV9QwwHUj+jMouwGR3eBbQRkTEjzFmtHT3WVW/U9U4d/QnnCfGBTNvPmeAl4EhQLw/g/MRb/b5fmCUqh4BUNUDfo4xo3mzzwrkc4fzA3/6Mb4Mp6pLcJ7PkpouwIfq+AkoICLFLmeboZgISgC7Pcb3uNNSXEZVzwHHgEJ+ic43vNlnTzE4ZxTBLN19dovMpVT1K38G5kPefM6VgEoislREfhKRa/0WnW94s88vAneJyB6c558M8E9oAXOx3/d0BcXD603GEZG7gGigRaBj8SURyQIMA3oFOBR/y4pTPdQSp9S3RERqqurRgEblW92ASar6log0xnnqYQ1VTQx0YMEiFEsEe4FSHuMl3WkpLiMiWXGKk4f9Ep1veLPPiEhbYBBwg6qe9lNsvpLePucFagCLReQPnLrUOUHeYOzN57wHmKOqZ1V1B/A7TmIIVt7scwwwE0BVlwGROJ2zhSqvvu8XIxQTwS9ARREpKyLZcRqD5yRbZg7Q0x2+FVikbitMkEp3n0WkLvA+ThII9npjSGefVfWYqhZW1ShVjcJpF7lBVVcEJtwM4c2x/RlOaQARKYxTVbTdn0FmMG/2eRfQBkBEquIkgoN+jdK/5gB3u1cPNQKOqeq+y1lhyFUNqeo5EXkQmIdzxcEEVV0vIoOBFao6BxiPU3zcitMo0zVwEV8+L/f5DSAP8InbLr5LVW8IWNCXyct9Dile7vM8oL2IbAASgCdVNWhLu17u8+PAByLyKE7Dca9gPrETkWk4ybyw2+7xApANQFXH4LSDdAK2AnHAPZe9zSB+v4wxxmSAUKwaMsYYcxEsERhjTJizRGCMMWHOEoExxoQ5SwTGGBPmLBEYvxCRBBFZ7fEXlcayUan1vOhvIhItIiPc4ZYico3HvD4icrcfY6lzKT1rikgxEfnSY3ya22vloxexjpoiMulit22CQ8jdR2AyrVOqWifQQVws9wa0pJvQWgKxwI/uvDEZvT0Ryer2f5WSOjjdg8y9yNU+Bnzgrv8q4GpVrXCRMf0mIiVFpLSq7rrI7ZtMzkoEJmDcM///iciv7t81KSxTXUSWu6WItSJS0Z1+l8f090UkIoXX/iEiQ0XkN3fZCh7bXST/PJuhtDv9NhFZJyJrRGSJO62liHzplmD6AI+622wmIi+KyBMiUkVElifbr9/c4foi8r2IrBSReSn1Eikik0RkjIj8DAwVkQYiskyc/vV/FJHK7l21g4E73O3fISK5xem7frm7bEq9rwLcAnzjDs8HSnjsw2IReccdXyciDdyYXhSRKSKyFJjivvYLgvzmS5OKQPe9bX/h8Ydzl+tq92+2Oy0XEOkOV8S5UxQgCrcvduBdoLs7nB3ICVTF+VHK5k4fDdydwjb/AAa5w3cDX7rDXwA93eF7gc/c4d+AEu5wAfd/S4/XvQg84bH+8+PufpV1h/8DPItzN+iPQBF3+h04d8Ymj3MS8CUQ4Y7nA7K6w22BT93hXsBIj9e9BtyVFC9Ov0K5k627LLDSY/z8e+uOLwY+cIebe7zvLwIrgZweyzYBvgj0sWR/Gf9nVUPGX1KqGsoGjBSROjiJolIKr1sGDBKRksB/VXWLiLQB6gO/uN1l5ARS6z9pmsf/t93hxsDN7vAUYKg7vBSYJCIzgf9ezM7hdHp2B/C6+/8OoDJOx3cL3DgjgNT6hPlEVRPc4fzAZLf0o7jdC6SgPXCDiDzhjkcCpYGNHssUI/1+d6aB0w++iOQTkQLu9DmqespjuQNA8XTWZYKQJQITSI8C+4HaONWUFzw8RlWnulUm1wFzReQBnCczTVbVp73YhqYyfOGCqn1EpKG7rZUiUt+73QBgBk4/Tv91VqVbRKQmsF5VG3vx+pMewy8D36nqTW6V1OJUXiPALaq6OY31nsJJEGlJ/r4kjZ9MNj3SXZ8JMdZGYAIpP7BPnX7je+CcMf+LiJQDtqvqCOBzoBawELhVRK50l7lCUn8G8x0e/5e5wz/yT113d+B/7nrKq+rPqvo8zlm0Z1e/ACdwure+gKpuwynVPIeTFAA2A0XE6SMfEckmItVTidNTfv7pVrhXGtufBwwQt7ghTg+zyf2OUx2Uljvc1zfF6cnyWCrLVQIyxdVcJmNZIjCBNBroKSJrgCpceAYKcDuwTkRW41SzfKiqG3Dq4OeLyFpgAU4VSEoKuss8jFMCAecJVve403u48wDecBuW1+EkizXJ1vUFcFNSQ2sK25oB3MU/feOfwenmfIi7j6uBCxrEUzAU+D8RWcW/S+3fAdWSGotxSg7ZgLUist4d/xdVPQlsS2ooT0W8u60xOH37p6YVECpPezMerPdRE7LEeSBNtKoeCnQsgSQiNwH1VfXZFOYtxmnwTvM5DSKSA/geaKqpX95qgpS1ERgT4lR1tohc7jO5SwMDLQmEJisRGGNMmLM2AmOMCXOWCIwxJsxZIjDGmDBnicAYY8KcJQJjjAlz/w9+q2PfMMm0sQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a function for plotting ROC curves\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_roc_curve(fpr, tpr):\n",
    "    \"\"\"\n",
    "    Plots a ROC curve given the false positive rate (fpr)\n",
    "    and true positive rate (tpr) of a model.\n",
    "    \"\"\"\n",
    "    # Plot roc curve\n",
    "    plt.plot(fpr, tpr, color=\"orange\", label=\"ROC\")\n",
    "    # Plot line with no predictive power (baseline)\n",
    "    plt.plot([0, 1], [0, 1], color=\"darkblue\", linestyle=\"--\", label=\"Guessing\")\n",
    "    \n",
    "    # Customize the plot\n",
    "    plt.xlabel(\"False positive rate (fpr)\")\n",
    "    plt.ylabel(\"True positive rate (tpr)\")\n",
    "    plt.title(\"Receiver Operating Characteristic (ROC) Curve\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "plot_roc_curve(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8669181034482759"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# area under the curve, max area = 1\n",
    "roc_auc_score(y_test, y_probs_positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZxN9f/A8dfbDIasobKPnbFn7MXImkR7qbT8JhLaVUpJUt9SISFEVEKrkhSypLRoFGLssiaGbIOxzLx/f5wzusYsd5g7d+7c9/PxmMec7Z7zPveee9/n8/mc8zmiqhhjjAleefwdgDHGGP+yRGCMMUHOEoExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhJBDiMid4jIPH/HkZOISLyIVPbDdsNFREUkNLu37QsiskZEos7jded9TIpIRxH54nxee75EJL+IrBORUtm53UBmiSAdIrJVRI67P0T/iMgUESnky22q6oeq2sGX2/AkIi1EZKGIHBGRQyLylYhEZNf2U4lnsYjc5zlNVQup6hYfba+6iHwiIvvc/V8lIo+JSIgvtne+3IRU9ULWoaq1VXVxBts5J/ld4DH5EvCKx/pVRI6636ldIjI85XstIl1EZJm73H4R+VBEyqVYprSITBKR3e6xu05EXhCRi1T1BPAuMCCDfQ2Izz47WCLI2LWqWghoADQEnvZzPOcltbNaEWkOzAO+BMoAlYCVwFJfnIHntDNrEakC/ArsAOqqalHgZiASKJzF2/Lbvvtr2yLSGCiqqr+kmFXf/U61Bm4F/s/jNTcB04CRQEmgNnAC+FFEirvLXAz8DBQAmqtqYaA9UAyo4q5qGnC3iORPI7Ys/exz2rGdaapqf2n8AVuBdh7jw4CvPcbzA68D24E9wDiggMf8bsAK4DCwGejkTi8KTAJ2A7uAoUCIO+8e4Ed3+G3g9RQxfQk85g6XAT4D4oC/gIc8lhsMfApMdbd/Xyr79wMwNpXp3wDvu8NRwE7gGWCf+57c4c174PHap4B/gA+A4sBsN+YD7nA5d/mXgEQgAYgHRrvTFajqDk8BxgBfA0dwvsxVPOLpAKwHDgFjge9T23d32amen2cq88Pdbd/t7t8+YKDH/CY4P0gH3c9yNJDPY74CfYGNwF/utDdxfnwOA8uBKz2WD3Hf583uvi0HygNL3HUddd+XW93lu+AcXweBn4B6KY7dp4BVOD+koXgcz27sMW4ce4Dh7vTt7rbi3b/meByT7jK1gfnAv+5rn0nj/RsETEwx7cxn6Y5/DIxxhwXYBjyZ4jV5gNXAEHd8KPAnkCeD7+9GoPV5fvZRwM60fg849/s1CDgOXOyxfEP3mMnrjv8fsBbnuJ8LVMzu37Q099ffAeTkvxQffDn34HvTY/4IYBZwMc5ZxFfA/9x5TXB+jNq7B3JZoKY7byYwHrgIuARYBtzvzjvzpQNa4fxoiDte3D3YyrjrXO4egPmAysAWoKPHgXoKuM5dtkCKfSuI86PbJpX9vhfY7Q5HAaeB4Tg/+q1xfpBqePEeJL/2Vfe1BYASwI3u9gsDnwBfeGx7MSl+uDk3Eex3399Q4ENghjuvpPulvMGd97D7HqSVCP4B7k3n8w93t/2OG3t9nB/VWu78RkAzd1vhOF/yR1LEPd99b5KT453uexAKPO7GEObOewLnGKuB86NYHyiR8j1wxxsCe4GmOAnkbpzjNb/HsbsCJ5EU8JiWfDz/DPRwhwsBzVLsc6jHtu7hv2OyME7SexwIc8ebpvH+fQI8kc5nWdNd16Me4wpUSmVdLwA/u8O/AC948f2dhcfJUSY/+ygyTgRnfb+AhUBPj+VfA8a5w92ATUAt97N/FvjJ379xZ2L1dwA5+c/94ONxzs4UWAAUc+cJzg+i59loc/478xsPjEhlnZfi/Jh4lhy6A4vcYc8vneCcobVyx3sCC93hpsD2FOt+Gpis/x2oS9LZt3LuPtVMZV4n4JQ7HIXzY36Rx/yPgee8eA+igJO4P3RpxNEAOOAxvpiME8FEj3mdgXXu8F3JPxYe79+OlOvzmH8Kt5SWxvxwd9vlPKYtA25LY/lHgJkp4r4qg2PsAE5VCTglmW5pLJcyEbwNvJhimfW4Z8Dusft/qRzPyT9kS3B+XEumsc9pJYLuwB9efn/mA71T2Y/D7nGjwHT+S15XuNPOOV6A3sBGd3hjyvWmsf0PgUHn+dlHkXEiWJJi/n389/1MPvaSv7vfANEey+YBjpFDSgXWRpCx69Spg4zCOWMp6U4vhXNWu1xEDorIQeBbdzo4Z2KbU1lfRSAvsNvjdeNxSgZnUeeImYHz5QO4HefgTl5PmeR1uOt5BifRJNuRzn4dAJKA0qnMK41TpD2zrKoe9RjfhlMqyeg9AIhT1YTkEREpKCLjRWSbiBzG+UEqlskGun88ho/hnNHixnRmn933b2c669lP6vvv1fbcxsbZ7oUEh4GX+e/4SHbWZyAi/UVkrds4eRCnmjD5NWkdM6mpCDye4vMvj/MepLrtFKKB6sA6EflNRLp4ud3MxHiA1OvbL8d5D2/FOaG5yJ2efMxldEx6+7kVxqk2S42360hPyvf3M6C5iJTGKc0n4VS/gvN5venxWf2LkyzKXmAMWcISgZdU9Xucs9HX3Un7cKppaqtqMfevqDqNYOAcJFXOXRM7cEoEJT1eV0RVa6ex6enATSJSEedL85nHev7yWEcxVS2sqp09w05nf47iVA/cnMrsW3BKP8mKi8hFHuMVgL+9eA9Si+FxnKqPpqpaBOcLA86XIt2YvbAbp6TjrFBEPMdT8R1ONdX5ehtYB1Rz9+UZ/tuPZGf2R0SuBJ7EeX+Lq2oxnOrD5NekdcykZgfwUorPv6CqTk9t2ymp6kZV7Y5zAvIq8Kn7GWf0/u/AqYb0xiqcZJPa9lVVP8Y5Bge5k9fjJO6zjkkRyYPzOSUfk98B17vT01ML5+KH1GT02R/FOclJjiGEs09wIMV7paoHcC6+uBXnpG2GezICzvt2f4rPq4Cq/pTBPmQLSwSZMxJoLyL1VTUJp+54hIhcAiAiZUWko7vsJOBeEWkrInnceTVVdTfOwfKGiBRx51URkdapbVBV/8D5wZ0IzFXV5DOcZcAREXlKRAqISIiI1HGv1PDWAJwrKx4SkcIiUlxEhuJU77yQYtkXRCSf+2PWBfjEi/cgNYVxksdB9+qP51PM34P3PzQpfQ3UFZHr3Ks4+gKXpbP880ALEXlNRC5z468qIlNFpJgX2yuMU80RLyI1gQe8WP40TkN5qIgMAop4zJ8IvCgi1cRRT0RKuPNSvi/vAL1FpKm77EUico2IeHXFi4jcKSKl3M8w+ZhKcmNLIu3PYDZQWkQeEed6/cIi0jSNZefgtCml5xWgp4hc5v5o9geeFZHbRSTM/Vwm4rxPI9zXDHfH33NPkJKPu+EiUi95HKdtJuUVS8ky+uw3AGHue5oXp04/1SuQUpiGU0WZfPVTsnHA0yJS291WURFJ7STMLywRZIKqxgHv898ZzFM4DUC/uFUD3+Gc7aKqy3AaXUfgnPV9j1M8BOdAyQfE4hSfPyX9Yuo0oB0eB5aqJuL8IDfAuWIoOVkUzcT+/Ah0xGlc3Y1T5dMQuEJVN3os+o8b5984VVO9VXVdRu9BGkbiNKztw/mSfpti/ps4JaADIjLK231x92cfztnkMJyifwTOlTEn0lh+M07SCwfWiMghnBJXDE67UEb645z5HcH5Yf4og+Xn4uzvBpz3OoGzqxeG47S/zMNJMJNw3itw6qTfc6sWblHVGJw2o9E4n80mnLp8b3XC2ed4nPf8NlU9rqrHcK7eWupuq5nni1T1CM4FENfiHBcbgTapbUBVfwcOpZMoUNU/caoHn3DHPwJ6AI/ifIax7nvQUlX3u8v8C7TAqef/VUSO4JQWDrnvAzify3vq3FOQ2nbT/exV9RDQB+c7tQunhJBeNWOyWUA14B9VPVMaUdWZOCWvGe73ZDVwtRfryxbJV6MYkypx7kSdqqrpVbHkSG7VwU6cy10X+TueYCQiHYA+qnpdNm4zP06VUCtV3Ztd2w1kgX0ThDEpuNVSv+JUPz2BU/+eVvWA8TFVnYdTwsnObZ7AubDDeMmqhkxu0xznqpZ9ONUX16nqcf+GZEzOZlVDxhgT5KxEYIwxQS7g2ghKliyp4eHh/g7DGGMCyvLly/epaqpdcwdcIggPDycmJsbfYRhjTEARkW1pzbOqIWOMCXKWCIwxJshZIjDGmCBnicAYY4KcJQJjjAlyPksEIvKuiOwVkdVpzBcRGSUim8R5aPTlvorFGGNM2nxZIpiC08NhWq7G6aWvGtALp293Y4wx2cxn9xGo6hIRCU9nkW44D0hXnC6Mi4lIabe//qy3aQJsnZbxcsYYk8McPR5C3KF8hNeqAY1GZvn6/dlGUJaz+2LfSRqPbRORXiISIyIxcXFx57e1rdPgwIrze60xxvjJwj9KUO/+VtzwQiRJSb7ZRkDcWayqE4AJAJGRkeffS17xBtBucRZFZYwxvnPwYAJPPPE9Eyf+SdWqxRgxsSN5Gpf3ybb8mQh24TwIO1k5d5oxxgS1xMQkWrSYxvr1B3jyycYMHtyCAgXy+mx7/kwEs4B+IjID56Hsh3zWPmCMMQFg//7jXHxxGCEheXjppSspX74wkZHpPXY7a/jy8tHpwM9ADRHZKSLRItJbRHq7i8wBtuA8Y/QdnOeDGmNM0FFVpk6NpXr1SUyc+CcA119fLVuSAPj2qqHuGcxXoK+vtm+MMYFgx47D9O49nzlz/qJZs9K0bFkm22MIiMZiY4zJjaZPX8v9988nMTGJkSPb0K9fQ0JCsv9iTksExhjjJ8WLh9G0aWkmTGhPpUrF/BaHJQJjjMkmp08nMWJEDCdPJjFwYDM6dapEx47hiIhf47JEYIwx2WDlyr1ER89l+fI93HJLDVQVEfF7EgDrfdQYY3zqxInTPPfcj0RGTmXHjiN88sm1zJjRJUckgGRWIjDGGB/auPEAr766jNtvr8nw4W0oUaKAv0M6hyUCY4zJYvHxJ/nyy03ccUcEdeqUYt26/6NyZf81BmfEqoaMMSYLzZ+/lbp1p9CjxxzWrt0PkKOTAFgiMMaYLHHgQALR0d/SocOn5MsXwvff30atWiX8HZZXrGrIGGMuUGJiEi1bTmPDhgM8/XRTBg1qTlhY4Py8Bk6kxhiTw+zbd4yLLy5ASEgeXn75SipUKMLll1/q77AyzaqGjDEmk1SV999fQ/Xq7zJx4ioArruuWkAmAbASgTHGZMq2bYe4//75zJ27lRYtytCqVTl/h3TBLBEYY4yXpk6N5YEH5qMKb711FX36NCRPnpxzY9j5skRgjDFeKlWqAC1blmX8+PZUrFjU3+FkGUsExhiThlOnEnnjjRhOnUriueea07FjJTp08H8ncVnNGouNMSYVf/yxh6ZNP+Tpp38gNnY/zrO0yHVJACwRGGPMWRISTvPMMz/QuPFU/v47ns8+68r06Tmrk7isZlVDxhjjYdOmA7z++m/cdVdt3ngjiuLFw/wdks9ZIjDGBL34+JPMnLmRHj1qU6dOKdav/z+/PjEsu1nVkDEmqM2d+xe1a0/m7ru/OdNJXDAlAbBEYIwJUvv3H+fuu+fQqdNnFCyYlx9+6B4wncRlNasaMsYEHaeTuOls2nSAgQOb8eyzzQKqk7isFrx7bowJOnFxxyhRwukk7tVXW1GxYhEaNLjE32H5nVUNGWNyPVVl8uQ/qV59Eu+843QS161bVUsCLisRGGNyta1bD9Gr1zzmz9/GlVeWo02b8v4OKcexRGCMybU++GANDzzwHSIwdmw77r+/fq7oJC6rWSIwxuRal156Ea1alWPcuPZUqFDE3+HkWJYIjDG5xqlTiQwb9huJiUkMGtSCDh3C6dAh3N9h5XjWWGyMyRV+/30PjRtP5dlnf2T9+gNnOokzGbNEYIwJaMePn2LAgCU0aTKVPXuOMXNmNz788Jpc3UlcVvNpIhCRTiKyXkQ2iciAVOZXEJFFIvKHiKwSkc6+jMcYk/ts2XKI4cNjuOeeOsTG3st111Xzd0gBx2eJQERCgDHA1UAE0F1EIlIs9izwsao2BG4DxvoqHmNM7nH48AmmTFkNQO3aJdm4MZqJEzsGRU+hvuDLEkETYJOqblHVk8AMoFuKZRRIbsovCvztw3iMMbnAnDlbqFNnCtHRc890EpebHhvpD75MBGWBHR7jO91pngYDd4rITmAO8GBqKxKRXiISIyIxcXFxvojVGJPD7dt3jB495nDNNZ9TuHA+li4N3k7ispq/G4u7A1NUtRzQGfhARM6JSVUnqGqkqkaWKlUq24M0xvhXcidxM2asY9Cg5vz+ew+aNSvj77ByDV/eR7AL8LyXu5w7zVM00AlAVX8WkTCgJLDXh3EZYwLEnj1HKVWqICEheXj99SgqVixCvXp2MpjVfFki+A2oJiKVRCQfTmPwrBTLbAfaAohILSAMsLofY4KcqjJp0p/UqPEuEyasBODaa6tYEvARn5UIVPW0iPQD5gIhwLuqukZEhgAxqjoLeBx4R0QexWk4vkftLhBjgtqWLQfp2XMeCxdup3XrcrRrV9HfIeV6GSYCEWkAXAmUAY4Dq4EFqnooo9eq6hycRmDPaYM8hmOBlpmM2RiTS7333mr69PmOkJA8jBvXnp4961kncdkgzaohEekhIjHAC0BxYBtwGGgHLBaRSSJSLnvCNMYEgzJlCnHVVRWIjb3XegrNRumVCC4GWqvq0dRmikgkUAvnslBjjMm0kycTeeWVX0lKUgYPbkn79uG0bx/u77CCTpolAlV9E0gQkYfSmB+jqvN9FpkxJlf77bfdNGr0Ac8//xNbthyyTuL8KN2rhlQ1EeiRTbEYY4LAsWOn6N9/Mc2aTePAgQRmzbqe99/vbJ3E+ZE3Vw39ICIjgY+AM9VEqrrKZ1EZY3Ktv/46xFtv/UHPnvV49dVWFC2a398hBT1vEkFj938jj2kKtMr6cIwxudGhQyf4/PMN3HtvXWrXLsmmTdGUL29PDMspMkwEqnpldgRijMmdvv56M/ffP5/du4/SvHkZatYsYUkgh8nwzmIRKS4iw0VkmYj8KiJviEjx7AjOGBO44uKOcccdX9Oly0yKFw/j559vp2ZN6yQuJ/KmamgG8Atwhzt+O057QQdfBWWMCWyJiUlcccV0/vrrEC+80IIBA5qSL1+Iv8MyafAmEZRV1ec9xl8QkdW+CsgYE7j++ecol1zidBL3xhtRhIcXoU4d6x8op/Om07kFInJT8oiI3ADY/QPGmDOSkpTx41dSvfokxo93Oonr0qWKJYEA4U2J4C7gQRE55Y7nBQ6JyD2AqurFvgrOGJPzbdp0gJ4957F48Q6uuqoCHTuG+zskk0neJIKSPo/CGBOQJk/+kz59FpAvXx7eeacD0dF17cawAORN1dA3qpro+ec5zdcBGmNyrgoVitCxYzixsfdy3331LAkEqDRLBO7DZMKAS0WkMJD8CRcBKmRDbMaYHObEidP8739OJ3FDhlxB27YVadvWnhcQ6NKrGuoLPAZcAqzhv0RwGBjn47iMMTnMr7/uJjr6W9as2c/dd9dGVa0EkEukmQhUdQQwQkQeUdWR2RiTMSYHOXr0JM89t5SRI5dTtmxhZs++nmuuqeLvsEwWSu/BNM0A0koCIlJIRCJ8FZgxJmfYtu0wY8euoHfv+qxZc48lgVwovaqh20XkNeAbYDnOQ+XDgKpAG/d/f59HaIzJdgcPJvDppxu47756RESUZNOm+yhXrrC/wzI+kl7V0EMiUhK4GeeZBKVxnlm8FnhPVRdnS4TGmGz15ZebeOCB+ezde4wrrihLzZolLAnkcuneR6Cq+4C33T9jTC62d+9RHnpoIR99tJ569Uoxa9b11klckPDmhjJjTC6XmJhEy5bT2b79CEOHXsGTTzYmb17rJC5YWCIwJoj9/Xc8l112ESEheXjzzasIDy9CRIR1JhBsvLmz2BiTyyQlKW+/vYKaNd9l3LgVAHTuXNmSQJDy5sE0BUTkaREZ545XFZGrfR+aMcYXNmz4lzZtPqJPn+9o2rQ0V19dyd8hGT/zpkTwLs5dxVe4438DL/ssImOMz0ya9Cf167/PqlVxvPtuR+bNu4lKlYr5OyzjZ960EVRT1e4icjOAqh4Tu6/cmIAUHl6Eq6+uxJgxbSldupC/wzE5hDeJ4KSIhAEKICKVgJM+jcoYkyVOnDjNiy/+AsDQodZJnEmdN1VDLwLfAuVE5D1gEfCMT6Myxlywn37aRYMG7/PSS7+we3c8qurvkEwOlWGJQFW/EZEYoAVOW8ETqrrX55EZY85LfPxJBg78kbfe+p3y5Qvz7bc30rGjNQibtHlz1dA8VY1T1S9V9QtV3Ssi87xZuYh0EpH1IrJJRAakscwtIhIrImtEZFpmd8AYc7bt2w8zfvxK+vZtyOrV91oSMBny2YNpRCQEGAO0B3YCv4nILFWN9VimGvA00FJVD4jIJee9J8YEsQMHEvjkk/X06lWfiIiSbNnSkzJlrDHYeMeXD6ZpAmxS1S0AIjID6AbEeizTExijqgcArMrJmMybOXMjffp8R1zcMVq3Lk+NGhdbEjCZkmbVkKqOUNXywFOqWkFVy7t/tb18UE1ZYIfH+E53mqfqQHURWSoiv4hIp9RWJCK9RCRGRGLi4uK82LQxud8//xzl5ptnccMNX3LZZRexbNmd1Khxsb/DMgHIm8bikSJSE4jAqSpKnp4V9fmhQDUgCigHLBGRuqp6MEUME4AJAJGRkXbpgwl6iYlJXHnldHbsOMLLL19J//6R1kmcOW8ZJgIReRboANQE5gIdgR+BjBLBLqC8x3g5d5qnncCvqnoK+EtENuAkht+8it6YILNz5xHKlClESEgeRo26ikqVilpX0eaCeXMfwa04TyTbrao9gPrARV687jegmohUchuebwNmpVjmC5zSAO5DcKoDW7wL3ZjgkZSkvPXW79Ss+S5vv+10Enf11ZUtCZgs4c2dxcdVNVFETrtXD/0DZHhroqqeFpF+OKWIEOBdVV0jIkOAGFWd5c7rICKxQCLOPQr7z3tvjMmF1q3bz333zWPp0l107BhOly6V/R2SyWW8SQR/iEgxnM7nYnCuGlrmzcpVdQ4wJ8W0QR7DinNl0mPeBmxMMJk4cRX9+i2gYMG8vPfe1fToEYF19WWyWrqJwO1cbrDbeDtGROYCRVT192yJzpggV6VKMa69tgqjR7fl0ku9qZE1JvMyemaxish8oI47vilbojImSCUknGbIkJ8BePnlK2nTpgJt2mR4/6YxF8SbxuIVItLQ55EYE+SWLnU6ifvf/34lLu6YdRJnso03bQQNcbqH2AwcxbnDWFX1cp9GZkyQOHLkJM888wNjxvxBxYpFmDv3Jjp0CPd3WCaIeJMIuvo8CmOC2M6dR5g48U8efPByXnrpCgoVyufvkEyQ8ebO4s3ZEYgxwWT//uN8/PF6HnigAbVqlWDLlvvsiWHGb7xpIzDGZBFV5dNP1xMRMZmHHlrI+vX/AlgSMH5licCYbLJ7dzw33jiLm2/+ivLlCxMTY53EmZzBmzYCRKQczkPsF4lIfiBUVY/6NjRjcg+nk7gZ7NoVz7BhrXj00UhCQ+08zOQM3nQ6939AP6AoUAWne4mxQDvfhmZM4Nux4zBlyxYmJCQPY8a0pVKlolSvbqUAk7N4c0ryENAMp2sJVHUDzsNqjDFpSExMYtSoszuJ69ixkiUBkyN5UzWUoKonk/s3cR9BaZ2dGJOGtWv3Ex09l59//purr67EtddW8XdIxqTLm0SwVESeBMJEpA3OIyxn+zYsYwLThAkrefDBhRQunI8PPujMHXfUsk7iTI7nTdXQk8ARYB3wMLAAGOjLoIwJVNWqFef666sSG3sPd95pPYWawOBNieAaYKKqvu3rYIwJNMePn2Lw4J8QEV55pZV1EmcCkjclgpuBTSIyWUQ6uW0ExgS9JUt2UL/++wwb9huHDp2wTuJMwMowEbiPp6wOfAXcC2wRkXG+DsyYnOrw4RP06TOf1q0/IjExiQULbuHtt9tbNZAJWF7dUKaqJ0TkS+A4zmMnbwF6+zIwY3Kqv/+OZ8qUNTz2WCOGDGnJRRdZJ3EmsHlzQ1l7nAfYtwN+BN4HbvdxXMbkKPv2HePjj9fTp09DatYswV9/9bQnhplcw5sSQS/gI+BBVT3u43iMyVFUlY8/Xs+DDy7g4METtGtXkerVL7YkYHIVb7qhvjk7AjEmp/n773geeGA+s2ZtJjLyUhYs6GR3BptcKc1EICLfq2prETkAeF4OkfyEMvtGmFwrMTGJVq2cTuJef701Dz/cyDqJM7lWeiWCNu7/ktkRiDE5wbZthyhXzukkbuzYdlSuXJSqVYv7OyxjfCrNUxxVTXIHJ6lqoucfMCl7wjMmeyQmJjF8eAy1ak0+00lchw7hlgRMUPCmsbie54h7Q1lj34RjTPZbvTqO6Oi5LFv2D126VOa666r5OyRjslV6bQRPAQOAwiLyb/JknPYCKxGYXGHcuBU89NBCihbNz7Rp13DbbTXtxjATdNIrEQwD3gD+h5MQAHCrhowJaKqKiFCrVgluvrkGI0e2oVSpgv4Oyxi/SC8RVFXVjSLyAVA7eWLy2ZKqrvJxbMZkuWPHTjFo0FJCQoRXX21N69blad26vL/DMsav0ksEA4BoYEwq8xRo5ZOIjPGRxYu3c99989i8+SB9+jQ4UyowJtilmQhUNdr9f2X2hWNM1jt06ARPPvk9EyasokqVYixceIt1FW2MhwzvkBGRG0SksDs8QEQ+FpH6vg/NmKyxe3c8U6fG0r9/JKtW3W1JwJgUvLlVcrCqHhGRFkBn4ENgvDcrd59fsF5ENonIgHSWu1FEVEQivQvbmPTFxR3jrbd+B6BmzRJs3dqL116LomDBvH6OzJicx5tEkHyVUBdgvKp+CeTP6EXu/QZjgKuBCKC7iESkslxhnEdg/upt0MakRVWZNm0ttWpN5vHHF7Nhg3Pls10RZEzavEkEu0VkDHAbMEdE8nn5uibAJlXdoqongRlAt1SWexF4FUjwMmZjUrVjx2GuvdMuI+AAABviSURBVHYmd9zxNVWrFuOPP+6yTuKM8YI3P+i3AN8DnVX1AE7fQ2lW83goC+zwGN/pTjtDRC4Hyqvq1+mtSER6iUiMiMTExcV5sWkTbE6fTiIq6iMWLdrOiBFtWLq0O7VrWzdZxnjDm26o40VkDRAlIlHAD6r6zYVuWETyAMOBe7yIYQIwASAyMtIeDGvO2Lr1EOXLFyY0NA/jx3egcuWiVK5czN9hGRNQvLlqqB/wCVDB/ftYRPp4se5dgOedOuXcackKA3WAxSKyFWgGzLIGY+ON06eTeP3136hVazJjxzqdxLVrV9GSgDHnwdsnlDVR1XgAEXkZ+AkYm8HrfgOqiUglnARwGx6PuFTVQ3h0cS0ii4H+qhqTmR0wwWfVqjiio78lJmYP3bpV5cYbq/s7JGMCmjeJQICTHuOn3GnpUtXTbmliLs4D799V1TUiMgSIUdVZ5xOwCW5jx/7Bww8vonjx/Hz0URduvrmG3R1szAXyJhF8APwqIp/hJIDrgPe8WbmqzgHmpJg2KI1lo7xZpwlOyd1B1KlTkttuq8mIEVGULGmXhBqTFbxpLB7mVttcgdPHUG9V/c3XgRkDcPToSZ59dimhocJrr0XRqlV5WrWyTuKMyUrePoQ1ATjh8d8Yn1uwYBt1677HyJHLOXEiEVW7YMwYX/DmqqGBwHSgNM6VP9NE5GlfB2aC18GDCdx331zatfuE0NA8LFlyG6NGtbW2AGN8xJs2gruAhqp6DEBEXgL+wHlgjTFZbs+eY8yYsY6nnmrC8883p0AB6x/IGF/yJhHsTrFcqDvNmCyzZ89RZsxYx8MPN6JGjYvZurWnNQYbk028SQT/AmtEZC5OY3EH4DcRGQ6gqo/5MD6Ty6kqH364locfXkh8/Ck6d65MtWrFLQkYk428SQRfu3/JfvFRLCbIbN9+mN695/PNN3/RvHkZJk3qSLVqxf0dljFBx5vLRydlRyAmuCR3Erd37zFGjbqKPn0aEBLi7UVsxpis5E2JwJgss2XLQSpWLEJoaB7eeacDVaoUIzy8qL/DMiao2SmYyRanTyfx6qu/EhExmTFjnE7i2rataEnAmBzA6xKBiORXVbuZzGTaihV7iY6ey++/7+H666tx883WSZwxOYk3N5Q1EZE/gY3ueH0RecvnkZlcYfTo32nceCq7dh3h00+78vnn3ShdupC/wzLGePCmamgUzvOK9wOo6kqgjS+DMoEvuTuIevVKcccdtYiNvde6izYmh/KmaiiPqm5LcXt/YloLm+AWH3+SgQN/JG/ePLz+unUSZ0wg8KZEsENEmgAqIiEi8giwwcdxmQA0b95W6tSZwltv/c6pU0nWSZwxAcKbEsEDONVDFYA9wHfuNGMAOHAggcceW8SUKWuoUeNiliy5jSuuKOfvsIwxXvLmhrK9OI+ZNCZVe/ce49NPN/D0000ZNKg5YWF2e4oxgSTDb6yIvIPTx9BZVLWXTyIyAeGff44yffpaHn000u0krhclShTwd1jGmPPgzanbdx7DYcD1wA7fhGNyOlXl/ffX8Oijizl27BRdulShWrXilgSMCWDeVA195DkuIh8AP/osIpNjbd16iPvvn8+8eVtp2bIsEyd2sE7ijMkFzqcytxJwaVYHYnK206eTaNPmI/btO86YMW3p3bsBefLYE8OMyQ28aSM4wH9tBHlwnk8wwJdBmZxj06YDVKpUlNDQPLz7bicqVy5KxYrWP5AxuUm69xGIcxdZfaCU+1dcVSur6sfZEZzxn1OnEnn55V+oXXvKmU7i2rSpYEnAmFwo3RKBqqqIzFHVOtkVkPG/33/fQ3T0XFas2MvNN1fn1ltr+DskY4wPeXNn8QoRaejzSEyOMGrU7zRpMpV//jnK55934+OPu3LppRf5OyxjjA+lWSIQkVBVPQ00xHlG8WbgKCA4hYXLsylGkw1UFRGhYcNLuOuu2rzxRhTFi4f5OyxjTDZIr2poGXA50DWbYjF+cOTISZ5+egn584fwxhttuPLKclx5pXUPYUwwSS8RCICqbs6mWEw2+/bbv7j//nns2HGERx5pdKZUYIwJLuklglIi8lhaM1V1uA/iMdlg//7jPPbYIt5/P5ZatS5m6dLbad68jL/DMsb4SXqJIAQohFsyMLnH/v3HmTlzE88914yBA5uRP791EmdMMEvvF2C3qg65kJWLSCfgTZykMlFVX0kx/zHgPuA0EAf8n6puu5BtmtTt3h3Phx+u5fHHI6le/WK2betljcHGGCD9y0cvqCQgIiHAGOBqIALoLiIRKRb7A4hU1XrAp8CwC9mmOZeq8u67f1Kr1mSee24pmzYdBLAkYIw5I71E0PYC190E2KSqW1T1JDAD6Oa5gKouUtVj7ugvgF2ukoX++usgHTp8SnT0XOrXL8XKlXdZJ3HGmHOkWTWkqv9e4LrLcnZ31TuBpuksHw18k9oMEekF9AKoUKHCBYYVHE6fTuKqqz5m//4E3n67Hb161bdO4owxqcoRrYQicicQCbRObb6qTgAmAERGRtqDcNOxceMBKld2OombPLkTVaoUo3z5Iv4OyxiTg3nTxcT52gWU9xgv5047i4i0AwYCXVX1hA/jydVOnUpk6NCfqVNnCqNH/wFAVFQFSwLGmAz5skTwG1BNRCrhJIDbgNs9F3D7MBoPdHKfjWzOQ0zMP0RHz2XVqjhuu60m3bvX9HdIxpgA4rNEoKqnRaQfMBfn8tF3VXWNiAwBYlR1FvAazr0Kn7h3tG5XVevSIhPefHM5jz22mMsuu4gvv7yOrl2r+jskY0yA8WkbgarOAeakmDbIY7idL7efmyV3BxEZeRnR0XUZNqwVxYrZJaHGmMzLEY3FxnuHD5/gqaeWEBYWyogRbWjZsiwtW5b1d1jGmADmy8Zik8XmzNlC7dpTmDBhFaGhgqpdQGWMuXBWIggA+/Yd45FHFvHhh2upXbsEn356O02blvZ3WMaYXMISQQA4cOAEX321meefb84zzzQjX74Qf4dkjMlFLBHkULt2HeHDD9fyxBONqVatONu29bLGYGOMT1gbQQ6jqrzzzioiIiYzePBPbN7sdBJnScAY4ytWIshBNm8+SM+ec1m0aAdRUeV5550OVK1qncSZ4HHq1Cl27txJQkKCv0MJWGFhYZQrV468efN6/RpLBDnE6dNJtG37Mf/+m8D48e2577561kmcCTo7d+6kcOHChIeH22NTz4Oqsn//fnbu3EmlSpW8fp0lAj9bv/5fqlQpRmhoHt5772qqVClGuXKF/R2WMX6RkJBgSeACiAglSpQgLi4uU6+zNgI/OXkykRde+Im6dacwZozTSVzr1uUtCZigZ0ngwpzP+2clAj9Ytmw30dFzWb16H7ffXos77qjl75CMMUHMSgTZbOTI5TRvPo0DBxL46qvr+fDDayhZsqC/wzLGACEhITRo0IA6depw7bXXcvDgwTPz1qxZw1VXXUWNGjWoVq0aL7744ll393/zzTdERkYSERFBw4YNefzxx/2xC+fFEkE2ST5gmjS5jJ4967Fmzb106VLFz1EZYzwVKFCAFStWsHr1ai6++GLGjBkDwPHjx+natSsDBgxg/fr1rFy5kp9++omxY8cCsHr1avr168fUqVOJjY0lJiaGqlUDpydgqxrysUOHTvDkk99ToEAoI0deRYsWZWnRwjqJMyZDyx+BAyuydp3FG0CjkV4t2rx5c1atWgXAtGnTaNmyJR06dACgYMGCjB49mqioKPr27cuwYcMYOHAgNWs6zwIJCQnhgQceyNrYfchKBD701VebiYiYzMSJf5I/f4h1EmdMgEhMTGTBggV07eo8HmXNmjU0atTorGWqVKlCfHw8hw8fZvXq1efMDyRWIvCBuLhjPPzwQqZPX0fduiX54otuNG5sncQZkylenrlnpePHj9OgQQN27dpFrVq1aN++fbbH4A9WIvCBQ4dOMGfOX7zwQgtiYnpYEjAmQCS3EWzbtg1VPdNGEBERwfLly89adsuWLRQqVIgiRYpQu3btc+YHEksEWWTHjsP873+/oqpUrep0EjdoUAvrKdSYAFSwYEFGjRrFG2+8wenTp7njjjv48ccf+e677wCn5PDQQw/x5JNPAvDEE0/w8ssvs2HDBgCSkpIYN26c3+LPLEsEFygpSRk3bgW1a09h6NCfz3QSV7Rofj9HZoy5EA0bNqRevXpMnz6dAgUK8OWXXzJ06FBq1KhB3bp1ady4Mf369QOgXr16jBw5ku7du1OrVi3q1KnDli1b/LwH3rM2gguwceMBevacy/ff76Rt2wpMmNCBypWL+TssY8x5io+PP2v8q6++OjNct25dFi9enOZru3TpQpcuXXwVmk9ZIjhPp08n0b79Jxw8eIJJkzpy77117NZ4Y0xAskSQSWvX7qdateKEhubhgw86U6VKMcqUKeTvsIwx5rxZG4GXTpw4zfPPL6VevfcYPdrpJO7KK8tZEjDGBDwrEXjhl1/+Jjp6LrGx++nRI4IePSL8HZIxxmQZSwQZeOON33jiie8pV64wc+bcwNVXV/Z3SMYYk6UsEaQhKUnJk0do3rwMvXvX55VXWlGkiF0SaozJfayNIIWDBxOIjv6Whx9eCECLFmUZO7a9JQFjgsSePXu4/fbbqVy5Mo0aNaJ58+bMnDnTp9uMiYnhoYce8uk20mOJwMMXX2wkImIy7723hsKF81knccYEGVXluuuuo1WrVmzZsoXly5czY8YMdu7c6dPtRkZGMmrUKJ9uIz1WNQTs3XuUfv0W8MknG2jQ4BJmz76Byy+/1N9hGRP0oqJmnDPtlltq0KdPQ44dO0Xnzp+dM/+ee+pwzz112LfvGDfdNOuseYsX35bu9hYuXEi+fPno3bv3mWkVK1bkwQcfZMqUKcTExDB69GjAuYGsf//+REVFMW/ePJ5//nlOnDhBlSpVmDx5MoUKFWLAgAHMmjWL0NBQOnTowOuvv84nn3zCCy+8QEhICEWLFmXJkiUsXryY119/ndmzZzN48GC2b9/Oli1b2L59O4888siZ0sKLL77I1KlTKVWqFOXLl6dRo0b0798/0+9rSpYIgMOHTzJ//jZeeukKnniiMXnzWv9AxgSjNWvWcPnll2fqNfv27WPo0KF89913XHTRRbz66qsMHz6cvn37MnPmTNatW4eInHna2ZAhQ5g7dy5ly5Y96wlontatW8eiRYs4cuQINWrU4IEHHmDFihV89tlnrFy5klOnTnH55ZdnWdfXQZsItm8/zAcfxPLMM02pWrU427ffT+HC+fwdljHGQ3pn8AUL5k13fsmSBTMsAWSkb9++/Pjjj+TLl4++ffumuswvv/xCbGwsLVu2BODkyZM0b96cokWLEhYWRnR09FndT7Rs2ZJ77rmHW265hRtuuCHVdV5zzTXkz5+f/Pnzc8kll7Bnzx6WLl1Kt27dCAsLIywsjGuvvfaC9s2TT9sIRKSTiKwXkU0iMiCV+flF5CN3/q8iEu7LeMC5Gmjs2D+oXXsyL7/8y5lO4iwJGGNq167N77//fmZ8zJgxLFiwgLi4OEJDQ0lKSjozLyEhAXDaFdq3b8+KFStYsWIFsbGxTJo0idDQUJYtW8ZNN93E7Nmz6dSpEwDjxo1j6NCh7Nixg0aNGrF///5z4sif/7+LU0JCQjh9+rSvdhnwYSIQkRBgDHA1EAF0F5GUd2JFAwdUtSowAnjVV/EArN9xEVFRH9G37wKaNy/DmjX3UrVqcV9u0hgTQK666ioSEhJ4++23z0w7duwYAOHh4axYsYKkpCR27NjBsmXLAGjWrBlLly5l06ZNABw9epQNGzYQHx/PoUOH6Ny5MyNGjGDlypUAbN68maZNmzJkyBBKlSrFjh07vIqtZcuWfPXVVyQkJBAfH8/s2bOzbL99WTXUBNikqlsARGQG0A2I9VimGzDYHf4UGC0ioj64XOd0otDx6aYcOhHH5MmduPvu2tZJnDHmLCLCF198waOPPsqwYcMoVarUmXr/li1bUqlSJSIiIqhVq9aZtoRSpUoxZcoUunfvzokTJwAYOnQohQsXplu3biQkJKCqDB8+HHCeXbBx40ZUlbZt21K/fn2+//77DGNr3LgxXbt2pV69elx66aXUrVuXokWLZs1+++oSSRG5Ceikqve54z2Apqraz2OZ1e4yO93xze4y+1KsqxfQC6BChQqNtm3blvmAlj/CjyvyUaXzIEqXtv6BjMmJ1q5dS61atfwdRo4VHx9PoUKFOHbsGK1atWLChAmpNm6n9j6KyHJVjUxtvQHRWKyqE4AJAJGRkeeXuRqN5IrAfba0McbQq1cvYmNjSUhI4O677870FU5p8WUi2AWU9xgv505LbZmdIhIKFAXObTkxxhjDtGnTfLJeX1419BtQTUQqiUg+4DZgVoplZgF3u8M3AQt90T5gjAkc9hNwYc7n/fNZIlDV00A/YC6wFvhYVdeIyBAR6eouNgkoISKbgMeAcy4xNcYEj7CwMPbv32/J4DypKvv37ycsLCxTr/NZY7GvREZGakxMjL/DMMb4wKlTp9i5c+eZa/RN5oWFhVGuXDny5s171vSAbyw2xgSHvHnzUqlSJX+HEXSs91FjjAlylgiMMSbIWSIwxpggF3CNxSISB5zHrcUAlAT2ZbhU7mL7HBxsn4PDhexzRVUtldqMgEsEF0JEYtJqNc+tbJ+Dg+1zcPDVPlvVkDHGBDlLBMYYE+SCLRFM8HcAfmD7HBxsn4ODT/Y5qNoIjDHGnCvYSgTGGGNSsERgjDFBLlcmAhHpJCLrRWSTiJzTo6mI5BeRj9z5v4pIePZHmbW82OfHRCRWRFaJyAIRqeiPOLNSRvvssdyNIqIiEvCXGnqzzyJyi/tZrxER33Rgn428OLYriMgiEfnDPb47+yPOrCIi74rIXvcJjqnNFxEZ5b4fq0Tkwp9Oo6q56g8IATYDlYF8wEogIsUyfYBx7vBtwEf+jjsb9rkNUNAdfiAY9tldrjCwBPgFiPR33NnwOVcD/gCKu+OX+DvubNjnCcAD7nAEsNXfcV/gPrcCLgdWpzG/M/ANIEAz4NcL3WZuLBE0ATap6hZVPQnMALqlWKYb8J47/CnQVgL7SfYZ7rOqLlLVY+7oLzhPjAtk3nzOAC8CrwK5oV9jb/a5JzBGVQ8AqOrebI4xq3mzzwoUcYeLAn9nY3xZTlWXAP+ms0g34H11/AIUE5HSF7LN3JgIygI7PMZ3utNSXUadB+gcAkpkS3S+4c0+e4rGOaMIZBnus1tkLq+qX2dnYD7kzedcHaguIktF5BcR6ZRt0fmGN/s8GLhTRHYCc4AHsyc0v8ns9z1D9jyCICMidwKRQGt/x+JLIpIHGA7c4+dQslsoTvVQFE6pb4mI1FXVg36Nyre6A1NU9Q0RaQ58ICJ1VDXJ34EFitxYItgFlPcYL+dOS3UZEQnFKU7uz5bofMObfUZE2gEDga6qeiKbYvOVjPa5MFAHWCwiW3HqUmcFeIOxN5/zTmCWqp5S1b+ADTiJIVB5s8/RwMcAqvozEIbTOVtu5dX3PTNyYyL4DagmIpVEJB9OY/CsFMvMAu52h28CFqrbChOgMtxnEWkIjMdJAoFebwwZ7LOqHlLVkqoarqrhOO0iXVU1kJ9z6s2x/QVOaQARKYlTVbQlO4PMYt7s83agLYCI1MJJBHHZGmX2mgXc5V491Aw4pKq7L2SFua5qSFVPi0g/YC7OFQfvquoaERkCxKjqLGASTvFxE06jzG3+i/jCebnPrwGFgE/cdvHtqtrVb0FfIC/3OVfxcp/nAh1EJBZIBJ5Q1YAt7Xq5z48D74jIozgNx/cE8omdiEzHSeYl3XaP54G8AKo6DqcdpDOwCTgG3HvB2wzg98sYY0wWyI1VQ8YYYzLBEoExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKByRYikigiKzz+wtNZNjytnhezm4hEisgodzhKRFp4zOstIndlYywNzqdnTREpLSKzPcanu71WPpqJddQVkSmZ3bYJDLnuPgKTYx1X1Qb+DiKz3BvQkm9CiwLigZ/ceeOyensiEur2f5WaBjjdg8zJ5GofA95x138Z0FhVq2Yypj9FpJyIVFDV7ZncvsnhrERg/MY98/9BRH53/1qkskxtEVnmliJWiUg1d/qdHtPHi0hIKq/dKiLDRORPd9mqHttdKP89m6GCO/1mEVktIitFZIk7LUpEZrslmN7Ao+42rxSRwSLSX0RqisiyFPv1pzvcSES+F5HlIjI3tV4iRWSKiIwTkV+BYSLSRER+Fqd//Z9EpIZ7V+0Q4FZ3+7eKyEXi9F2/zF02td5XAW4EvnWH5wFlPfZhsYi86Y6vFpEmbkyDReQDEVkKfOC+9isC/OZLkwZ/971tf8Hxh3OX6wr3b6Y7rSAQ5g5Xw7lTFCActy924C3gDnc4H1AAqIXzo5TXnT4WuCuVbW4FBrrDdwGz3eGvgLvd4f8DvnCH/wTKusPF3P9RHq8bDPT3WP+ZcXe/KrnDTwHP4twN+hNQyp1+K86dsSnjnALMBkLc8SJAqDvcDvjMHb4HGO3xupeBO5PjxelX6KIU664ELPcYP/PeuuOLgXfc4VYe7/tgYDlQwGPZlsBX/j6W7C/r/6xqyGSX1KqG8gKjRaQBTqKonsrrfgYGikg54HNV3SgibYFGwG9udxkFgLT6T5ru8X+EO9wcuMEd/gAY5g4vBaaIyMfA55nZOZxOz24FXnH/3wrUwOn4br4bZwiQVp8wn6hqojtcFHjPLf0obvcCqegAdBWR/u54GFABWOuxTGky7ndnOjj94ItIEREp5k6fparHPZbbC5TJYF0mAFkiMP70KLAHqI9TTXnOw2NUdZpbZXINMEdE7sd5MtN7qvq0F9vQNIbPXVC1t4g0dbe1XEQaebcbAHyE04/T586qdKOI1AXWqGpzL15/1GP4RWCRql7vVkktTuM1AtyoquvTWe9xnASRnpTvS/L40RTTw9z1mVzG2giMPxUFdqvTb3wPnDPms4hIZWCLqo4CvgTqAQuAm0TkEneZiyXtZzDf6vH/Z3f4J/6r674D+MFdTxVV/VVVB+GcRXt29QtwBKd763Oo6macUs1zOEkBYD1QSpw+8hGRvCJSO404PRXlv26F70ln+3OBB8UtbojTw2xKG3Cqg9Jzq/v6K3B6sjyUxnLVgRxxNZfJWpYIjD+NBe4WkZVATc49AwW4BVgtIitwqlneV9VYnDr4eSKyCpiPUwWSmuLuMg/jlEDAeYLVve70Hu48gNfchuXVOMliZYp1fQVcn9zQmsq2PgLu5L++8U/idHP+qruPK4BzGsRTMQz4n4j8wdml9kVARHJjMU7JIS+wSkTWuONnUdWjwObkhvI0JLjbGofTt39a2gC55WlvxoP1PmpyLXEeSBOpqvv8HYs/icj1QCNVfTaVeYtxGrzTfU6DiOQHvgeu0LQvbzUBytoIjMnlVHWmiFzoM7krAAMsCeROViIwxpggZ20ExhgT5CwRGGNMkLNEYIwxQc4SgTHGBDlLBMYYE+T+Hy0RpDUxG8UAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot perfect ROC curve and AUC score\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_test)\n",
    "plot_roc_curve(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perfect AUC score\n",
    "roc_auc_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Confusion Matrix**\n",
    "\n",
    "A confusion matrix is a quick way to compare the labels a model predicts and the actual labels it was supposed to predict.\n",
    "\n",
    "In essence, giving you an idea of where the model is getting confused."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/confusion-matrix.jpg\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[23,  6],\n",
       "       [ 6, 26]])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "confusion_matrix(y_test, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Predicted Labels</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Actual Labels</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Predicted Labels   0   1\n",
       "Actual Labels           \n",
       "0                 23   6\n",
       "1                  6  26"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualize confusion matrix with pd.crosstab()\n",
    "pd.crosstab(y_test, y_preds, rownames=[\"Actual Labels\"], colnames=[\"Predicted Labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "22 + 7 + 8 + 24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEACAYAAACatzzfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAASmklEQVR4nO3de2wU5b/H8U9bW2vaXVIugSMtpayXgAp6JFW8a7kVFQJyAG2wCmJRW7RQFQ3RmICB/kBqLUY5CwSjJk29y0X4FZWA4R9L4vHSYmwrLfUOmrUVLNud88cvNC4L3d22yzztvF9kYnxmZ+ZJSD58851nZuIsy7IEADBWvN0TAAB0jaAGAMMR1ABgOIIaAAxHUAOA4QhqADDceXZP4PgHa+2eAgzjmr3e7inAUP72lh4df/K3hoh/mzh4VI+u1ZtsD2oAOGcCHXbPoFsIagDOYQXsnkG3ENQAnCNAUAOA0SwqagAwXIc/JqcNBAKqrKzUm2++qSNHjmjQoEHKyclRUVGRUlNTJUn33XefDhw4EHLsW2+9pSuuuKLL8xPUAJwjRjcTvV6vysrKtHDhQk2YMEGNjY0qLy/Xd999p02bNkmS6urqdO+99+r2228POtbj8YQ9P0ENwDli0PqwLEter1dz587VsmXLJEnXXXed0tLSVFxcrNraWg0cOFC///67brzxRl155ZVRX4OgBuAcMbiZ2NbWpunTpys3NzdofNSo/6zDbmpq0i+//CJJuvTSS7t1DZ5MBOAYlhWIeItUamqqVqxYoauvvjpovLq6WpJ00UUXqa6uTklJSSovL9c111yjK664QosWLVJjY2NE1yCoAThHIBDx5vP5dOTIkZDN5/OFvcwXX3yhjRs3auLEifJ4PKqrq1N7e7uSk5NVUVGhVatWqampSXl5efr111/Dni/O7i+88Ag5Tscj5Dibnj5C/nfd3oh/u/Hf/6eKioqQ8cLCQhUVFZ31uJqaGi1evFhDhgzRG2+8obS0NH377bc6duyYrr322s7fNTc3Kzc3VwsXLlRxcXGXc6FHDcA5omhp5Ofna+bMmSHjbrf7rMfs2LFDy5cv18iRI+X1epWWliZJuuSSS0J+m5GR0Vlth0NQA3COKG4mut3uLkP5dFu2bNGaNWuUnZ2tDRs2yOVySfrPqpD3339f6enpGj9+fNAxJ06c6AzzrtCjBuAcViDyLQpVVVVavXq1cnNz5fV6O0NakuLi4rRp0yY9//zzCvzjH4qvv/5aTU1Nys7ODnt+KmoAzhGD5XlHjx7VqlWrNHz4cOXl5embb74J2j9ixAgVFhZqyZIlKikp0V133aUffvhBL774okaPHq0ZM2aEvQZBDcAxrMDJXj/nvn37dPz4cbW0tCgvLy9kf2lpqWbMmKENGzbolVdeUWFhoZKTkzVp0iQtXbpUCQkJYa/Bqg8Yh1UfOJuervo4cfCDiH+b/N/Te3St3kRFDcA5eHseABiOL7wAgOGoqAHAcHzhBQAMF6MPB8QaQQ3AOaioAcBslsXNRAAwGxU1ABiOVR8AYDgqagAwHKs+AMBwtD4AwHC0PgDAcAQ1ABiO1gcAGI6biQBgOFofAGA4Wh8AYDgqagAwHEENAIaz91ve3UZQA3AOP6s+AMBs3EwEAMPRowYAw9GjBgDDUVEDgOEIagAwm9XBx20BwGxU1ABgOJbnAYDhAqz6AACz9ffWR0tLixobG9Xa2qr4+Hi5XC5lZWVp2LBhsZwfAPSe/nozcffu3XrxxRfV0NAg67TF4nFxccrMzNRjjz2mqVOnxmySANAr+mNF/d5772n58uXKzc1VUVGRMjMzlZKSIklqbW3V4cOHtWvXLhUXF+vkyZO68847z8mkAaBb+mOPeuPGjbr77rv17LPPnnH/mDFjlJubq2effVavvvoqQQ3AbH101Ud8VztbWlo0ceLEsCeZOHGimpube21SABATASvyzSBdBnVGRob2798f9iSffvopNxUBGM8KBCLeTNJl62Px4sV6/PHH9csvv2jy5MnKyspSamqqJKmtra2zR71t2zY999xz52TCANBt/XHVxx133KH4+HiVlZVp+/btiouLC9pvWZbS09P1/PPPa+bMmTGdKAD0mGEtjUiFXZ43bdo0TZs2Tc3NzWpoaFBra6ssy+pcRz1ixIhzMU8A6DnDWhqRiviBl4yMDGVkZMRyLgAQWzGqqAOBgCorK/Xmm2/qyJEjGjRokHJyclRUVNTZLv7yyy9VWlqqr776SikpKZo1a5aKioqUmJgY9vw8Qg7AOWK0PM/r9aqsrEwLFy7UhAkT1NjYqPLycn333XfatGmTDh8+rPvuu09XXXWVysrKVF9fr/Xr16u1tVXPPPNM2PMT1ACcIwYVtWVZ8nq9mjt3rpYtWyZJuu6665SWlqbi4mLV1tbq9ddfl8vl0ssvv6ykpCTdfPPNSk5O1sqVK1VQUKChQ4d2eY0ul+cBQH9i+Tsi3iLV1tam6dOn64477ggaHzVqlCSpqalJn332mW699VYlJSV17p86dao6OjoiWgJNRQ3AOWJQUaempmrFihUh49XV1ZIkj8ejH3/8UVlZWUH7Bw4cqNTUVDU2Noa9BkENwDmi6FH7fD75fL6QcbfbLbfb3eWxX3zxhTZu3KiJEyd2/vbUTcV/SklJUWtra9i5ENQAnCOKinrr1q2qqKgIGS8sLFRRUdFZj6upqdHixYuVnp6ulStXqr29vcvrxMeH70AT1AAcw4oiqPPz88/4IF9X1fSOHTu0fPlyjRw5Ul6vV2lpaWpra5Okzv/+U2trq1wuV9i5ENQAnCOKm4SRtDj+acuWLVqzZo2ys7O1YcOGzgBOSUnR0KFDdfjw4aDfHz16VG1tbSG96zNh1QcA54jR2/Oqqqq0evVq5ebmyuv1hlTJ119/vT755JOgNsiuXbuUkJCg7OzssOenogbgHDFY9XH06FGtWrVKw4cPV15enr755pug/SNGjNADDzyg7du368EHH1R+fr6+//57vfDCC5ozZ44uvPDCsNcgqAE4xumfE+wN+/bt0/Hjx9XS0qK8vLyQ/aWlpZoxY4Y2b96s0tJSLVmyRGlpabr//vu7vCn5T3FWLGYeheMfrLXz8jCQa/Z6u6cAQ/nbW3p0vG/R5Ih/6/7f3T26Vm+iogbgHP31NacA0F9Y/n7+mlMA6PP6Zk4T1ACcI5oHXkxCUANwDoIaAAxH6wMAzEbrAwAMZ/kJagAwG60PADBbjL5tG3MENQDnIKgBwGxU1ABgOMtv9wy6h6AG4BhU1ABgOIIaAExnxdk9g24hqAE4BhU1ABjOClBRA4DRAh0ENQAYjdYHABiO1gcAGM7qmy/PI6gBOAcVNQAYjpuJAGA4KmoAMJzFk4kAYDaW5wGA4QJU1ABgNlofAGA4Vn0AgOFY9QEAhqNHDQCGo0cNAIbjXR8AYDhaHwBguAA3E7vHNXu93VOAYY7/sM/uKaCfoqIGAMNxMxEADEdFDQCG66OLPghqAM7REYi3ewrdQlADcIw++pZTghqAc1iKfY+6trZWs2fP1p49ezRs2LDO8UmTJqmpqSnk9wcOHNDAgQO7PCdBDcAxAjFuUtfX16ugoEB+vz9ovK2tTc3NzVq2bJmys7OD9rnd7rDnJagBOEYgRhW13+9XZWWl1q1bp8TExJD9hw4dkmVZysnJkcfjifr8fbOzDgDdYCku4i0aNTU1Wrt2rRYsWKCSkpKQ/bW1tUpOTtbIkSO7NW8qagCO0RFFAPt8Pvl8vpBxt9sd0q7weDyqrq7WoEGD9M4774Qcc+jQIQ0YMEBLly7VZ599po6ODt1yyy16+umnNWTIkLBzIagBOEY0qz62bt2qioqKkPHCwkIVFRUFjQ0ePLjLc9XV1em3337TxRdfrPnz56uhoUHl5eW699579e677yo5ObnL4wlqAI4RTVDn5+dr5syZIeOR3Pw73YoVK2RZlsaNGydJGj9+vDwej+655x598MEHmjNnTpfHE9QAHCOa3vOZWhzdNXbs2JCxq6++Wi6XS3V1dWGP52YiAMcIxEW+9Za//vpLb7/9dkggBwIBnTx5UmlpaWHPQVADcIyA4iLeesv555+v1atXh/S7P/74Y504cSJkXfWZ0PoA4BgdNlwzISFBDz/8sFavXq2VK1fqtttu07fffquXXnpJOTk5uuaaa8Keg6AG4BiBOHtec3r//fcrNTVVr732mqqqqjRgwADNmzcvZPXI2cRZlr2fezwvabidl4eB+MILziZx8KgeHV/1X3kR//Z/fnyjR9fqTVTUAByDt+cBgOH66LdtCWoAzhHNI+QmIagBOAYVNQAYjh41ABiOj9sCgOFofQCA4Wh9AIDhOqioAcBsVNQAYDiCGgAMx6oPADAcqz4AwHC0PgDAcHZ8OKA3ENQAHIPWBwAYjtYHABiOVR8AYLhAH41qghqAY3AzEQAMR48aAAzHqg8AMBw9agAwXN+MaYIagIPQowYAw3X00ZqaoAbgGFTUAGA4biYCgOH6ZkwT1AAchNYHABiOm4kAYDh61ABguL4Z0wQ1AAfptxX1zz//HNUJhw4d2u3JAEAs9dubiTk5OeroiPwtrrW1tT2aEADEitVfK+qqqioVFBSovb1dy5Yt03nn0S0B0Df121Ufo0eP1pYtWzRnzhz9+uuvevjhh8/FvACg1/XV1kd8JD/yeDxaunSpvF6vjh07Fus5AUBMBCwr4s0kEfcx5s2bp4svvjiWcwGAmDIrfiMXcVAnJCQoOzs7lnMBgJjqq8vzImp9AEB/YEXxp7tqa2t12WWX6aeffgoa379/v+666y6NGzdOt912mzZv3hzxOVnCAcAx/DGuqOvr61VQUCC/3x80fvDgQS1evFi5ubl69NFHVVNTo9LSUlmWpYULF4Y9L0ENwDFitY7a7/ersrJS69atU2JiYsj+8vJyjRkzRv/6178kSTfddJP8fr9eeeUVzZ8/X0lJSV2en9YHAMcIRLFFo6amRmvXrtWCBQtUUlIStO/vv//W559/rsmTJweNT5kyRT6fTwcPHgx7foIagGNYlhXxFg2Px6Pq6moVFhYqISEhaF9zc7NOnjyprKysoPHMzExJUmNjY9jz0/oA4BjRrPrw+Xzy+Xwh4263W263O2hs8ODBZz3Pn3/+KUlKTU0NGk9JSZEktba2hp0LQQ3AMaJ5hHzr1q2qqKgIGS8sLFRRUVHE5wlXncfHh29sENQAHCOaijo/P18zZ84MGT+9mg7H5XJJktra2oLGT1XSp/Z3haAG4BjR9J7P1OLojhEjRighIUFNTU1B46f+//Te9ZlwMxGAY8Rq1UdXzj//fI0fP167d+8O+odi165dcrlcuvzyy8Oeg6AG4Bjn4snEM3nooYd08OBBFRcXa+/evSorK9OmTZtUUFCgCy64IOzxBDUAxwjIinjrTRMmTNBLL72k+vp6PfLII/rwww/1xBNPaNGiRREdH2dFu2Cwl52XNNzOy8NAx3/YZ/cUYKjEwaN6dPyt6ZMi/u0nR/7do2v1Jm4mAnCMfvspLgDoL0z7IECkCGoAjtE3Y5qgBuAgffXDAQQ1AMcgqAHAcB1W3/wOOUENwDFY9QEAhrP5sZFuI6gBOAY9agAwHBU1ABiuo1ffi3fuENQAHIMnEwHAcKz6AADDUVEDgOGoqAHAcFTUAGA4HiEHAMPR+gAAw1lU1ABgNh4hBwDD8Qg5ABiOihoADNcRoEcNAEZj1QcAGI4eNQAYjh41ABiOihoADMfNRAAwHK0PADAcrQ8AMByvOQUAw7GOGgAMR0UNAIYL8JpTADAbNxMBwHB9NajjrL46cwBwiHi7JwAA6BpBDQCGI6gBwHAENQAYjqAGAMMR1ABgOIIaAAxHUAOA4QhqADAcQW2Abdu26fbbb9fYsWOVm5ur9957z+4pwSC1tbW67LLL9NNPP9k9FdiEoLbZjh07VFJSohtuuEEbNmxQdna2nnzySX300Ud2Tw0GqK+vV0FBgfx+v91TgY1414fNJk2apMsvv1zr16/vHHvsscd06NAh7dy508aZwU5+v1+VlZVat26dEhMT9ccff2jv3r0aNmyY3VODDaiobdTc3KympiZNnjw5aHzKlClqaGhQc3OzTTOD3WpqarR27VotWLBAJSUldk8HNiOobdTQ0CBJysrKChrPzMyUJDU2Np7zOcEMHo9H1dXVKiwsVEJCgt3Tgc14H7WN/vzzT0lSampq0HhKSookqbW19ZzPCWYYPHiw3VOAQaiobRTu9kB8PH89AAhqW7lcLklSW1tb0PipSvrUfgDORlDb6FRvuqmpKWj88OHDQfsBOBtBbaPMzEylp6eHrJnevXu3Ro4cqQsvvNCmmQEwCTcTbfbII4/oqaee0oABA3TLLbdoz5492rlzZ9C6agDORlDbbNasWWpvb9fmzZtVVVWljIwMrVmzRtOmTbN7agAMwZOJAGA4etQAYDiCGgAMR1ADgOEIagAwHEENAIYjqAHAcAQ1ABiOoAYAwxHUAGC4/wcKmiqYDVOlEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Make our confusion matrix more visual with Seaborn's heatmap()\n",
    "import seaborn as sns\n",
    "\n",
    "# Set the font scale \n",
    "sns.set(font_scale=1.5)\n",
    "\n",
    "# Create a confusion matrix\n",
    "conf_mat = confusion_matrix(y_test, y_preds)\n",
    "\n",
    "# Plot it using Seaborn\n",
    "sns.heatmap(conf_mat);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: In the original notebook, the function below had the \"True label\" as the x-axis label and the \"Predicted label\" as the y-axis label. But due to the way confusion_matrix() outputs values, these should be swapped around. The code below has been corrected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOQAAADfCAYAAADm6n/jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAfU0lEQVR4nO3de1yP9//H8ceng1JRGnPqIGJIaOYQM4dyiubwS/mhGfadzZlhGDNfu81hTpnNML4OYyw2m7A5TL7DZtPP99sojJL0saSTzqnP5/dHX5/t8634lE99rup1/2td19XVs/F0Xe/ren+uS6XVarUIIRTBzNQBhBB/kkIKoSBSSCEURAophIJIIYVQEAtTB6goOd+uNnWEaqNOwDpTR6hWCvITSl0nR0ghFEQKKYSCSCGFUBAppBAKIoUUQkGkkEIoiBRSCAWRQgqhIFJIIRRECimEgkghhVAQKaQQCiKFFEJBpJBCKIgUUggFkUIKoSBSSCEURAophIJIIYVQECmkEAoihRRCQaSQQiiIFFIIBZFCCqEgpT4ouXXr1qhUqjLtTKVSERUV9dShhKipSi3ksGHDylxIIcTTKbWQK1asqMwcQgjK8W6PlJQUzp8/j1qtxs/PDxsbG1JTU2nRokVF5BOiRilTIbdv305ISAh5eXmoVCo8PT3Jyspi2rRpjBo1infffVdOc4V4CgZfZT18+DCrVq3C19eXkJAQtFotAB4eHvTr1499+/axe/fuCgsqRE1gcCG3b99Ojx49WLNmDV26dNEtb9y4MRs2bKBXr16EhoZWSEghagqDC3nz5k369u1b6vo+ffoQHx9vlFBC1FQGjyFtbW3JyMgodb1arcbGxsYooaqT89fi2XrqEtF37qNSqWjv8ixTBr5Ae9eGum1+uZHAJ99HcP1uCrZWlvRr35ypA1/AxsrShMmrjvr1HXl/2Xz8h/Sndm1rLl36jYXvLOfCL/9n6mhlZvARsmfPnuzdu5fk5ORi665evcqePXvo3r27UcNVdRdv3mXKtu/IyMlnysDOTOr3PPHJD5i4KYzfbt8Disr4xpZjPCzUMH1QZ4Z0asnBn6OZ/NkxNBqtiX8D5bOzs+X0D18xMsCfzVt2seS9D2nSpDEnjn+Jh8dzpo5XZirto6szT5CYmEhAQAAPHz6kc+fOnDx5kgEDBlBQUEB4eDh2dnaEhobi7Oxc0ZkNooRXmget+4oH2Xl8NXcktWsVnYwkZ2Qz/MMDtHGqz+bX/fjf9V+Tnp3LV3NHYm1ZtM3+81Es//ocGycO5MXWpv//qeRXmv996Tzmvz0NH98Afjx7AYCGDRvw+7WfOPjVEcZPmGHihMUZ5ZXmDRs25ODBg/Tu3Zuff/4ZrVbLd999x7lz5/Dx8VFUGZXgQXYe1+8m069Dc10ZAZ6pY0On5o34961E8h4WUM/WmhFdW+vKCNCpeSMAfr9b/GxE6HslOJCjx07pygiQmJjEvLeXcfYvy6qKMt2HfPbZZ1mxYgVarZbU1FQKCwtxdHTE3Ny83AESEhKIjY0lMzMTMzMz6tSpg5ubG40aNSr3PpXA1tqSQ3MD9cr4SFp2LhbmKqwsLfjkb4OKrb+mTgGgkYNdheesypo1c8bJqTGr13yiW2Zra0NWVjafbt5pwmTlV+aZOunp6Zw7d46EhATMzc1xcXGhW7du2NmV7S/P8ePHCQkJISYmhv8+a1apVLi6ujJz5kwGDhxY1oiKYG5mhmsD+2LLr6uT+detRLq3ciq2Tp2awa837rI27GfcG9Wjb7tmlZC06mrp7gbAvaT7rFy+iNdeG4O9fV1u3IhlztylhB05YeKEZVemQm7cuJGtW7eSl5ent9zW1pZ58+YRFBRk0H4OHTrE/PnzGTRoENOmTcPV1RVbW1sAMjMziYuL4/vvv2fWrFk8fPgQf3//ssRUrOy8hyzaFw7A+D4d9dalZ+fi98E+AKwtLXh7WHesLMv872WNYu9Q9A/e0iXzeFjwkFmzl1CoKeSt2W9w8MA2/AaP4dQPP5o4ZdkYfFFn586dLF++nG7dujF27FicnZ3RarXExsayc+dO/v3vf7N69WoGDx78xH35+fnRtWtXlixZ8tjtlixZQkREBGFhYYb9Nn+hhIs6f5WTX8D07d/x6827TOjbkemDOuutf5Cdx0/X7/CwUMPes5e5pk5m5RgffNu7mSjxn5R6UWfMmP9h5z82kJSUTBuPnqSlpQNgb1+X61fPExMbh3f3J/99rGxGuaizd+9eevTowY4dO/D19eW5556jdevWDBo0iD179vD888/z6aefGrSvhIQEfH19n7idr69vtZhs8CAnjze3HuXXm3cZ1rkV0wa+UGybujZWDOjYgiGdWrJ9sj+NHexYffgnE6StOrKysgH4+tAxXRkB0tMfcDjsOJ2eb4+tbdW6N25wIe/evYuPj0+J68zNzRk8eDBxcXEG7cvZ2ZmzZ88+cbvw8PAqf3EnJTOHv316hH/dSuR/urZmyciXnjgB39rSgp5tXfgjLYvUrNxKSlr1qBP+ACAp6X6xdUlJ9zEzM8POzrayYz0VgwcpLVu2JDIyktGjR5e4/vbt2zRr1sygfb3xxhvMnTuXe/fu0b9/f9zc3HQXhbKysnRjyLCwMJYuXWpoRMXJys3nza3HuKZOZmzPdsx52Vtvfey9NCZ/doxXe3cgqHtbvXXZeQ9RqaCWuTxlpTSXr1wlNzeXtm1bFVvXrJkLOTk5JCVVrVtHBhdy8eLFTJgwgQYNGjBx4kQcHBwAyM7O5sCBAxw4cICNGzcatK8hQ4ZgZmbG+vXrOXLkSLEjhlarxcnJiQ8++IDhw4eX4ddRluVfn+OaOpnRLxYvI4DzM3XJzM3nwE/RjOjyHJYWRbeP1KkZnIyMpVPzxtha16rs2FVGdnYOh8NOMPTlAbRt24qoqOtA0e0Q/yH9+Obb79FoNCZOWTalXtQp6Zk6Wq1Wt8zBwQGVSkV6ejoajQZra2scHBw4ffp0mQLEx8cTExNDZmYmWq1Wdx/SxcWlnL9SEVNf1IlJTGXE6gPUqV2LOf7eWJgVP00d3KklRyJ+55194bR3eRa/591Jz85j37krPCzUsGOKP+6NHE2QXp9SL+oAuLo68dO5IwB8tHEb+fn5TJv6Gra2tenSbRCxsbdNnLC4x13UMfkzdZydnavlDJ+ImLsAZOTks+TLMyVuM7hTSwZ3aomFhRk7Tkey5vDP1K5lSZeWTZg28AVcGzhUZuQqKS7uDj16+rP8g3d4a/YbqFQqzp79hbcXLFNkGZ/E4NseVY2pj5DViZKPkFWRUW57PI5GoyEjI4MTJ6rezAghlMTgizoZGRm8++67/Pjjj2RnZxeb7vZIdHS00cIJUdMYfIT88MMPOXbsGO7u7nh7e6PVahkyZAje3t5YWFhgZWXFRx99VJFZhaj2DD5ChoeH069fPz766CNSU1Px9vYmODiY9u3bEx0dzZgxY4iJianIrEJUewYfIVNSUujRowcA9erVo2HDhkRGRgLQpk0bAgICOHz4cMWkFKKGMLiQtra2ejdZXVxcuH79uu7rVq1aoVarjZtOiBrG4EK2b9+eY8eOUVhYCIC7uzsXL17UXdyJjY2lVi2ZVSLE0zC4kBMmTCAiIoIBAwaQnp7O8OHDiYmJYcKECbz33nvs2rWLrl27VmRWIao9gwvp7e3Nli1bcHNzo27durRv3565c+fyr3/9i3379tG2bVsWLFhQkVmFqPaeeqZOfn4+ubm51K1b11iZjEJm6hiPzNQxrnLNZTVUrVq1ZOwohJHIG5SFUBCTf9pDCPEneYOyEAoiz4cQQkGkkEIoiBRSCAWRQgqhIFJIIRRECimEghh1YgDIIzyEeBoGTwzQarUcP36cgoICXnzxRZo3b45GoyE+Pp4zZ85gZ2fHyJEjKyW0ENWVwRMDtmzZgpWVFQcPHsTNTf+NTHfu3GH06NEys0eIp2TwGPLzzz9n3LhxxcoI4OTkxNixYwkNDTVqOCFqGoMLmZmZ+dhPdWg0GvLz840SSoiayuBCduzYkd27d5OYmFhs3c2bN9mxYwddunQxajghahqDPw85e/ZsgoOD8fPzo1evXjg7O5OXl8etW7c4e/YsderUYd68eRWZVYhqz+BCtmvXjtDQUDZs2EB4eDjZ2UVvr7Wzs8Pf358ZM2ZU+ZerCmFqZXpigLu7Oxs2bECr1ZKamopKpaJevXoVlU2IGqfMj/BISUnh/PnzqNVq/Pz8dOVs0aJFReQTokYpUyG3b99OSEgIeXl5qFQqPD09ycrKYtq0aYwaNYp3331X7kUK8RQMvsp6+PBhVq1aha+vLyEhIboHJHt4eNCvXz/27dvH7t27KyyoEDWBwYXcvn07PXr0YM2aNXq3Nxo3bsyGDRvo1auXTAwQ4ikZfMp68+ZNAgICSl3fp08fli9fbpRQxiDPEjWeHPWPpo5QY5TpZTsZGRmlrler1djY2BgllBA1lcGF7NmzJ3v37iU5ObnYuqtXr7Jnzx66d+9u1HBC1DQGv0ogMTGRgIAAHj58SOfOnTl58iQDBgygoKCA8PBw7OzsCA0NxdnZuaIzG8SiVlNTR6g25JTVuCzrNy91XZne7XHv3j3Wrl3LqVOndKevtWvX5qWXXmLOnDmKKSNIIY1JCmlcRivkI48mAxQWFuLo6Ii5uTlQ9OIdpbznQwppPFJI43pcIQ0eQ/r4+HDq1Cmg6B0ejo6ONGjQQFfGsLAwevbs+ZRRhajZSr3tkZKSws2bN3VfJyQk8Ntvv5X42jmNRsOJEyfk85BCPKVST1mzsrIYNGgQSUlJBu1Iq9Xi5+fH2rVrjRqwvOSU1XjklNW4yj2GvHLlCtevX0er1bJw4UICAwPx8vIqtp2ZmRmOjo54e3tjYfHUr5w0Cimk8UghjetxhXxsezw8PPDw8ACKbvz379+fVq1aGTedEELH4Is6U6dOJT8/n1mzZulNDli5ciXTp0/XG28KIcrH4EJevHiR0aNHc+7cOVJTU3XLGzRoQEREBAEBAVy9erVCQgpRUxh8HzI4OJgHDx6wc+dOHBwc9Nalp6cTHBxMw4YN2bp1a4UELSsZQxqPjCGNyyj3IaOjowkKCipWRgB7e3sCAwOJjIwsX0IhBFCGQlpYWOidqv63zMxMNBqNUUIJUVMZXMiuXbvy+eefEx8fX2xdYmIin3/+uTyXVYinZPAYMiYmhpEjR6LRaHjppZdo1qwZgO5lOyqViv379yvmYVcyhjQeGUMal9Eml8fFxbFu3Tr++c9/6p7Lam1tTY8ePZg9e7ZiyghSSGOSQhpXhX3aQ6PRUK9ePd0EcyWRQhqPFNK4yj1TpzSPPu0hhDCuUgvp4+PDwoUL8fHx0X39JCqVipMnTxovnRA1TKmFbNKkid5Dq5o0aVIpgYSoyco1hqwKZAxpPDKGNC6jzNQRQlS8Uk9ZX3nllXLtcNeuXeUOI0RNV2oh79y5U2xZcnIyeXl52Nvb4+rqikajISEhgdTUVBwcHBR1H1KIqqjUQv7www/Fvp45cyYrVqzg5Zdfxszsz7PdsLAwFi1axJgxYyouqRA1gMFjyHXr1hEUFMSwYcP0yggwZMgQRo8eTUhIiNEDClGTGFzI27dv6+avlqRRo0bcu3fPGJmEqLEMLqSbmxtHjhyhsLCw2Lq8vDwOHjzIc889Z9RwQtQ0Bk+de/3115k9ezajR49mxIgRODs7k5ubS1xcHF988QVqtZrNmzdXZFYhqr0yTQz46quvWLNmDcnJybpXl2u1Wpo2bcrixYvp3bt3ReUsM5kYYDwyMcC4jPppD41Gw5UrV0hISEClUuHs7Ezbtm2fOqSxSSGNRwppXEb9tIeZmRkNGzZEo9HQvHlzrKys0Gg0xa68itLVr+/I+8vm4z+kP7VrW3Pp0m8sfGc5F375P1NHU7xzFyLYvOMLoq7dQGWmooNHa6b97RU6tGuj2yYlNY2QzTs5ffZn8vLyaPOcO7PeGK+3jVKVqUURERGMGDGCXr16MWrUKC5fvswvv/xC7969OXr0aEVlrFbs7Gw5/cNXjAzwZ/OWXSx570OaNGnMieNf4uEhF8Ue59dLkbzx1mIyMrOY/vo43hw/hviEu7w6dR6/RV0DICsrm3FT5vH9D/8kaPhgpv7tFe4lJTNx+gJ+j7ll2l/AAAYXMjIykvHjx5OVlcW4ceN4dKZrb2+PhYUFc+bM4cyZMxUWtLqYN3cKz7VqwbDhr/L3ZWv5aOM2evcdDsCctyabOJ2yrQzZTKNnG7B36zpeGTWcCWMC2LNlHbWtrQnZvBOAbXtCuXX7DhtXvceUiWMJDhzGrk8+BGD7ngOmjG8QgwsZEhKCk5MT33zzDa+//rpuuaenJ99++y0tWrSQq6wGeCU4kKPHTvHj2Qu6ZYmJScx7exln/7JM6Et/kMG1G7EM6NuT2tbWuuX1Hevxgpcn/74chVar5ZujJ3ipe2de6Oj55zbPODJn6mt06uBhiuhlYvAY8tKlS0yePBlra2tycnL01tnZ2REYGMiGDRuMHrA6adbMGSenxqxe84luma2tDVlZ2Xz6n3/hRcnsbG0I+2KrXhkfSUt7gLm5OQl3E0lMSmb8mJFA0R2AnJxcbGxqM2rEkMqOXC5lGkM+7u3IeXl58lzWJ2jp7gbAvaT7rFy+iOSkaNJTf+dq1FmGDO5n4nTKZm5ujqtzU55t8Ize8ms3Yrn0WxQdPdsSF58AgGM9e1Zv/AzvAQF06TeCQYETCD/7sylil5nBhezQoQNhYWElrsvOziY0NBRPT88S14si9g72ACxdMo9Bfj7Mmr2EceOnk52Tw8ED2/DpK2+gLovs7BwWLlsNwMSxI8nIzAJg49bd/POnX5g/4w0+WDwHa2srpi9Yxk+/XjJlXIMYfMo6ffp0goODGTt2LD4+PqhUKiIjI/n999/ZvXs3arWapUuXGvyDExMTyxS0YcOGZdpeiaysis4wHBzq0sajJ2lp6QCEhZ3g+tXzvP/+fE51l3t+hsjJzWXq20u5diOG14KD6OzVnm+/OwVARmYmYV98hn3dOgD07tGVQYETWP/pP/DuXPz9pkpicCG9vLzYvHkzS5YsYeXKlUDRJ0Cg6A1Ya9eupVu3bgb/YB8fnxLnxZYmOjra4G2VKiur6Fm2Xx86pisjQHr6Aw6HHeeV4JG6MaUo3YOMTKbMW8KlyCiGD+nPjEnjAHTjS99ePXRlBKhbx44+L3bjm2Mnyc7OwcamtklyG8LgQqalpdGjRw9OnDhBVFQUt2/fRqPR0LRpU9q1a1fmNyeHhoYyadIk8vPzeeuttxTz5uWKpE74A4CkpPvF1iUl3cfMzAw7O1sp5GMkp6YxadY7XP09hpFDB/Hu3Gm6aZwN/zO+dCzhhVCO9ezRarVk51STQg4bNoyRI0cyZcoUvTcrl1ebNm34xz/+QWBgIElJSUyeXP3vwV2+cpXc3Fzati3+FupmzVzIyckhKSm5hO8UUHSG8aiMrwQNZ9701/XWuzdvRq1altyIjSv2vQl3E7GqVYt6/xnHK5XBF3VSU1Np0KCBUX94ixYtmD17Np999hkpKSlG3bcSZWfncDjsBIP9fPVK2ayZM/5D+vHt4eNypfox3l/7CVd/j2HsyKHFyghgU9uaPi9248z5X7gR82cp76j/4PTZn+nTs5sin7L/VwZPLn/nnXe4fv06mzZton79+kYLUFhYSEREBO7u7kZ9GrpSJ5e7ujrx07kjAHy0cRv5+flMm/oatra16dJtELGxt02csDglTC6/ees2Q8dMom4dO+ZNex1zi+LF8h/Ql4S7ifzvazMAGBM4FEsLS/aEHiInN4/92zbg3LRxZUcvxiif9li8eDFhYWHk5+fj4uLCM888U2xCuUqlYudOZdzgVmohAdzcXFj+wTv4+vREpVJx9uwvvL1gGVev3jB1tBIpoZD7vz7CstUbH7vN5XPHAIhPuMu6Tdv56ddLaLVanu/QjremTKRFM5fKiPpERilk3759Dfph//1wLFNRciGrGiUUsjox+tuvqgIppPFIIY3rqT4P+fDhQ27cuEFBQQHu7u7Urq3cS8ZCVHWPLeSOHTv4+OOPyczMBIrmso4ePbrG3DcUorKV2qpDhw6xYsUKmjZtytChQzEzM+PChQvs2LGDwsJCFi5cWJk5hagRSh1DBgYGYmZmxs6dO7GysgKKPs4ya9YsTp8+za+//vrYT3+YmowhjUfGkMZVrrdf3bx5E39/f10Zoei2xquvvkp+fj4xMTHGTSmEKL2QOTk51KlTp9hyJycntFotDx48qNBgQtREpRZSo9HoJu3+1aOpR2X5pIYQwjDy7EYhFOSx9y7S0tJQq9V6y9LTiz7Hl5KSUmwdQJMmTYwYT4iapdSrrK1bty7xlBWKrraWtE6lUhEVFWXchOUkV1mNR66yGle5ZuoMHz68QsIIIUonc1nFE8kR0rjKdR9SCFH5pJBCKIgUUggFkUIKoSBSSCEURAophIJIIYVQECmkEAoihRRCQaSQQiiIFFIIBZFCCqEgUkghFEQKKYSCSCGFUBAppBAKIoUUQkGkkEIoiBRSCAWRQgqhINX2IVdCVEVyhBRCQaSQQiiIFFIIBZFCCqEgUkghFEQKKYSCSCGFUBAppBAKIoUUQkGkkEIoiBTSRMLCwhg8eDDt27dn0KBBHDp0yNSRqrzo6Gg8PDz4448/TB2l3KSQJnD06FHmzJnDiy++yMcff0yXLl14++23+e6770wdrcq6efMmkyZNoqCgwNRRnopMLjeBfv360a5dO9atW6dbNnPmTK5du8axY8dMmKzqKSgoYP/+/axZswZLS0vS0tI4c+YMjRo1MnW0cpEjZCWLj4/n9u3b9O/fX2/5gAEDiImJIT4+3kTJqqaIiAhWr17NhAkTmDNnjqnjPDUpZCWLiYkBwM3NTW+5q6srALGxsZWeqSpr0aIFJ0+eZOrUqZibm5s6zlOzMHWAmiYjIwMAOzs7veW2trYAZGZmVnqmqqx+/fqmjmBUcoSsZE8aspuZyR9JTSZ/+pWsTp06AGRlZektf3RkfLRe1ExSyEr2aOx4+/ZtveVxcXF660XNJIWsZK6urjg5ORW753j8+HGaNWtGkyZNTJRMKIFc1DGBKVOmsGDBAuzt7enduzenTp3i2LFjevclRc0khTSBESNGkJ+fz/bt2wkNDcXZ2ZmVK1fi5+dn6mjCxGSmjhAKImNIIRRECimEgkghhVAQKaQQCiKFFEJBpJBCKIjch1SA+fPn8/XXXz9xu+HDh7NixYpKSFS6vn370rRpU3bv3l0p31dZ+1MKKaQCBAUF4e3trfs6IiKC/fv3ExQURKdOnXTLXVxcTBFPVCIppAJ4eXnh5eWl+7qwsJD9+/fTsWNHhg4dasJkorLJGFIIBZFCVkF9+/Zl0aJFLFy4kPbt2/PSSy+RkpJC3759CQ4OLnH7/15+6dIlxo8frzs6T5gwgcjIyDJn0Wq1fPHFFwQEBODl5YWnpycDBw5ky5YtJX4YOzQ0FB8fHzw9PRk5ciQ//vhjsW2Mla0qkkJWUUeOHOHatWssXLiQwMBAHB0dDf7ec+fOERwcTEZGBjNmzODNN99ErVYzZswYLl68WKYc69ev57333sPd3Z0FCxYwe/ZsrKysWLNmDXv37tXb9vLly7z//vv4+fkxe/ZsHjx4wKRJkzh//nyFZKuStEJxDh48qG3VqpX24MGDJa7v06ePtnXr1to//vij2PKxY8eWuP2j5YWFhVofHx/tqFGjtAUFBbptsrKytP369dMOHTr0sdn+uq/8/Hzt888/r501a5beNhkZGdp27dppJ02apPd9rVq10oaHh+uWpaamart06aIdPnx4mbOV9rtWdXKErKJcXFxo2LBhmb8vKiqK+Ph4fH19SU9PJyUlhZSUFHJzc+nTpw/R0dEkJiYatC9LS0vOnz/P3//+d73lqamp2NnZkZ2drbe8VatW9OrVS/e1g4MD/v7+XLlyhaSkJKNmq6rkKmsV9cwzz5Tr+x49OmTVqlWsWrWqxG3UarXBZbe0tCQ8PJxTp04RGxtLXFwc6enpQPEHepX0eJJHt3ISEhJQq9VGzVYVSSGrqLI8g7SwsFD33xqNBoAZM2bQsWPHErdv3ry5QfvVarVMnjyZ06dP06lTJ7y8vAgKCqJz586MGzfO4H1A0dP2jJmtqpJCViNmZmbk5+frLSsoKCA1NVV3JGratCkANjY2dO/eXW/byMhI0tPTsba2NujnXbx4kdOnTzN58mRmzJih9zPT0tJwdnbW2z4hIaHYPm7dugWAs7Oz7h8OY2SrqmQMWY3Ur1+f2NhYcnNzdct++OEH8vLydF+3a9eOBg0asHv3br1HUWZmZjJz5kwWLFhg8NE3LS0NAHd3d73lX375JTk5OcVefHPlyhWioqJ0X9+/f59vv/2WF154gXr16hk1W1UlR8hqZMiQISxbtozXXnuNl19+mbi4OL788kvdURGKxnyLFi1i1qxZjBgxgoCAAKysrAgNDUWtVrN69WosLAz7a+Hl5YWdnR3Lly8nISEBe3t7Lly4wNGjR7Gysir27Fl7e3smTpzI+PHjMTc3Z8+ePRQUFLBgwQKjZ6uqqvdvV8OMHj2atLQ0Dhw4wLJly2jdujUbN25k+/btelc8Bw4ciL29PZs2beKTTz7BzMyMli1bsmnTJvr06WPwz6tfvz5btmxh9erVbNq0iVq1auHm5sbatWuJjIxk165d3L9/X/e4/549e+Lp6cm2bdtIS0ujQ4cOrF+/nnbt2hk9W1UlD7kSQkFkDCmEgkghhVAQKaQQCiKFFEJBpJBCKIgUUggFkUIKoSBSSCEURAophIL8P69MSKYH9IQSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_conf_mat(conf_mat):\n",
    "    \"\"\"\n",
    "    Plots a confusion matrix using Seaborn's heatmap().\n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(3,3))\n",
    "    ax = sns.heatmap(conf_mat,\n",
    "                     annot=True, # Annotate the boxes with conf_mat info\n",
    "                     cbar=False)\n",
    "    plt.xlabel(\"True label\")\n",
    "    plt.ylabel(\"Predicted label\")\n",
    "    \n",
    "plot_conf_mat(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEWCAYAAAAw6c+oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deVyU5d4/8M/MsMoAAnoQUBDJpUSNIs0l18ClzMRQj4IGLrihmZhRHj3PcyoTUVPLtNJS43iUXHItxcxfUulReTIVTREFlcUFkWGfmfv3x8jUOMAswmx83r3uVy+u+5p7vgPy5dru6xYJgiCAiIgMJjZ3AERE1ooJlIjISEygRERGYgIlIjISEygRkZGYQImIjMQESkSNTlDeN3cIjUJkC+tAv86ehxL5HXOH0aCmt9+ETy9PNHcYjSI93MXcITSKr7PXIipwhrnDaFAt/Dzx0fH3GuRa8rtjAGV+/ZXErWDnta1B3s8U7MwdQEMokd/Bg+oCc4fR4GzxMwFAwXVXc4fQaAqu3zZ3CBZLobgFKG7WX0mitKqkZE2xEpEVEx7+Vx+RjvOWhgmUiExCCQEClPXWYQIlIqqFXFBCKdSfQMU6zlsaJlAiMgkFBCh1tDB1dfEtDZcxEZFJKB8mUF2HsTIzM9G5c2fk52vO9IeFhaFjx45ax71799R1fv/9d0RHRyMkJAR9+vTBihUrUF1drfM92QIlIpNQCgIUulZNGrmqMisrC3FxcZDL5RrlpaWlyM3Nxbx589C9e3eNc25ubgCA69ev4/XXX0dISAg++ugjZGVlYeXKlZDJZFi0aFG978sESkQmoXx41Edk4DXlcjm2bduG5cuXw97eXuv8pUuXIAgCBg0ahKCgoFqv8dlnn8HV1RVr166Fg4MD+vXrBycnJ7z33nuIi4uDt7d3ne/PLjwRmYQCgl6HIU6fPo3k5GTExsYiISFB63xmZiacnJzQtm3bOq+Rnp6OAQMGwMHBQV02ZMgQKBQKHD9+vN73ZwIlIpOQC0C1jkNuYA8+KCgIaWlpmDVrFiQSidb5S5cuwd3dHW+++SZCQ0MREhKCuXPn4vZt1Q0P5eXlyMvLQ2BgoMbrPD09IZVKkZ2dXe/7swtPRCahgAgKHZ100cPzeXl5UCgUGufc3NzU45Y1WrRoUe/1Ll68iDt37qB9+/aIjo7G1atXsXr1akyYMAG7du1CSUkJAEAqlWq91sXFBTKZrN7rM4ESkUkoBdWhqw4AjB8/Hjdvat72OWvWLMTHxxv0ngsXLoQgCOjWrRsAIDQ0FEFBQRg3bhz27NmDfv361ft6sbj+TjoTKBGZhFKPFqj44fmUlJRaW6CG6tq1q1bZs88+C1dXV1y8eBEvvfQSANVs/aNkMhlcXevft4EJlIhMQp8ufE0C9fHxeez3Kysrw8GDB9G5c2d06tRJXa5UKlFdXQ0PDw+4uLjA29sb169f13jt3bt3UVpaqjU2qh0vEZEJyAUxqnUccqHhUpKjoyM+/PBDfPzxxxrlP/zwAyoqKtTrQnv37o2jR4+iqqpKXef777+HRCLRWjv6KLZAicgkFBBDoaPNpuu8ISQSCWbMmIEPP/wQ7733HgYOHIg//vgDa9aswaBBg9CjRw8AwOTJk7F//35MnToVEydOxLVr17BixQqMHj0avr6+9b4HEygRmYRqEqn+LryuSSZDxcTEQCqVYvPmzUhNTYW7uzvGjh2rMRkVFBSEjRs3IikpCbNnz4aHhwdiYmL0mrBiAiUik9BnEklp8L1If4qIiEBERIRWeWRkJCIjI+t9bWhoKLZv327wezKBEpFJKCCGQscYZ0N24U2BCZSITEIJMZQ6EqSu85aGCZSITKJaEKNK0L7d8q/EDTgLbwpMoERkEkqIdI5xPs4YqDkwgRKRSSj1WMbELjwRUS0Ugh6TSOzCExFp4yQSEZGRlAKgMPFC+sbGBEpEJlEt2KFaqD/l6DpvaawrWiKyWpxEIiIykkIQ6ezC6zpvaZhAicgkVOtAdbVAmUCJiLQo9VjGpOQyJiIibdWCBNU6buXUdd7SMIESkUmotrNjF56IyGBKiHRvqMwESkSkzdSP9DAFJlAiMglBEOucJBI4iUREpE2fxxrrOm9pmECJyCRUjzWuf5a9IR9rbApMoERkEko9uvBcB0pEVAvuB0pEZCRBj0d6CBwDJSLSptpMRFcLlAmUiEiLUtBjIT0TKBGRNrke98LLeS886ZL5T0eUXhMj9KtyjfJ7JyTI+tgBsksSZDSfimYDHRE0uxJ2zTRffzddguz1DnhwQQKRCHDvpkBQfCXcuylN+CmohrunHDGJeVAW9sTOS0W48rszNn7gg4tnXMwdmkWxxWciWVe0NuDmDnvc/MZBq/zeCQnOTHGGUC3CE3Mr8WJUX9xMtUdGnDOEv+TFov9KkDHNGdUlIgTNrkTg9EqU5Ypx6vVmKP6dP05Tc3ZRIHnXFfQdfh9w/js2J7VCi1bVWJqahYCO5bov0IQoIFJvqlznYWWTSPyNMxFBAVz91AGZ/3Ss9fzl5Y5w8hHw7FdlaPP3akz+MArt51eiOMMOd9P/7NZcWuoIp1YCuv+7DAETqtE2thrPpZRB4gxkrar92tR4Rs8sROugSiyaGAix62zs3tAS80Y+oTo347aZo7MsgvDnOGhdh2BlD5UzewLdt28fXnrpJXTt2hVDhw7F7t27zR1Sg1NUAicim+HqJ47wGS6Ho7dS67yDhwC/UdWQOP1Z7hGqAADI/lAl0OpiQHZJDO/Bckic/6zn2EKAR6gc93+zrvEj6ycgbHQRTh5xw7kTUnVp0W17fP6/vjh3gl34v6pZSK/rsCZmHQM9cOAAEhISMHHiRPTp0wdpaWlYsGABnJycMGTIEHOG1qCUlYC8VIQuyeXwHiLH8XDNXyyJIxCyXru7V3JR9Y/JyUeVcO2kQK99pRrJs0b1fRFEzJ8m5d2mCi19q5H6aUsAgCAIcGqmQEWZBPs2tTBzdJZHdStn/QmSt3IaYOXKlRg6dCgSExMBAC+88AKKi4uxatUqm0qgdlKg1/5SiPX8bpffEuH7k0fxxxInuLRXoOUgOQBAJAGaBWj3cUouiXE/QwKv3oqGDJt08GtXBQC4f8cOk/9xC0Lhs/j2igy3sh2w7p++OHHY3cwRWhZbvJXTbNHm5uYiJycH4eHhGuWDBw/G1atXkZuba6bIGp5IDL2TZ3UxkB4uRXLsWiiqgI6JlZDUM7QpLwPOv6Pq97edVNUA0ZK+pG6qP1gT5+ej+6AHELkuRFJ8G1SUi7F44zWEvFBi5ggti/LhnUi6DmtitgR69epVAEBgYKBGeUBAAAAgOzvb5DFZiuBl5Xhr0yxI2ymRMcUZBYdrz76KcuC3Wc6QXZKg7aQqeDzHFqgp2TuohlZc3BV4c8QTEDWLwJEdnkiIeAKlxRLEJOaZOULLohCgexbeyiaRzNaFLylR/XWWSqUa5S4uqvFBmUym97Wmt9/UcIGZwO/2M+Dt0hILnvqf2iv0VP2v72vPY0qXechfIceKOes0qsjul2Lh8CUoOnkJQ2IG4M310yESWcdf7wU2slxVqPgewv14NG/9GnYV/QsAcFiZCgBQFr8NN89dOCT/CiIxJ5MAbqjcoAQd6xXEYv2/kZ9enogH1QWPG5LJFFe7oLI0D0sv1D3Ou+Cp7/BR9gjY9XREXooD/pU+FA4equ9Z1V0RzsQ5Q3ZRAr/IKlS/uQdJmXtMFf5jSwt2NXcIDaJjSClW7wdSPjiKzcsicViZijBxJABg0ru3MHqmgDE+41F0297MkRrPO6Alvs5e2yDXssVbOc2W7l1dVb9EpaWlGuU1Lc+a801B6VUxjoe7IPc/2r9o8lIRIBIgdhAefg118vSfUIUnF1fCShqeNufaRSdUVYgQ0LFS61yrNlWoLBeh+C5v9qshhxhyQcdh/pWVBjFbtDVjnzk5ORrl169f1zjfFDj7KyGXiXBzmz2U1X+Wl98SofCwHTxCFbB72Au8+J4TZBclaBNVhQ5vaf/ikulUlkvw6yE39HjxAQI6VKjLvdtU4vnwB/jlkDuUSv51qyHosQaUXXg9BQQEoHXr1vjuu+8QFhamLj906BDatm0LX19fc4VmcmI7oGNiBc4nOuP0683Q6uVqbLFPxcnVzSASAx3fUSXK0iwx8vfaw85NgGsnBfL2av/4fIbLTR1+k/bFe77o2qsUSd9kQZB9itemF+LVSXdQWSHGl0tamTs8i6IUdHfRlZxE0t/MmTORmJgId3d39O/fH0eOHMHBgwexcuVKc4ZlFj7D5RDbl+PaRgf8keSIW9L98HhOgaDZlXBpq/pXVXRKtVJe/kCECwtrWU0PwGc4l86YUsENB8x5+QlMejcPfVt+gXFzSnHuhAs+/5cv8nN4a+1f6bNMydqWMZk1gUZERKCqqgobN25Eamoq2rRpg6VLl2LYsGHmDKvR9TlUWmu59xA5vIeoWpALnvpGa5Kp9ZhqtB5TXdtLyYzycxzxflxb9J+SigifSHOHY7FscRLJ7CPcY8eOxdixY80dBhE1MkGPBCowgRIRaZMrxZArddwLr+O8pWECJSKT4BgoEZGR2IUnIjKSEnosYzJNKA2GCZSITIKz8ERERlIqxVDomCRSchKJiEgbJ5GIiIzELjwRkZEEQaRzlt1mZuEnTJhg8MVEIhE2bbKuzY2JyDSaVAv0xo0bpoyDiGydoEcL01Z2Y/rhhx9MGQcR2TiFIIJCx/6oCltpgdansLAQeXl5aNeuHRwdHWFnZ2fQIziIqOmxxVl4g7Le6dOnERERgX79+mHs2LE4d+4cTp48if79++PAgQONFSMR2YCaSSRdhzXRO4GePXsWMTExKC0txcSJE9Xl7u7usLOzQ0JCAo4dO9YoQRKR9au5F76+w2YT6KpVq9C6dWt8++23mDp1qvqpml26dMGePXsQFBSE9evXN1qgRGTdBEG/w5ronUAzMjIQEREBJycnreePS6VSjB49GpcvX27wAInINjR2Fz4zMxOdO3dGfn6+Rvnx48cxatQodOvWDQMHDsTGjRu1Xvv7778jOjoaISEh6NOnD1asWIHqat1PfzBoDNTBwaHOc5WVlVAqrW0vFSIyFcXDe+F1HcbIyspCXFwc5HLNhyqeOXMG06ZNQ7t27bBmzRoMHz4cSUlJ2LBhg7rO9evX8frrr8PR0REfffQRYmNj8eWXX2LJkiU631fvWfhu3bph3759tS6wLysrQ2pqKrp06aLv5YioiRGgu4tuaA9eLpdj27ZtWL58Oezt7bXOr169Gk899RSWLVsGAOjbty/kcjnWrVuH6OhoODg44LPPPoOrqyvWrl0LBwcH9OvXD05OTnjvvfcQFxcHb2/vOt9f73Q/e/ZsXLhwAVFRUdi9ezdEIhHOnj2LzZs3Y8SIEbhx4wamTZtm4McnoqZCEPTpxht2zdOnTyM5ORmxsbFISEjQOFdZWYlTp04hPDxco3zw4MF48OABzpw5AwBIT0/HgAEDNHrYQ4YMgUKhwPHjx+t9f70TaEhICNavX4/8/HwsXboUgiBg5cqV+OCDD1BRUYEVK1bg+eef1/dyRNTU6DP+aeAYaFBQENLS0jBr1ixIJBKNc7m5uaiurkZgYKBGeUBAAAAgOzsb5eXlyMvL06rj6ekJqVSK7Ozset/foIX0vXv3xuHDh3HhwgXk5ORAqVTCz88PwcHBsLPjviREVDcBurvoNefz8vKgUCg0zrm5ucHNzU2jrEWLFnVeq6SkBIBqkvuvXFxcAAAymazOOjX1ZDJZvfEanPVEIhFatWoFhUIBsViMNm3aMHkSkU6CUgRBx62cNefHjx+PmzdvapybNWsW4uPj9X8/HeMBYrFYrzr1MSjz/fLLL0hOTsaFCxc0ykNDQ/HOO+/gySefNORyRNSEGLKdXUpKSq0tUEO4uroCAEpLSzXKa1qVrq6u6pbno3Vq6tVcoy56J9Djx48jLi4OUqkUUVFR8Pf3hyAIyM7Oxt69ezFu3Dh8/fXX6Ny5s76XJKImRJ+F8jXnfXx8Hvv9/P39IZFIkJOTo1Fe83VgYCBcXFzg7e2N69eva9S5e/cuSktLtcZGH6X3JNLq1avh7++PQ4cO4d1330V0dDQmTJiAxYsX47vvvoOnpyeSkpL0vRwRNTGmvhfe0dERoaGhOHTokEZX/fvvv4erqyuCg4MBqOZ2jh49iqqqKo06EokE3bt3r/c99E6gFy9exJgxY+Du7q51rkWLFhg3bhx+++03fS9HRE3Ow1n2+o4G3o1p+vTpOHPmDObOnYtjx47ho48+woYNGxAXFwdnZ2cAwOTJk3H79m1MnToVR48eVS+iHz16NHx9feu9vt4J9G9/+xuKiorqPK9QKNC8eXN9L0dETYw57oXv2bMn1qxZg6ysLMycORN79+7FW2+9hSlTpqjrBAUFYePGjSgrK8Ps2bPx5ZdfIiYmBu+++67O6+s9Bjpt2jS8//77CA0NxQsvvKBxLjMzE5s2beJCeiKqkyGz8MaIiIhARESEVnlYWBjCwsLqfW1oaCi2b99u8Hsa/EykqVOnon379ggMDIRIJMLNmzdx/vx5uLu749y5cwYHQERNhCELQa2EQc9E8vDwAKCa3v/999/V5a1atQIAnDp1qqHjIyJboc8kkZXtB8pnIhGRadhgC7RBH2R07969hrwcEdkckY7Duhh0J9LWrVvx008/oaysTGPvT4VCgdLSUly5coXjoERUOwGAri2DrawFqncC/fzzz7F8+XI4ODhAKpWiqKgIrVq1wv3791FeXg4nJydER0c3ZqxEZM302W3JysZA9e7C79y5E08++SR+/vlnbNu2DYIgYPPmzTh16hQWLVqEyspKdOvWrTFjJSIr1qSfiXTz5k2MGDECUqkUbdq0gbu7O06dOgWJRIJx48Zh2LBh2LRpU2PGSkTWTNDzsCJ6J1A7Ozv1PnqAalPSS5cuqb/u0aMHrl271qDBEZEN0XUbpxEbKpub3gk0KCgIGRkZ6q8DAwM1JoyKi4s1bsYnIvorkaDfYU30TqARERHYuXMnEhISUFZWhoEDB+LUqVP4+OOPceDAAWzatAmdOnVqzFiJyJopRfodVkTvWfi///3vyM/PR0pKCuzs7BAeHo7+/fvj448/BqDaEv/RhzoREWmwshamLgatA507dy7i4+PVj/BYt24d/vvf/6K4uBghISHw8vJqlCCJyAbY4J1IBj/M6NHnHz333HMNFgwR2bCmlEDr2o2pPiKRiEuZiKh2NriQ3qDdmIiIjKbPLLuttEC5GxMRNaim1IW3JunhLii4Xv/jR63NAiWQFmxbn6nG97f+z9whNBqb+2wSvwa7lD7rPK1tHahNJFAisgJNaQyUiKjBWVkLUxcmUCIyDY6BEhEZR6RUHbrqWBOjHulRWFiI3377DSUlJaiqqtLYnZ6IqFZNeTs7ADh9+jQiIiLQr18/jB07FufOncPJkyfRv39/HDhwoLFiJCIb0KR3Yzp79ixiYmJQWlqKiRMnqsvd3d1hZ2eHhIQEHDt2rFGCJCIb0JT3A121ahVat26Nb7/9FlOnToXwcO/9Ll26YM+ePQgKCsL69esbLVAisnJNuQufkZGBiIgIODk5QSTS/CshlUoxevRoXL58ucEDJCLbIIIeXXhzB2kgg2bhHRwc6jxXWVnJySQiqlOTnoXv1q0b9u3bV+u5srIypKamokuXLg0WGBHZmKbchZ89ezYuXLiAqKgo7N69GyKRCGfPnsXmzZsxYsQI3LhxA9OmTWvMWInImtlgAtW7Cx8SEoL169dj8eLFWLp0KQBg5cqVAICWLVtixYoVeP755xsnSiKyek1+M5HevXvj8OHDOH/+PHJzc6FUKuHn54fg4GCtneqJiGydwVlPJBIhODgYwcHBjREPEdmqpnwvvL6P+Ni8ebPRwRCR7RIJeszC22oCre0RH0qlEkVFRaisrISfnx/at2/foMERkQ1pyi3Quh7xoVAocOTIESxcuBCTJk1qsMCIyMbY4DORjNqN6a8kEgnCw8MRGRmJ5OTkhoiJiGyRDS5jeuwEWqNt27a4ePFiQ12OiGyMLe7G1CBrj6qqqrBnzx54eXk1xOWIyBYpHx666liRx56Fr6qqQnZ2Nh48eID4+PgGC4yIbEuTXkhf2yw8oBoDbdeuHV5++WWMGzeuwQIjIhtkZQlSF70T6I4dO+Dh4dGYsRCRLbPBZUx6TyKNHDkSa9eubcxYiMiGNelJpKKiIrRo0aIxYyEiW9aUW6Avv/wyUlNTcefOncaMh4hsVM2GyroOa6J3C1QsFuPKlSvo168f/P394eXlBbFYM/+KRCJs2rSpwYMkIhtggy1QvRNoenq6ehKpsrISt27darSgiMj2iKD7mUc2+0ykuu6FJyLSiw22QOscA01MTMRvv/1myliIyIbZ4lM560ygu3btQk5OjiljISJbZoObifA5HERkErb4WGMmUCIyDRscA603gZ46dQoKhcKgC7766quPFRAR2Sgb3FC53gS6fft2bN++Xa8LCYIAkUjEBEpEtWtqLdDRo0fj6aefNlUsRGTDmtx2dqGhoRg+fLipYiEiWyZA94bJtpRAiYgaSpNrgRIRNZimNAY6cuRI+Pv7mzIWIrJhIkGASKg/Q+o6/yi5XI5nnnkGlZWVGuXNmjVDRkYGAOD48eNYuXIlrly5Ai8vL0RFRSE2Ntaw4OtQZwJdsmRJg7wBERGARmmBZmdno7KyEkuXLkXbtm3V5TU7xZ05cwbTpk3D0KFDMWfOHJw+fRpJSUkQBAGTJk0y7M1qwS48EZlEY4yBXrx4EWKxGIMHD4azs7PW+dWrV+Opp57CsmXLAAB9+/aFXC7HunXrEB0dDQcHB8Pe8BEN9lx4IqL6iAQ9NlQ2MIFmZmbC39+/1uRZWVmJU6dOITw8XKN88ODBePDgAc6cOfM4HwcAEygRmUojbCZy6dIlODg4YNKkSQgJCcFzzz2HRYsWQSaTITc3F9XV1QgMDNR4TUBAAABV9/9xsQtvYdw95YhJzIOysCd2XirCld+dsfEDH1w842Lu0Jq0j+a3xs2rTli244pGefyw9vjj/2p+NpEAVDee9HnpPv7x+TV1vXMnXPDlhz64/FszSJvL0WtwMaIT8uHuZdit0tbMkC58Xl6e1m3kbm5ucHNz0yi7ePEiZDIZIiMjMW3aNJw7dw5r1qxBdnY23nzzTQCAVCrVeI2Li+rnJZPJHuPTqDCBWhBnFwWSd12Bl3c14DwDm9/diVdi7mBpahZmD2uP65e0uynU+L77tycOprRA156av3CCAOT84YReQ+6jz0vFELknQyhOAAD8rXWVut5vP0vx7rh2cHFTYOzsAoglAnZ93hL/l+6KlXsuw7V5E0miBkwijR8/Hjdv3tQ4NWvWLMTHx2uUrVy5Eu7u7ujYsSMA4LnnnoOXlxfmz5+P9PT0et/q0UcSGYMJ1IKMnlmI1kGVmD8qCCt/mY3dG37CsT3N8dWvmRg94zaWzeGyMlNSKICtq7zx9fJWtZ4vyHVARZkEPQc/wKBRRRC36gtlfpFWvbUL/SAWAyv3XIZvW1Vi7TWkGNNf7Iitq7wxdXHTeDyOIS3QlJSUWlugj+revbtWWf/+/TW+Li0t1fi6puXp6uqqI2LdLGYMNDMzE507d0Z+fr65QzETAWGji3DyiBvOnfizy1F02x6f/68vzp1gF96UqipEmDm4I7Yk+2DQa0Vo4VOlVefaJScAgH/7ijqvk5/rgGsXnTHotXvq5Kl6TSV6hD1AWqpnwwdvqZQCRDoOKFUZ1MfHB61bt9Y4Hk2gd+/eRWpqKnJzczXKKypUPw8vLy9IJBKtjeFrvn50bNQYFpFAs7KyEBcXB7lcbu5QzMa7TRVa+lbjzP9TJU9BEODUTPUXeN+mFjj4by9zhtfkVFWKUVYixjvrrmH+qhxIJNp1rj9MoG0eJtDyUu1EejfPHgDQtpP2Od+2lSi+Z4fCm/YNGLkFa+BJJJFIhEWLFuHrr7/WKD9w4AAkEgl69eqF0NBQHDp0CMJfFuh///33cHV1RXBw8GN+IDN34eVyObZt24bly5fD3r6J/COqg187Vevk/h07TP7HLQiFz+LbKzLcynbAun/64sRhdzNH2LQ0c1Xgy/RMSOr5Dbl+yQnNpAp89k8/HNvTHOWl0fAJeBKvL8hD/1fvAwCcmql2zygv1W6rlBSpLl502w5/86tu+A9hYWqWMemqoy9PT0+MHz8eW7ZsgVQqRWhoKE6fPo1169Zh/PjxCAgIwPTp0xETE4O5c+di5MiRyMjIwIYNGzBv3rxalz4Zyqwt0NOnTyM5ORmxsbFISEgwZyhmJ3VTtTYnzs9H90EPIHJdiKT4NqgoF2PxxmsIeaHEzBE2LWIx6k2egCqBlskkkD2QYP7qHMzbMAPOLkosmdEWad+oHgHu36ECzVwVOL6/Of56l2JVhQinflSNwVVVWkRHsPE1wjKmBQsWYO7cudi/fz+mTp2K3bt3Y/bs2UhMTAQA9OzZE2vWrEFWVhZmzpyJvXv34q233sKUKVMa5COZtQUaFBSEtLQ0eHl5YefOneYMxezsHVR/ml3cFZjUpxN2FUXgyI6t+PWwO776ORMxiXnI+OnxB72p4QyNugul4h5eibkDABC3GoB+/eMQN7ATvviXLwaMLIK9g4BRUwuxZbkPPpwZgLHxBVAqRPgqqRUqylWJUyKxsh00jNQYdyLZ29tjypQp9SbEsLAwhIWFGXZhPZn1T1+LFi3g5cWxPQCoKFP9KNIPuENW/OfftdIHEvx62A3tu5arx0TJMrw84a46edZwdBYwaFQRim7bI+cP1RjpuLkFGDmlEP9vT3NMG9QJM8I7QiwCRs8oBIAmtIxJ0O+wIjaxjOnr7LXmDuGxCVW/QbgXiZemjcXw+W8AAA4rUwEAypJlQOnn2FO8HiJJS3OG2XRJZgAOLSFudbDeauJWl+HR7nsAX6DS6RuIW3WEGMCM9cDf//c+bl7OQ8s2LeAd0BIb3/03xJJv4RN6BmKnx7sn2xrwqZwWKipwBgqu3zZ3GI/F0VmBb86LcPJACv41OR2HlakIE3L9/sMAAA7zSURBVEcCAN5ddw09wkR41XE6lEqRmSN9fN/f+j9zh2A4xVNAVTaU+e0BAHfy7JH493bo98p9RL1ZAECVPJX57ZFzxg9AS/zN5VUo8+U4uqs5PL3l6NZLBvcnVJdT5gNnjzyB9l1FsLvfWedG7WYj8YO45Y8Ncilb3FC5iYxeW77Kcgl+PeSGHi8+QECHP5e8eLepxPPhD/DLIXebSJ62ooVPNUofSHAwxQulJX/+GhXesMfh7Z7o1rsEnn9TLcvb+XlLfPKuHxR/WaV3Is0N509KMfz1O49e2obp0323rgxqEy1QW/HFe77o2qsUSd9kQZB9itemF+LVSXdQWSHGl0tqvxuGzGfW+zfxP5MCMfeV9hg6/i4qsAPfrukAsZ2AWR/cUNcbPbMQ700JxD8mtEOfYcXIz3XAzs9a4tn+DzAwQvvOJVvFFig1qoIbDpjz8hM4+4sLhNIvMG5OAa6ed8LcV55Afo6jucOjR/QaWozFG6/CqZkSG973xTcr9uLJZ0ux8tvL8G//5w7pL7xUjMS111BUaI91i31x7NvmiJxeiEVfZNe6QN9mNcIyJnNjC9TC5Oc44v24tug/JRURPpHmDoce2nzyQq3lvYY8QK8hDwD8OQZam/6v3lcvrm+q2AJtRBEREbh06RJatWJXlcgmKQT9DivCFigRmYQttkCZQInIRPRZKG9dGZQJlIhMQ48WqJXlTyZQIjKRRnissbkxgRKRSYgUgEjHJJHIyrYFYAIlIpMQCQJEOsZAdZ23NEygRGQa7MITERmLs/BEREbhOlAiImPps2Eyx0CJiLSJFIIes/BMoERE2jiJRERkHC5jIiIyGmfhiYiMo3x46KpjRZhAicgk2IUnIjKWUgCUOpqYSiZQIiJt7MITERlHBD268JxEIiKqBe9EIiIyEhMoEZGR9HnqJm/lJCKqhR7LmNgCJSKqDbvwRERGEqB7nad15U8mUCIyEbZAiYiMxARKRGQkhVJ16KpjRZhAicg0BKXq0FXHijCBEpGJcD9QIiLjKKF7Ft66GqBMoERkIpxEIiIyEhMoEZGRFArVoauOFWECJSIT4SQSEZFx2IUnIjISZ+GJiIwkKCFwIT0RkRF4KycRkZEEpe7HGrMFSkRUC04iEREZR1AKEHS0QAVdk0wWhgmUiEyDLVAiIiMpBT2WMTGBEhFpEZQKCDpu1RSUvJWTiEibIOixoTJboCbXws/T3CE0Cu+AluYOoXFI/MwdQeOxtc8mbtVgl/Ly9dA5SeTl69Fg72cKIkGwspRPRGQhxOYOgIjIWjGBEhEZiQmUiMhITKBEREZiAiUiMhITKBGRkZhAiYiMxARKRGQkJlAiIiMxgVqYffv24aWXXkLXrl0xdOhQ7N6929whkZ4yMzPRuXNn5OfnmzsUMhEmUAty4MABJCQkoE+fPvjkk0/QvXt3LFiwAN999525QyMdsrKyEBcXB7lcbu5QyIR4L7wFCQsLQ3BwMFauXKkue+ONN3Dp0iUcPHjQjJFRXeRyObZt24bly5fD3t4e9+/fx7Fjx9CqVcNtwkGWiy1QC5Gbm4ucnByEh4drlA8ePBhXr15Fbm6umSKj+pw+fRrJycmIjY1FQkKCucMhE2MCtRBXr14FAAQGBmqUBwQEAACys7NNHhPpFhQUhLS0NMyaNQsSicTc4ZCJ2cR+oLagpKQEACCVSjXKXVxcAAAymczkMZFuLVq0MHcIZEZsgVoIXUPRYjF/VESWhr+VFsLV1RUAUFpaqlFe0/KsOU9EloMJ1ELUjH3m5ORolF+/fl3jPBFZDiZQCxEQEIDWrVtrrfk8dOgQ2rZtC19fXzNFRkR14SSSBZk5cyYSExPh7u6O/v3748iRIzh48KDGulAishxMoBYkIiICVVVV2LhxI1JTU9GmTRssXboUw4YNM3doRFQL3olERGQkjoESERmJCZSIyEhMoERERmICJSIyEhMoEZGRmECJiIzEBGoh3n77bXTs2FHjePLJJ/HMM88gMjISu3btMkkcAwcORHR0tPrr6OhoDBw40ODryGQy3Lt3r8Hiqvn+PG6dhnydqa5HlosL6S1MYmIiPDw8AKh2aJLJZNizZw/efvttFBUVITY21qTxTJs2DeXl5Qa95ty5c5g+fTqSk5PRo0ePRoqMyPyYQC3Miy++iNatW2uUvfbaaxg2bBg++eQTREVFwcHBwWTx9O7d2+DX/PHHHygsLGyEaIgsC7vwVsDJyQkDBw6ETCbD5cuXzR0OET3EBGolRCIRAEChUABQjVUuXLgQ77zzDrp27Yq+ffuqxxwzMjIQExODkJAQhISEIDY2FmfPntW65oEDBzBixAh07doVL7/8Mo4cOaJVp7Yx0KysLMyZMwc9evTAs88+i+joaJw6dQoAsGbNGiQmJgIAJkyYoPHa/Px8vPXWW3j++efRpUsXvPrqq9izZ4/We547dw6xsbEICQnBCy+8gPXr1+vccLouv/zyCyZPnowePXqgc+fOeOGFF7Bo0SI8ePBAq25GRgZGjRqFLl26IDw8HF999ZVWHX0/AzUN7MJbAaVSiZMnT8LBwQFBQUHq8v3796Ndu3Z45513cOfOHXh6eiI9PR1xcXHo1KkT5syZg6qqKuzcuRPjx4/Hl19+idDQUADAzp07kZiYiJCQEMyfPx/Xr1/HG2+8AZFIBD8/vzpjuXbtGkaPHg07OztERUXB09MT//nPfxATE4OUlBSEhYXh9u3b2LZtG6ZNm4YuXboAAAoKChAZGQlBEBAdHQ13d3ccOXIE8+fPR2FhISZPngwAuHz5MqKjo+Hm5oYZM2aguroaGzduRFVVlcHft+PHj2PKlCl45plnMHv2bIhEIqSnp2Pbtm0oLi7GqlWrNOrHxsYiLCwMERERSEtLw5IlS1BSUoL4+HiDPgM1IQJZhAULFggdOnQQzp8/L9y9e1e4e/euUFhYKGRkZAhz5swROnToIHzwwQfq+gMGDBA6deok5Ofnq8sUCoUwaNAgYezYsYJcLleXl5aWCmFhYcKIESMEQRAEuVwu9OzZUxg1apRQVVWlrrdjxw6hQ4cOQlRUlLosKipKGDBggPrrOXPmCF27dhWuXbumLrt3757w7LPPCrNnz9a4zq+//qrx+bp37y4UFBSoy5RKpfDmm28KwcHBwp07dwRBEIT4+Hjh6aefFm7duqWud+XKFSE4OFjo0KGDXt/DGpMmTRIGDBggVFZWatQbPXq0EBISovW6pUuXanwvJ0yYIAQHBwv37t0z6DM8GgfZLnbhLczIkSPRs2dP9OzZE3369MGYMWNw5MgRREdHY968eRp1/f394e3trf76woULyM3NxYsvvoji4mLcu3cP9+7dQ0VFBQYMGIDMzEwUFBTg/PnzuHv3LiIiImBvb69+/YgRI+Du7l5nbEqlEseOHUO/fv3UTwsFAA8PD/z73//GwoUL63xdWloaQkNDYWdnp46rqKgI4eHhqKqqQnp6OpRKJX766Sf069cPPj4+6tcHBQWhT58+Bn8v169fjx07dmhMuhUVFUEqlaKsrEyr/l9bkGKxGFFRUaiqqsLPP/+s92egpoVdeAuzbNky9ZMexWIx3NzcEBQUBEdHR626Xl5eGl/XPA4kKSkJSUlJtV7/1q1byM/PB6BKwH8lkUg0EuOj7t+/j7KyslrrdOjQoc7XFRUVoaSkBGlpaUhLS6u1Tl5envr6j8YFAO3atcMPP/xQ53vURiKRIDc3F6tWrcKVK1eQk5ODgoKCWus2b94cnp6eGmVt2rQBANy8eVPvz0BNCxOohXnmmWe0ljHV5dHnkCuVSgDAnDlz8PTTT9f6mnbt2qmTSEVFhdb5mmvUpmYCq2ZCS181rxs8eDDGjh1ba52aZGVMXHXZsGEDkpKSEBgYiNDQUISHh6Nbt27YsmUL9u7dq1G3vs8kFosN/gzUNDCB2pCayZ9mzZqhV69eGufOnj2L4uJiODk5qX/Rax5YV0MQBNy8eRPt27ev9foeHh5wcnLSevAdoEpWt2/fxttvv611ztPTE87OzpDL5Vpx3bp1CxcuXICzszM8PDwglUq14gKAGzdu1PPJtVVWVmLNmjXo0aMHNm7cCDu7P/+pPzp5BADFxcWQyWSQSqXqsuzsbACqlrq+n4GaFo6B2pDg4GC0bNkSW7Zs0Xg8skwmwxtvvIHExERIJBI89dRT8PPzw9atWzXuMtq/fz+KiorqvL6dnR169+6NY8eOaXRXi4uLsWHDBuTm5gL48xn2Na1GOzs79O3bF8eOHcPFixc1rvnhhx9i5syZKCoqgkgkQlhYGH766SeN9a43btzAjz/+aND3oqKiAuXl5Wjbtq1G8szMzMTJkycBAHK5XF2uVCrxzTffqL+Wy+XYtGkTmjVrhp49e+r9GahpYQvUhtjb22PhwoWYO3cuIiIi8Nprr8HR0RGpqam4desWkpOT1cnkH//4B2bOnIkxY8Zg1KhRKCgoQEpKCpo3b17ve8ybNw+RkZGIjIzE+PHjIZVKsX37dpSVleGNN94AAPVY4tatW3Hnzh0MHz4cCQkJOHHiBMaPH4/x48fD19cXP/74I44ePYoxY8aoW71z5szBjz/+iKioKLz++uuQSCTYsmULXFxcDFrK5O7ujm7dumHnzp2QSqUIDAzE5cuXkZqaqk7wpaWl6kkzZ2dnrF69Gnl5efD398eBAweQkZGBxYsXw9XVFQD0/gzUdDCB2pghQ4bA3d0dn376KdauXQuxWIz27dvj008/xYABA9T1BgwYgPXr12PNmjVYsWIFvL298f777yMlJaXe6wcFBWHbtm1YsWIFvvjiC4jFYnTt2hVLly5VJ5CePXti6NChOHr0KH799VeEh4fD398f27dvx+rVq9UJt02bNkhMTNTYvMTHxwdbt25FUlISvvjiCzg4OCAyMhKAalbdEKtWrcKSJUuwY8cOVFVVwc/PD1OnTkVQUBDi4+Px66+/YvDgwQAANzc3LF26FB988AFSUlIQEBCAZcuW4ZVXXlFfT9/PQE0HHypHRGQkjoESERmJCZSIyEhMoERERmICJSIyEhMoEZGRmECJiIzEBEpEZCQmUCIiIzGBEhEZiQmUiMhI/x8eDdntRj5oTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "plot_confusion_matrix(clf, X, y);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Classification Report**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Precision - Indicates the proportion of positive identifications (model predicted class 1) which were actually correct. A model which produces no false positives has a precision of 1.0.\n",
    "- Recall - Indicates the proportion of actual positives which were correctly classified. A model which produces no false negatives has a recall of 1.0. \n",
    "- F1 score - A combination of precision and recall. A perfect model achieves an F1 score of 1.0.\n",
    "- Support - The number of samples each metric was calculated on. Accuracy - The accuracy of the model in decimal form. Perfect accuracy is equal to 1.0.\n",
    "- Macro avg - Short for macro average, the average precision, recall and F1 score between classes. \n",
    "- Macro avg doesn’t class imbalance into effort, so if you do have class imbalances, pay attention to this metric.\n",
    "- Weighted avg - Short for weighted average, the weighted average precision, recall and F1 score between classes. Weighted means each metric is calculated with respect to how many samples there are in each class. This metric will favour the majority class (e.g. will give a high value when one class out performs another due to having more samples)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.79      0.79        29\n",
      "           1       0.81      0.81      0.81        32\n",
      "\n",
      "    accuracy                           0.80        61\n",
      "   macro avg       0.80      0.80      0.80        61\n",
      "weighted avg       0.80      0.80      0.80        61\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.395"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(0.79 *0.79)/(0.79+0.79)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.0</th>\n",
       "      <th>1.0</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro avg</th>\n",
       "      <th>weighted avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.99990</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499950</td>\n",
       "      <td>0.99980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.99990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1-score</th>\n",
       "      <td>0.99995</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>0.499975</td>\n",
       "      <td>0.99985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>9999.00000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.9999</td>\n",
       "      <td>10000.000000</td>\n",
       "      <td>10000.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0.0  1.0  accuracy     macro avg  weighted avg\n",
       "precision     0.99990  0.0    0.9999      0.499950       0.99980\n",
       "recall        1.00000  0.0    0.9999      0.500000       0.99990\n",
       "f1-score      0.99995  0.0    0.9999      0.499975       0.99985\n",
       "support    9999.00000  1.0    0.9999  10000.000000   10000.00000"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Where precision and recall become valuable\n",
    "disease_true = np.zeros(10000)\n",
    "disease_true[0] = 1 # only one positive case\n",
    "\n",
    "disease_preds = np.zeros(10000) # model predicts every case as 0\n",
    "\n",
    "pd.DataFrame(classification_report(disease_true,\n",
    "                                   disease_preds,\n",
    "                                   output_dict=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To summarize classification metrics:\n",
    "\n",
    "- **Accuracy** is a good measure to start with if all classes are balanced (e.g. same amount of samples which are labelled with 0 or 1).\n",
    "- **Precision** and recall become more important when classes are imbalanced.\n",
    "- If false positive predictions are worse than false negatives, aim for higher precision.\n",
    "- If false negative predictions are worse than false positives, aim for higher recall.\n",
    "- **F1-score** is a combination of precision and recall."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.2 Regression model evaluation metrics\n",
    "\n",
    "Model evaluation metrics documentation - https://scikit-learn.org/stable/modules/model_evaluation.html\n",
    "\n",
    "1. R^2 (pronounced r-squared) or coefficient of determination.\n",
    "2. Mean absolute error (MAE)\n",
    "3. Mean squared error (MSE)\n",
    "\n",
    "**R^2**\n",
    "\n",
    "What R-squared does: \n",
    "- Compares your models predictions to the mean of the targets. Values can range from negative infinity (a very poor model) to 1. \n",
    "- For example, if all your model does is predict the mean of the targets, it's R^2 value would be 0. \n",
    "- And if your model perfectly predicts a range of numbers it's R^2 value would be 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8654448653350507"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# Fill an array with y_test mean\n",
    "y_test_mean = np.full(len(y_test), y_test.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21.488235294117644"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model only predicting the mean gets an R^2 score of 0\n",
    "r2_score(y_test, y_test_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model predicting perfectly the correct values gets an R^2 score of 1\n",
    "r2_score(y_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean absolue error (MAE)**\n",
    "\n",
    "MAE is the average of the aboslute differences between predictions and actual values. It gives you an idea of how wrong your models predictions are."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.136382352941176"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "mae = mean_absolute_error(y_test, y_preds)\n",
    "mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual values</th>\n",
       "      <th>predicted values</th>\n",
       "      <th>differences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>23.6</td>\n",
       "      <td>23.081</td>\n",
       "      <td>-0.519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>32.4</td>\n",
       "      <td>30.574</td>\n",
       "      <td>-1.826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>13.6</td>\n",
       "      <td>16.759</td>\n",
       "      <td>3.159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>22.8</td>\n",
       "      <td>23.460</td>\n",
       "      <td>0.660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>16.1</td>\n",
       "      <td>16.893</td>\n",
       "      <td>0.793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>17.9</td>\n",
       "      <td>13.159</td>\n",
       "      <td>-4.741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>9.6</td>\n",
       "      <td>12.476</td>\n",
       "      <td>2.876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>17.2</td>\n",
       "      <td>13.612</td>\n",
       "      <td>-3.588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>22.5</td>\n",
       "      <td>20.205</td>\n",
       "      <td>-2.295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>21.4</td>\n",
       "      <td>23.832</td>\n",
       "      <td>2.432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     actual values  predicted values  differences\n",
       "173           23.6            23.081       -0.519\n",
       "274           32.4            30.574       -1.826\n",
       "491           13.6            16.759        3.159\n",
       "72            22.8            23.460        0.660\n",
       "452           16.1            16.893        0.793\n",
       "..             ...               ...          ...\n",
       "412           17.9            13.159       -4.741\n",
       "436            9.6            12.476        2.876\n",
       "411           17.2            13.612       -3.588\n",
       "86            22.5            20.205       -2.295\n",
       "75            21.4            23.832        2.432\n",
       "\n",
       "[102 rows x 3 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(data={\"actual values\": y_test,\n",
    "                        \"predicted values\": y_preds})\n",
    "df[\"differences\"] = df[\"predicted values\"] - df[\"actual values\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mean squared error (MSE)**\n",
    "\n",
    "MSE is the average of the square value of aboslute differences between predictions and actual values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.867437068627442"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "y_preds = model.predict(X_test)\n",
    "mse = mean_squared_error(y_test, y_preds)\n",
    "mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.867437068627442"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate MSE by hand\n",
    "squared = np.square(df[\"differences\"])\n",
    "squared.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2.3 Finally using the scoring parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81967213, 0.90163934, 0.83606557, 0.78333333, 0.78333333])"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# By default cross_val_score uses the scoring provided in the given estimator, \n",
    "# which is usually the simplest appropriate scoring method. \n",
    "# E.g. for most classifiers this is accuracy score and for regressors this is r2 score.\n",
    "cv_acc = cross_val_score(clf, X, y, cv=5, scoring=None)\n",
    "cv_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy is: 82.48%\n"
     ]
    }
   ],
   "source": [
    "# Cross-validated accuracy\n",
    "print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cross-validated accuracy is: 82.48%\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_acc = cross_val_score(clf, X, y, cv=5, scoring=\"accuracy\")\n",
    "print(f'The cross-validated accuracy is: {np.mean(cv_acc)*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8085601538512754"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Precision\n",
    "cv_precision = cross_val_score(clf, X, y, cv=5, scoring=\"precision\")\n",
    "np.mean(cv_precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8424242424242424"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recall\n",
    "cv_recall = cross_val_score(clf, X, y, cv=5, scoring=\"recall\")\n",
    "np.mean(cv_recall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.841476533416832"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_f1 = cross_val_score(clf, X, y, cv=5, scoring=\"f1\")\n",
    "np.mean(cv_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How about our regression model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6243870737930857"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_r2 = cross_val_score(model, X, y, cv=5, scoring=None)\n",
    "np.mean(cv_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.77231143, 0.86035935, 0.74664002, 0.47632078, 0.26630379])"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "cv_r2 = cross_val_score(model, X, y, cv=5, scoring=\"r2\")\n",
    "cv_r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-2.13045098, -2.49771287, -3.45471287, -3.81509901, -3.11813861])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean absolute error\n",
    "cv_mae = cross_val_score(model, X, y, cv=5, scoring=\"neg_mean_absolute_error\")\n",
    "cv_mae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-21.12863512415064"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Mean squared error\n",
    "cv_mse = cross_val_score(model, X, y, cv=5, scoring=\"neg_mean_squared_error\")\n",
    "np.mean(cv_mse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 Using different evalution metrics as Scikit-Learn functions\n",
    "\n",
    "**Classification evaluation functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier metrics on the test set\n",
      "Accuracy: 85.25%\n",
      "Precision: 0.8484848484848485\n",
      "Recall: 0.875\n",
      "F1: 0.8615384615384615\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = heart_disease.drop(\"target\", axis=1)\n",
    "y = heart_disease[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make some predictions\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "# Evaluate the classifier\n",
    "print(\"Classifier metrics on the test set\")\n",
    "print(f\"Accuracy: {accuracy_score(y_test, y_preds)*100:.2f}%\")\n",
    "print(f\"Precision: {precision_score(y_test, y_preds)}\")\n",
    "print(f\"Recall: {recall_score(y_test, y_preds)}\")\n",
    "print(f\"F1: {f1_score(y_test, y_preds)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Regression evaluation functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Regression model metrics on the test set\n",
      "R^2: 0.8654448653350507\n",
      "MAE: 2.136382352941176\n",
      "MSE: 9.867437068627442\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "X = boston_df.drop(\"target\", axis=1)\n",
    "y = boston_df[\"target\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=100)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions using our regression model\n",
    "y_preds = model.predict(X_test)\n",
    "\n",
    "# Evaluate the regression model\n",
    "print(\"Regression model metrics on the test set\")\n",
    "print(f\"R^2: {r2_score(y_test, y_preds)}\")\n",
    "print(f\"MAE: {mean_absolute_error(y_test, y_preds)}\")\n",
    "print(f\"MSE: {mean_squared_error(y_test, y_preds)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Improving a model\n",
    "\n",
    "First predictions = baseline predictions. First model = baseline model.\n",
    "\n",
    "From a data perspective:\n",
    "- Could we collect more data? (generally, the more data, the better)\n",
    "- Could we improve our data?\n",
    "\n",
    "From a model perspective:\n",
    "- Is there a better model we could use?\n",
    "- Could we improve the current model?\n",
    "\n",
    "Hyperparameters vs. Parameters\n",
    "- Parameters = model find these patterns in data\n",
    "- Hyperparameters = settings on a model you can adjust to (potentially) improve its ability to find patterns\n",
    "\n",
    "Three ways to adjust hyperparameters:\n",
    "1. By hand\n",
    "2. Randomly with RandomSearchCV\n",
    "3. Exhaustively with GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/hyperparameter-tuning.jpg\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "clf = RandomForestClassifier(n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.1 Tuning hyperparameters by hand\n",
    "\n",
    "Let's make 3 sets, training, validation and test."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/train-validate-test.jpg\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/tuning-hyperparameters-by-hand.jpg\" />"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're going to try and adjust:\n",
    "\n",
    "- max_depth\n",
    "- max_features\n",
    "- min_samples_leaf\n",
    "- min_samples_split\n",
    "- n_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preds(y_true, y_preds):\n",
    "    \"\"\"\n",
    "    Performs evaluation comparison on y_true labels vs. y_pred labels\n",
    "    on a classification.\n",
    "    \"\"\"\n",
    "    accuracy = accuracy_score(y_true, y_preds)\n",
    "    precision = precision_score(y_true, y_preds)\n",
    "    recall = recall_score(y_true, y_preds)\n",
    "    f1 = f1_score(y_true, y_preds)\n",
    "    metric_dict = {\"accuracy\": round(accuracy, 2),\n",
    "                   \"precision\": round(precision, 2),\n",
    "                   \"recall\": round(recall, 2),\n",
    "                   \"f1\": round(f1, 2)}\n",
    "    print(f\"Acc: {accuracy * 100:.2f}%\")\n",
    "    print(f\"Precision: {precision:.2f}\")\n",
    "    print(f\"Recall: {recall:.2f}\")\n",
    "    print(f\"F1 score: {f1:.2f}\")\n",
    "    \n",
    "    return metric_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.81\n",
      "Recall: 0.88\n",
      "F1 score: 0.85\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.82, 'precision': 0.81, 'recall': 0.88, 'f1': 0.85}"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Shuffle the data\n",
    "heart_disease_shuffled = heart_disease.sample(frac=1)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split the data into train, validation & test sets\n",
    "train_split = round(0.7 * len(heart_disease_shuffled)) # 70% of data\n",
    "valid_split = round(train_split + 0.15 * len(heart_disease_shuffled)) # 15% of data\n",
    "X_train, y_train = X[:train_split], y[:train_split] # training set\n",
    "X_valid, y_valid = X[train_split:valid_split], y[train_split:valid_split] # validation set\n",
    "X_test, y_test = X[valid_split:], y[valid_split:] # test set\n",
    "\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "# Make baseline predictions - Practice exam\n",
    "y_preds = clf.predict(X_valid)\n",
    "\n",
    "# Evaluate the classifier on validation set\n",
    "baseline_metrics = evaluate_preds(y_valid, y_preds)\n",
    "baseline_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 80.43%\n",
      "Precision: 0.83\n",
      "Recall: 0.80\n",
      "F1 score: 0.82\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.8, 'precision': 0.83, 'recall': 0.8, 'f1': 0.82}"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make baseline predictions - Final exam\n",
    "y_preds = clf.predict(X_test)\n",
    "\n",
    "# Evaluate the classifier on test set\n",
    "baseline_metrics = evaluate_preds(y_test, y_preds)\n",
    "baseline_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 82.22%\n",
      "Precision: 0.84\n",
      "Recall: 0.84\n",
      "F1 score: 0.84\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "# Create a second classifier with different hyperparameters\n",
    "clf_2 = RandomForestClassifier(n_estimators=100)\n",
    "clf_2.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions with different hyperparameters\n",
    "y_preds_2 = clf_2.predict(X_valid)\n",
    "\n",
    "# Evalute the 2nd classsifier\n",
    "clf_2_metrics = evaluate_preds(y_valid, y_preds_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2 Hyperparameter tuning with RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.3s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    2.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.6s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   3.3s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   3.0s\n",
      "[CV] n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=1200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=5, total=   2.7s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.3s\n",
      "[CV] n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30 \n",
      "[CV]  n_estimators=100, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=30, total=   0.2s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.5s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.5s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=2, min_samples_leaf=2, max_features=sqrt, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20 \n",
      "[CV]  n_estimators=100, min_samples_split=6, min_samples_leaf=1, max_features=auto, max_depth=20, total=   0.2s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=1, max_features=sqrt, max_depth=5, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=10, min_samples_split=4, min_samples_leaf=2, max_features=auto, max_depth=10, total=   0.0s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=500, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.9s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None \n",
      "[CV]  n_estimators=200, min_samples_split=6, min_samples_leaf=2, max_features=sqrt, max_depth=None, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10 \n",
      "[CV]  n_estimators=200, min_samples_split=4, min_samples_leaf=4, max_features=auto, max_depth=10, total=   0.4s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   2.3s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.8s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.9s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.9s\n",
      "[CV] n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20 \n",
      "[CV]  n_estimators=1000, min_samples_split=4, min_samples_leaf=2, max_features=sqrt, max_depth=20, total=   1.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  50 out of  50 | elapsed:   36.5s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "grid = {\"n_estimators\": [10, 100, 200, 500, 1000, 1200],\n",
    "        \"max_depth\": [None, 5, 10, 20, 30],\n",
    "        \"max_features\": [\"auto\", \"sqrt\"],\n",
    "        \"min_samples_split\": [2, 4, 6],\n",
    "        \"min_samples_leaf\": [1, 2, 4]}\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup RandomizedSearchCV\n",
    "rs_clf = RandomizedSearchCV(estimator=clf,\n",
    "                            param_distributions=grid, \n",
    "                            n_iter=10, # number of models to try\n",
    "                            cv=5,\n",
    "                            verbose=2)\n",
    "\n",
    "# Fit the RandomizedSearchCV version of clf\n",
    "rs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 200,\n",
       " 'min_samples_split': 6,\n",
       " 'min_samples_leaf': 2,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': None}"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 81.97%\n",
      "Precision: 0.77\n",
      "Recall: 0.86\n",
      "F1 score: 0.81\n"
     ]
    }
   ],
   "source": [
    "# Make predictions with the best hyperparameters - Final exam\n",
    "rs_y_preds = rs_clf.predict(X_test)\n",
    "\n",
    "# Evaluate the predictions\n",
    "rs_metrics = evaluate_preds(y_test, rs_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3 Hyperparameter tuning with GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': [10, 100, 200, 500, 1000, 1200],\n",
       " 'max_depth': [None, 5, 10, 20, 30],\n",
       " 'max_features': ['auto', 'sqrt'],\n",
       " 'min_samples_split': [2, 4, 6],\n",
       " 'min_samples_leaf': [1, 2, 4]}"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2700"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(n_estimators) - 6\n",
    "# len(max_depth) - 5\n",
    "# len(max_features) - 2\n",
    "# len(min_samples_split) - 3\n",
    "# len(min_samples_leaf) - 3\n",
    "# cross validation - 5\n",
    "\n",
    "6*5*2*3*3*5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reduce search space of hyperparameter\n",
    "grid_2 = {'n_estimators': [100, 200, 500],\n",
    "          'max_depth': [None],\n",
    "          'max_features': ['auto', 'sqrt'],\n",
    "          'min_samples_split': [6],\n",
    "          'min_samples_leaf': [1, 2]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(n_estimators) - 3\n",
    "# len(max_depth) - 1\n",
    "# len(max_features) - 2\n",
    "# len(min_samples_split) - 1\n",
    "# len(min_samples_leaf) - 2\n",
    "# cross validation - 5\n",
    "\n",
    "3*1*2*1*2*5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.3s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.5s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.5s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.5s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   1.0s\n",
      "[CV] max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=auto, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   1.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   1.0s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=1, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=100, total=   0.2s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=200, total=   0.4s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n",
      "[CV] max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500 \n",
      "[CV]  max_depth=None, max_features=sqrt, min_samples_leaf=2, min_samples_split=6, n_estimators=500, total=   0.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  60 out of  60 | elapsed:   30.9s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "# Split into X & y\n",
    "X = heart_disease_shuffled.drop(\"target\", axis=1)\n",
    "y = heart_disease_shuffled[\"target\"]\n",
    "\n",
    "# Split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Instantiate RandomForestClassifier\n",
    "clf = RandomForestClassifier(n_jobs=1)\n",
    "\n",
    "# Setup GridSearchCV\n",
    "gs_clf = GridSearchCV(estimator=clf,\n",
    "                      param_grid=grid_2, \n",
    "                      cv=5,\n",
    "                      verbose=2)\n",
    "\n",
    "# Fit the GridSearchCV version of clf\n",
    "gs_clf.fit(X_train, y_train);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': None,\n",
       " 'max_features': 'sqrt',\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 6,\n",
       " 'n_estimators': 200}"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    }
   ],
   "source": [
    "gs_y_preds = gs_clf.predict(X_test)\n",
    "\n",
    "# evaluate the predictions\n",
    "gs_metrics = evaluate_preds(y_test, gs_y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAINCAYAAADrzqHiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde3yP9f/H8efns9nBPtqGzdmMObNyKOccKowOmkQ66OCQdHAK00FHpWjOpCIkx4qcRYmUH0L5ppDJFmKY2Rx2+nx+fyyrTxsb29vnMx73283ttr2v93Vdr+uz9217uq73dV0Wh8PhEAAAAIyxuroAAACAax2BCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABjm6eoC8iIh4Yzsdh4XdkGJEjadOJHs6jJQSDBekFeMFVwOxkt2VqtFgYF+OS4rFIHLbncQuP6DzwOXg/GCvGKs4HIwXvKOS4oAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgWKG4SxEAcO07d+6MkpMTlZGR5upSkAfHjlllt9tdXcZV4eFRRDabv3x9c37kQ14QuAAALpeWlqqkpAQFBJRUkSLeslgsri4JufD0tCo9/doPXA6HQ2lpKTp16rg8PYuoSBGvK9oOlxQBAC6XlHRKNpu/vLx8CFtwKxaLRV5ePvLz81dy8qkr3g6BCwDgcunpqfL29nV1GcBF+fj4Ki0t9YrXJ3ABAFzObs+Q1erh6jKAi7JaPWS3Z1z5+gVYCwAAV4xLiXBn+R2fBC4AAADDuEsRAODWit3gKx9v1/65Op+SrqTT5y57vfvuu0sNG96iYcNeMlBV3h05clhdutytl156Te3addCKFUs1cuSr+vzz5QoOLuXS2q4XBC4AgFvz8fbUXYOWuLSGpWPuUZJLKyhYTZo019SpMxQYWNzVpVw3CFwAAFxnAgMDFRgY6OoyrisELgAADEpLS9Xo0W/pq69WydPTU61b36GnnnpGRYv6KSMjQ59+Oktr1qzUoUOHZLVaVLVqdfXq1Vf16zeUJKWknNeECWO1adMGnTqVoDJlyurOOzupe/eHs/aRmHhKU6dO1MaN3+rs2bOqXr2G+vZ9RuHhN+VY038vKb755is6ceKE2rS5TZ98MlNHj/6lkJBQ9e37jBo1apK13l9/HdHkyeO1ZctmpaenKTy8np55ZoBCQyub/RCvAQQuAAAMWrt2jerWvVEvv/yGjhw5pPffn6y//jqs0aPHa/Lkcfryyy/05JPPqHLlKoqPj9fHH3+gl18epkWLlsnHx0fjxo3R1q3/p6ef7q/AwOLavPl7TZ48ToGBgYqIuFMpKSl67rmnlJBwUk8+2U/Fi5fU4sWfqX//pzRp0geqWbN2nurcvXuXjh37Sz17Pik/P5s+/HCqXnxxiL74YqVsNptOnTqlvn2fkK+vrwYPHiZfXx998slMPfVUT82YMUelS5cx/EkWbgQuAAAMCggI0Jgx4+Xt7SNJ8vT01OjRb2vfvj06fjxeffr0U+fO92f19/b20gsvDNGBA/tVs2Zt7dy5XQ0bNtJtt7WVJNWv31BFixaVv3+AJGn16hXav3+fPvhgpmrUqCVJaty4qXr16qH335+ksWMn56nO5ORkTZ8+R2XLlpMk+fr66umne2vHjm1q0aKV5s+fo9OnE/X++zMUHFxKnp5WNWzYSF273quZMz/S0KEvFthndi0icAEAJEmpGWkKCiqW7+2cT0tV0qmUAqjo2tCkSfOssCVJzZu30ujRb+vnn3fq1VffkiQlJCQoNvag/vwzVps2bZQkpaVlvsS7fv2GWrz4M8XHH1WTJs3UpElzPfpoz6zt/fjjFgUFBSssrJrS09Oz2ps2ba5PPvk4azu5KVGiZFbYkqSgoGBJ0rlz5//ez1ZVr15TxYuX+Hs/Vnl4eOrmmxtp69b/u4JP5vpC4AIASJK8PIro/vl9872dBV2nKEkErgv+eydgQEDmmank5GT99ttujRnztn79dbd8fHwUGlpZpUqVliQ5HJn9n312kIKCgrVmzUpFR7+r6Oh3VadOuAYNGqaqVaspMTFRx44dVatWjXPcf2Ji3t7/5+Pj4/S91Wr9u47MF1SfPp2oP/+My3E/np7EidzwCQEAYFBSkvMDJRISTkqSfH2LatCgZxQWVl2zZy9QSEglWa1W/fDDd1q//uus/l5eXurR4wn16PGE/vrrL23atEEzZ36k119/SbNmzZfNZlOlSqF68cVXc9y/v3+Ajh+Pz/dx+PnZ1KDBzerb9xlJkoeHVRkZ9nxv93rBk+YBADBo27Ytysj45x1833yzVpJUp05dJSYmqmvX7goNrZx1Rmnz5u8lZZ5ZSklJ0QMPRGru3E8kSaVLl1bnzvfr9tvb6tixo5Kkm26qr7/+OqKSJYNUo0atrH8bN36rhQvnFdjZp5tuqq/Y2IMKCQlVjRq1VLNm5n6WLPlCX321ukD2cS3jDBcAAAbFxx/ViBFR6tTpPu3bt1cffDBFHTrcpYoVK8nPz08ff/yhLJbMlyOvX79Oy5d/KUk6d+6cvL29VatWbc2Y8YGKFPFUlSpVFRt7UCtWLFOrVrdJkjp0uFuLFi1Q//5P6eGHH1NQULA2bdqo+fPn6LHHehXYOyq7dXtQq1cv14AB/dS1a3f5+/tr+fKlWr16hYYPH1Eg+7iWEbgAADCoU6f7lJR0WlFRg+Tt7aMuXbqpV6++8vT01FtvjdHkyeP14otDVbSon6pWra6JE6dp8ODn9PPPO9WkSTMNHjxc/v4Bmjv3E508eUKBgcV1112d1LPnk5KkokWLavLkDzR16kRNmBCts2fPqmzZchow4Hl17ty1wI4jKChYU6ZM1/vvT9SoUW8qPT1NFStW0iuvvKnbb29XYPu5VlkcjgvT8tzXiRPJstvdvsyrJiiomOLjr6WXTMAkxgvyKiioWIFNmr/cMffXXwdVunRIjssK87sUr2Wenlalp19fc7guNU4lyWq1qEQJW47LOMMFAHBrSafPXVPvMcT1iUnzAAAAhnGGC3BTgf5e8vTyzvd27OmpBVAN3F1BjRdc+zw8LAUykb4QzEhyKwQuwE15enkr5s3O+d5O5Rc+k3gI5TWvIMZL5ljBtc5isSjlyP58b8e7TBVJhK684pIiAACAYQQuAAAAwwhcAAAAhjGHCzDAHZ4bBABwH/xFAAzw8fbUXYOW5GsbS8fcU0DVAABcjUuKAAAAhhG4AAAADOOSIgDArbnDQ13TU1OUkFjwDxF++une8vDw1LhxkyVJMTH79cYbI3TgwH5VrBiimTPn5Wk7Z84ka8aMD7Vhwzc6efKEypYtp3vv7aJOnToXyENOkX8ErqusIH5xpKanKiioWL5rOZ+WqqRTPBATQMGyp17+76hjx6zy9PznoovD7lCGPfOhmgX1EOD8yHworPm3Nsyc+aGOHDmskSPfVWBgiWzLrVaLrNbsAeqVV17Q7t2/qFevPgoJqaStW7coOvodnT2brB49HjdeN3JH4LrKCupp0PfP75vvWhZ0naIknkAOoIBZvby06Z7L+z3nO+BZJZ9Nz/reFlZFsl9/TzFPTExUlSphatKkeY7LrVaLfo875dR28MDv+uGHTXpm4Mu6qVErSVLbu6rr0JHjmjlzhpq1iXTqH1YhwEjtuDTmcAEAYJDD4dD8+XPUvXtntWnTTN26RWrRouyXCps3b6ht27Zo587tat68oVasWJq37cuh1rffqdp16ju1lylXUWfPnlFSUmKBHAfyhzNcAAAYNHnyeC1Y8Km6d39E9es31K5dP2ncuDGyWj2c+k2dOkPjxr2rjIwMDRw4TOXKlc/T9iuFVtUTfQZma/9x6yb5BxSXzXZDgRwH8ifPgWvZsmWaMmWK4uLiVK5cOfXp00edOnW6aP+TJ0/q3Xff1caNG5Wamqp69eopKipKlSpVKoi6AQBwe0lJSVqw4FPdf3939enTT5J0882NFB9/TDt3bnfqW6dOXRUtalNGRrrq1Kmbr/2uWv6Zfv1lpx56tB+T5t1EngLXihUrNHjwYPXo0UPNmzfX2rVrNXToUPn4+Kh9+/bZ+jscDvXr10+xsbF6/vnnFRAQoPHjx+uRRx7R0qVL5e/vX+AHAgCAu/nll13KyMhQy5atndqHDn1RUuZdigVtzcovNGfmZDVq0krtOkTmvsIVcjgcTjc6XAm7wyF7xvUxVy9PgSs6OloRERGKioqSJLVo0UKJiYkaN25cjoHrjz/+0Pbt2zVq1Kiss2BVqlTR7bffrq+//lr33ntvAR4CAADu6fTpzPlTgYHFje/Lbrdr3ifva8XShWra/Db1eXqY0bNbFotF+0/G5msbVYpXlF3XR+DKNZrGxcUpNjZWbdu2dWpv166dYmJiFBcXl22dlJTMO9/8/Pyy2i6c1Tp16lS2/gAAXIv8/GySpISEBKf2Q4f+1I4dP8rhKJiwkZ6eronRr2nF0oXqcFcX9X12uDw8PHJfEVdNrme4YmJiJEmhoaFO7SEhIZKkAwcOqEKFCk7LatSooUaNGmnSpEmqXLmyAgMD9fbbb6to0aK6/fbbC6p2AADcWq1adeTp6alNmzY4zcuaNWu6duz4UcHBpQpkPx9Mfkdb/2+jHnr0KbXveF+BbBMFK9fAlZSUJEmy2WxO7RfOXiUnJ+e43iuvvKKePXuqQ4cOkiQvLy9NmjQpWzgDAOBaFRgYqM6du2ru3Nny9PTUjTfW088/79TKlcs0dOiLWrlyWb73sePHzdq0ca3qN2yqsKq19Pve3U7LK4VWlWeRIvneD/In18CV2+lOqzX7Vcn9+/erW7duqlixooYPHy4fHx8tWLBAzz77rD788EM1bNjwsoosUcKWeyfDUtMy5FXk2js9WxBPrIf74+fsvq7V3y0F4cKE7IzUlL+f9O46GakpVzxB/LnnBqhEiRJasuRzzZkzU+XLV9ALL4xQx453adWq5bJY/jlWiyVzbtTl7Gvr/22QJG3f9r22b/s+2/JxU+erRImgK6r9asjvxPuryWq1XvHv01wDV7FimRs+c+aMU/uFM1sXlv/bxx9/LEmaPn161tytZs2aqXv37ho5cqQ+//zzyyryxIlk2V38xOGgoGK6a9CSfG9n6Zh7CqCaghMfn+TqEq5J7hZw+Dm7r2v1d0tBSE+3S5JOJqbqarxWx6Tu3R9R9+6POLWlp9s1YcL7WV9L0tixk52+/6+cwknvp4ao91NDCrLcq+pix+qO7Hb7JX+fWq2Wi54kyjVWXpi7FRvrfCfCwYMHnZb/2+HDh1WlShWnxz9YLBY1aNBAv//+e267BADgupeRkaH09PRs/zIyMrL+2TMyXF0m8ijXM1whISEqX768Vq1apTvuuCOrfc2aNapUqZLKli2bbZ3Q0FB98cUXOn36tG644Z8n3P70008qV65cAZUOAMC167nn+mZ7OOp/lQwqpbGT516lipAfeXoOV79+/RQVFSV/f3+1atVK69at08qVKxUdHS0p86nysbGxCgsLk81m06OPPqovv/xSjz/+uHr37i0fHx8tWbJEW7ZsyVoHrmdPTc33pa/08ylKSCrcp/oBwB0NGTJcZ8+edWrz8LAq7ug/l7SKeDIZvrDIU+CKjIxUamqqpk+froULF6pChQoaNWpU1h2I69evV1RUlGbNmqVGjRqpfPnymjt3rkaPHq2oqChZLBZVq1ZNM2bMUNOmTY0eEPLO6uWlTfd0ztc2mi35TCJwAUCBq1ixUrY2T0+rith4nmVhlOd3KXbr1k3dunXLcVlkZKQiI51fH1ClShVNmTIlf9UBAABcAwrPvZgAAACFFIELAADAsDxfUgQAAChIDoejQB586rA7lOHi53XmhsAFAABcwmKxKPn3/fneji2siuTmgYtLigAAAIZxhgsA4NaKBXjLp4iXS2s4n5aqpFMpLq0hr3b/slMjXxmol14bp+o167q6HLezYsVSjRz5qj7/fLmCg0tdtf0SuAAAbs2niJfun9/XpTUs6DpFSSocgQvuicAFXONSM9Ly/UaBwvS/ewBwRwQu4Brn5VEk32cH+N89cGXuu+8utWzZRnv3/qZffvmfOna8W4MGDdW+fXs0ffoH2rVrp5KSklS8eAm1anWbnnzyaXl7e0uSmjdvqMGDo7R79/+0YcN6ZWRkqGnTZop84En5+wdm7WPdV0u1culCnTh+TFXCaujWNhHZ6vh9724tmjdDMTF75GG1qEHdOnqyx0MqExwsSVr59Tca98FHeuuFKE2aMVMH4/5U+bJlNOjJ3rJYLBr/4XTFxMaqXOnSeuaJx9Qg/OKXKndt36mFsz/Vnwdj5eHpoZp1aqvbY4+obIXyWX22fr9Zi+ct0OHYP1WsqJ/aNG2ux7o8IK8i/7yqaMOWzfps1TLFxB5Uenq6ygSXUqe2Ebr79naSpJ27f9Hgka+o/+O9NXfQM0pOTtKoUdG66ab6+uGH7zRz5nT9/vte2WzF1KpVG/Xu3U9Fixb9p85dP2vBgk+1b98eBQYWV5cu3dSt20NX+JPOHZPmAQAwaNGieapTJ1xvvTVaEREdFR9/TP369VZqaqpeeOEVjR49Xm3a3KGFC+dq0aJ5TutOnTpBkvT662/pqaee1XffbdCnM/95i8ualV9oxrRo3VS/kQYOfV1h1Wpp+vvvOW1j10/b9NpLz8rD00NPPj1MgwcP074DB9Rv2Is6eeqf1wSlpqbprfETdV/HDnptyCCdP5+iEe+O0WvvjdWdd9yuN4cNkcPh0GtjxiolNedXuh078pfee32kKlcN06ARL6jXs/10+M9DenfE63I4Mu8i3PTNtxr7xtuqEBKiUaPG6IG779Xyr9dq5ORxWdv5/setem38GNWoHKbXBgzRiOcGq3RQsMZ//KF+27/PaZ+ffLFI/fsPUv/+z6tWrTratGmjhgwZoJIlg/T662+rV6++Wr16pd566zWn9d59d6Tatm2vd94Zq9q162rixLHavPn7vP5YLxtnuAAAMKhcufLq06df1vebN3+vatWq6/XX384643LzzY20bdv/aefO7XrwwR5ZfcPCqmn48BF/95H27Nmtr7/5WlLmM6wWf/aJGjdrrYcezdx+3Rtv1rmzZ7Tuq6VZ21gw9yOVK19Jg4aNlNVqVViFANUsE6iH+j2n+Yu/VN9HH5EkZdjteqzb/WrfppUkKfbQYU3+eJaG9Ourjre3kSSdO39eL78zRoeO/KUbQmpmO9b9e/cpNSVV93S9T4ElikuSSgSV1I+bt+j8ufPy8fXRvBmzVO+Whnpy0HOqUryi6gaVVlCJkhoR/Y7+t/c31alWQ7GHD6ndra3V96FHs7Zdq2o1RT75uH76dbdqVKma1X7PHe3VuvVtSk+3S5KmT5+mGjVq6o03RmX1cTgcmjdvjs6ePZPV1q/fc7rrrk6SpDp1wrVx43pt375VjRubeeczgQsAAIOqVq3m9H3jxk3VuHFTpaen68CBGB06FKf9+39XQkKCAgOLO/WtW/dGp++Dg0sp5fx5SdKRw3E6nZigBjc3c+rTqGmrrMB1/vw5/RGzV/d1fUxW6z8XtUoFBalurRra8ctup3VrVv0nyAT6+0uSalX7p+2GYpnzQZPPnFFOwmpUUxEvL73Uf7AaNW+qGxs2UM3wOqpSPfMzOBz3p04eP6F7u3dVRkaG0tPTlZGRoZvr3qginp76cdfPqlOthrr9HYTOnT+nuCOHdejoX9obk/m8rvSMdKd9VgmplPV1Ssp57d37m3r3fsqpz5133qM777zHqe3GG2/K+trHx0eBgcWVlJSc43EVBAIXAAAG+foWdfrebrfr/fcn6fPPF+rcubMKDi6lWrVqy9vbW47/PLvzwnyuCywWixyOzDM5ycmnJUnFbghw6hMQWCLr67NnkuVwOOQf4BzkJKm4f4COHjvu1FbU1zdbPx8f72xtFxNUqpReGvWGvlz4ub5ZvVarlixTUZuf7ujYQV0e6a6k00mSpI/GT9ZH4ydnW//EqZOSpMSk04qePk3fb9siWSwqX7qM6lSvIUnZPiNfH5+sr0+fPi2Hw6HAwEDlxsfH+VitVmvWZ2sCgQsAgKvok08+1oIFn+r554fr1ltby2azSZJ69XrksrZTrFjmGajTpxKc2pOTTmd9XbSoTRaLRYl/B5l/O5GQIP8b8ncHc06qVK+mAS8OU3pamvb88qvWrVytJfMXqlKVUJUpX06S9FDvx1W9Vi2V9y+ts3F/Zq3r//cZtJGTxinuyGG9EzVCtapWk1eRIjqfkqIV36y75L79/DI/y4SEU07tZ8+e0a5dP6t2bdc9l4xJ8wAAXEU//7xTVapUVYcOd2WFrfj4Y9q/f/9lnWEpXaa8SpQI1v/9sN6pffu2fyZ++/j6qlLlatr8/XrZ7f9s+9jx4/rfb3tUt2aNfB3Lf63+cpmee7SX0tLS5FmkiGrfFK6ez2Ze3jtx/LjKVSivG/z9dfzoMVWuFqaaNWupeuUq8i9WTB/O+0Sxhw9Jkv639ze1bNREN9WqnXXn4pafdkjSJT+jokWLKiysmr7/foNT+4YN6zVo0DNKTjZ3yTA3nOECAOAqqlmztmbO/Ehz5sxUrVp19OefcZo9e4bS0lJ17ty5PG/HYrGo60O9NHncm/ro/fd0S+MW2rd3t9atWerU7/4HHtc7b0ZpzNsv6PZ2dyvmV4s+mDpBRX19df9ddxbosdW+MVxzP5qp6NffUtu7Osjq4aF1K1ariJeX6t1ys6weHurySHfNmPS+LBar2rdur2N792rW5wuUfPaMwv6ej1W9cpjWbtqgsJBQlQgsrl/2/aa5X34hi8Wi8ymXfkRNz559FBU1WK+99pLateug+Pijmjp1otq376jSpUsX6PFeDgIXAABX0cMPP6bExFNasOBTJScnq1Sp0mrXroOsVqtmz/5YZ84kZ10ay03T5rfJarFq8Wez9d23q1W+YmU93meAJo19I6tP3Rtv1tAXR+mzBR9r/JhX5evrq4bhtdXn4YdUonjuc50uR/mQihr8yov6bM48TRz1nuwZGQqtGqZhb4xQ6bJlJEltItrJt2hRLVv0hdatWKWiPj4Kr1FLj9/fXcUDMusZ+uTTmjDzI42f+WHmdkuX0YDH+2jtpg3atefXS9bQvHlLvfXWGM2Y8YGiogYpICBQd911rx59tGeBHuvlsjgc/51+5n5OnEiW3cVvAQ8KKqa7Bi3J93aWjrlHMW92ztc2Kr/wWYG85mJB1ynadE/+amm25DPFxyflu5ZrTUGMl4IYK1LBjJcFXafwczaE3y2ZfAc8qyC/f/7428KqZN3mz7sU/+HpadXvcady73gJYRUClHJkf75r8S5TRftPxuZrG1WKV1Ty7/mv5d/jxaS//jqo0qVDLrrcarWoRImcwzJnuAAAbi3pVApvOkChx6R5AAAAwwhcAAAAhhG4AAAADGMOF4Bc2VNTFRSU/wckpp9PUUJSzi+9BYBrGYELQK6sXl75vqNVyryrVQQuANchLikCAAAYRuACAAAwjMAFAABgGIELAADAMCbNAwDcWmAxL3n6eLu0hqt9h+2RI4fVpcvdeuml19SuXYeL9uv/1AOqXbeBevUdfNVqK2gP9n9K9WvX1aBe+X+tlDsjcAEA3Jqnj3eB3CWbH1f7DtsSJUpq6tQZKl++wlXbJ8wicAEA4Ga8vLxUp05dV5eBAkTgAgDAkLS0NE2dOlFr167SmTNn1KRJc9WpU1cTJkTru++2SZKefrq3SpcuozNnzmjLlh/UoMEtGjDg+WyXFH//fZ8mTRqrn3/+SbZiN+j+7j1z3X9qSormzJqi7du+V9LpRAUFl9Z9nTvrvjbNsvoknk7S+7PnaNOWrTp7/pyqV66sPo88pLo1a2T1OZWYqI/mztf/bd+hEwkJ8vXxUYOGjdSpRzcFlQqWJL0x9AWVDA7WubNn9fP2Hap9Y7gGv/Kizp49q4UzP9GWTT/o3JmzKl8pRPc/8qDq1Lvxn88pI11T58zU2k0bdT7lvGpXq6HnHuulssGlCuTn4A4IXAAAGDJq1Bv65pu16tWrr0JCQrVkyWd6//1J2fp99dUqRUTcqVGjomWxWLItj48/pqef7qWKFUP01HMv6OzZM5r3yTSdTky45P5nfzxJ//tpmx58pK9uCAjUzzu2aMKEsbIpTe3btFJKaqoGjnhVCYmn1fvh7ioeEKAlq7/SwBGvafybr6lm1TA5HA4NeX2kzp47pz4PP6TigQHa/8dBfTR3gRKST2nIay9n7e/79RvU4rbWGjTiBVlkkT0jQ2+/8IqOHjmiLg93V+myZfX1ytV6d8TrejX6HVUpXlGS9PWmjWoYfpOGPvm0Ek6d0pQ5H2vkpHGa+OrIfH3+7oTABQCAAYcO/anVq1eof//n1bnz/ZKkRo2aqEePbjpwIMapr7e3jwYNGqYiRYpIypw0/28LFsxVRoZd0dETFH86s61M2Qp6ZXi/S9bw2+6fVCe8gRo3ay1JqlX7JpUtFSj/GzJf1bVm/QbtPxirqe+8pRphVTJrrF9PTw6J0gdzPtV7r7ys+BMn5evrq2d7Pq46NapLkurVqa2/Tp/Tl19+4bS/Il5eeqxfH3n+fRzb/2+r9u/Zq8GvvKh6tzSUJNUMr6OX+z+v3T/t0m03t5IkBZcsqdcGDJGnZ2YsOXT0L81Z8pnOp6TIx9u1N0wUFAIXAAAGbN++TQ6HQ61atclqs1qtat36dh04MM2pb6VKoVlhKyc//bRDdeveKH//AMWfPiVJCqtaUyVKBl+yhlq1b9K6r5bq5Ml43VivkW6q31iPP95LKUf2Z9a4a5dKFi+usNBKSs/IyFqvScMGmvPZF0pLS1NwyRIa9/orcjgcOnLsmP48fESxhw7r5593Kj093Wl/5SpWyApbkrR3968qUqSIbrq5QVabh4eH3pzwntN6NatUywpbklQ6OPO4ks+eIXABAICLO3Uq83JfQECgU3vx4iWy9fX1LXrJbZ0+fVoVKmS/YzEgMPu2/u2hx/qpeIkgbdq4VrOmT9Cs6RNUt264+j/2sMJCKykxKUnxJ07otvu65bh+YlKSShYvrq++3ahpn8zRseMndIPNpqqVQ11F7YAAACAASURBVOXj4yOHw+HU38fXx+n7pNNJKuZ/Q46XSZ3W+0+osv7d32F35NS9UCJwAQBgQMmSQZKkhIQElSxZMqv9QhC7HAEBATp58mS29uSk05dcr0gRL93T+SHd0/khHY8/qh0//qClX8zRG2PH6+Nx78lWtKhCypfT8OeeyXF9/2LF9PPuXzVy/ATdd2dH3X/3nQoqkRnypi1aql27fr7k/ov6FVXS6aRs7fv37lORIkWy5nBdD3jSPAAABoSH3yQPDw999916p/aNG7+97G01aHCzfv55p06cOJ7VdijuDx07euSi66Smpmrws49oxdIFkqSSQaV0R/tOatu2neKPn5Ak3Vi7to7GH1fJ4oGqEVYl69+mLVv12fIV8vT01P/27JXd7tBj3e7PClsZGRnauvX/sp3h+q/qtWoqLTVVu7bvzGqzZ2Ro8jvRWrVk6WV/DoUZZ7gAADCgXLnyateugyZNGqfU1FSFhIRqxYql2rdvT66X2P7r/vsf0LJlS/Tss0/pznsfUkZ6hhbM/chp3tN/eXl5qUpYDX2+cJY8PT1VoWIVHTkcp+XLl6pl08aSpIg2rfT58pUaOOI1PdQ5UiVLFNcP237Ugi+X6dGuXWSxWFSzapgkadwHH6ld61ZKSk7W5ytWad++vXI4HEpNSZHXReZZ1W90s6pUq6opY8aqy8MPqmRwkL5Z/ZUSTpxQRKe7L+szKOwIXAAAGDJo0FD5+vpq1qzpOn/+vJo3b6l77ums1atXXNZ2/P0DNHnyh5ow4T29P3GUvH18dec9XbX5+/WXXO+x3gNU7AZ/Lf9ygRJPJegG/wDdfXcn9bi7vSSpqK+vJox8Te/PnqNJM2bq7PlzKluqlJ7r9bgiO0RIyrwjsX/vnlqwZKm+/u57BQb4q16d2nrg7dEaOnSQfvtlt8Lr18tx/1YPDw19Y4TmzZilBTM/UUpKikLDqihq5KuqUCnksj6Dwo7ABQBwa+nnUzJfrePiGi7X6dOJ2rz5B/Xs2VcDBgzJan/ppWEqX7581vcTJ07Ltm6ZMmWzHox6Qbly5TV69Fj9Hncqqy3izi6XrMHHx1cPPdpPDz36z+MjwioEZN2lKEnFAwIU9cylHy9xb0Q73RvRzqnNu0wVzVmxOOv7F0e9meO6fjabnnjmKT3xzFM5Lp8zdnK2tna3tla7W1tfsqbChsAFAHBrCUmpV/U9hgXF29tb0dHvaM2aOurcuau8vb21Zctmffvt1xo27CVXl4erjMAFAIAB3t4+io6eqGnTpuj1119WSsp5hYSE6sUXX1XbthGuLg9XGYELAABDatSopffem+DqMuAGeCwEAACAYQQuAAAAwwhcAADXszvk0LXzGhdce3J7yGtuCFwAAJdznElWRu7dAJdJS0uVh8eVT30ncAEAXC71m291KjlB6eJMF9yLw+FQamqKTp2Kl80WcMXb4S5FAIDL2ffH6PzS5TrZuqUsfjYlHT4ou93u6rLcjtVqVfLps/naxuHDiUpPvPwXaP+Xp+Ogks+cyF8t5x1KOZP/Wk4bHi8eHp4qVixQvr5+V7wNAhcAwC3Y98fo/P4YSVL9JZ8pPj7JxRW5n6CgYho2aEm+trF0zD2KebNzvmup/MJnun9+33xtY0HXKdp0T/5rKQzjhUuKAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMPyHLiWLVumjh07Kjw8XBEREVq8ePEl+9vtdk2ZMkW33XabwsPDddddd2n58uX5LhgAAKCw8cxLpxUrVmjw4MHq0aOHmjdvrrVr12ro0KHy8fFR+/btc1xn5MiRmj9/vgYOHKgaNWpo+fLlGjRokGw2m1q2bFmgBwEAAODO8hS4oqOjFRERoaioKElSixYtlJiYqHHjxuUYuGJjYzVnzhy99tpr6tKliySpSZMm+uOPP7Rx40YCFwAAuK7kGrji4uIUGxurgQMHOrW3a9dOK1euVFxcnCpUqOC0bO3atfLx8VGnTp2c2j/55JMCKBkAAKBwyXUOV0xMjCQpNDTUqT0kJESSdODAgWzr7NmzR6Ghofr+++919913q1atWmrbtq1WrFhREDUDAAAUKrkGrqSkJEmSzWZzavfz85MkJScnZ1vn5MmTOnLkiIYPH66HHnpIH374oWrXrq0BAwZo8+bNBVE3AABAoZHrJUWHw3HJ5VZr9syWlpamkydPaurUqWrdurUkqXHjxoqJidHEiRPVuHHjyyqyRAlb7p3gMkFBxVxdAgoRxgvyirGCy+Hu4yXXwFWsWOYBnDlzxqn9wpmtC8v/zc/PTx4eHmrWrFlWm9VqVdOmTbVo0aLLLvLEiWTZ7ZcOfqa5+w/SleLjk1xdgtthvFwc48UZY+XiGCvZMV4uzh3Gi9VquehJolwvKV6YuxUbG+vUfvDgQafl/xYSEiK73a709HSn9rS0NFkslrxVDQAAcI3INXCFhISofPnyWrVqlVP7mjVrVKlSJZUtWzbbOi1atJDD4dDKlSuz2tLT07Vx40Y1aNCgAMoGAAAoPPL0HK5+/fopKipK/v7+atWqldatW6eVK1cqOjpaUuYk+djYWIWFhclms6lJkyZq2bKl3njjDZ09e1aVKlXSp59+qkOHDmnMmDFGDwgAAMDd5ClwRUZGKjU1VdOnT9fChQtVoUIFjRo1Sh06dJAkrV+/XlFRUZo1a5YaNWokSRo/frzGjRunadOmKTExUbVq1dL06dNVp04dc0cDAADghvIUuCSpW7du6tatW47LIiMjFRkZ6dTm4+OjoUOHaujQofmrEAAAoJDL88urAQAAcGUIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMy3PgWrZsmTp27Kjw8HBFRERo8eLFed7JkSNH1KBBA02ePPmKigQAACjM8hS4VqxYocGDB6t58+aaNGmSbrnlFg0dOlSrVq3KdV2Hw6Hhw4crOTk538UCAAAURp556RQdHa2IiAhFRUVJklq0aKHExESNGzdO7du3v+S6n376qWJiYvJfKQAAQCGV6xmuuLg4xcbGqm3btk7t7dq1U0xMjOLi4i657ujRo/X666/nv1IAAIBCKtfAdeHsVGhoqFN7SEiIJOnAgQM5rme32zVs2DBFRETo1ltvzW+dAAAAhVaulxSTkpIkSTabzandz89Pki46N2vmzJn6888/NXXq1PzWCAAAUKjlGrgcDscll1ut2U+S7d+/X2PHjtX48eNVrFixK6/ubyVK2HLvBJcJCsr/zxjXD8YL8oqxgsvh7uMl18B1ITCdOXPGqf3Cma3/BqqMjAxFRUWpffv2atasmdLT07OW2e12paeny9MzT3P1s5w4kSy7/dLBzzR3/0G6Unx8kqtLcDuMl4tjvDhjrFwcYyU7xsvFucN4sVotFz1JlOscrgtzt2JjY53aDx486LT8giNHjuinn37S4sWLVbt27ax/kjRhwoSsrwEAAK4XuZ5qCgkJUfny5bVq1SrdcccdWe1r1qxRpUqVVLZsWaf+wcHBWrRoUbbt3HfffXrggQfUuXPnAigbAACg8MjTtb1+/fopKipK/v7+atWqldatW6eVK1cqOjpaknTy5EnFxsYqLCxMNptNdevWzXE7wcHBF10GAABwrcrTk+YjIyP16quv6rvvvlO/fv20detWjRo1Sh06dJAkrV+/Xl27dtUvv/xitFgAAIDCKM+z17t166Zu3brluCwyMlKRkZGXXH/Pnj2XVxkAAMA1Is8vrwYAAMCVIXABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYFieA9eyZcvUsWNHhYeHKyIiQosXL75k//j4eL344otq3bq16tWrp8jISK1cuTLfBQMAABQ2nnnptGLFCg0ePFg9evRQ8+bNtXbtWg0dOlQ+Pj5q3759tv6pqanq2bOnkpKS9Oyzzyo4OFirV69W//79lZGRoTvvvLPADwQAAMBd5SlwRUdHKyIiQlFRUZKkFi1aKDExUePGjcsxcG3YsEG//fabFi5cqPDwcElSs2bNdPjwYX3wwQcELgAAcF3J9ZJiXFycYmNj1bZtW6f2du3aKSYmRnFxcdnW8fPzU9euXVW3bl2n9sqVKys2NjafJQMAABQuuZ7hiomJkSSFhoY6tYeEhEiSDhw4oAoVKjgta9KkiZo0aeLUlpaWpm+//VZVq1bNV8EAAACFTa5nuJKSkiRJNpvNqd3Pz0+SlJycnKcdvfvuu/rjjz/Uu3fvy60RAACgUMv1DJfD4bjkcqv10pnN4XDo3Xff1cyZM/XEE0/o9ttvv7wKJZUoYcu9E1wmKKiYq0tAIcJ4QV4xVnA53H285Bq4ihXLPIAzZ844tV84s3VheU5SU1M1bNgwLV++XE888YSGDBlyRUWeOJEsu/3Swc80d/9BulJ8fJKrS3A7jJeLY7w4Y6xcHGMlO8bLxbnDeLFaLRc9SZRr4Lowdys2NlbVq1fPaj948KDT8v9KTk5Wnz59tH37dg0fPlw9evS47MIBAACuBbnO4QoJCVH58uW1atUqp/Y1a9aoUqVKKlu2bLZ1MjIy1LdvX/3000+Kjo4mbAEAgOtanp7D1a9fP0VFRcnf31+tWrXSunXrtHLlSkVHR0uSTp48qdjYWIWFhclms2nevHnasmWLunbtqtKlS2vnzp1Z27JYLLrxxhvNHA0AAIAbylPgioyMVGpqqqZPn66FCxeqQoUKGjVqlDp06CBJWr9+vaKiojRr1iw1atRIq1evliTNnz9f8+fPd9qWh4eHdu/eXcCHAQAA4L7yFLgkqVu3burWrVuOyyIjIxUZGZn1/axZs/JfGQAAwDUizy+vBgAAwJUhcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwjMAFAABgGIELAADAMAIXAACAYQQuAAAAwwhcAAAAhhG4AAAADCNwAQAAGEbgAgAAMIzABQAAYBiBCwAAwDACFwAAgGEELgAAAMMIXAAAAIYRuAAAAAwjcAEAABhG4AIAADCMwAUAAGAYgQsAAMAwAhcAAIBhBC4AAADDCFwAAACGEbgAAAAMI3ABAAAYRuACAAAwLM+Ba9myZerYsaPCw8MVERGhxYsXX7L/mTNn9Oqrr6pZs2aqV6+eevXqpT/++CO/9QIAABQ6eQpcK1as0ODBg9W8eXNNmjRJt9xyi4YOHapVq1ZddJ0BAwZo1apVGjx4sEaNGqWjR4/qkUceUVJSUoEVDwAAUBh45qVTdHS0IiIiFBUVJUlq0aKFEhMTNW7cOLVv3z5b/23btunbb7/VBx98oFtvvVWS1LBhQ912222aO3euevfuXYCHAAAA4N5yPcMVFxen2NhYtW3b1qm9Xbt2iomJUVxcXLZ1Nm3aJD8/PzVr1iyrrXjx4rr55pu1YcOGAigbAACg8Mg1cMXExEiSQkNDndpDQkIkSQcOHMhxnZCQEHl4eDi1V6xYMcf+AAAA17JcLylemHNls9mc2v38/CRJycnJ2dZJTk7O1v/COjn1z43VarnsdUwIDvQtkO14+gflextBRYsXQCWSd3D+a3GXn4+7KYjxUhBjRSqY8VIQY0VivOSE3y05Y6zkjN8tOXOH8XKpGiwOh8NxqZWXLl2qwYMHa926dSpfvnxW+8GDB9W2bdsc53E9/vjjSktL0+zZs53ao6OjNWvWLO3YseNKjgMAAKBQyvWSYrFixSRlPubh3y6cqbqw/N9sNlu2/he2kdOZLwAAgGtZroHrwtyt2NhYp/aDBw86Lf/vOnFxcfrvybODBw/m2B8AAOBalmvgCgkJUfny5bM9c2vNmjWqVKmSypYtm22d5s2b6/Tp0/r++++z2k6ePKlt27apadOmBVA2AABA4eHxyiuvvJJbp2LFimnKlClKSEiQxWLR9OnTtXjxYo0YMUJVq1bVyZMntWfPHtlsNnl5ealcuXLasmWLPv30UwUEBOjw4cMaPny4HA6HRo4cKR8fn6twaAAAAO4h10nzF8ybN0/Tp0/XkSNHVKFCBfXu3VudOnWSJH3++eeKiorSrFmz1KhRI0lSYmKi3n77ba1du1Z2u10NGjTQsGHDVLlyZXNHAwAA4IbyHLgAAABwZfL88moAAABcGQIXAACAYQQuAAAAwwhcAAAAhhG43Jzdbnd1CQAAIJ8IXG6uZcuWGj16tPbv3+/qUgAAwBXisRBubsyYMVq6dKmOHj2qOnXqqHPnzurYsWOO77AELtizZ4/OnTuX4xnS+vXru6AiuIupU6fmua/FYlGfPn0MVgNcPwhchYDD4dAPP/ygL774IutBsm3atNG9996rFi1ayGKxuLpEuIn//e9/eu6553T48OFsyxwOhywWi3799VcXVAZ3UaNGjTz3Zbxcvx577LE8/22xWCz66KOPDFdU+BG4CpmzZ89q/fr1mjt3rrZt26agoCB17txZDzzwgIKDg11dHlzsgQce0PHjx9WvXz+VLl1aVmv2WQO33HKLCyoDUJiMGjVKM2bM0A033KDq1avn2n/27NlXoarCjcBViMTHx2vZsmVauXKldu3apXLlyqlly5b67rvvdOzYMb311ltq3769q8uEC4WHh+u9997T7bff7upSABRys2fP1ttvv60ZM2bwH7UCQOByc+fOndOaNWv05ZdfavPmzSpSpIjatm2rzp07Z7230uFwqGfPntqzZ4++++47F1cMV2rVqpVGjBih1q1bu7oUuKm2bdte1jSE1atXG6wG7m7QoEH67bfftHz5cleXUuh5uroAXFrTpk11/vx5hYeHa8SIEerQoYNsNptTH4vFonr16mnPnj0uqhLu4oEHHtC0adPUuHFj+fr6urocuKH69esz7xO5SktLU5EiRTRkyBCNHTtW+/fvV5UqVVxdVqHGGS43984776hz5865DvTk5GT5+PjI05MMfT17+eWXtXLlStntdlWrVi1b6GJyK4C8aN68uSZOnKibbrpJEydOVJcuXVSqVClXl1Wo8dfZzQ0ZMkS//fabPv74Yz366KOSMm/5nz17th577LGsIPbfs164Ph04cMDpLrS0tDQXVoPCICUlRfv27VNaWpou/P/bbrfr3Llz2rZtmwYMGODiCuEKp0+f1tGjRyVJkyZN0q233krgyifOcLm5H374Qb1791bVqlX1+eefS5J27dqlwYMH69ixY5o5c6bCw8NdXCWAwmjLli3q37+/EhISclzu5+enbdu2XeWq4A4efvhh7dixQ8HBwTp8+LCCgoLk5eWVY1+LxaK1a9de5QoLHwKXm+vatavKlCmjMWPGyMPDI6vdbrdr4MCBOnHiBLfjIpvff/9dW7ZsUXJysgIDA9WgQQNVrlzZ1WXBzXTv3l0JCQkaOHCgvvzyS1mtVkVGRmrDhg2aO3eu5syZo3r16rm6TLjAsWPHNHv2bJ06dUqLFi1S69atVbx48Yv2f+ONN65idYUTlxTd3N69e9W/f3+nsCVJVqtV999/v55++mkXVQZ3ZLfb9fLLL+uzzz7Tv/8vZbFYdM899+itt95iwjSy/Prrr3rjjTd0xx13KCkpSfPmzVPLli3VsmVLpaamasqUKZo2bZqry4QLBAcHa9CgQZKkTZs26dlnn72sh+YiO96l6OZsNptiY2NzXHbo0CHuRIOTadOmafHixRo0aJC+/fZb/fLLL1q/fr0GDhyo5cuX68MPP3R1iXAjdrs9a15OSEiI9u3bl7WsXbt22r17t6tKgxv5+uuvCVsFgMDl5tq2bauxY8dqw4YNTu0//PCDxo0bxwMu4WTRokV68skn1bNnT5UqVUoeHh4qXbq0evXqpT59+mjRokWuLhFupGLFilkhKzQ0VOfOnVNMTIwkKSMjQ2fOnHFlecA1hUuKbm7AgAHatWuXevfuLW9vbxUvXlwJCQlKSUlR3bp19fzzz7u6RLiR+Ph4NWjQIMdl9evX5/IQnNx555169913Zbfb9eCDD6pOnTp688039cgjj2jKlCkKCwtzdYnANYPA5eZsNpvmzZunb7/9Vj/++KMSExNls9nUoEEDtWnTJsd35eH6VaFCBe3YsUNNmjTJtmzHjh0KCgpyQVVwV7169dLJkye1fft2PfjggxoxYkTW2VCbzaYpU6a4ukTgmsFdisA15OOPP9Z7772ngQMHqkOHDipZsqSOHz+u5cuXKzo6Wn369FG/fv1cXSbcWHJysmJiYlS5cmWe7wcUIAJXIbBq1Spt3bo1xwcT7vj/9u41qMpqjQP4/2WjInIZxAAtHAgYpa0yAVKIF6QNNoyQUkRDaRSgEpYXMknNJrUEwSba3BzQcbRkmj1O3nJEFAbRgkTcNFwt8gaDTSMXuch9nw+e9jkMeDyno7wL9v/3Cdfiw/+LwzPP+6y1rl5FQUGBzAlJFP39/diyZQuOHz8+6DSiTqdDcHAwEhIS2BWlQQoLC1FcXIzNmzcDAH755Rd9cf7iiy/KnI5o7OAnRcGlpaVBrVbD3NwcfX19GDduHIyNjdHU1AQjIyOEhobKHZEEolAokJiYiKioKFy+fBn37t2DhYUF5s6dCxcXF7njkWBOnz6NuLg4LFiwQL82ceJEDAwMIDIyEhkZGVi4cKGMCYnGDna4BKdSqeDp6Yndu3cjJSUFjY2NSExMREVFBVatWoX33nsPb731ltwxiWgUCg4OhpeXF7Zt2zZkb+fOnSgvL+fJVqLHhB0uwd25cwdBQUGQJAlKpRKnT58GAMyaNQtr1qyBRqNhwWXglixZgpSUFMycORMBAQGPvNg0Nzd3hJKR6G7duoUtW7YMu6dSqfTPiRHR/48Fl+BMTU31MzfTp09HfX09urq6YGJiAldXV9TX18uckOTm7u6OSZMm6X/mTfL037K2tkZlZeWws1q1tbWwtLSUIRXR2MSCS3CzZ8/G8ePH4e3tDUdHRygUChQXF8PX1xfXr19/6GOiZDh2796t/zkhIUHGJDTaBAUFITU1FaampvD394e1tTWampqQn58PtVqN8PBwuSMSjRmc4RJcSUkJIiMjsWDBAmRkZGDrT1jr5wAACrNJREFU1q04d+4cvL29UVhYCJVKhaSkJLljkkA6OjrQ0dEBGxsb9Pb24ttvv0VjYyMCAgIeeikqGabe3l7ExcXh7NmzQ061BgQEYO/evRg3bpyMCYnGDhZco0BVVRWuXbuGZcuWobu7G7t27UJZWRnmzJmD+Ph4tv1Jr7y8HNHR0QgLC0NcXBw+/fRTfPfdd7CwsEBHRwfUajX8/PzkjkmCuXbtmv5iZXNzc3h4ePDtPKLHjAWX4LKzs/HSSy/B0dFR7ig0Crzzzju4f/8+kpKSMGXKFHh7eyMkJATbt2/H9u3bUV1dDY1GI3dMElBfXx+am5thZWUFY2NOmxA9brwBUXBqtRo3b96UOwaNEuXl5YiJiYG9vT0uXbqE7u5uvPLKKwCAwMBA/UPFRH+pqKhAZGQk3N3dsWjRItTW1iI+Ph5paWlyRyMaU1hwCc7JyQm3bt2SOwaNEkZGRpgwYQIAoKioCBYWFpgzZw6AB0+2mJiYyBmPBFNWVobw8HC0tLQgOjpa/5KFnZ0dUlNTceTIEZkTEo0d7BsLTqVSYe/evbh48SJmzpwJU1PTQfuSJGH16tUypSPRzJo1CxqNBiYmJjhz5gx8fX0hSRLu3r2LrKwszJ49W+6IJJDk5GTMmzcPmZmZ6Ovr03e11q9fj66uLuTk5PCkItFjwhkuwT1qcFWSJFRXV49QGhJdZWUloqKi0NzcjMmTJ+PIkSNwcHCAt7c3dDodDhw4gOeee07umCQINzc3fP3111i0aBH6+/uhVCpx9OhRKJVKlJSUYPXq1dBqtXLHJBoT2OESXE1NjdwRaBRRKpXIy8tDXV0dXFxc9B3RnTt3wt3dHZMnT5Y5IYlk0qRJuHv37rB7f/zxh/5CXSL6/3GGi2iMMTMzg5ub26DPzyqVisUWDeHn54evvvoKVVVV+jVJkvDnn39i3759WLRokYzpiMYWflIU3LvvvvvI3zlw4MAIJCFR8S1F+rtaWlrw9ttv49dff4WtrS0aGxvh7OyMhoYG2NjYICcnh4U60WPCT4qC6+3tHbLW2dmJuro6mJqaIiAgQIZUJBK+pUh/l1qtxieffILff/8dxcXFcHR0hJmZGd544w2EhIQMOaRDRH8fO1yjVGtrK6Kjo7F06VKsXLlS7jgkMJ1OxyKMhuXm5ob09HT4+PjIHYVozOMM1yhlaWmJVatW4eDBg3JHIcHk5ORg48aN+n+XlpYiICAAx44dkzEVicjNzQ2XL1+WOwaRQeAnxVHuYSeMyDB98803+PzzzxEaGqpfs7Ozg6enJ7Zu3QpJkvQ3zxMplUpkZ2fj7NmzcHV1Hfaevx07dsiUjmhs4SdFwZWVlQ1ZGxgYQGNjI9RqNaZMmcLboElvyZIlCA4ORmxs7JC91NRU5Obm4uTJkzIkIxE96iFzSZJw/vz5EUpDNLaxwyW48PBwSJI0aA7nrxp56tSp2LJli5zxSDB37tyBu7v7sHseHh7Iysoa4UQksvz8fLkjEBkMFlyCO3To0JA1SZJgZmaGGTNmwMiIY3j0L9OmTUNJSQm8vb2H7F25cgW2trYypCIiIhZcgvPy8kJXVxcqKyvh4eEB4EEX4+eff4ajoyMfI6ZBwsLCkJSUhL6+Pvj7+2Py5Mlobm5Gfn4+9u/fj3Xr1skdkYjIIHGGS3C3b99GREQEdDqdvv1/6dIlREVFwdnZGdnZ2exa0CCJiYk4fPgw+vv79WsKhQIrVqzA5s2bZUxGRGS4WHAJLjY2FvX19UhJSYGDg4N+vaGhAWvXroWTkxOSk5PlC0hCamtrg1arRUtLC8zNzTFnzhzeGE5EJCMOAAmutLQUGzZsGFRsAcDTTz+N2NhY/Pjjj/IEI6GZm5vDyckJ9vb28PLywsSJE+WORERk0DjDJTidToeenp6H7nd1dY1gGhoN8vPzsWfPHty8eROSJEGj0SA9PR2WlpbYuXMnFAqF3BGJiAwOO1yCmzt3LtLT09Hc3Dxo/d69e8jMzISXl5dMyUhE+fn5iI2NhbOzM3bt2oWBgQEAwLx583DixAns27dP5oRERIaJM1yCu379Ol5//XX09/fD3d0d1tbWaGpqwtWrV6FQKHDkyBE4OTnJHZMEsXz5cri6uuKLL75Af38/lEoljh49CqVSiYyMDBw7dgy5ublyxyQiMjjscAnO0dERp06dQlhYmH4QuqmpCSEhITh27BiLLRqkrq4OgYGBw+55eHigsbFxhBMRERHAGa5RwdbWFu+//77+nbP29na0t7fDzs5O5mQkGisrK9y4cQPz588fsnfjxg1YWVnJkIqIiNjhElxnZyc2bNiAsLAw/ZpWq8XixYvx8ccf/8eBejI8gYGBSElJwblz59Db2wvgwcsENTU1SE9Px8svvyxzQiIiw8QZLsHt2rULJ0+exEcffYRXX30VANDR0YG8vDwkJiYiLCwM69evlzkliaK7uxuxsbG4ePEijI2N0dfXBwsLC7S1teH5559Hdna2vlNKREQjhwWX4BYuXIgNGzZg+fLlQ/Y0Gg0yMjL4AC0NcenSJRQXF6OlpQVmZmbw8vKCr6+v/gF0IiIaWZzhElxbWxusra2H3Zs6dSru3r07wolIZDExMYiIiICPjw98fHzkjkNERP/EGS7BzZgxA99///2we8ePH4eLi8sIJyKR/fTTT2DTmohIPOxwCS4mJgZr1qxBQ0MDVCqV/h6ugoICaLVapKenyx2RBDJ//nz88MMP8PT0hLEx/3sTEYmCM1yjQEFBAdRqNaqrq/Xdi5kzZ+KDDz6An5+fzOlIJPHx8Th16hRMTEzg7Ow8ZEBekiTs379fpnRERIaLBdco0dzcjJ6eHvT390OhUECn06GzsxNXrlxBaGio3PFIECtWrHjk7xw+fHgEkhAR0b9jwSW42tpafPjhh/jtt9+G3ZckCVVVVSOcikRXV1eH0tJStLa2wtraGi+88AKeeeYZuWMRERksDnkIbs+ePWhpacHmzZtRUFCA8ePHY/Hixbhw4QIKCwtx6NAhuSOSQLq7u7Fp0ybk5eUNGp43MjLCa6+9hs8++4xXQxARyYCnFAWn1Wqxbt06REREIDAwEPfv30d4eDgyMzPh7+/Pz0M0SEJCAoqKirBt2zYUFRWhsrISFy5cQHx8PE6cOIG0tDS5IxIRGSQWXILr6emBg4MDAMDBwQE1NTX6vZCQEGi1WpmSkYjOnDmDjRs34s0338RTTz0FhUIBGxsbrFy5EuvWrYNGo5E7IhGRQWLBJbhp06ahvr4ewIOCq729HQ0NDQCACRMmoLW1Vc54JJje3l7Y29sPu+fs7Iy2trYRTkRERAALLuGpVCokJycjLy8Ptra2ePbZZ5GSkoK6ujocPHjwoX9cyTAtW7YMWVlZ6OrqGrQ+MDCAnJwcLF26VKZkRESGjacUBXf//n1s2rQJ3d3dyMrKQlFREdauXYuenh4oFAp8+eWXCAgIkDsmCUKtVuPw4cMwNjaGn58fbGxs0NLSgqKiIjQ0NCAoKAjjx48H8OCE644dO2ROTERkGFhwjRI9PT36P5S3b99GRUUFlEolpk+fLnMyEsn/chGuJEk4f/78E0xDRER/YcFFRERE9IRxhouIiIjoCWPBRURERPSEseAiIiIiesJYcBERERE9YSy4iIiIiJ6wfwDiQ2smlHDV6AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "compare_metrics = pd.DataFrame({\"baseline\": baseline_metrics,\n",
    "                                \"clf_2\": clf_2_metrics,\n",
    "                                \"random search\": rs_metrics,\n",
    "                                \"grid search\": gs_metrics})\n",
    "\n",
    "compare_metrics.plot.bar(figsize=(10, 8));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Saving and loading trained machine learning models\n",
    "\n",
    "Two ways to save and load machine learning models:\n",
    "1. With Python's pickle module\n",
    "2. With the joblib module\n",
    "\n",
    "**Pickle**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# Save an extisting model to file\n",
    "pickle.dump(gs_clf, open(\"gs_random_random_forest_model_1.pkl\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a saved model\n",
    "loaded_pickle_model = pickle.load(open(\"gs_random_random_forest_model_1.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.79, 'precision': 0.74, 'recall': 0.82, 'f1': 0.78}"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make some predictions\n",
    "pickle_y_preds = loaded_pickle_model.predict(X_test)\n",
    "evaluate_preds(y_test, pickle_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Joblib**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['gs_random_forest_model_1.joblib']"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "\n",
    "# Save model to file\n",
    "dump(gs_clf, filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import a saved joblib model\n",
    "loaded_joblib_model = load(filename=\"gs_random_forest_model_1.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 78.69%\n",
      "Precision: 0.74\n",
      "Recall: 0.82\n",
      "F1 score: 0.78\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.79, 'precision': 0.74, 'recall': 0.82, 'f1': 0.78}"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make and evaluate joblib predictions\n",
    "joblib_y_preds = loaded_joblib_model.predict(X_test)\n",
    "evaluate_preds(y_test, joblib_y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Putting it all together!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Black</td>\n",
       "      <td>35820.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32042.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>NaN</td>\n",
       "      <td>White</td>\n",
       "      <td>155144.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>66604.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>31570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>215883.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Blue</td>\n",
       "      <td>248360.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12732.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Make Colour  Odometer (KM)  Doors    Price\n",
       "0     Honda  White        35431.0    4.0  15323.0\n",
       "1       BMW   Blue       192714.0    5.0  19943.0\n",
       "2     Honda  White        84714.0    4.0  28343.0\n",
       "3    Toyota  White       154365.0    4.0  13434.0\n",
       "4    Nissan   Blue       181577.0    3.0  14043.0\n",
       "..      ...    ...            ...    ...      ...\n",
       "995  Toyota  Black        35820.0    4.0  32042.0\n",
       "996     NaN  White       155144.0    3.0   5716.0\n",
       "997  Nissan   Blue        66604.0    4.0  31570.0\n",
       "998   Honda  White       215883.0    4.0   4001.0\n",
       "999  Toyota   Blue       248360.0    4.0  12732.0\n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make              object\n",
       "Colour            object\n",
       "Odometer (KM)    float64\n",
       "Doors            float64\n",
       "Price            float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Steps we want to do (all in one cell):\n",
    "\n",
    "- Fill missing data\n",
    "- Convert data to numbers\n",
    "- Build a model on the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22188417408787875"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Getting data ready\n",
    "import pandas as pd\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "# Modelling\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "\n",
    "# Setup random seed\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "\n",
    "# Import data and drop rows with missing labels\n",
    "data = pd.read_csv(\"data/car-sales-extended-missing-data.csv\")\n",
    "data.dropna(subset=[\"Price\"], inplace=True)\n",
    "\n",
    "# Define different features and transformer pipeline\n",
    "categorical_features = [\"Make\", \"Colour\"]\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\")),\n",
    "    (\"onehot\", OneHotEncoder(handle_unknown=\"ignore\"))])\n",
    "\n",
    "door_feature = [\"Doors\"]\n",
    "door_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=4))\n",
    "])\n",
    "\n",
    "numeric_features = [\"Odometer (KM)\"]\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    (\"imputer\", SimpleImputer(strategy=\"mean\"))\n",
    "])\n",
    "\n",
    "# Setup preprocessing steps (fill missing values, then convert to numbers)\n",
    "preprocessor = ColumnTransformer(\n",
    "                    transformers=[\n",
    "                        (\"cat\", categorical_transformer, categorical_features),\n",
    "                        (\"door\", door_transformer, door_feature),\n",
    "                        (\"num\", numeric_transformer, numeric_features)\n",
    "                    ])\n",
    "\n",
    "# Creating a preprocessing and modelling pipeline\n",
    "model = Pipeline(steps=[(\"preprocessor\", preprocessor),\n",
    "                        (\"model\", RandomForestRegressor())])\n",
    "\n",
    "# Split data\n",
    "X = data.drop(\"Price\", axis=1)\n",
    "y = data[\"Price\"]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "# Fit and score the model\n",
    "model.fit(X_train, y_train)\n",
    "model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It's also possible to use GridSearchCV or RandomizedSesrchCV with our Pipeline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.8s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.6s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.6s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.9s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.8s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   3.0s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.7s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.6s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.6s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.4s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.6s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.5s\n",
      "[CV] model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=None, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.6s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=2, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=mean, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.2s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=100, preprocessor__num__imputer__strategy=median, total=   0.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.3s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=mean, total=   1.9s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.0s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   2.1s\n",
      "[CV] model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median \n",
      "[CV]  model__max_depth=5, model__max_features=auto, model__min_samples_split=4, model__n_estimators=1000, preprocessor__num__imputer__strategy=median, total=   1.9s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  80 out of  80 | elapsed:  1.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5,\n",
       "             estimator=Pipeline(steps=[('preprocessor',\n",
       "                                        ColumnTransformer(transformers=[('cat',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value='missing',\n",
       "                                                                                                        strategy='constant')),\n",
       "                                                                                         ('onehot',\n",
       "                                                                                          OneHotEncoder(handle_unknown='ignore'))]),\n",
       "                                                                         ['Make',\n",
       "                                                                          'Colour']),\n",
       "                                                                        ('door',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer(fill_value=4,\n",
       "                                                                                                        strategy='constant'))]),\n",
       "                                                                         ['Doors']),\n",
       "                                                                        ('num',\n",
       "                                                                         Pipeline(steps=[('imputer',\n",
       "                                                                                          SimpleImputer())]),\n",
       "                                                                         ['Odometer '\n",
       "                                                                          '(KM)'])])),\n",
       "                                       ('model', RandomForestRegressor())]),\n",
       "             param_grid={'model__max_depth': [None, 5],\n",
       "                         'model__max_features': ['auto'],\n",
       "                         'model__min_samples_split': [2, 4],\n",
       "                         'model__n_estimators': [100, 1000],\n",
       "                         'preprocessor__num__imputer__strategy': ['mean',\n",
       "                                                                  'median']},\n",
       "             verbose=2)"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use GridSearchCV with our regression Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "pipe_grid = {\n",
    "    \"preprocessor__num__imputer__strategy\": [\"mean\", \"median\"],\n",
    "    \"model__n_estimators\": [100, 1000],\n",
    "    \"model__max_depth\": [None, 5],\n",
    "    \"model__max_features\": [\"auto\"],\n",
    "    \"model__min_samples_split\": [2, 4]    \n",
    "}\n",
    "\n",
    "gs_model = GridSearchCV(model, pipe_grid, cv=5, verbose=2)\n",
    "gs_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3339554263158365"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}