Classification_UnbalancedDataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    },
    "colab": {
      "name": "Classification_Solution.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vPyNZdle_BCT"
      },
      "source": [
        "# Binary Classification Project with Unbalanced Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XYJk8JUQ_BCh"
      },
      "source": [
        "**In this project I have two csv files, one is for training the model and one for testing. I need to develop a model that can help us to come up with the probability for majority and minority class.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v8p-AX9f_BCj"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score, recall_score\n",
        "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "import tensorflow as tf\n",
        "from sklearn.metrics import classification_report,confusion_matrix\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation,Dropout\n",
        "from tensorflow.keras.callbacks import EarlyStopping"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xADbiB72_BCk"
      },
      "source": [
        "# Visualization of the dataset\n",
        "df1=pd.read_csv('exercise_02_train.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_iSsMeBv_BCl",
        "outputId": "a760c087-71a2-40bd-d7fc-cec382921e20"
      },
      "source": [
        "#shape of the df\n",
        "df1.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(40000, 101)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2gGTzg9j_BCn",
        "outputId": "6b580947-d0d2-4dc7-a405-a805486b0a97"
      },
      "source": [
        "# Check at first if we have any missing values\n",
        "df1.isnull().sum().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "818"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7zSI-hzE_BCo",
        "outputId": "24e95c5b-6c05-4cb9-8bfc-608c04115442"
      },
      "source": [
        "#max amount of missing value in our df1 columns\n",
        "Missing = df1.isna().sum()/(len(df1))*100\n",
        "print(\"Column with lowest amount of missings contains {} % missings.\".format(Missing.min()))\n",
        "print(\"Column with highest amount of missings contains {} % missings.\".format(Missing.max()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Column with lowest amount of missings contains 0.0 % missings.\n",
            "Column with highest amount of missings contains 0.0375 % missings.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fCEBKSJO_BCp"
      },
      "source": [
        "**The percentage of missing values in df1 columns were not numerous, so I will drop them all.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nfe0AGMO_BCq",
        "outputId": "8ead37a5-e8e3-405f-a2a1-d7ae6b395c04"
      },
      "source": [
        "#Determine columns with missings\n",
        "df1.loc[:, df1.isnull().any()].columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10',\n",
              "       'x11', 'x12', 'x13', 'x14', 'x15', 'x16', 'x17', 'x18', 'x19', 'x20',\n",
              "       'x21', 'x22', 'x23', 'x24', 'x25', 'x26', 'x27', 'x28', 'x29', 'x30',\n",
              "       'x31', 'x32', 'x33', 'x34', 'x35', 'x36', 'x37', 'x38', 'x39', 'x40',\n",
              "       'x41', 'x42', 'x43', 'x44', 'x45', 'x46', 'x47', 'x48', 'x49', 'x50',\n",
              "       'x51', 'x52', 'x53', 'x54', 'x55', 'x56', 'x57', 'x58', 'x59', 'x60',\n",
              "       'x61', 'x62', 'x63', 'x64', 'x65', 'x66', 'x67', 'x68', 'x69', 'x70',\n",
              "       'x71', 'x72', 'x73', 'x74', 'x75', 'x76', 'x77', 'x78', 'x79', 'x80',\n",
              "       'x81', 'x82', 'x83', 'x84', 'x85', 'x86', 'x87', 'x88', 'x89', 'x90',\n",
              "       'x91', 'x92', 'x93', 'x94', 'x95', 'x96', 'x97', 'x98', 'x99'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EtFnuF2L_BCs"
      },
      "source": [
        "**looks like we have missing value in all columns of df1**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y_F9klbW_BCt"
      },
      "source": [
        "#lets drop the missing values from our datasets\n",
        "df1.dropna(inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJ6r3zNV_BCt",
        "outputId": "0ea18a40-dc7a-4014-b7b3-c3ba0d4b82d6"
      },
      "source": [
        "df1.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(39182, 101)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 143
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4w4_ILXT_BCu",
        "outputId": "baf1d4c5-556c-4891-bac4-6881cf1217d7"
      },
      "source": [
        "#check for any categorical columns in df1\n",
        "cols = df1.columns\n",
        "num_cols = df1._get_numeric_data().columns\n",
        "list(set(cols) - set(num_cols))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['x45', 'x34', 'x68', 'x93', 'x35', 'x41']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 145
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CeFAw-W_BCv"
      },
      "source": [
        "**As can be seen there are 6 columns with categorical values are available**\n",
        "\n",
        "**Columns X34,35,68,93. We need to encode them by get.dummies function**\n",
        "\n",
        "**Columns X41,45. There are % and $ inside the cells and thats why we found them as categorical.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGCJ7Bnp_BCv",
        "outputId": "5b69fdb4-6f46-4095-f059-21d2078c5f76"
      },
      "source": [
        "#Cleaning the X34,35,68,93\n",
        "print(df1['x34'].unique())\n",
        "print(df1['x35'].unique())\n",
        "print(df1['x68'].unique())\n",
        "print(df1['x93'].unique())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Toyota' 'Honda' 'bmw' 'volkswagon' 'tesla' 'nissan' 'chrystler' 'ford'\n",
            " 'mercades' 'chevrolet']\n",
            "['wed' 'thurday' 'wednesday' 'thur' 'tuesday' 'monday' 'friday' 'fri']\n",
            "['Jun' 'sept.' 'Oct' 'July' 'Aug' 'Apr' 'May' 'Mar' 'Nov' 'Feb' 'Dev'\n",
            " 'January']\n",
            "['asia' 'america' 'euorpe']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-VH7o4wJ_BCw"
      },
      "source": [
        "#x35 need to rearrange\n",
        "df1[\"x35\"].replace({\"wed\": \"wednesday\", \"thur\": \"thurday\", \"fri\": \"friday\"}, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uuGWHCN2_BCw"
      },
      "source": [
        "#use get.dummies function for encoding the categorical values\n",
        "df1 = pd.get_dummies(data = df1, columns = ['x34', 'x35', 'x68', 'x93'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luiPUywa_BCx"
      },
      "source": [
        "#removing dollar sign from column x41\n",
        "df1['x41'] = df1['x41'].str.replace('$', '').astype(float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RdRUsNDR_BCx"
      },
      "source": [
        "#removing percentage sign from column x45\n",
        "df1['x45'] = df1['x45'].str.replace('%', '').astype(float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tAnoZhMn_BCx"
      },
      "source": [
        "df1[\"dependent\"] = df1[\"y\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BILYvgZB_BCy"
      },
      "source": [
        "df1.drop(['y'], inplace=True, axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G3vUSyRP_BCy",
        "outputId": "dd231492-8ab3-493b-82e6-364e03f82062"
      },
      "source": [
        "df1.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x0</th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x4</th>\n",
              "      <th>x5</th>\n",
              "      <th>x6</th>\n",
              "      <th>x7</th>\n",
              "      <th>x8</th>\n",
              "      <th>x9</th>\n",
              "      <th>...</th>\n",
              "      <th>x68_Jun</th>\n",
              "      <th>x68_Mar</th>\n",
              "      <th>x68_May</th>\n",
              "      <th>x68_Nov</th>\n",
              "      <th>x68_Oct</th>\n",
              "      <th>x68_sept.</th>\n",
              "      <th>x93_america</th>\n",
              "      <th>x93_asia</th>\n",
              "      <th>x93_euorpe</th>\n",
              "      <th>dependent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.198560</td>\n",
              "      <td>74.425320</td>\n",
              "      <td>67.627745</td>\n",
              "      <td>-3.095111</td>\n",
              "      <td>-6.822327</td>\n",
              "      <td>19.048071</td>\n",
              "      <td>-0.362378</td>\n",
              "      <td>-10.699174</td>\n",
              "      <td>-22.699791</td>\n",
              "      <td>-1.561262</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-29.662621</td>\n",
              "      <td>24.320711</td>\n",
              "      <td>-48.205182</td>\n",
              "      <td>1.430339</td>\n",
              "      <td>-6.552206</td>\n",
              "      <td>4.263074</td>\n",
              "      <td>6.551412</td>\n",
              "      <td>4.265483</td>\n",
              "      <td>1.245095</td>\n",
              "      <td>2.246814</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>15.493759</td>\n",
              "      <td>-66.160459</td>\n",
              "      <td>50.512903</td>\n",
              "      <td>-2.265792</td>\n",
              "      <td>14.428578</td>\n",
              "      <td>2.509323</td>\n",
              "      <td>-6.707536</td>\n",
              "      <td>3.820842</td>\n",
              "      <td>-11.100833</td>\n",
              "      <td>-1.459825</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-19.837651</td>\n",
              "      <td>33.210943</td>\n",
              "      <td>53.405563</td>\n",
              "      <td>1.079462</td>\n",
              "      <td>11.364251</td>\n",
              "      <td>-1.064581</td>\n",
              "      <td>9.308857</td>\n",
              "      <td>9.266076</td>\n",
              "      <td>14.552959</td>\n",
              "      <td>-2.012755</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>11.896655</td>\n",
              "      <td>-26.717872</td>\n",
              "      <td>-17.758176</td>\n",
              "      <td>1.692017</td>\n",
              "      <td>21.553537</td>\n",
              "      <td>-5.852097</td>\n",
              "      <td>-0.857435</td>\n",
              "      <td>-2.186940</td>\n",
              "      <td>18.075272</td>\n",
              "      <td>-1.404618</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 127 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          x0         x1         x2        x3         x4         x5        x6  \\\n",
              "0   0.198560  74.425320  67.627745 -3.095111  -6.822327  19.048071 -0.362378   \n",
              "1 -29.662621  24.320711 -48.205182  1.430339  -6.552206   4.263074  6.551412   \n",
              "2  15.493759 -66.160459  50.512903 -2.265792  14.428578   2.509323 -6.707536   \n",
              "3 -19.837651  33.210943  53.405563  1.079462  11.364251  -1.064581  9.308857   \n",
              "4  11.896655 -26.717872 -17.758176  1.692017  21.553537  -5.852097 -0.857435   \n",
              "\n",
              "          x7         x8        x9  ...  x68_Jun  x68_Mar  x68_May  x68_Nov  \\\n",
              "0 -10.699174 -22.699791 -1.561262  ...        1        0        0        0   \n",
              "1   4.265483   1.245095  2.246814  ...        0        0        0        0   \n",
              "2   3.820842 -11.100833 -1.459825  ...        0        0        0        0   \n",
              "3   9.266076  14.552959 -2.012755  ...        1        0        0        0   \n",
              "4  -2.186940  18.075272 -1.404618  ...        0        0        0        0   \n",
              "\n",
              "   x68_Oct  x68_sept.  x93_america  x93_asia  x93_euorpe  dependent  \n",
              "0        0          0            0         1           0          0  \n",
              "1        0          1            0         1           0          1  \n",
              "2        1          0            1         0           0          1  \n",
              "3        0          0            0         1           0          0  \n",
              "4        0          0            0         1           0          0  \n",
              "\n",
              "[5 rows x 127 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8BcBNVg_BCy",
        "outputId": "11eb1489-bffc-49fa-934b-2dadd0a40318"
      },
      "source": [
        "df1.corr()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x0</th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x4</th>\n",
              "      <th>x5</th>\n",
              "      <th>x6</th>\n",
              "      <th>x7</th>\n",
              "      <th>x8</th>\n",
              "      <th>x9</th>\n",
              "      <th>...</th>\n",
              "      <th>x68_Jun</th>\n",
              "      <th>x68_Mar</th>\n",
              "      <th>x68_May</th>\n",
              "      <th>x68_Nov</th>\n",
              "      <th>x68_Oct</th>\n",
              "      <th>x68_sept.</th>\n",
              "      <th>x93_america</th>\n",
              "      <th>x93_asia</th>\n",
              "      <th>x93_euorpe</th>\n",
              "      <th>dependent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>x0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.218484</td>\n",
              "      <td>-0.156282</td>\n",
              "      <td>-0.146731</td>\n",
              "      <td>-0.261757</td>\n",
              "      <td>-0.078714</td>\n",
              "      <td>0.000105</td>\n",
              "      <td>-0.003964</td>\n",
              "      <td>0.110925</td>\n",
              "      <td>-0.003594</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.100942</td>\n",
              "      <td>-0.078656</td>\n",
              "      <td>-0.136981</td>\n",
              "      <td>0.059919</td>\n",
              "      <td>0.108867</td>\n",
              "      <td>0.150035</td>\n",
              "      <td>-0.003411</td>\n",
              "      <td>0.002935</td>\n",
              "      <td>-0.000099</td>\n",
              "      <td>-0.065957</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x1</th>\n",
              "      <td>-0.218484</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.025459</td>\n",
              "      <td>-0.032857</td>\n",
              "      <td>-0.019945</td>\n",
              "      <td>0.083369</td>\n",
              "      <td>-0.003246</td>\n",
              "      <td>0.004735</td>\n",
              "      <td>-0.112988</td>\n",
              "      <td>0.005602</td>\n",
              "      <td>...</td>\n",
              "      <td>0.015071</td>\n",
              "      <td>0.041255</td>\n",
              "      <td>0.044045</td>\n",
              "      <td>-0.012699</td>\n",
              "      <td>-0.026627</td>\n",
              "      <td>-0.038342</td>\n",
              "      <td>0.006459</td>\n",
              "      <td>-0.003355</td>\n",
              "      <td>-0.003566</td>\n",
              "      <td>0.103643</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x2</th>\n",
              "      <td>-0.156282</td>\n",
              "      <td>-0.025459</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.089697</td>\n",
              "      <td>-0.067693</td>\n",
              "      <td>0.003510</td>\n",
              "      <td>0.006680</td>\n",
              "      <td>-0.002142</td>\n",
              "      <td>-0.000591</td>\n",
              "      <td>0.004686</td>\n",
              "      <td>...</td>\n",
              "      <td>0.066880</td>\n",
              "      <td>0.018834</td>\n",
              "      <td>0.063460</td>\n",
              "      <td>-0.035703</td>\n",
              "      <td>-0.067728</td>\n",
              "      <td>-0.088896</td>\n",
              "      <td>-0.001778</td>\n",
              "      <td>-0.001132</td>\n",
              "      <td>0.004485</td>\n",
              "      <td>0.088139</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x3</th>\n",
              "      <td>-0.146731</td>\n",
              "      <td>-0.032857</td>\n",
              "      <td>-0.089697</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.019363</td>\n",
              "      <td>-0.006893</td>\n",
              "      <td>-0.008622</td>\n",
              "      <td>0.004259</td>\n",
              "      <td>-0.023798</td>\n",
              "      <td>-0.005182</td>\n",
              "      <td>...</td>\n",
              "      <td>0.071132</td>\n",
              "      <td>0.050875</td>\n",
              "      <td>0.089833</td>\n",
              "      <td>-0.036193</td>\n",
              "      <td>-0.067031</td>\n",
              "      <td>-0.094722</td>\n",
              "      <td>-0.002673</td>\n",
              "      <td>0.000717</td>\n",
              "      <td>0.002620</td>\n",
              "      <td>-0.094680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x4</th>\n",
              "      <td>-0.261757</td>\n",
              "      <td>-0.019945</td>\n",
              "      <td>-0.067693</td>\n",
              "      <td>0.019363</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.005555</td>\n",
              "      <td>-0.001393</td>\n",
              "      <td>-0.001745</td>\n",
              "      <td>0.072124</td>\n",
              "      <td>0.002550</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.004541</td>\n",
              "      <td>0.013689</td>\n",
              "      <td>0.009049</td>\n",
              "      <td>-0.007848</td>\n",
              "      <td>-0.000622</td>\n",
              "      <td>-0.010242</td>\n",
              "      <td>-0.009434</td>\n",
              "      <td>0.007236</td>\n",
              "      <td>0.001228</td>\n",
              "      <td>-0.008274</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x68_sept.</th>\n",
              "      <td>0.150035</td>\n",
              "      <td>-0.038342</td>\n",
              "      <td>-0.088896</td>\n",
              "      <td>-0.094722</td>\n",
              "      <td>-0.010242</td>\n",
              "      <td>0.009803</td>\n",
              "      <td>-0.000781</td>\n",
              "      <td>-0.003594</td>\n",
              "      <td>0.086612</td>\n",
              "      <td>-0.007332</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.169128</td>\n",
              "      <td>-0.032223</td>\n",
              "      <td>-0.113382</td>\n",
              "      <td>-0.018864</td>\n",
              "      <td>-0.046754</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.006898</td>\n",
              "      <td>-0.006953</td>\n",
              "      <td>0.001934</td>\n",
              "      <td>0.006291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x93_america</th>\n",
              "      <td>-0.003411</td>\n",
              "      <td>0.006459</td>\n",
              "      <td>-0.001778</td>\n",
              "      <td>-0.002673</td>\n",
              "      <td>-0.009434</td>\n",
              "      <td>-0.001090</td>\n",
              "      <td>0.003370</td>\n",
              "      <td>0.004743</td>\n",
              "      <td>-0.006255</td>\n",
              "      <td>0.008306</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.006914</td>\n",
              "      <td>-0.003686</td>\n",
              "      <td>-0.000261</td>\n",
              "      <td>0.000951</td>\n",
              "      <td>0.004715</td>\n",
              "      <td>0.006898</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.810336</td>\n",
              "      <td>-0.056328</td>\n",
              "      <td>0.008155</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x93_asia</th>\n",
              "      <td>0.002935</td>\n",
              "      <td>-0.003355</td>\n",
              "      <td>-0.001132</td>\n",
              "      <td>0.000717</td>\n",
              "      <td>0.007236</td>\n",
              "      <td>0.002490</td>\n",
              "      <td>0.000401</td>\n",
              "      <td>-0.003845</td>\n",
              "      <td>0.001455</td>\n",
              "      <td>-0.006724</td>\n",
              "      <td>...</td>\n",
              "      <td>0.003987</td>\n",
              "      <td>0.002555</td>\n",
              "      <td>0.000109</td>\n",
              "      <td>-0.000438</td>\n",
              "      <td>0.000249</td>\n",
              "      <td>-0.006953</td>\n",
              "      <td>-0.810336</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.539390</td>\n",
              "      <td>-0.004508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>x93_euorpe</th>\n",
              "      <td>-0.000099</td>\n",
              "      <td>-0.003566</td>\n",
              "      <td>0.004485</td>\n",
              "      <td>0.002620</td>\n",
              "      <td>0.001228</td>\n",
              "      <td>-0.002675</td>\n",
              "      <td>-0.005527</td>\n",
              "      <td>-0.000263</td>\n",
              "      <td>0.006510</td>\n",
              "      <td>-0.000479</td>\n",
              "      <td>...</td>\n",
              "      <td>0.003143</td>\n",
              "      <td>0.000944</td>\n",
              "      <td>0.000189</td>\n",
              "      <td>-0.000620</td>\n",
              "      <td>-0.007201</td>\n",
              "      <td>0.001934</td>\n",
              "      <td>-0.056328</td>\n",
              "      <td>-0.539390</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.004038</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dependent</th>\n",
              "      <td>-0.065957</td>\n",
              "      <td>0.103643</td>\n",
              "      <td>0.088139</td>\n",
              "      <td>-0.094680</td>\n",
              "      <td>-0.008274</td>\n",
              "      <td>-0.090497</td>\n",
              "      <td>0.002031</td>\n",
              "      <td>0.001280</td>\n",
              "      <td>-0.008889</td>\n",
              "      <td>0.011759</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.012539</td>\n",
              "      <td>0.019024</td>\n",
              "      <td>0.010139</td>\n",
              "      <td>0.028432</td>\n",
              "      <td>0.023149</td>\n",
              "      <td>0.006291</td>\n",
              "      <td>0.008155</td>\n",
              "      <td>-0.004508</td>\n",
              "      <td>-0.004038</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>127 rows × 127 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                   x0        x1        x2        x3        x4        x5  \\\n",
              "x0           1.000000 -0.218484 -0.156282 -0.146731 -0.261757 -0.078714   \n",
              "x1          -0.218484  1.000000 -0.025459 -0.032857 -0.019945  0.083369   \n",
              "x2          -0.156282 -0.025459  1.000000 -0.089697 -0.067693  0.003510   \n",
              "x3          -0.146731 -0.032857 -0.089697  1.000000  0.019363 -0.006893   \n",
              "x4          -0.261757 -0.019945 -0.067693  0.019363  1.000000  0.005555   \n",
              "...               ...       ...       ...       ...       ...       ...   \n",
              "x68_sept.    0.150035 -0.038342 -0.088896 -0.094722 -0.010242  0.009803   \n",
              "x93_america -0.003411  0.006459 -0.001778 -0.002673 -0.009434 -0.001090   \n",
              "x93_asia     0.002935 -0.003355 -0.001132  0.000717  0.007236  0.002490   \n",
              "x93_euorpe  -0.000099 -0.003566  0.004485  0.002620  0.001228 -0.002675   \n",
              "dependent   -0.065957  0.103643  0.088139 -0.094680 -0.008274 -0.090497   \n",
              "\n",
              "                   x6        x7        x8        x9  ...   x68_Jun   x68_Mar  \\\n",
              "x0           0.000105 -0.003964  0.110925 -0.003594  ... -0.100942 -0.078656   \n",
              "x1          -0.003246  0.004735 -0.112988  0.005602  ...  0.015071  0.041255   \n",
              "x2           0.006680 -0.002142 -0.000591  0.004686  ...  0.066880  0.018834   \n",
              "x3          -0.008622  0.004259 -0.023798 -0.005182  ...  0.071132  0.050875   \n",
              "x4          -0.001393 -0.001745  0.072124  0.002550  ... -0.004541  0.013689   \n",
              "...               ...       ...       ...       ...  ...       ...       ...   \n",
              "x68_sept.   -0.000781 -0.003594  0.086612 -0.007332  ... -0.169128 -0.032223   \n",
              "x93_america  0.003370  0.004743 -0.006255  0.008306  ... -0.006914 -0.003686   \n",
              "x93_asia     0.000401 -0.003845  0.001455 -0.006724  ...  0.003987  0.002555   \n",
              "x93_euorpe  -0.005527 -0.000263  0.006510 -0.000479  ...  0.003143  0.000944   \n",
              "dependent    0.002031  0.001280 -0.008889  0.011759  ... -0.012539  0.019024   \n",
              "\n",
              "              x68_May   x68_Nov   x68_Oct  x68_sept.  x93_america  x93_asia  \\\n",
              "x0          -0.136981  0.059919  0.108867   0.150035    -0.003411  0.002935   \n",
              "x1           0.044045 -0.012699 -0.026627  -0.038342     0.006459 -0.003355   \n",
              "x2           0.063460 -0.035703 -0.067728  -0.088896    -0.001778 -0.001132   \n",
              "x3           0.089833 -0.036193 -0.067031  -0.094722    -0.002673  0.000717   \n",
              "x4           0.009049 -0.007848 -0.000622  -0.010242    -0.009434  0.007236   \n",
              "...               ...       ...       ...        ...          ...       ...   \n",
              "x68_sept.   -0.113382 -0.018864 -0.046754   1.000000     0.006898 -0.006953   \n",
              "x93_america -0.000261  0.000951  0.004715   0.006898     1.000000 -0.810336   \n",
              "x93_asia     0.000109 -0.000438  0.000249  -0.006953    -0.810336  1.000000   \n",
              "x93_euorpe   0.000189 -0.000620 -0.007201   0.001934    -0.056328 -0.539390   \n",
              "dependent    0.010139  0.028432  0.023149   0.006291     0.008155 -0.004508   \n",
              "\n",
              "             x93_euorpe  dependent  \n",
              "x0            -0.000099  -0.065957  \n",
              "x1            -0.003566   0.103643  \n",
              "x2             0.004485   0.088139  \n",
              "x3             0.002620  -0.094680  \n",
              "x4             0.001228  -0.008274  \n",
              "...                 ...        ...  \n",
              "x68_sept.      0.001934   0.006291  \n",
              "x93_america   -0.056328   0.008155  \n",
              "x93_asia      -0.539390  -0.004508  \n",
              "x93_euorpe     1.000000  -0.004038  \n",
              "dependent     -0.004038   1.000000  \n",
              "\n",
              "[127 rows x 127 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUxg2mAC_BCz"
      },
      "source": [
        "#There are 127 diffrent features, but we need to reduce the dimentionality of the data \n",
        "#by removing those features that doesnt provide that much information\n",
        "\n",
        "#Correlation with output variable\n",
        "cor = df1.corr()\n",
        "cor_target = abs(cor[\"dependent\"])\n",
        "#Selecting highly correlated features\n",
        "relevant_features = cor_target[cor_target>0.1]\n",
        "relevant_features.sort_values()\n",
        "\n",
        "dfpred = df1[relevant_features.index.values]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bbg7g3r1_BC0",
        "outputId": "bdabb9b1-ad72-425b-9255-5be17de1a701"
      },
      "source": [
        "dfpred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x1</th>\n",
              "      <th>x37</th>\n",
              "      <th>x41</th>\n",
              "      <th>x58</th>\n",
              "      <th>x70</th>\n",
              "      <th>x75</th>\n",
              "      <th>x97</th>\n",
              "      <th>dependent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>74.425320</td>\n",
              "      <td>-10.839200</td>\n",
              "      <td>449.48</td>\n",
              "      <td>2.078396</td>\n",
              "      <td>41.040206</td>\n",
              "      <td>40.617107</td>\n",
              "      <td>-2.125570</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>24.320711</td>\n",
              "      <td>57.917006</td>\n",
              "      <td>-525.06</td>\n",
              "      <td>-2.696257</td>\n",
              "      <td>36.204784</td>\n",
              "      <td>-49.303165</td>\n",
              "      <td>-36.030599</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-66.160459</td>\n",
              "      <td>-12.991058</td>\n",
              "      <td>-599.50</td>\n",
              "      <td>-2.417447</td>\n",
              "      <td>49.538602</td>\n",
              "      <td>-19.706659</td>\n",
              "      <td>26.212736</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>33.210943</td>\n",
              "      <td>37.658926</td>\n",
              "      <td>-220.71</td>\n",
              "      <td>4.443710</td>\n",
              "      <td>82.130387</td>\n",
              "      <td>-7.301283</td>\n",
              "      <td>19.221130</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-26.717872</td>\n",
              "      <td>-59.497091</td>\n",
              "      <td>-1405.59</td>\n",
              "      <td>-2.421952</td>\n",
              "      <td>-19.154066</td>\n",
              "      <td>-2.751656</td>\n",
              "      <td>-5.703269</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39995</th>\n",
              "      <td>63.861763</td>\n",
              "      <td>11.733697</td>\n",
              "      <td>-2113.20</td>\n",
              "      <td>9.469915</td>\n",
              "      <td>-30.688388</td>\n",
              "      <td>49.415653</td>\n",
              "      <td>-21.035463</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39996</th>\n",
              "      <td>-88.026423</td>\n",
              "      <td>-1.972438</td>\n",
              "      <td>935.41</td>\n",
              "      <td>-8.679121</td>\n",
              "      <td>40.089671</td>\n",
              "      <td>-8.803315</td>\n",
              "      <td>-1.333318</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39997</th>\n",
              "      <td>64.294604</td>\n",
              "      <td>-28.895908</td>\n",
              "      <td>338.20</td>\n",
              "      <td>-2.099754</td>\n",
              "      <td>-43.621345</td>\n",
              "      <td>68.031607</td>\n",
              "      <td>-5.452678</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39998</th>\n",
              "      <td>-29.455176</td>\n",
              "      <td>-9.029036</td>\n",
              "      <td>-169.39</td>\n",
              "      <td>-8.545977</td>\n",
              "      <td>18.107577</td>\n",
              "      <td>2.375774</td>\n",
              "      <td>20.135669</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39999</th>\n",
              "      <td>13.095298</td>\n",
              "      <td>68.550878</td>\n",
              "      <td>-768.26</td>\n",
              "      <td>-5.407202</td>\n",
              "      <td>-14.951734</td>\n",
              "      <td>-61.383565</td>\n",
              "      <td>12.565180</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>39182 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "              x1        x37      x41       x58        x70        x75  \\\n",
              "0      74.425320 -10.839200   449.48  2.078396  41.040206  40.617107   \n",
              "1      24.320711  57.917006  -525.06 -2.696257  36.204784 -49.303165   \n",
              "2     -66.160459 -12.991058  -599.50 -2.417447  49.538602 -19.706659   \n",
              "3      33.210943  37.658926  -220.71  4.443710  82.130387  -7.301283   \n",
              "4     -26.717872 -59.497091 -1405.59 -2.421952 -19.154066  -2.751656   \n",
              "...          ...        ...      ...       ...        ...        ...   \n",
              "39995  63.861763  11.733697 -2113.20  9.469915 -30.688388  49.415653   \n",
              "39996 -88.026423  -1.972438   935.41 -8.679121  40.089671  -8.803315   \n",
              "39997  64.294604 -28.895908   338.20 -2.099754 -43.621345  68.031607   \n",
              "39998 -29.455176  -9.029036  -169.39 -8.545977  18.107577   2.375774   \n",
              "39999  13.095298  68.550878  -768.26 -5.407202 -14.951734 -61.383565   \n",
              "\n",
              "             x97  dependent  \n",
              "0      -2.125570          0  \n",
              "1     -36.030599          1  \n",
              "2      26.212736          1  \n",
              "3      19.221130          0  \n",
              "4      -5.703269          0  \n",
              "...          ...        ...  \n",
              "39995 -21.035463          0  \n",
              "39996  -1.333318          0  \n",
              "39997  -5.452678          0  \n",
              "39998  20.135669          0  \n",
              "39999  12.565180          0  \n",
              "\n",
              "[39182 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmLx-ITo_BC0"
      },
      "source": [
        "#defining dependent and independent variables\n",
        "X = dfpred.iloc[:, :-1].values\n",
        "y = dfpred.iloc[:, -1].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GpeIkRu6_BC1"
      },
      "source": [
        "#Normalization\n",
        "sc=StandardScaler()\n",
        "x=sc.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r4UEme5C_BC1"
      },
      "source": [
        "#Let's implement 3 diffrent classifiers\n",
        "\n",
        "classifiers = {\n",
        "    \"LogisiticRegression\": LogisticRegression(),\n",
        "    \"KNearest\": KNeighborsClassifier(),\n",
        "    \"DecisionTreeClassifier\": DecisionTreeClassifier()\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0kGJgDge_BC2",
        "outputId": "61305669-7f79-464b-bee7-c1b2de087064"
      },
      "source": [
        "#accuracy of the classifiers without tunnig\n",
        "\n",
        "for key, classifier in classifiers.items():\n",
        "    classifier.fit(x, y)\n",
        "    training_score = cross_val_score(classifier, x, y, cv=5)\n",
        "    print(\"Classifiers: \", classifier.__class__.__name__, \"Has a training score of\", \n",
        "          round(training_score.mean(), 2) * 100, \"% accuracy score\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Classifiers:  LogisticRegression Has a training score of 82.0 % accuracy score\n",
            "Classifiers:  KNeighborsClassifier Has a training score of 81.0 % accuracy score\n",
            "Classifiers:  DecisionTreeClassifier Has a training score of 75.0 % accuracy score\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WsDuJhRk_BC2"
      },
      "source": [
        "# Find the best parameter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "u5chITAe_BC3",
        "outputId": "7ce3b13c-f741-418a-dd13-7d7063b82571"
      },
      "source": [
        "# Use GridSearchCV to find the best parameters.\n",
        "\n",
        "#Logistic Regression \n",
        "log_reg_params = {\"penalty\": ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}\n",
        "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n",
        "grid_log_reg.fit(x, y)\n",
        "log_reg = grid_log_reg.best_estimator_\n",
        "\n",
        "\n",
        "#KNears best estimator\n",
        "knears_params = {\"n_neighbors\": list(range(2,5,1)), 'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']}\n",
        "grid_knears = GridSearchCV(KNeighborsClassifier(), knears_params)\n",
        "grid_knears.fit(x, y)\n",
        "knears_neighbors = grid_knears.best_estimator_\n",
        "\n",
        "#DecisionTree Classifier\n",
        "tree_params = {\"criterion\": [\"gini\", \"entropy\"], \"max_depth\": list(range(2,4,1)), \n",
        "              \"min_samples_leaf\": list(range(5,7,1))}\n",
        "grid_tree = GridSearchCV(DecisionTreeClassifier(), tree_params)\n",
        "grid_tree.fit(x, y)\n",
        "tree_clf = grid_tree.best_estimator_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "fsgWVwyQ_BC4",
        "outputId": "4cbeb80f-8687-4496-8d06-6c397f9009a2"
      },
      "source": [
        "# accuracy and recall based on best estimators\n",
        "\n",
        "log_reg_score = cross_val_score(log_reg, x, y, cv=5, scoring='accuracy')\n",
        "log_reg_recall = cross_val_score(log_reg, x, y, cv=5, scoring='recall')\n",
        "print('Logistic Regression Cross Validation accuracy: ', round(log_reg_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('Logistic Regression Cross Validation recall: ', round(log_reg_recall.mean() * 100, 2).astype(str) + '%')\n",
        "\n",
        "knears_score = cross_val_score(knears_neighbors, x, y, cv=5, scoring='accuracy')\n",
        "knears_score_recall = cross_val_score(knears_neighbors, x, y, cv=5, scoring='recall')\n",
        "print('Knears Neighbors Cross Validation Score', round(knears_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('Knears Neighbors Cross Validation Score', round(knears_score_recall.mean() * 100, 2).astype(str) + '%')\n",
        "\n",
        "tree_score = cross_val_score(tree_clf, x, y, cv=5, scoring='accuracy')\n",
        "tree_score_recall = cross_val_score(tree_clf, x, y, cv=5, scoring='recall')\n",
        "print('DecisionTree Classifier Cross Validation Score', round(tree_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('DecisionTree Classifier Cross Validation Score', round(tree_score_recall.mean() * 100, 2).astype(str) + '%')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Logistic Regression Cross Validation accuracy:  82.04%\n",
            "Logistic Regression Cross Validation recall:  28.85%\n",
            "Knears Neighbors Cross Validation Score 81.01%\n",
            "Knears Neighbors Cross Validation Score 23.25%\n",
            "DecisionTree Classifier Cross Validation Score 80.17%\n",
            "DecisionTree Classifier Cross Validation Score 12.5%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3P4jYfDB_BC5"
      },
      "source": [
        "**We have a good accuracy but a very bad recall. In other words, the model predicts well the true negative but badly the true positive. This is due to the unbalanced data. Lets apply random undersampling to make the data balance. I am going to consider 50% minority class and 50% majority class**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p1DKfbu9_BC6",
        "outputId": "fd9a3824-3ea8-497f-d28c-9229f9bfd7de"
      },
      "source": [
        "#imbalanced data\n",
        "colors = [\"#0101DF\", \"#DF0101\"]\n",
        "\n",
        "sns.countplot('dependent', data=df1, palette=colors)\n",
        "plt.title('Class Distributions \\n (0: No accident || 1: accident)', fontsize=14)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Class Distributions \\n (0: No accident || 1: accident)')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 161
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEoCAYAAACZ5MzqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5xd093H8c83IehFQxNKhKSkleglmOKpXpQ2QqtUUdqnQtNG3Z7qTemjpVRLb0qJeyRUmyr6SJWmqVK0boO4JSJTgqCEuFYlTfyeP9Y6bCdnZs7szJkzY77v1+u85py11957nTMz53v2XvuspYjAzMysjAHNboCZmfVdDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwi1idIWiDpG81uR2ckjZAUkloasO1jJN1deDxV0uXdvZ+87YY9D3t9cYhY00laV9LJkv4haYmkRyRdKWnnZretIr+hVm4vSrpf0q8kfaCq6sPAesDsOrfblXD8CfDhLjS7LpKukXRqVXGXnof1Xw4RaypJI4DbgB2BI4H3AB8F/gCc0bSG1fYl0hvraGAisBS4VtI3KxUiYnlE/DMilnXXTiUNkDQwIl6IiKe6a7sdacTzsNcnh4g122RAQEtEXBQR8yJibkScCry3vZUkfU3SnZL+lY9czpE0uLD8LZIukPSEpJfykcNhheUHSLovL1skaaakVTpp6zP5jfXBiLg6IvYDTgB+KGmTvN3XnAaStKqkUyQ9mo+yHpZ0Ql52DbAR8OPKUU4u30/SC5J2zqevlgKjq09nFZ7LUZIez+ucJ2mNwrIVjjKKp8EkTSUd3RxcONIaUet0lqQPSbopv2aPSzpJ0qCqfU2W9ANJT+bX/ieSBhTq7J5/b/+WtFjSXyWt28nrbr2YQ8SaRtLawHjg1Ih4oXp5RDzdweovA4cBmwGfBbYCflFY/n3g3cAngE2BLwCP5P22AKcB3wPeSTry+WPJp/FT0v/Rbu0s/x/gU8DewCjgM8C8vGx3YCFwLOkIZ73CeqsDRwEHAGOAB9vZ/odJYbsD8GlgHHBiF9r/FeAG4LxCGx6uriRpGHAlcDuwOelIbB/gh1VVPwcsA94PHEL6HX0mb+NtwHRgGulo7kPABV1oq/VCnX3yMmukTUhHIXO7umJE/LzwcIGkw4HLJE2IiJdJn/Bvj4ibK3UK9TcE/gXMiIjnSW/Qd5RoPxHxlKQngLe3U2Uj4D7gukgD1T0E/D2vu1jScuD5iPhn1XoDgUMj4tZKgaRa218O7J9D+G5J3wLOlXRkRPyrjvY/K2kp8GKxDTX2dRDwGHBQfn3nSjoCOFPSdyLixVxvTkR8N9+/T9KXSAH3a2B9YFXg4oiohOIKR1bWt/hIxJqp5rtiXStK20uaJWmhpOeBS4FBwNtyldOBvSTdkU+pFDukZ5GC4wFJF0qaIOnNZdtCeh7tjWQ6FRhLekM9TdLHi6d3OrCM+jq176w6iruB9DpsXMe6XTEauCEHSMX1eV+bFNtTtd6jwDr5/h3An0lhd4mkAyUN7eZ2Wg9ziFgzzSe9+Y7uykqSNiJ1vM8F9gS2JJ2ugvSmRkRcSToK+AkwBPiDpPPysueBLYC9SEcGRwL3Slq/q09A0hBgKHB/reURcRswAvg26f9tGjCrjiBZEhHLu9qeGl5mxbBetcR2OgrKYvl/aiwbAKmznnS6bRwpbCYC8yW12/dlvZ9DxJomIhYDM4FDJL2penmxo7xKCyksvhoRN0TEfaRTJdXbfzIiLsgd4BOBCZJWy8uWRcRfIqJyRdgbSf0nXfV10hv1Ze1ViIjnI+K3EXEg8HFge1799L6UdOqqrHdLemPh8TZ5m//Ijxfx2r4WWPGChXraMAf4r6rw+0DVvjoVyQ0R8T3gfaQjlc/Uu771Pu4TsWY7iNRH0CrpO6RPqAI+QjpC2LDGOvNJH4AOk3Qp6Y3zsGIFSceSLh2+h/R3vjtwf0QskfQJ0umea4HFeV9vpvO+mcG5c7hyumgCsC9weES01VpB0tdIfQmzSZ/SPws8R+pQh9RX80FJvyQdfTzZSRuqrQJMyc93fdLVYmcX+kP+Avxc0idJHfoHAMN5bR/RAmArpcutXyC9JtUmk17jyZJOJvUBnUC6KOLFGvVXIGkb0kUMM4HHSR30w0kBZX2UQ8SaKiIekLQF6XTPicAw4CnS+fMD2lnnTklfAb5Fugrr78A3gN8Uqi0BjgdGAi8BNwK75GXPkK6m+i7wBtIn6S9GxHWdNPfswrYfy9vcLiKu7WCd54Fvkq7MCtLVTTsV3ni/C5yZ27AaXe8n+ispKK/Oz+US4PDC8imkI60p+fFk4HekU3wVPyGdZpsDrEF6zV4jIh6RtBPwY1IgPgP8ivR7q9ezwLbAocBg0lVgx0XEL7uwDetl5JkNzcysLPeJmJlZaQ4RMzMrzSFiZmalOUTMzKw0h4iZmZXmEOnn8siwUzqv2X9I2i6PYDukgzp7VEbdbYb8e5taVbZA0nbNaVH3qTWCcI06LZURh3ugPZ+QNLvO4Wr6Hb8o/ZikdYCvkb5rUSw/SNIDecjvWyV9sMS2p+Z/8qOqyjt9g+4F/k76lnePzN1RlF+bPRq4/ZMltebf7YJG7WclNW1CLNWYJCwiLicNdPm5nm5PX+AQ6d++CNwcEa+M+yTpM8DJwA9I3yj+O3ClpFrfHO/MS8DhfW2QvYhYmucNeT1+iaoyftf5zW5Ie3rphFjnkYb1tyoOkf7ts8CMqrKvAVMj4uw8OdShpG9nH1hi+1eThtT4TkeVOpvsqEb9gZLOzUdL/5Y0X9Lh1acb8ui8dylNBvV48fSPpDUlnS7psbzfuTlAax4tSdpX0oNKU+NeDqwwkZKkXfKR20u5bcfrtZM2LVCaQOpMSc/lEYi/WVye7/42738B3SwiDo2IX5CGpy9N0gmS5uXXf4GkH0lavarOx/Pv9d+SnpL0+0odSYOUJq96MP9+7pf0P3lZrQmxxku6N7+21wHvqNGm9ytNcvWi0kRlp0tas7C8w0mz1M4kYdkMoEV58jF7lUOkn1KaEGoM0FooG0QaEfdPVdX/RJpkqFJvap1vcC8DRwBfllRzaHLVP9lR0QDSBFN7kUYA/l/S8Bv7F7Z7AGk4kfNIw37sTBoeBEnK+/xwXmcMKTyXttPGrUlDup9FGtb996SJpIp1dgQuBE4lTZT1BWAP0hFd0VeBu0ijCJ8I/EjSf+Vl78s/K9Pwvo8myG+gx3RS7V+k5ziaNP7Z3qTfQ2Ub40mDUs4i/U19hDRES+U9Zxpp3LGv8ep0w8+0057hwP/lbY0lTT72o6o67yb9nc4gDTC5e65b3d/X7qRZdDBJWEQ8RBrvq9vnuO/zIsK3fngj/YMFMLJQtn4u+1BV3e8C8wqPfwhc1cn2pwKX5/tXA9Pz/e3yPobkx8cDbcCAwrr7kcanekMXns8JwJ8LjxcCJ7RT92OkgBvdzvLqNv4KmFVV55z07/PK42uB71TV2Y00oGFleKEFwK+r6swHjio8DmCPOp7vMaQjxmLZAtJYXvW8Xt8AFrSz7F7gkC7+PX0ZaCs8/lvld16jbmUcsfHtLB+Rl7fkxz8gHTmpUOeoXGdEfnw+cG47f+Pr5MfXkOZEKdaZBZxT9Rp+o5123UYa66vp/7+96eYBGPuvyjzcL9VYVt0X8Jq5JCINn94VhwM3SvpJjWWdTXZUPclRapD0ZVKfzkak57IqeQpZpQsGhgFXtdOezYHHIqLeGRVHk44+im4gfXqu2JI0Eu63CmUDctveRjolSI3nU5y0qVeIiE07q5M7/w8j/Y7eRBpKvjic/OakDxK1bE4K8avrbNJo4MbI7+TZDVV1tgQ2qZySrDQz/9wYeCLfX5nX/9+8+n9jmU9n9V+VIcfXqipbzquzA1asQzqULyUibiGNLltr7u96Jzt6dYX0RvFz0pvUjqRPnJPJE1LR+Ui4XR0pt576A0hzto8t3N5D+tS9qFCv3Umb+gqlId2nk4Z034UUCkdR/2RXjXr9z+G1r/97Sa9/8SqvlXn91+a1v0vDQ8H3Z/8gzWsxhjyfQ0QslXQr6XTPbwt1P0YKgZXx7byf8VXlc0jT2A4oHI10NtnRB4CbIuLUSkGxzyUiHpf0CGlu71k11r8NWE/S6DqPRuaQ5iwpqn58G7BptDOvSBf8h5WbpKonbAs8EhHHVQqUZpssup30+p/Nim4jvXF/BPhjHfubA3xakgpHI7Ve/8264fWvOUFXviBg47wfK+hTn4Cs++Q37D+T3pCLfgbsJ+mLkkYrTUC0PnBGpYKkH0pq71RRe/trI3VMf6Vq0eS8/cl5fx+n88mO7gO2kLSTpFFKk1lVd3geT5q06quS3iFprKSv52VXATcBl0jaUdJISR+TtFs7+zsF+KikI/P+vgR8qqrOscBnJR0r6V2SNlX6QuKPVtxchxYAO0h6m6S1OqvcVZI2kTSW9JoPyq/L2KqryO6VdEgHm7kPGCbpc5LeLulA0sUQRccDe0r6vqQxkjbLv4s3RMR84CLgHEmfzq//ByV9vp39nUHqJ/m5pHfmU2lfrqpzIul04hmSNs/P8xOSzqz3tckWkCYJG6bXfpdpG1I/3d+6uL3Xv2Z3yvjWvBvpVNBjwMCq8oNI/0xLgFtZsaN9Ku10ylbVubyqbB3SJE2vdFrn8g+R3tSXkE6bnQSs1sG2BwHnAk+Trug5l9T5v6Cq3kTSp9ilwD+BKYVlg0mfkheR+oXmAHvlZdvVaOP+pPnY/026susQCh3ruc444DrgRdJRXiuFDmpqdNqSOntPLTzehdTZ/p+OXmNKdqzn/UWN24hCnQCO6WQ7P8yv3QvApaRLwKtfj0/mv58lpFOlM4DV87LVSFdYPZKX/6PyWlHVsZ7LPk6amfEl0hv552q0u4V0ZPMc6eqxu4Bj23uta/2dksLijryfKJSfCZzR7P/Z3njzpFT9nKQbgMkRcUGz22L1y5fgjog0f3ylbAGwX0Rc05xWvT4pfVn2XlKoPdDs9vQ2Pp1lB+C/A7OOjAQOcoDU5o71fi4i7qSdy2jNDCLiZuDmZrejt3KImPVN15D6dYp+TuoXMesx7hMxM7PS+t2RyJAhQ2LEiBHNboaZWZ9y6623PhkRK4zI3e9CZMSIEbS2tnZe0czMXiHpwVrlvirHzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxK63ffWF9ZG254fbObYL3QQw9VTxBp1j/4SMTMzEpziJiZWWkOETMzK61hISJpdUk3S7pD0j2SvpfLR0q6SdJ8Sb+RNCiXr5Yft+XlIwrbOjKXz5O0Y6F8fC5rk3REo56LmZnV1sgjkSXA9hHxXmAsMF7SNsCJwEkRMQp4GpiY608Eno6ITYCTcj0kjQH2BjYDxgOTJQ2UNBA4DdgJGAPsk+uamVkPaViIRPJCfrhqvgWwPXBxLp8G7Jbv75ofk5fvIEm5fHpELImIB4A2YKt8a4uI+yNiKTA91zUzsx7S0D6RfMQwG3gCmAX8A3gmIpblKguBYfn+MOBhgLz8WeCtxfKqddorNzOzHtLQEImI5RExFtiAdOQwula1/FPtLOtq+QokTZLUKql10aJFnTfczMzq0iNXZ0XEM8A1wDbAYEmVLzluADya7y8EhgPk5W8BFhfLq9Zpr7zW/s+KiJaIaBk6dIUpgs3MrKRGXp01VNLgfH8N4KPAXOBqYI9cbQJwWb4/Iz8mL/9LREQu3ztfvTUSGAXcDNwCjMpXew0idb7PaNTzMTOzFTVy2JP1gGn5KqoBwEURcbmkOcB0Sd8HbgfOzfXPBS6Q1EY6AtkbICLukXQRMAdYBhwcEcsBJB0CzAQGAlMi4p4GPh8zM6ui9GG//2hpaYnW1tbS63vsLKvFY2fZ652kWyOipbrc31g3M7PSHCJmZlaaQ8TMzEpziJiZWWkOETMzK80hYmZmpTlEzMysNIeImZmV5hAxM7PSHCJmZlaaQ8TMzEpziJiZWWkOETMzK80hYmZmpTlEzMysNIeImZmV5hAxM7PSHCJmZlaaQ8TMzEpziJiZWWkOETMzK80hYmZmpTlEzMysNIeImZmV1rAQkTRc0tWS5kq6R9JXcvkxkh6RNDvfdi6sc6SkNknzJO1YKB+fy9okHVEoHynpJknzJf1G0qBGPR8zM1tRI49ElgFfj4jRwDbAwZLG5GUnRcTYfLsCIC/bG9gMGA9MljRQ0kDgNGAnYAywT2E7J+ZtjQKeBiY28PmYmVmVhoVIRDwWEbfl+88Dc4FhHayyKzA9IpZExANAG7BVvrVFxP0RsRSYDuwqScD2wMV5/WnAbo15NmZmVkuP9IlIGgFsDtyUiw6RdKekKZLWymXDgIcLqy3MZe2VvxV4JiKWVZWbmVkPaXiISHoTcAlwWEQ8B5wObAyMBR4DflqpWmP1KFFeqw2TJLVKal20aFEXn4GZmbWnoSEiaVVSgFwYEZcCRMTjEbE8Il4GziadroJ0JDG8sPoGwKMdlD8JDJa0SlX5CiLirIhoiYiWoUOHds+TMzOzhl6dJeBcYG5E/KxQvl6h2qeAu/P9GcDeklaTNBIYBdwM3AKMyldiDSJ1vs+IiACuBvbI608ALmvU8zEzsxWt0nmV0rYFPg/cJWl2Lvs26eqqsaRTTwuAAwAi4h5JFwFzSFd2HRwRywEkHQLMBAYCUyLinry9bwHTJX0fuJ0UWmZm1kMaFiIRcT21+y2u6GCd44Hja5RfUWu9iLifV0+HmZlZD/M31s3MrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU1LEQkDZd0taS5ku6R9JVcvrakWZLm559r5XJJOkVSm6Q7JW1R2NaEXH++pAmF8i0l3ZXXOUWSGvV8zMxsRY08ElkGfD0iRgPbAAdLGgMcAVwVEaOAq/JjgJ2AUfk2CTgdUugARwNbA1sBR1eCJ9eZVFhvfAOfj5mZVWlYiETEYxFxW77/PDAXGAbsCkzL1aYBu+X7uwLnR3IjMFjSesCOwKyIWBwRTwOzgPF52ZoRcUNEBHB+YVtmZtYDeqRPRNIIYHPgJmDdiHgMUtAA6+Rqw4CHC6stzGUdlS+sUW5mZj2k4SEi6U3AJcBhEfFcR1VrlEWJ8lptmCSpVVLrokWLOmuymZnVqaEhImlVUoBcGBGX5uLH86ko8s8ncvlCYHhh9Q2ARzsp36BG+Qoi4qyIaImIlqFDh67ckzIzs1c08uosAecCcyPiZ4VFM4DKFVYTgMsK5fvmq7S2AZ7Np7tmAuMkrZU71McBM/Oy5yVtk/e1b2FbZmbWA1Zp4La3BT4P3CVpdi77NnACcJGkicBDwJ552RXAzkAb8CKwP0BELJZ0HHBLrndsRCzO9w8EpgJrAFfmm5mZ9ZCGhUhEXE/tfguAHWrUD+DgdrY1BZhSo7wVeNdKNNPMzFaCv7FuZmalOUTMzKw0h4iZmZXmEDEzs9IcImZmVppDxMzMSnOImJlZaXWFiKSr6ikzM7P+pcMvG0paHXgDMCQPOVL58uCawPoNbpuZmfVynX1j/QDgMFJg3MqrIfIccFoD22VmZn1AhyESEScDJ0s6NCJ+0UNtMjOzPqKusbMi4heS3g+MKK4TEec3qF1mZtYH1BUiki4ANgZmA8tzcWVKWjMz66fqHcW3BRiTR9o1MzMD6v+eyN3A2xrZEDMz63vqPRIZAsyRdDOwpFIYEZ9sSKvMzKxPqDdEjmlkI8zMrG+q9+qsvza6IWZm1vfUe3XW86SrsQAGAasC/4qINRvVMDMz6/3qPRJ5c/GxpN2ArRrSIjMz6zNKjeIbEf8HbN/NbTEzsz6m3tNZuxceDiB9b8TfGTEz6+fqvTprl8L9ZcACYNdub42ZmfUp9faJ7N/ohpiZWd9T76RUG0j6naQnJD0u6RJJG3SyzpRc/+5C2TGSHpE0O992Liw7UlKbpHmSdiyUj89lbZKOKJSPlHSTpPmSfiNpUNeeupmZrax6O9bPA2aQ5hUZBvw+l3VkKjC+RvlJETE2364AkDQG2BvYLK8zWdJASQNJ85bsBIwB9sl1AU7M2xoFPA1MrPO5mJlZN6k3RIZGxHkRsSzfpgJDO1ohIq4FFte5/V2B6RGxJCIeANpIlxBvBbRFxP0RsRSYDuwqSaSrwy7O608DdqtzX2Zm1k3qDZEnJf135ehA0n8DT5Xc5yGS7synu9bKZcOAhwt1Fuay9srfCjwTEcuqys3MrAfVGyJfAPYC/gk8BuwBlOlsP500L8nYvJ2f5nLVqBslymuSNElSq6TWRYsWda3FZmbWrnpD5DhgQkQMjYh1SKFyTFd3FhGPR8TyiHgZOJtXv/W+EBheqLoB8GgH5U8CgyWtUlXe3n7PioiWiGgZOrTDs3BmZtYF9YbIeyLi6cqDiFgMbN7VnUlar/DwU6R5SiB12u8taTVJI4FRwM3ALcCofCXWIFLn+4w8OdbVpCMigAnAZV1tj5mZrZx6v2w4QNJalSCRtHZn60r6NbAdMETSQuBoYDtJY0mnnhYABwBExD2SLgLmkL7MeHBELM/bOQSYCQwEpkTEPXkX3wKmS/o+cDtwbp3PxczMukm9IfJT4O+SLiYFwF7A8R2tEBH71Chu940+Io6vtc18GfAVNcrvx4NAmpk1Vb3fWD9fUivpsloBu0fEnIa2zMzMer16j0TIoeHgMDOzV5QaCt7MzAwcImZmthIcImZmVppDxMzMSnOImJlZaQ4RMzMrzSFiZmalOUTMzKw0h4iZmZXmEDEzs9IcImZmVppDxMzMSnOImJlZaQ4RMzMrzSFiZmalOUTMzKw0h4iZmZXmEDEzs9IcImZmVppDxMzMSnOImJlZaQ4RMzMrzSFiZmalNSxEJE2R9ISkuwtla0uaJWl+/rlWLpekUyS1SbpT0haFdSbk+vMlTSiUbynprrzOKZLUqOdiZma1NfJIZCowvqrsCOCqiBgFXJUfA+wEjMq3ScDpkEIHOBrYGtgKOLoSPLnOpMJ61fsyM7MGa1iIRMS1wOKq4l2Bafn+NGC3Qvn5kdwIDJa0HrAjMCsiFkfE08AsYHxetmZE3BARAZxf2JaZmfWQnu4TWTciHgPIP9fJ5cOAhwv1FuayjsoX1iivSdIkSa2SWhctWrTST8LMzJLe0rFeqz8jSpTXFBFnRURLRLQMHTq0ZBPNzKxaT4fI4/lUFPnnE7l8ITC8UG8D4NFOyjeoUW5mZj2op0NkBlC5wmoCcFmhfN98ldY2wLP5dNdMYJyktXKH+jhgZl72vKRt8lVZ+xa2ZWZmPWSVRm1Y0q+B7YAhkhaSrrI6AbhI0kTgIWDPXP0KYGegDXgR2B8gIhZLOg64Jdc7NiIqnfUHkq4AWwO4Mt/MzKwHNSxEImKfdhbtUKNuAAe3s50pwJQa5a3Au1amjWZmtnJ6S8e6mZn1QQ4RMzMrzSFiZmalOUTMzKw0h4iZmZXmEDEzs9IcImZmVppDxMzMSmvYlw3NrOddv+GGzW6C9UIfeOihhm3bRyJmZlaaQ8TMzEpziJiZWWkOETMzK80hYmZmpTlEzMysNIeImZmV5hAxM7PSHCJmZlaaQ8TMzEpziJiZWWkOETMzK80hYmZmpTlEzMysNIeImZmV1pQQkbRA0l2SZktqzWVrS5olaX7+uVYul6RTJLVJulPSFoXtTMj150ua0IznYmbWnzXzSOQjETE2Ilry4yOAqyJiFHBVfgywEzAq3yYBp0MKHeBoYGtgK+DoSvCYmVnP6E2ns3YFpuX704DdCuXnR3IjMFjSesCOwKyIWBwRTwOzgPE93Wgzs/6sWSESwJ8k3SppUi5bNyIeA8g/18nlw4CHC+suzGXtlZuZWQ9p1hzr20bEo5LWAWZJureDuqpRFh2Ur7iBFFSTADb0HNRmZt2mKUciEfFo/vkE8DtSn8bj+TQV+ecTufpCYHhh9Q2ARzsor7W/syKiJSJahg4d2p1PxcysX+vxEJH0RklvrtwHxgF3AzOAyhVWE4DL8v0ZwL75Kq1tgGfz6a6ZwDhJa+UO9XG5zMzMekgzTmetC/xOUmX/v4qIP0q6BbhI0kTgIWDPXP8KYGegDXgR2B8gIhZLOg64Jdc7NiIW99zTMDOzHg+RiLgfeG+N8qeAHWqUB3BwO9uaAkzp7jaamVl9etMlvmZm1sc4RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrDSHiJmZleYQMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyutz4eIpPGS5klqk3REs9tjZtaf9OkQkTQQOA3YCRgD7CNpTHNbZWbWf/TpEAG2Atoi4v6IWApMB3ZtcpvMzPqNvh4iw4CHC48X5jIzM+sBqzS7AStJNcpihUrSJGBSfviCpHkNbVX/MQR4stmN6A1U6y/Rms1/nxXd8we6Ua3Cvh4iC4HhhccbAI9WV4qIs4CzeqpR/YWk1ohoaXY7zGrx32fP6Ouns24BRkkaKWkQsDcwo8ltMjPrN/r0kUhELJN0CDATGAhMiYh7mtwsM7N+o0+HCEBEXAFc0ex29FM+RWi9mf8+e4AiVuiHNjMzq0tf7xMxM7MmcohYKR5uxnorSVMkPSHp7ma3pT9wiFiXebgZ6+WmAuOb3Yj+wiFiZXi4Geu1IuJaYHGz29FfOESsDA83Y2aAQ8TKqWu4GTN7/XOIWBl1DTdjZq9/DhErw8PNmBngELESImIZUBluZi5wkYebsd5C0q+BG4B3SlooaWKz2/R65m+sm5lZaT4SMTOz0hwiZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWLWRZKOkfSNJux3O0mXr8T63+7O9piBQ8SsP3GIWLdziJjVQdL/5gNCiIkAAAJZSURBVPlT/gy8M5dtLOmPkm6VdJ2kTXP5VEln5LL7JH0ilw+U9GNJt0i6U9IBuXw7SddIuljSvZIulKS8bHwuux7YvdCeN+Z5M26RdLukXXP5fpIuze2aL+lHufwEYA1JsyVd2IMvnb3O9fk51s0aTdKWpKFdNif9z9wG3Eqaw/vLETFf0tbAZGD7vNoI4MPAxsDVkjYB9gWejYj3SVoN+JukP+X6mwObkcYg+xuwraRW4Oy8zTbgN4Vm/S/wl4j4gqTBwM054ADG5u0tAeZJ+kVEHCHpkIgY260vjvV7DhGzzn0Q+F1EvAggaQawOvB+4Lf5oAFgtcI6F0XEy8B8SfcDmwLjgPdI2iPXeQswClgK3BwRC/P2Z5NC6AXggYiYn8t/CUzK644DPlnom1kd2DDfvyoins3rzAE24rVD95t1G4eIWX2qxwcaADzTwSf76vpBGkL/0IiYWVwgaTvSUUPFcl7932xvXCIBn46IeVXb2rqDbZl1O/eJmHXuWuBTktaQ9GZgF+BF4AFJewIoeW9hnT0lDZC0MfB2YB5pwMoDJa2a13mHpDd2sN97gZF5GwD7FJbNBA4t9J1sXsfz+E9l32bdxSFi1omIuI3UHzEbuAS4Li/6HDBR0h3APbx2iuB5wF+BK0n9Ji8B5wBzgNsk3Q2cSQdHCXmdScAfcsf6g4XFxwGrAnfmbR1Xx1M5K9d3x7p1G4/ia9bNJE0FLo+Ii5vdFrNG85GImZmV5iMRMzMrzUciZmZWmkPEzMxKc4iYmVlpDhEzMyvNIWJmZqU5RMzMrLT/B/sr6k50e7cDAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtOCTgj4_BC6",
        "outputId": "5be1e195-3664-41c2-b96e-87971f358e85"
      },
      "source": [
        "#lets make our dataset balance\n",
        "df_1= df1[df1['dependent'] == 1]\n",
        "df_0 = df1[df1['dependent'] == 0]\n",
        "\n",
        "balanced_df=pd.concat([df_0.iloc[:df_1.shape[0],:], df_1]) # 50% of 0 50% of 1\n",
        "balanced_df=balanced_df.sample(balanced_df.shape[0])\n",
        "\n",
        "balanced_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x0</th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x4</th>\n",
              "      <th>x5</th>\n",
              "      <th>x6</th>\n",
              "      <th>x7</th>\n",
              "      <th>x8</th>\n",
              "      <th>x9</th>\n",
              "      <th>...</th>\n",
              "      <th>x68_Jun</th>\n",
              "      <th>x68_Mar</th>\n",
              "      <th>x68_May</th>\n",
              "      <th>x68_Nov</th>\n",
              "      <th>x68_Oct</th>\n",
              "      <th>x68_sept.</th>\n",
              "      <th>x93_america</th>\n",
              "      <th>x93_asia</th>\n",
              "      <th>x93_euorpe</th>\n",
              "      <th>dependent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5632</th>\n",
              "      <td>2.341898</td>\n",
              "      <td>-30.506207</td>\n",
              "      <td>-27.939673</td>\n",
              "      <td>-3.413023</td>\n",
              "      <td>0.644538</td>\n",
              "      <td>19.858541</td>\n",
              "      <td>-4.890062</td>\n",
              "      <td>7.060795</td>\n",
              "      <td>-10.093390</td>\n",
              "      <td>-0.584804</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8533</th>\n",
              "      <td>9.400499</td>\n",
              "      <td>20.093036</td>\n",
              "      <td>22.047442</td>\n",
              "      <td>0.363747</td>\n",
              "      <td>-34.346754</td>\n",
              "      <td>-16.930969</td>\n",
              "      <td>14.269300</td>\n",
              "      <td>-3.137979</td>\n",
              "      <td>-15.634431</td>\n",
              "      <td>1.661638</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3461</th>\n",
              "      <td>-2.581860</td>\n",
              "      <td>21.100633</td>\n",
              "      <td>29.987554</td>\n",
              "      <td>-1.123336</td>\n",
              "      <td>-10.590672</td>\n",
              "      <td>-16.751599</td>\n",
              "      <td>-10.883603</td>\n",
              "      <td>-0.398871</td>\n",
              "      <td>-4.284934</td>\n",
              "      <td>-5.367186</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23165</th>\n",
              "      <td>7.343586</td>\n",
              "      <td>-5.372281</td>\n",
              "      <td>-61.570623</td>\n",
              "      <td>-1.900518</td>\n",
              "      <td>-10.230765</td>\n",
              "      <td>13.179263</td>\n",
              "      <td>7.407237</td>\n",
              "      <td>-0.114064</td>\n",
              "      <td>-3.436136</td>\n",
              "      <td>2.908474</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2525</th>\n",
              "      <td>0.899738</td>\n",
              "      <td>25.465650</td>\n",
              "      <td>-43.862205</td>\n",
              "      <td>-0.200567</td>\n",
              "      <td>-1.439579</td>\n",
              "      <td>27.291545</td>\n",
              "      <td>9.955198</td>\n",
              "      <td>-1.410723</td>\n",
              "      <td>-15.266531</td>\n",
              "      <td>-0.508082</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7245</th>\n",
              "      <td>21.209138</td>\n",
              "      <td>-27.714749</td>\n",
              "      <td>24.720043</td>\n",
              "      <td>1.034420</td>\n",
              "      <td>-21.831644</td>\n",
              "      <td>-11.092761</td>\n",
              "      <td>3.668053</td>\n",
              "      <td>-3.915528</td>\n",
              "      <td>-3.599257</td>\n",
              "      <td>-0.020990</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16196</th>\n",
              "      <td>-26.063152</td>\n",
              "      <td>51.053721</td>\n",
              "      <td>109.108932</td>\n",
              "      <td>1.758557</td>\n",
              "      <td>26.870361</td>\n",
              "      <td>-6.518332</td>\n",
              "      <td>9.664408</td>\n",
              "      <td>15.065577</td>\n",
              "      <td>1.433051</td>\n",
              "      <td>1.150273</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6663</th>\n",
              "      <td>6.176946</td>\n",
              "      <td>3.224787</td>\n",
              "      <td>79.291491</td>\n",
              "      <td>-0.912219</td>\n",
              "      <td>-31.876495</td>\n",
              "      <td>18.090395</td>\n",
              "      <td>4.904954</td>\n",
              "      <td>-12.377386</td>\n",
              "      <td>-16.802863</td>\n",
              "      <td>4.459321</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21415</th>\n",
              "      <td>28.778754</td>\n",
              "      <td>-21.301771</td>\n",
              "      <td>-32.052868</td>\n",
              "      <td>1.460705</td>\n",
              "      <td>-10.072068</td>\n",
              "      <td>19.396735</td>\n",
              "      <td>0.364654</td>\n",
              "      <td>0.040774</td>\n",
              "      <td>-26.088268</td>\n",
              "      <td>0.826504</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25179</th>\n",
              "      <td>-9.850242</td>\n",
              "      <td>-35.250467</td>\n",
              "      <td>-23.976480</td>\n",
              "      <td>0.851244</td>\n",
              "      <td>20.530569</td>\n",
              "      <td>-3.182587</td>\n",
              "      <td>-4.449550</td>\n",
              "      <td>0.835611</td>\n",
              "      <td>-17.282931</td>\n",
              "      <td>-0.183770</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>15932 rows × 127 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "              x0         x1          x2        x3         x4         x5  \\\n",
              "5632    2.341898 -30.506207  -27.939673 -3.413023   0.644538  19.858541   \n",
              "8533    9.400499  20.093036   22.047442  0.363747 -34.346754 -16.930969   \n",
              "3461   -2.581860  21.100633   29.987554 -1.123336 -10.590672 -16.751599   \n",
              "23165   7.343586  -5.372281  -61.570623 -1.900518 -10.230765  13.179263   \n",
              "2525    0.899738  25.465650  -43.862205 -0.200567  -1.439579  27.291545   \n",
              "...          ...        ...         ...       ...        ...        ...   \n",
              "7245   21.209138 -27.714749   24.720043  1.034420 -21.831644 -11.092761   \n",
              "16196 -26.063152  51.053721  109.108932  1.758557  26.870361  -6.518332   \n",
              "6663    6.176946   3.224787   79.291491 -0.912219 -31.876495  18.090395   \n",
              "21415  28.778754 -21.301771  -32.052868  1.460705 -10.072068  19.396735   \n",
              "25179  -9.850242 -35.250467  -23.976480  0.851244  20.530569  -3.182587   \n",
              "\n",
              "              x6         x7         x8        x9  ...  x68_Jun  x68_Mar  \\\n",
              "5632   -4.890062   7.060795 -10.093390 -0.584804  ...        0        0   \n",
              "8533   14.269300  -3.137979 -15.634431  1.661638  ...        1        0   \n",
              "3461  -10.883603  -0.398871  -4.284934 -5.367186  ...        0        0   \n",
              "23165   7.407237  -0.114064  -3.436136  2.908474  ...        0        0   \n",
              "2525    9.955198  -1.410723 -15.266531 -0.508082  ...        0        0   \n",
              "...          ...        ...        ...       ...  ...      ...      ...   \n",
              "7245    3.668053  -3.915528  -3.599257 -0.020990  ...        0        0   \n",
              "16196   9.664408  15.065577   1.433051  1.150273  ...        0        0   \n",
              "6663    4.904954 -12.377386 -16.802863  4.459321  ...        0        0   \n",
              "21415   0.364654   0.040774 -26.088268  0.826504  ...        0        0   \n",
              "25179  -4.449550   0.835611 -17.282931 -0.183770  ...        0        0   \n",
              "\n",
              "       x68_May  x68_Nov  x68_Oct  x68_sept.  x93_america  x93_asia  \\\n",
              "5632         0        0        0          0            0         1   \n",
              "8533         0        0        0          0            0         1   \n",
              "3461         0        0        0          0            0         1   \n",
              "23165        0        0        0          0            0         1   \n",
              "2525         0        0        1          0            1         0   \n",
              "...        ...      ...      ...        ...          ...       ...   \n",
              "7245         0        0        0          1            0         1   \n",
              "16196        0        0        0          0            0         1   \n",
              "6663         0        0        0          1            0         1   \n",
              "21415        0        0        1          0            0         1   \n",
              "25179        0        0        0          0            0         1   \n",
              "\n",
              "       x93_euorpe  dependent  \n",
              "5632            0          0  \n",
              "8533            0          0  \n",
              "3461            0          1  \n",
              "23165           0          1  \n",
              "2525            0          0  \n",
              "...           ...        ...  \n",
              "7245            0          1  \n",
              "16196           0          1  \n",
              "6663            0          0  \n",
              "21415           0          1  \n",
              "25179           0          1  \n",
              "\n",
              "[15932 rows x 127 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3okKphZv_BC7",
        "outputId": "055eefa3-a1fa-4267-8a6c-3851826e8db2"
      },
      "source": [
        "#balanced data\n",
        "colors = [\"#0101DF\", \"#DF0101\"]\n",
        "\n",
        "sns.countplot('dependent', data=balanced_df, palette=colors)\n",
        "plt.title('Class Distributions \\n (0: No accident || 1: accident)', fontsize=14)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Class Distributions \\n (0: No accident || 1: accident)')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 163
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEoCAYAAACpaN3LAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5gcVZ3G8e8broKXBBkQk2AQolx2NeAIeEfREBANImjUXSJmN6iAsqwiuCoKsoLLroDIJZKQgBeMqEsWUYwBxAu34SJCQsgIgQwgGQkgGAkGf/vHOS2VTvdUT5jqmWTez/P001WnTp06XT3Tvz51qs9RRGBmZtaXEYNdATMzG/ocLMzMrJSDhZmZlXKwMDOzUg4WZmZWysHCzMxKOVjYkCJpqaRPDXY9ykgaJykkdVZQ9hcl3VFYny3p8oE+Ti67stdhGxYHC2sbSdtKOlPS7yWtkvSApJ9IOmCw61aTPzhrj5WS7pH0HUlvrMu6DNgOuK3FcvsTBE8H3tKPardE0jWSzq5L7tfrsOHLwcLaQtI44BZgP+AE4FXA24EfA+cNWsUa+1fSB+guwDTgaeBaSZ+uZYiIZyLiDxGxeqAOKmmEpI0i4smIeGSgyu1LFa/DNkwOFtYu5wACOiNibkQsjohFEXE28OpmO0k6VtLtkv6cWyIXSBpZ2P4iSRdLWi7pqdwSOKaw/QhJd+dtvZKulLRxSV0fyx+g90XE1RHxYeBU4CuSdsrlrnH5RtImks6S9GBuNS2TdGredg3wMuC/aq2WnP5hSU9KOiBfdnoa2KX+MlThtXxO0sN5nwslPa+wba1WQ/HylaTZpNbKkYWW07hGl6EkvVnSDfmcPSzpa5I2rTvWOZL+U9If87k/XdKIQp6D8/v2F0krJP1C0rYl592GMAcLq5ykrYBJwNkR8WT99oh4tI/d/wYcA+wGfBDYE/h6YfuXgX8EDgR2Bj4CPJCP2wl8A/gS8EpSS+an6/gy/pv0/3JQk+2fAN4DTAHGA+8HFudtBwM9wEmkFst2hf02Bz4HHAHsCtzXpPy3kILqvsB7gYnAaf2o/yeB64ALC3VYVp9J0mjgJ8CtwO6kltUHgK/UZf0QsBp4PXAU6T16fy7jJcAlwBxS6+zNwMX9qKsNQWXfsMwGwk6kVsWi/u4YEWcUVpdKOg64TNLUiPgb6Rv7rRFxYy1PIf/2wJ+BeRHxBOmD+LfrUH8i4hFJy4GXN8nyMuBu4JeRBly7H/hN3neFpGeAJyLiD3X7bQQcHRE31xIkNSr/GeDwHGzvkPQZYKakEyLizy3U/3FJTwMri3VocKyPAw8BH8/nd5Gk44HzJX0+IlbmfAsj4gt5+W5J/0oKZN8FXgpsAlwaEbXgt1ZLydYvbllYOzT89GtpR+ltkuZL6pH0BPBDYFPgJTnLucD7JP02XwopdgzPJwWIeyV9W9JUSS9Y17qQXkezkTdnAxNIH5zfkPTO4mWZPqymtc7l2+taZdeRzsOOLezbH7sA1+VAUfOrfKydivWp2+9BYJu8/Fvg56Sg9gNJH5PUMcD1tDZzsLB2WEL6kN2lPztJehmpA3wRcCjwGtJlJkgfXkTET0jf6k8HtgZ+LOnCvO0JYA/gfaRv+icAd0l6aX9fgKStgQ7gnkbbI+IWYBzwWdL/1RxgfgsBY1VEPNPf+jTwN9YOypusQzl9BcRi+l8bbBsBqdOcdJlsIimoTAOWSGraN2VDn4OFVS4iVgBXAkdJen799mKHdZ1OUlD4t4i4LiLuJl3iqC//jxFxce6IngZMlbRZ3rY6Iq6KiNodWFuS+jf6699JH8iXNcsQEU9ExPcj4mPAO4G38ey38adJl5zW1T9K2rKwvncu8/d5vZc1+0Jg7RsHWqnDQuB1dUHujXXHKhXJdRHxJeC1pJbH+1vd34Ye91lYu3ycdA2/S9LnSd84BbyV9I1/+wb7LCF9oTlG0g9JH5DHFDNIOol0S+6dpL/ng4F7ImKVpANJl2muBVbkY72A8r6TkbmTtnaZZypwGHBcRHQ32kHSsaRr/beRvnV/EPgTqWMbUl/KmyR9i9Sa+GNJHeptDMzKr/elpLuzvlnor7gKOEPSu0kd60cAY1mzD2cpsKfSbcxPks5JvXNI5/gcSWeS+mhOJd2csLJB/rVI2pt0M8GVwMOkjvKxpEBk6ykHC2uLiLhX0h6kyzSnAaOBR0jXt49oss/tkj4JfIZ019NvgE8B3ytkWwWcAuwAPAVcD7wrb3uMdPfSF4AtSN+M/yUifllS3W8Wyn4ol7lPRFzbxz5PAJ8m3QkVpLuJ9i98wH4BOD/XYTP634/zC1JAvDq/lh8AxxW2zyK1nGbl9XOAH5EuzdWcTro8thB4HumcrSEiHpC0P/BfpMD3GPAd0vvWqseBNwBHAyNJd12dHBHf6kcZNsTIM+WZmVkZ91mYmVkpBwszMyvlYGFmZqUcLMzMrJSDhZmZlXKwGAbyKKazynMOH5L2yaOtbt1HnkNqI8QOhvy+za5LWyppn8Gp0cBpNNptgzydtdFx21CfAyXd1uIQLcOST8wGTtI2wLGk3ykU0z8u6d48DPXNkt60DmXPzv/Mn6tLL/0gHgJ+Q/rFc1vmjSjK5+aQCss/U1JXfm+XVnWc52jQJl1Sg4moIuJy0mCNH2p3fdYXDhYbvn8BboyIv49pJOn9wJnAf5J+Xfsb4CeSGv2KusxTwHHr20BxEfF0nrNiQ/yhUW1sqosGuyLNDNFJly4kDTVvDThYbPg+CMyrSzsWmB0R38wTEB1N+qXyx9ah/KtJw0h8vq9MZRPqNMi/kaSZufXzF0lLJB1Xf5kgjyT7O6UJhx4uXraR9EJJ50p6KB93UQ6UDVs/kg6TdJ/SdKqXA2tN1iPpXbkl9lSu2ylac2KgpUqTFJ0v6U95tNxPF7fnxe/n4y9lgEXE0RHxddKQ6etM0qmSFufzv1TSVyVtXpfnnfl9/YukRyT9Xy2PpE2VJki6L78/90j6RN7WaNKlSZLuyuf2l8ArGtTp9UoTKa1UmgzrXEkvLGzvc2ImNZmIKpsHdCpPcGVrcrDYgClNOrQr0FVI25Q0euvP6rL/jDSRTS3f7BY/yP4GHA98VFLD4bLV+oQ6RSNIkxi9jzRa7X+Qhpw4vFDuEaQhNC4kDXVxAGlIDCQpH/MteZ9dSUHy6SZ13Is0zPgM0lDj/0earKiYZz/g28DZpMmYPgIcQmqhFf0b8DvSiLenAV+V9Lq87bX5uTZ162sZBPmD8osl2f5Meo27kMb2mkJ6H2plTCINrDif9Df1VtKwJLXPlTmkMbWO5dkpah9rUp+xwP/msiaQJrj6al2efyT9nc4jDZJ4cM5b3x/XdGIm+piIKiLuJ41lNeDzn28QIsKPDfRB+kcKYIdC2ktz2pvr8n4BWFxY/wqwoKT82cDleflq4JK8vE8+xtZ5/RSgGxhR2PfDpLGXtujH6zkV+HlhvQc4tUned5AC2S5NttfX8TvA/Lo8F6R/kb+vXwt8vi7PQaRB+WpD5ywFvluXZwnwucJ6AIe08Hq/SGoBFtOWksapauV8fQpY2mTbXcBR/fx7+ijQXVj/de09b5C3NkbWpCbbx+XtnXn9P0ktIRXyfC7nGZfXLwJmNvkb3yavX0Oaj6OYZz5wQd05/FSTet1CGsdq0P9/h9rDAwlu2GpzND/VYFv9tfo15jGINKR3fxwHXC/p9AbbyibUqZ9IJ1VI+iipz+VlpNeyCXnaUaWO+9HAgib12R14KCJanZ1vF1Jroug60rfhmteQRm39TCFtRK7bS0iX8mjweooTAw0JEbFzWZ7cCX8M6T16Pml48+IQ57uTvjA0sjspWF/dYpV2Aa6P/ImdXVeX5zXATrVLibVq5ucdgeV5+bmc/7/w7P+NFfgy1IatNgz2qLq0Z3h2prmabUhN8HUSETeRRkJtNC90qxPqPLtD+kA4g/RhtB/pG+Q55EmPKB+1tb+juraSfwRpPu8JhcerSN+iewv5mk4MtL5QGmb8EtIw4+8iffh/jtYnVKrq/F/Amuf/1aTzX7yr6rmc/61Y8720zC2LDdvvSXMq7EqeSyAinpZ0M+kyzfcLed9B+rB/Lj6bjzOpLn0haerTEYXWRdmEOm8EboiIs2sJxT6RiHhY0gOkeZ/nN9j/FmA7Sbu02LpYSJovo6h+/RZg52gyp0U//JXnNhFSO7wBeCAiTq4lKM1cWHQr6fx/k7XdQvqAfivw0xaOtxB4ryQVWheNzv9uA3D+G04ClTvmd8zHsTrr1bcd65/8wfxz0gdv0f8AH5b0L5J2UZrk5qXAebUMkr4iqdklnmbH6yZ1EH+ybtM5ufxz8vHeSfmEOncDe0jaX9J4pQmT6jseTyFNjPRvkl4haYKkf8/bFgA3AD+QtJ+kHSS9Q9JBTY53FvB2SSfk4/0r8J66PCcBH5R0kqR/kLSz0g/3vrp2cX1aCuwr6SWSRpVl7i9JO0maQDrnm+bzMqHurq27JB3VRzF3A6MlfUjSyyV9jHRTQtEpwKGSvixpV0m75fdii4hYAswFLpD03nz+3yTpn5sc7zxSP8YZkl6ZL4F9tC7PaaTLgOdJ2j2/zgMlnd/qucmWkiaiGq01fwu0N6kf7df9LG94GOxOEz+qfZAu4TwEbFSX/nHSP80q4GbW7vCeTZPO0bo8l9elbUOaCOjvncc5/c2kD+9VpMtdXwM266PsTYGZwKOkO2hmkjrhl9blm0b6Vvo08AdgVmHbSNK33l5Sv81C4H152z4N6ng4aa7uv5DupDqKQgd3zjMR+CWwktRq66LQUUyDzlNSp+vZhfV3kTq9/9rXOWYdO7jz8aLBY1whTwBfLCnnK/ncPQn8kHRrdf35eHf++1lFusQ5D9g8b9uMdEfTA3n772vniroO7pz2TtIsf0+RPrA/1KDenaSWyp9Id2v9Djip2blu9HdKCgq/zceJQvr5wHmD/T87VB+e/GgYkHQdcE5EXDzYdbHW5Vtbx0WaW7yWthT4cERcMzi12jAp/aj0LlLwunew6zMU+TLU8HAEfq/N+rID8HEHiubcwT0MRMTtNLk91cwgIm4EbhzsegxlDhZmQ9c1pH6XojNI/RZmbeU+CzMzK7VBtiy23nrrGDdu3GBXw8xsvXLzzTf/MSIajiC9QQaLcePG0dXVVZ7RzMz+TtJ9zbb5DhkzMyvlYGFmZqUcLMzMrJSDhZmZlao0WORBxe6UdIek70raPA8odoPSNJnfqw1uJmmzvN6dt48rlHNCTl+cZyszM7M2qixY5Kk0P0Eaa+UfSEMCTyGNHPm1iBhPGiSuNrnMNODRiNiJNMjcabmcXfN+u5GGvj5H0lAf3tnMbINS9WWojYHnSdoY2II0+unbgEvz9jmkaSkBJud18vZ98zzKk0lTN67K47Z0A3tWXG8zMyuoLFhExAPA6aQhnx8CHicNZfxYRKzO2XpIU2OSn5flfVfn/C8upjfYx8zM2qDKy1CjSK2CHUiTsGwJ7N8ga228kUbTKkYf6fXHmy6pS1JXb69nRTQzG0hV/oL77cC9EdELIOmHwOuBkZI2zq2HMaTJ1CG1GMYCPfmy1YuAFYX0muI+fxcRM0iztNHZ2fmcB7zafvtfPdcibAN0//31kw6236+2336wq2BD0Bvvv7/S8qvss7gf2FvSFrnvYV/STGVXA4fkPFOBy/LyvLxO3n5VpFEO5wFT8t1SO5AmZ/dQwmZmbVRZyyIibpB0KWny89Wkyd1nAD8GLpH05Zw2M+8yE7hYUjepRTEll3OnpLmkQLMaODIinqmq3mZmtrZKBxKMiBOBE+uS76HB3UwR8RRwaJNyTiFNDm9mZoPAv+A2M7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSlQULSa+UdFvh8SdJx0jaStJ8SUvy86icX5LOktQt6XZJexTKmprzL5E0tflRzcysCpUFi4hYHBETImIC8BpgJfAj4HhgQUSMBxbkdYD9gfH5MR04F0DSVqSpWfciTcd6Yi3AmJlZe7TrMtS+wO8j4j5gMjAnp88BDsrLk4GLIrkeGClpO2A/YH5ErIiIR4H5wKQ21dvMzGhfsJgCfDcvbxsRDwHk521y+mhgWWGfnpzWLN3MzNqk8mAhaVPg3cD3y7I2SIs+0uuPM11Sl6Su3t7e/lfUzMyaakfLYn/gloh4OK8/nC8vkZ+X5/QeYGxhvzHAg32kryEiZkREZ0R0dnR0DPBLMDMb3toRLD7As5egAOYBtTuapgKXFdIPy3dF7Q08ni9TXQlMlDQqd2xPzGlmZtYmG1dZuKQtgHcARxSSTwXmSpoG3A8cmtOvAA4Aukl3Th0OEBErJJ0M3JTznRQRK6qst5mZranSYBERK4EX16U9Qro7qj5vAEc2KWcWMKuKOpqZWTn/gtvMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEpVGiwkjZR0qaS7JC2S9DpJW0maL2lJfh6V80rSWZK6Jd0uaY9COVNz/iWSpjY/opmZVaHqlsWZwE8jYmfg1cAi4HhgQUSMBxbkdYD9gfH5MR04F0DSVsCJwF7AnsCJtQBjZmbtUVmwkPRC4M3ATICIeDoiHgMmA3NytjnAQXl5MnBRJNcDIyVtB+wHzI+IFRHxKDAfmFRVvc3MbG1VtixeDvQCF0q6VdIFkrYEto2IhwDy8zY5/2hgWWH/npzWLN3MzNqkymCxMbAHcG5E7A78mWcvOTWiBmnRR/qaO0vTJXVJ6urt7V2X+pqZWRNVBoseoCcibsjrl5KCx8P58hL5eXkh/9jC/mOAB/tIX0NEzIiIzojo7OjoGNAXYmY23FUWLCLiD8AySa/MSfsCC4F5QO2OpqnAZXl5HnBYvitqb+DxfJnqSmCipFG5Y3tiTjMzszbZuOLyjwa+LWlT4B7gcFKAmitpGnA/cGjOewVwANANrMx5iYgVkk4Gbsr5ToqIFRXX28zMCioNFhFxG9DZYNO+DfIGcGSTcmYBswa2dmZm1ir/gtvMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEpVGiwkLZX0O0m3SerKaVtJmi9pSX4eldMl6SxJ3ZJul7RHoZypOf8SSVObHc/MzKrRjpbFWyNiQkTUplc9HlgQEeOBBXkdYH9gfH5MB86FFFyAE4G9gD2BE2sBxszM2mMwLkNNBubk5TnAQYX0iyK5HhgpaTtgP2B+RKyIiEeB+cCkdlfazGw4qzpYBPAzSTdLmp7Tto2IhwDy8zY5fTSwrLBvT05rlm5mZm2yccXlvyEiHpS0DTBf0l195FWDtOgjfc2dUzCaDrD99tuvS13NzKyJSlsWEfFgfl4O/IjU5/BwvrxEfl6es/cAYwu7jwEe7CO9/lgzIqIzIjo7OjoG+qWYmQ1rlQULSVtKekFtGZgI3AHMA2p3NE0FLsvL84DD8l1RewOP58tUVwITJY3KHdsTc5qZmbVJlZehtgV+JKl2nO9ExE8l3QTMlTQNuB84NOe/AjgA6AZWAocDRMQKSScDN+V8J0XEigrrbWZmdSoLFhFxD/DqBumPAPs2SA/gyCZlzQJmDXQdzcysNf4Ft5mZlXKwMDOzUg4WZmZWysHCzMxKtRQsJC1oJc3MzDZMfd4NJWlzYAtg6/wbh9qvqV8IvLTiupmZ2RBRduvsEcAxpMBwM88Giz8B36iwXmZmNoT0GSwi4kzgTElHR8TX21QnMzMbYlr6UV5EfF3S64FxxX0i4qKK6mVmZkNIS8FC0sXAjsBtwDM5OQAHCzOzYaDV4T46gV3zkBxmZjbMtPo7izuAl1RZETMzG7pabVlsDSyUdCOwqpYYEe+upFZmZjaktBosvlhlJczMbGhr9W6oX1RdETMzG7pavRvqCZ6d93pTYBPgzxHxwqoqZmZmQ0erLYsXFNclHUSaT9vMzIaBdRp1NiL+F3hbK3klbSTpVkmX5/UdJN0gaYmk70naNKdvlte78/ZxhTJOyOmLJe23LnU2M7N11+plqIMLqyNIv7to9TcXnwQWkQYfBDgN+FpEXCLpPGAacG5+fjQidpI0Jed7v6RdgSnAbqQxqn4u6RUR8Uz9gczMrBqttizeVXjsBzwBTC7bSdIY4J3ABXldpBbJpTnLHOCgvDw5r5O375vzTwYuiYhVEXEv0I0vgZmZtVWrfRaHr2P5ZwDHAbU+jxcDj0XE6rzeA4zOy6OBZfl4qyU9nvOPBq4vlFncx8zM2qDVyY/GSPqRpOWSHpb0g9xq6GufA4HlEXFzMblB1ijZ1tc+xeNNl9Qlqau3t7evqpmZWT+1ehnqQmAeqc9gNPB/Oa0vbwDeLWkpcAnp8tMZwEhJtRbNGODBvNwDjAXI218ErCimN9jn7yJiRkR0RkRnR0dHiy/LzMxa0Wqw6IiICyNidX7MBvr8RI6IEyJiTESMI3VQXxURHwKuBg7J2aYCl+XleXmdvP2qPHDhPGBKvltqB2A8cGOL9TYzswHQarD4o6R/yrfBbiTpn4BH1vGYnwGOldRN6pOYmdNnAi/O6ccCxwNExJ3AXGAh8FPgSN8JZWbWXq2ODfUR4Gzga6T+gt8ALXd6R8Q1wDV5+R4a3M0UEU8BhzbZ/xTglFaPZ2ZmA6vVYHEyMDUiHgWQtBVwOimImJnZBq7Vy1CvqgUKgIhYAexeTZXMzGyoaTVYjJA0qraSWxattkrMzGw91+oH/n8Dv5F0KanP4n24D8HMbNho9RfcF0nqIv1WQsDBEbGw0pqZmdmQ0fKlpBwcHCDMzIahdRqi3MzMhhcHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpSoLFpI2l3SjpN9KulPSl3L6DpJukLRE0vckbZrTN8vr3Xn7uEJZJ+T0xZL2q6rOZmbWWJUti1XA2yLi1cAEYJKkvYHTgK9FxHjgUWBazj8NeDQidiJN33oagKRdgSnAbsAk4BxJG1VYbzMzq1NZsIjkyby6SX4EaZjzS3P6HOCgvDw5r5O37ytJOf2SiFgVEfcC3TSYw9vMzKpTaZ+FpI0k3QYsB+YDvwcei4jVOUsPMDovjwaWAeTtjwMvLqY32MfMzNqg0mAREc9ExARgDKk1sEujbPlZTbY1S1+DpOmSuiR19fb2rmuVzcysgbbcDRURjwHXAHsDIyXVJl0aAzyYl3uAsQB5+4uAFcX0BvsUjzEjIjojorOjo6OKl2FmNmxVeTdUh6SRefl5wNuBRcDVwCE521Tgsrw8L6+Tt18VEZHTp+S7pXYAxgM3VlVvMzNbW8vTqq6D7YA5+c6lEcDciLhc0kLgEklfBm4FZub8M4GLJXWTWhRTACLiTklzSVO6rgaOjIhnKqy3mZnVqSxYRMTtwO4N0u+hwd1MEfEUcGiTsk4BThnoOpqZWWv8C24zMyvlYGFmZqUcLMzMrJSDhZmZlXKwMDOzUg4WZmZWysHCzMxKOViYmVkpBwszMyvlYGFmZqUcLMzMrJSDhZmZlXKwMDOzUg4WZmZWysHCzMxKOViYmVkpBwszMytV5RzcYyVdLWmRpDslfTKnbyVpvqQl+XlUTpeksyR1S7pd0h6Fsqbm/EskTW12TDMzq0aVLYvVwL9HxC7A3sCRknYFjgcWRMR4YEFeB9gfGJ8f04FzIQUX4ERgL9J0rCfWAoyZmbVHZcEiIh6KiFvy8hPAImA0MBmYk7PNAQ7Ky5OBiyK5HhgpaTtgP2B+RKyIiEeB+cCkquptZmZra0ufhaRxwO7ADcC2EfEQpIACbJOzjQaWFXbryWnN0s3MrE0qDxaSng/8ADgmIv7UV9YGadFHev1xpkvqktTV29u7bpU1M7OGKg0WkjYhBYpvR8QPc/LD+fIS+Xl5Tu8BxhZ2HwM82Ef6GiJiRkR0RkRnR0fHwL4QM7Nhrsq7oQTMBBZFxP8UNs0Danc0TQUuK6Qflu+K2ht4PF+muhKYKGlU7tiemNPMzKxNNq6w7DcA/wz8TtJtOe2zwKnAXEnTgPuBQ/O2K4ADgG5gJXA4QESskHQycFPOd1JErKiw3mZmVqeyYBERv6JxfwPAvg3yB3Bkk7JmAbMGrnZmZtYf/gW3mZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVcrAwM7NSDhZmZlbKwcLMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFmZmVqnIO7lmSlku6o5C2laT5kpbk51E5XZLOktQt6XZJexT2mZrzL5E0tdGxzMysWlW2LGYDk+rSjgcWRMR4YEFeB9gfGJ8f04FzIQUX4ERgL2BP4MRagDEzs/apLFhExLXAirrkycCcvDwHOKiQflEk1wMjJW0H7AfMj4gVEfEoMJ+1A5CZmVWs3X0W20bEQwD5eZucPhpYVsjXk9OapZuZWRsNlQ5uNUiLPtLXLkCaLqlLUldvb++AVs7MbLhrd7B4OF9eIj8vz+k9wNhCvjHAg32kryUiZkREZ0R0dnR0DHjFzcyGs3YHi3lA7Y6mqcBlhfTD8l1RewOP58tUVwITJY3KHdsTc5qZmbXRxlUVLOm7wD7A1pJ6SHc1nQrMlTQNuB84NGe/AjgA6AZWAocDRMQKSScDN+V8J0VEfae5mZlVrLJgEREfaLJp3wZ5AziySTmzgFkDWDUzM+unodLBbWZmQ5iDhZmZlXKwMDOzUg4WZmZWysHCzMxKOViYmVkpBwszMyvlYGFmZqUcLMzMrJSDhZmZlXKwMDOzUg4WZmZWysHCzMxKOViYmVkpBwszMyvlYGFmZqUcLMzMrNR6EywkTZK0WFK3pNnL+2AAAAQOSURBVOMHuz5mZsPJehEsJG0EfAPYH9gV+ICkXQe3VmZmw8d6ESyAPYHuiLgnIp4GLgEmD3KdzMyGjfUlWIwGlhXWe3KamZm1wcaDXYEWqUFarJFBmg5Mz6tPSlpcea2Gj62BPw52JYYCNfpLtMHkv82agfnjfFmzDetLsOgBxhbWxwAPFjNExAxgRjsrNVxI6oqIzsGuh1k9/222z/pyGeomYLykHSRtCkwB5g1ynczMho31omUREaslHQVcCWwEzIqIOwe5WmZmw8Z6ESwAIuIK4IrBrscw5ct7NlT5b7NNFBHluczMbFhbX/oszMxsEDlYWJ88zIoNRZJmSVou6Y7Brstw4WBhTXmYFRvCZgOTBrsSw4mDhfXFw6zYkBQR1wIrBrsew4mDhfXFw6yYGeBgYX0rHWbFzIYHBwvrS+kwK2Y2PDhYWF88zIqZAQ4W1oeIWA3UhllZBMz1MCs2FEj6LnAd8EpJPZKmDXadNnT+BbeZmZVyy8LMzEo5WJiZWSkHCzMzK+VgYWZmpRwszMyslIOFWQOSvijpU4Nw3H0kXf4c9v/sQNbHrMbBwmzD4mBhlXCwMMsk/Ueeu+PnwCtz2o6SfirpZkm/lLRzTp8t6bycdrekA3P6RpL+S9JNkm6XdERO30fSNZIulXSXpG9LUt42Kaf9Cji4UJ8t87wNN0m6VdLknP5hST/M9Voi6as5/VTgeZJuk/TtNp46GwbWmzm4zaok6TWk4Ux2J/1f3ALcTJrj+aMRsUTSXsA5wNvybuOAtwA7AldL2gk4DHg8Il4raTPg15J+lvPvDuxGGl/r18AbJHUB38xldgPfK1TrP4CrIuIjkkYCN+ZABjAhl7cKWCzp6xFxvKSjImLCgJ4cMxwszGreBPwoIlYCSJoHbA68Hvh+bgQAbFbYZ25E/A1YIukeYGdgIvAqSYfkPC8CxgNPAzdGRE8u/zZSsHkSuDciluT0bwHT874TgXcX+k42B7bPywsi4vG8z0LgZaw5nLzZgHKwMHtW/dg3I4DH+vimXp8/SMO6Hx0RVxY3SNqH1AqoeYZn//+ajbkj4L0RsbiurL36KMusEu6zMEuuBd4j6XmSXgC8C1gJ3CvpUAAlry7sc6ikEZJ2BF4OLCYNuvgxSZvkfV4hacs+jnsXsEMuA+ADhW1XAkcX+jZ2b+F1/LV2bLOB5GBhBkTELaT+gtuAHwC/zJs+BEyT9FvgTtacVnYx8AvgJ6R+jaeAC4CFwC2S7gDOp49v/Xmf6cCPcwf3fYXNJwObALfnsk5u4aXMyPndwW0DyqPOmq0DSbOByyPi0sGui1k7uGVhZmal3LIwM7NSblmYmVkpBwszMyvlYGFmZqUcLMzMrJSDhZmZlXKwMDOzUv8P0P5Qfq/RHikAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7TNcG58G_BC7"
      },
      "source": [
        "#Correlation with output variable\n",
        "cor = balanced_df.corr()\n",
        "cor_target = abs(cor[\"dependent\"])\n",
        "\n",
        "#Selecting highly correlated features\n",
        "relevant_features = cor_target[cor_target>0.1]\n",
        "relevant_features.sort_values()\n",
        "\n",
        "dfpred = balanced_df[relevant_features.index.values]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A5iYwIl__BC8",
        "outputId": "c2c17d86-79c2-4da2-cf59-baa7f6cc6830"
      },
      "source": [
        "dfpred"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x5</th>\n",
              "      <th>x10</th>\n",
              "      <th>x20</th>\n",
              "      <th>x21</th>\n",
              "      <th>x22</th>\n",
              "      <th>x33</th>\n",
              "      <th>x37</th>\n",
              "      <th>...</th>\n",
              "      <th>x73</th>\n",
              "      <th>x75</th>\n",
              "      <th>x78</th>\n",
              "      <th>x79</th>\n",
              "      <th>x83</th>\n",
              "      <th>x85</th>\n",
              "      <th>x96</th>\n",
              "      <th>x97</th>\n",
              "      <th>x99</th>\n",
              "      <th>dependent</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5632</th>\n",
              "      <td>-30.506207</td>\n",
              "      <td>-27.939673</td>\n",
              "      <td>-3.413023</td>\n",
              "      <td>19.858541</td>\n",
              "      <td>-86.875647</td>\n",
              "      <td>0.003640</td>\n",
              "      <td>41.389960</td>\n",
              "      <td>87.042929</td>\n",
              "      <td>-5.640067</td>\n",
              "      <td>45.330522</td>\n",
              "      <td>...</td>\n",
              "      <td>40.881151</td>\n",
              "      <td>20.615232</td>\n",
              "      <td>-1.605192</td>\n",
              "      <td>-35.409911</td>\n",
              "      <td>1.154636</td>\n",
              "      <td>36.257851</td>\n",
              "      <td>-3.912497</td>\n",
              "      <td>-33.280921</td>\n",
              "      <td>1.284467</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8533</th>\n",
              "      <td>20.093036</td>\n",
              "      <td>22.047442</td>\n",
              "      <td>0.363747</td>\n",
              "      <td>-16.930969</td>\n",
              "      <td>-4.852505</td>\n",
              "      <td>1.737856</td>\n",
              "      <td>28.967661</td>\n",
              "      <td>77.000176</td>\n",
              "      <td>-2.377054</td>\n",
              "      <td>41.567437</td>\n",
              "      <td>...</td>\n",
              "      <td>-11.697267</td>\n",
              "      <td>-8.918759</td>\n",
              "      <td>-8.634336</td>\n",
              "      <td>57.081723</td>\n",
              "      <td>5.342754</td>\n",
              "      <td>-25.399170</td>\n",
              "      <td>8.651768</td>\n",
              "      <td>-1.766345</td>\n",
              "      <td>-0.526032</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3461</th>\n",
              "      <td>21.100633</td>\n",
              "      <td>29.987554</td>\n",
              "      <td>-1.123336</td>\n",
              "      <td>-16.751599</td>\n",
              "      <td>3.786722</td>\n",
              "      <td>16.802476</td>\n",
              "      <td>-22.622090</td>\n",
              "      <td>24.686648</td>\n",
              "      <td>0.016123</td>\n",
              "      <td>-24.231346</td>\n",
              "      <td>...</td>\n",
              "      <td>-25.892340</td>\n",
              "      <td>-37.761717</td>\n",
              "      <td>1.390277</td>\n",
              "      <td>-20.749920</td>\n",
              "      <td>-0.981127</td>\n",
              "      <td>-21.328386</td>\n",
              "      <td>-24.935859</td>\n",
              "      <td>44.049856</td>\n",
              "      <td>-1.484568</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23165</th>\n",
              "      <td>-5.372281</td>\n",
              "      <td>-61.570623</td>\n",
              "      <td>-1.900518</td>\n",
              "      <td>13.179263</td>\n",
              "      <td>-39.503843</td>\n",
              "      <td>1.854998</td>\n",
              "      <td>-12.404893</td>\n",
              "      <td>-20.127922</td>\n",
              "      <td>-5.643627</td>\n",
              "      <td>5.984061</td>\n",
              "      <td>...</td>\n",
              "      <td>24.684419</td>\n",
              "      <td>-42.327945</td>\n",
              "      <td>-2.006549</td>\n",
              "      <td>-16.684458</td>\n",
              "      <td>11.947687</td>\n",
              "      <td>28.132576</td>\n",
              "      <td>-34.618253</td>\n",
              "      <td>-27.759254</td>\n",
              "      <td>0.094354</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2525</th>\n",
              "      <td>25.465650</td>\n",
              "      <td>-43.862205</td>\n",
              "      <td>-0.200567</td>\n",
              "      <td>27.291545</td>\n",
              "      <td>-55.118966</td>\n",
              "      <td>-0.383934</td>\n",
              "      <td>-6.865923</td>\n",
              "      <td>-20.728290</td>\n",
              "      <td>-6.304919</td>\n",
              "      <td>-11.191012</td>\n",
              "      <td>...</td>\n",
              "      <td>9.267191</td>\n",
              "      <td>56.268463</td>\n",
              "      <td>2.890149</td>\n",
              "      <td>-13.844349</td>\n",
              "      <td>1.123736</td>\n",
              "      <td>-36.136721</td>\n",
              "      <td>37.885762</td>\n",
              "      <td>-3.781596</td>\n",
              "      <td>1.970603</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7245</th>\n",
              "      <td>-27.714749</td>\n",
              "      <td>24.720043</td>\n",
              "      <td>1.034420</td>\n",
              "      <td>-11.092761</td>\n",
              "      <td>-59.711586</td>\n",
              "      <td>-18.295318</td>\n",
              "      <td>-4.071317</td>\n",
              "      <td>-23.339082</td>\n",
              "      <td>-5.364521</td>\n",
              "      <td>21.584645</td>\n",
              "      <td>...</td>\n",
              "      <td>-21.638387</td>\n",
              "      <td>-16.101341</td>\n",
              "      <td>3.212601</td>\n",
              "      <td>10.797591</td>\n",
              "      <td>8.801407</td>\n",
              "      <td>18.144028</td>\n",
              "      <td>-6.116260</td>\n",
              "      <td>-16.930609</td>\n",
              "      <td>-0.415506</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16196</th>\n",
              "      <td>51.053721</td>\n",
              "      <td>109.108932</td>\n",
              "      <td>1.758557</td>\n",
              "      <td>-6.518332</td>\n",
              "      <td>-28.118554</td>\n",
              "      <td>-16.350501</td>\n",
              "      <td>-45.125592</td>\n",
              "      <td>-28.012390</td>\n",
              "      <td>-2.981930</td>\n",
              "      <td>-39.112791</td>\n",
              "      <td>...</td>\n",
              "      <td>6.134711</td>\n",
              "      <td>-59.591535</td>\n",
              "      <td>2.456997</td>\n",
              "      <td>-4.503880</td>\n",
              "      <td>5.662627</td>\n",
              "      <td>-1.446337</td>\n",
              "      <td>13.374964</td>\n",
              "      <td>12.420319</td>\n",
              "      <td>1.403669</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6663</th>\n",
              "      <td>3.224787</td>\n",
              "      <td>79.291491</td>\n",
              "      <td>-0.912219</td>\n",
              "      <td>18.090395</td>\n",
              "      <td>57.758797</td>\n",
              "      <td>32.406733</td>\n",
              "      <td>9.697422</td>\n",
              "      <td>52.570274</td>\n",
              "      <td>4.793865</td>\n",
              "      <td>-26.144282</td>\n",
              "      <td>...</td>\n",
              "      <td>-34.117106</td>\n",
              "      <td>26.829676</td>\n",
              "      <td>0.250067</td>\n",
              "      <td>0.600720</td>\n",
              "      <td>-2.799914</td>\n",
              "      <td>-19.453914</td>\n",
              "      <td>-6.331658</td>\n",
              "      <td>0.467210</td>\n",
              "      <td>0.759307</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21415</th>\n",
              "      <td>-21.301771</td>\n",
              "      <td>-32.052868</td>\n",
              "      <td>1.460705</td>\n",
              "      <td>19.396735</td>\n",
              "      <td>37.397261</td>\n",
              "      <td>17.543043</td>\n",
              "      <td>58.611508</td>\n",
              "      <td>-55.558027</td>\n",
              "      <td>1.304704</td>\n",
              "      <td>45.066049</td>\n",
              "      <td>...</td>\n",
              "      <td>47.791873</td>\n",
              "      <td>-13.462052</td>\n",
              "      <td>0.338621</td>\n",
              "      <td>-20.594125</td>\n",
              "      <td>-0.729124</td>\n",
              "      <td>20.480892</td>\n",
              "      <td>24.522073</td>\n",
              "      <td>13.454860</td>\n",
              "      <td>0.744973</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25179</th>\n",
              "      <td>-35.250467</td>\n",
              "      <td>-23.976480</td>\n",
              "      <td>0.851244</td>\n",
              "      <td>-3.182587</td>\n",
              "      <td>-4.554338</td>\n",
              "      <td>2.680315</td>\n",
              "      <td>-64.769801</td>\n",
              "      <td>-30.629431</td>\n",
              "      <td>0.116481</td>\n",
              "      <td>-28.879162</td>\n",
              "      <td>...</td>\n",
              "      <td>22.503449</td>\n",
              "      <td>-26.838240</td>\n",
              "      <td>0.230255</td>\n",
              "      <td>1.455464</td>\n",
              "      <td>14.455825</td>\n",
              "      <td>17.015399</td>\n",
              "      <td>-11.704959</td>\n",
              "      <td>22.208947</td>\n",
              "      <td>1.172967</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>15932 rows × 32 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "              x1          x2        x3         x5        x10        x20  \\\n",
              "5632  -30.506207  -27.939673 -3.413023  19.858541 -86.875647   0.003640   \n",
              "8533   20.093036   22.047442  0.363747 -16.930969  -4.852505   1.737856   \n",
              "3461   21.100633   29.987554 -1.123336 -16.751599   3.786722  16.802476   \n",
              "23165  -5.372281  -61.570623 -1.900518  13.179263 -39.503843   1.854998   \n",
              "2525   25.465650  -43.862205 -0.200567  27.291545 -55.118966  -0.383934   \n",
              "...          ...         ...       ...        ...        ...        ...   \n",
              "7245  -27.714749   24.720043  1.034420 -11.092761 -59.711586 -18.295318   \n",
              "16196  51.053721  109.108932  1.758557  -6.518332 -28.118554 -16.350501   \n",
              "6663    3.224787   79.291491 -0.912219  18.090395  57.758797  32.406733   \n",
              "21415 -21.301771  -32.052868  1.460705  19.396735  37.397261  17.543043   \n",
              "25179 -35.250467  -23.976480  0.851244  -3.182587  -4.554338   2.680315   \n",
              "\n",
              "             x21        x22       x33        x37  ...        x73        x75  \\\n",
              "5632   41.389960  87.042929 -5.640067  45.330522  ...  40.881151  20.615232   \n",
              "8533   28.967661  77.000176 -2.377054  41.567437  ... -11.697267  -8.918759   \n",
              "3461  -22.622090  24.686648  0.016123 -24.231346  ... -25.892340 -37.761717   \n",
              "23165 -12.404893 -20.127922 -5.643627   5.984061  ...  24.684419 -42.327945   \n",
              "2525   -6.865923 -20.728290 -6.304919 -11.191012  ...   9.267191  56.268463   \n",
              "...          ...        ...       ...        ...  ...        ...        ...   \n",
              "7245   -4.071317 -23.339082 -5.364521  21.584645  ... -21.638387 -16.101341   \n",
              "16196 -45.125592 -28.012390 -2.981930 -39.112791  ...   6.134711 -59.591535   \n",
              "6663    9.697422  52.570274  4.793865 -26.144282  ... -34.117106  26.829676   \n",
              "21415  58.611508 -55.558027  1.304704  45.066049  ...  47.791873 -13.462052   \n",
              "25179 -64.769801 -30.629431  0.116481 -28.879162  ...  22.503449 -26.838240   \n",
              "\n",
              "            x78        x79        x83        x85        x96        x97  \\\n",
              "5632  -1.605192 -35.409911   1.154636  36.257851  -3.912497 -33.280921   \n",
              "8533  -8.634336  57.081723   5.342754 -25.399170   8.651768  -1.766345   \n",
              "3461   1.390277 -20.749920  -0.981127 -21.328386 -24.935859  44.049856   \n",
              "23165 -2.006549 -16.684458  11.947687  28.132576 -34.618253 -27.759254   \n",
              "2525   2.890149 -13.844349   1.123736 -36.136721  37.885762  -3.781596   \n",
              "...         ...        ...        ...        ...        ...        ...   \n",
              "7245   3.212601  10.797591   8.801407  18.144028  -6.116260 -16.930609   \n",
              "16196  2.456997  -4.503880   5.662627  -1.446337  13.374964  12.420319   \n",
              "6663   0.250067   0.600720  -2.799914 -19.453914  -6.331658   0.467210   \n",
              "21415  0.338621 -20.594125  -0.729124  20.480892  24.522073  13.454860   \n",
              "25179  0.230255   1.455464  14.455825  17.015399 -11.704959  22.208947   \n",
              "\n",
              "            x99  dependent  \n",
              "5632   1.284467          0  \n",
              "8533  -0.526032          0  \n",
              "3461  -1.484568          1  \n",
              "23165  0.094354          1  \n",
              "2525   1.970603          0  \n",
              "...         ...        ...  \n",
              "7245  -0.415506          1  \n",
              "16196  1.403669          1  \n",
              "6663   0.759307          0  \n",
              "21415  0.744973          1  \n",
              "25179  1.172967          1  \n",
              "\n",
              "[15932 rows x 32 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 188
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6kGNv_qd_BC9"
      },
      "source": [
        "#defining dependent and independent variables\n",
        "X = dfpred.iloc[:, :-1].values\n",
        "y = dfpred.iloc[:, -1].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iO-KvNv8_BC9"
      },
      "source": [
        "#Normalization\n",
        "sc=StandardScaler()\n",
        "x=sc.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oS0eqq9A_BC_"
      },
      "source": [
        "#Let's implement 3 diffrent classifiers\n",
        "\n",
        "classifiers = {\n",
        "    \"LogisiticRegression\": LogisticRegression(),\n",
        "    \"KNearest\": KNeighborsClassifier(),\n",
        "    \"DecisionTreeClassifier\": DecisionTreeClassifier()\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifbneBNr_BC_",
        "outputId": "2d7e3946-a3ad-4328-b957-fd646802a3f6"
      },
      "source": [
        "#accuracy of the classifiers without tunnig\n",
        "\n",
        "for key, classifier in classifiers.items():\n",
        "    classifier.fit(x, y)\n",
        "    training_score = cross_val_score(classifier, x, y, cv=5)\n",
        "    print(\"Classifiers: \", classifier.__class__.__name__, \"Has a training score of\", \n",
        "          round(training_score.mean(), 2) * 100, \"% accuracy score\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Classifiers:  LogisticRegression Has a training score of 81.0 % accuracy score\n",
            "Classifiers:  KNeighborsClassifier Has a training score of 93.0 % accuracy score\n",
            "Classifiers:  DecisionTreeClassifier Has a training score of 74.0 % accuracy score\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UxDLe-K1_BDA",
        "outputId": "b2c38277-0319-4436-8ec4-3edab2365add"
      },
      "source": [
        "# Use GridSearchCV to find the best parameters.\n",
        "\n",
        "#Logistic Regression \n",
        "log_reg_params = {\"penalty\": ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}\n",
        "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n",
        "grid_log_reg.fit(x, y)\n",
        "log_reg = grid_log_reg.best_estimator_\n",
        "\n",
        "\n",
        "#KNears best estimator\n",
        "knears_params = {\"n_neighbors\": list(range(2,5,1)), 'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']}\n",
        "grid_knears = GridSearchCV(KNeighborsClassifier(), knears_params)\n",
        "grid_knears.fit(x, y)\n",
        "knears_neighbors = grid_knears.best_estimator_\n",
        "\n",
        "#DecisionTree Classifier\n",
        "tree_params = {\"criterion\": [\"gini\", \"entropy\"], \"max_depth\": list(range(2,4,1)), \n",
        "              \"min_samples_leaf\": list(range(5,7,1))}\n",
        "grid_tree = GridSearchCV(DecisionTreeClassifier(), tree_params)\n",
        "grid_tree.fit(x, y)\n",
        "tree_clf = grid_tree.best_estimator_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n",
            "C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
            "Traceback (most recent call last):\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 1304, in fit\n",
            "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
            "  File \"C:\\Users\\Amir\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py\", line 443, in _check_solver\n",
            "    \"got %s penalty.\" % (solver, penalty))\n",
            "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
            "\n",
            "  FitFailedWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dv_hzWbG_BDA",
        "outputId": "ef094d06-2395-4770-a649-22e6535456e7"
      },
      "source": [
        "# accuracy and recall based on best estimators on balanced dataset\n",
        "\n",
        "log_reg_score = cross_val_score(log_reg, x, y, cv=5, scoring='accuracy')\n",
        "log_reg_recall = cross_val_score(log_reg, x, y, cv=5, scoring='recall')\n",
        "print('Logistic Regression Cross Validation accuracy: ', round(log_reg_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('Logistic Regression Cross Validation recall: ', round(log_reg_recall.mean() * 100, 2).astype(str) + '%')\n",
        "\n",
        "knears_score = cross_val_score(knears_neighbors, x, y, cv=5, scoring='accuracy')\n",
        "knears_score_recall = cross_val_score(knears_neighbors, x, y, cv=5, scoring='recall')\n",
        "print('Knears Neighbors Cross Validation Score', round(knears_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('Knears Neighbors Cross Validation recall', round(knears_score_recall.mean() * 100, 2).astype(str) + '%')\n",
        "\n",
        "tree_score = cross_val_score(tree_clf, x, y, cv=5, scoring='accuracy')\n",
        "tree_score_recall = cross_val_score(tree_clf, x, y, cv=5, scoring='recall')\n",
        "print('DecisionTree Classifier Cross Validation Score', round(tree_score.mean() * 100, 2).astype(str) + '%')\n",
        "print('DecisionTree Classifier Cross Validation recall', round(tree_score_recall.mean() * 100, 2).astype(str) + '%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Logistic Regression Cross Validation accuracy:  81.02%\n",
            "Logistic Regression Cross Validation recall:  81.07%\n",
            "Knears Neighbors Cross Validation Score 92.29%\n",
            "Knears Neighbors Cross Validation recall 88.9%\n",
            "DecisionTree Classifier Cross Validation Score 67.4%\n",
            "DecisionTree Classifier Cross Validation recall 67.62%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tnQgzEHG_BDD"
      },
      "source": [
        "**As can be seen from the results our recall increased and both logestic regression and knearest neighbors working properly with available dataset.**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yut4QlKu_BDD"
      },
      "source": [
        "# Applying Neural Network Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiupWSOE_BDD"
      },
      "source": [
        "#developing NN model by using Sequential function and three dense layer\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Dense(units=30,activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "\n",
        "model.add(Dense(units=15,activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "\n",
        "model.add(Dense(units=1,activation='sigmoid'))\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SR1MUcEg_BDE"
      },
      "source": [
        "X = dfpred.iloc[:, :-1].values\n",
        "y = dfpred.iloc[:, -1].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKNJv_WB_BDE"
      },
      "source": [
        "#Normalization\n",
        "sc=StandardScaler()\n",
        "x=sc.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HVKpUXSa_BDF"
      },
      "source": [
        "#for checking how my NN model is working, I have splited my x into test and train but the test x here is working as validation set\n",
        "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UHk7WIdX_BDF"
      },
      "source": [
        "#defining early_stoping layer to find the optimum number of epoch\n",
        "early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zh2hlrBa_BDG",
        "outputId": "e07c7c7a-2031-47a5-fdb2-62ccd8b6705c"
      },
      "source": [
        "model.fit(x=X_train, \n",
        "          y=y_train, \n",
        "          epochs=600,\n",
        "          validation_data=(X_test, y_test), verbose=1,\n",
        "          callbacks=[early_stop]\n",
        "          )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.6886 - val_loss: 0.5049\n",
            "Epoch 2/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.5225 - val_loss: 0.3805\n",
            "Epoch 3/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.4502 - val_loss: 0.3347\n",
            "Epoch 4/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.4175 - val_loss: 0.3083\n",
            "Epoch 5/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.3917 - val_loss: 0.2908\n",
            "Epoch 6/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.3715 - val_loss: 0.2711\n",
            "Epoch 7/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3493 - val_loss: 0.2531\n",
            "Epoch 8/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3353 - val_loss: 0.2404\n",
            "Epoch 9/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3294 - val_loss: 0.2302\n",
            "Epoch 10/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3149 - val_loss: 0.2195\n",
            "Epoch 11/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3132 - val_loss: 0.2148\n",
            "Epoch 12/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3067 - val_loss: 0.2085\n",
            "Epoch 13/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.3004 - val_loss: 0.2062\n",
            "Epoch 14/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2928 - val_loss: 0.1999\n",
            "Epoch 15/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2913 - val_loss: 0.1955\n",
            "Epoch 16/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2946 - val_loss: 0.1958\n",
            "Epoch 17/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2927 - val_loss: 0.1931\n",
            "Epoch 18/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2909 - val_loss: 0.1913\n",
            "Epoch 19/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2799 - val_loss: 0.1893\n",
            "Epoch 20/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2888 - val_loss: 0.1881\n",
            "Epoch 21/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2899 - val_loss: 0.1898\n",
            "Epoch 22/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2731 - val_loss: 0.1867\n",
            "Epoch 23/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2703 - val_loss: 0.1846\n",
            "Epoch 24/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2776 - val_loss: 0.1841\n",
            "Epoch 25/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2711 - val_loss: 0.1845\n",
            "Epoch 26/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2744 - val_loss: 0.1832\n",
            "Epoch 27/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2785 - val_loss: 0.1874\n",
            "Epoch 28/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2765 - val_loss: 0.1844\n",
            "Epoch 29/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2750 - val_loss: 0.1847\n",
            "Epoch 30/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2741 - val_loss: 0.1814\n",
            "Epoch 31/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2711 - val_loss: 0.1831\n",
            "Epoch 32/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2716 - val_loss: 0.1825\n",
            "Epoch 33/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2726 - val_loss: 0.1846\n",
            "Epoch 34/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2745 - val_loss: 0.1831\n",
            "Epoch 35/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2661 - val_loss: 0.1807\n",
            "Epoch 36/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2740 - val_loss: 0.1814\n",
            "Epoch 37/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2711 - val_loss: 0.1811\n",
            "Epoch 38/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2690 - val_loss: 0.1817\n",
            "Epoch 39/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2731 - val_loss: 0.1841\n",
            "Epoch 40/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2650 - val_loss: 0.1835\n",
            "Epoch 41/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2744 - val_loss: 0.1821\n",
            "Epoch 42/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2735 - val_loss: 0.1839\n",
            "Epoch 43/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2647 - val_loss: 0.1844\n",
            "Epoch 44/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2690 - val_loss: 0.1842\n",
            "Epoch 45/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2676 - val_loss: 0.1803\n",
            "Epoch 46/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2623 - val_loss: 0.1791\n",
            "Epoch 47/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2653 - val_loss: 0.1824\n",
            "Epoch 48/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2632 - val_loss: 0.1817\n",
            "Epoch 49/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2696 - val_loss: 0.1809\n",
            "Epoch 50/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2629 - val_loss: 0.1795\n",
            "Epoch 51/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2661 - val_loss: 0.1816\n",
            "Epoch 52/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2681 - val_loss: 0.1802\n",
            "Epoch 53/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2701 - val_loss: 0.1809\n",
            "Epoch 54/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2612 - val_loss: 0.1820\n",
            "Epoch 55/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2649 - val_loss: 0.1799\n",
            "Epoch 56/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2664 - val_loss: 0.1799\n",
            "Epoch 57/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2667 - val_loss: 0.1818\n",
            "Epoch 58/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2649 - val_loss: 0.1801\n",
            "Epoch 59/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2709 - val_loss: 0.1825\n",
            "Epoch 60/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2634 - val_loss: 0.1799\n",
            "Epoch 61/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2685 - val_loss: 0.1813\n",
            "Epoch 62/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2657 - val_loss: 0.1810\n",
            "Epoch 63/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2636 - val_loss: 0.1775\n",
            "Epoch 64/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2605 - val_loss: 0.1778\n",
            "Epoch 65/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2676 - val_loss: 0.1796\n",
            "Epoch 66/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2597 - val_loss: 0.1805\n",
            "Epoch 67/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2611 - val_loss: 0.1783\n",
            "Epoch 68/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2687 - val_loss: 0.1781\n",
            "Epoch 69/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2650 - val_loss: 0.1760\n",
            "Epoch 70/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2651 - val_loss: 0.1778\n",
            "Epoch 71/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2634 - val_loss: 0.1767\n",
            "Epoch 72/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2593 - val_loss: 0.1770\n",
            "Epoch 73/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2677 - val_loss: 0.1807\n",
            "Epoch 74/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2675 - val_loss: 0.1808\n",
            "Epoch 75/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2596 - val_loss: 0.1789\n",
            "Epoch 76/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2678 - val_loss: 0.1807\n",
            "Epoch 77/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2592 - val_loss: 0.1808\n",
            "Epoch 78/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2704 - val_loss: 0.1797\n",
            "Epoch 79/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2720 - val_loss: 0.1827\n",
            "Epoch 80/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2564 - val_loss: 0.1768\n",
            "Epoch 81/600\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2649 - val_loss: 0.1774\n",
            "Epoch 82/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2646 - val_loss: 0.1766\n",
            "Epoch 83/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2627 - val_loss: 0.1781\n",
            "Epoch 84/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2634 - val_loss: 0.1793\n",
            "Epoch 85/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2632 - val_loss: 0.1775\n",
            "Epoch 86/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2633 - val_loss: 0.1792\n",
            "Epoch 87/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2580 - val_loss: 0.1754\n",
            "Epoch 88/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2554 - val_loss: 0.1751\n",
            "Epoch 89/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2649 - val_loss: 0.1777\n",
            "Epoch 90/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2641 - val_loss: 0.1784\n",
            "Epoch 91/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2596 - val_loss: 0.1783\n",
            "Epoch 92/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2642 - val_loss: 0.1808\n",
            "Epoch 93/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2587 - val_loss: 0.1789\n",
            "Epoch 94/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2665 - val_loss: 0.1800\n",
            "Epoch 95/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2609 - val_loss: 0.1809\n",
            "Epoch 96/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2574 - val_loss: 0.1812\n",
            "Epoch 97/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2579 - val_loss: 0.1806\n",
            "Epoch 98/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2620 - val_loss: 0.1809\n",
            "Epoch 99/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2551 - val_loss: 0.1818\n",
            "Epoch 100/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2610 - val_loss: 0.1802\n",
            "Epoch 101/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2636 - val_loss: 0.1825\n",
            "Epoch 102/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2621 - val_loss: 0.1815\n",
            "Epoch 103/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2667 - val_loss: 0.1835\n",
            "Epoch 104/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2609 - val_loss: 0.1840\n",
            "Epoch 105/600\n",
            "399/399 [==============================] - 0s 1ms/step - loss: 0.2660 - val_loss: 0.1836\n",
            "Epoch 106/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2558 - val_loss: 0.1802\n",
            "Epoch 107/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2617 - val_loss: 0.1829\n",
            "Epoch 108/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2668 - val_loss: 0.1831\n",
            "Epoch 109/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2577 - val_loss: 0.1822\n",
            "Epoch 110/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2549 - val_loss: 0.1817\n",
            "Epoch 111/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2627 - val_loss: 0.1822\n",
            "Epoch 112/600\n",
            "399/399 [==============================] - 1s 2ms/step - loss: 0.2573 - val_loss: 0.1809\n",
            "Epoch 113/600\n",
            "399/399 [==============================] - 1s 1ms/step - loss: 0.2627 - val_loss: 0.1833\n",
            "Epoch 00113: early stopping\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x2ceda093588>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3y9h0oTW_BDG",
        "outputId": "90c5c8a2-136f-453a-df6e-63cdfe7638ee"
      },
      "source": [
        "#check training loss and validation logg for overfitting\n",
        "#looks like our model working well\n",
        "model_loss = pd.DataFrame(model.history.history)\n",
        "model_loss.plot()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x2cedb2daa20>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1f3/8deZyWTfICsk7IR9N2wuKK64UpcqiFqtlbovbf1Vv63W2l371W9r0ZZa16pAlSpWCrWKIApIgLCTEAKEhCUb2ZeZzJzfH2cSkpCEISQMd/g8H488YO7cufO5c2fe99xzN6W1RgghhPXZ/F2AEEKIriGBLoQQAUICXQghAoQEuhBCBAgJdCGECBBB/nrj+Ph43b9/f3+9vRBCWNKGDRuKtdYJbT3nt0Dv378/GRkZ/np7IYSwJKXU/vaeky4XIYQIED4FulJqhlIqSymVo5R6oo3nX1RKZXr/spVSZV1fqhBCiI6csMtFKWUH5gGXAfnAeqXUEq31jsZxtNaPNRv/IWB8N9QqhBCiA770oU8CcrTWuQBKqQXATGBHO+PPBn7WNeUJIQKNy+UiPz+furo6f5dyRgsNDSU1NRWHw+Hza3wJ9BTgQLPH+cDktkZUSvUDBgCft/P8XGAuQN++fX0uUggROPLz84mKiqJ///4opfxdzhlJa01JSQn5+fkMGDDA59f50ofe1ife3hW9ZgHva63dbT2ptZ6vtU7XWqcnJLR51I0QIsDV1dURFxcnYd4BpRRxcXEnvRXjS6DnA32aPU4FDrYz7izgvZOqQAhx1pEwP7HOfEa+BPp6IE0pNUApFYwJ7SVtvPlQoAew5qSrOAnr95Xy/PJduD1y2V8hhGjuhIGutW4AHgSWAzuBRVrr7UqpZ5VS1zUbdTawQHfzBdYz88qYt2IPNc6G7nwbIUQAi4yM9HcJ3cKnM0W11kuBpa2GPd3q8TNdV1b7woLtANQ63USF+r73VwghAp3lzhQNc5hAr3G2ud9VCCF8prXm8ccfZ9SoUYwePZqFCxcCcOjQIaZNm8a4ceMYNWoUX375JW63mzvvvLNp3BdffNHP1R/Pb9dy6azwYAl0IQLFzz/ezo6DFV06zRG9o/nZtSN9Gnfx4sVkZmayefNmiouLmThxItOmTePdd9/liiuu4Cc/+Qlut5uamhoyMzMpKChg27ZtAJSVnXknxFuvhd7Y5eKSPnQhxKlZvXo1s2fPxm63k5SUxIUXXsj69euZOHEir7/+Os888wxbt24lKiqKgQMHkpuby0MPPcSyZcuIjo72d/nHsWAL3ZQsLXQhrM/XlnR3ae8YjmnTprFq1So++eQTbr/9dh5//HHuuOMONm/ezPLly5k3bx6LFi3itddeO80Vd8xyLfTwZjtFhRDiVEybNo2FCxfidrspKipi1apVTJo0if3795OYmMg999zD3XffzcaNGykuLsbj8XDjjTfyi1/8go0bN/q7/ONYroV+rMtFAl0IcWquv/561qxZw9ixY1FK8dxzz5GcnMybb77J888/j8PhIDIykrfeeouCggLuuusuPB4PAL/5zW/8XP3xLBfoslNUCHGqqqqqAHM25vPPP8/zzz/f4vnvfOc7fOc73znudWdiq7w563W5OKQPXQgh2mK5QA8NNiXXypmiQgjRguUCPdhuw25T0kIXQohWLBfoSinCHXYJdCGEaMVygQ7mSBc5bFEIIVqyZKCHB9vlsEUhhGjFkoEeFhwkXS5CCNGKJQPdtNDlKBchRPfr6Nrp+/btY9SoUaexmo5ZNtClhS6EEC1Z7kxRgFCHnaLKen+XIYQ4Vf9+Ag5v7dppJo+GK3/b7tM//vGP6devH/fffz8AzzzzDEopVq1axdGjR3G5XPzyl79k5syZJ/W2dXV13HfffWRkZBAUFMQLL7zA9OnT2b59O3fddRdOpxOPx8MHH3xA7969ufnmm8nPz8ftdvPUU09xyy23nNJsg0UDXVroQojOmjVrFo8++mhToC9atIhly5bx2GOPER0dTXFxMVOmTOG66647qRs1z5s3D4CtW7eya9cuLr/8crKzs/nzn//MI488wpw5c3A6nbjdbpYuXUrv3r355JNPACgvL++SeZNAF0L4Twct6e4yfvx4CgsLOXjwIEVFRfTo0YNevXrx2GOPsWrVKmw2GwUFBRw5coTk5GSfp7t69WoeeughAIYNG0a/fv3Izs5m6tSp/OpXvyI/P58bbriBtLQ0Ro8ezY9+9CN+/OMfc80113DBBRd0ybxZsg89zBEkp/4LITrtpptu4v3332fhwoXMmjWLd955h6KiIjZs2EBmZiZJSUnU1dWd1DTbu7b6rbfeypIlSwgLC+OKK67g888/Z8iQIWzYsIHRo0fz5JNP8uyzz3bFbFm3hV7rcqO1PqlNIiGEANPtcs8991BcXMzKlStZtGgRiYmJOBwOVqxYwf79+096mtOmTeOdd97h4osvJjs7m7y8PIYOHUpubi4DBw7k4YcfJjc3ly1btjBs2DB69uzJbbfdRmRkJG+88UaXzJclAz0s2I5HQ32Dh1DvTaOFEMJXI0eOpLKykpSUFHr16sWcOXO49tprSU9PZ9y4cQwbNuykp3n//fdz7733Mnr0aIKCgnjjjTcICQlh4cKF/P3vf8fhcJCcnMzTTz/N+vXrefzxx7HZbDgcDl555ZUumS/V3mZCd0tPT9cZGRmdeu3rX+3l5x/vYNNTl9EjIriLKxNCdKedO3cyfPhwf5dhCW19VkqpDVrr9LbGt2QfetNNLuT0fyGEaGLRLhdTtuwYFUKcDlu3buX2229vMSwkJIR169b5qaK2WTPQHXIbOiGszGoHNIwePZrMzMzT+p6d6Q63dpeLBLoQlhMaGkpJSUmnAutsobWmpKSE0NDQk3qdNVvo3kCXS+gKYT2pqank5+dTVFTk71LOaKGhoaSmpp7UaywZ6I0tdLnJhRDW43A4GDBggL/LCEg+dbkopWYopbKUUjlKqSfaGedmpdQOpdR2pdS7XVtmS+EOsx6SLhchhDjmhC10pZQdmAdcBuQD65VSS7TWO5qNkwY8CZyntT6qlErsroKhWZeLHOUihBBNfGmhTwJytNa5WmsnsABofV3Je4B5WuujAFrrwq4tsyXZKSqEEMfzJdBTgAPNHud7hzU3BBiilPpKKbVWKTWjrQkppeYqpTKUUhmnskNEDlsUQojj+RLobR0s2vp4oyAgDbgImA28qpSKPe5FWs/XWqdrrdMTEhJOttYmNpsiJMgmR7kIIUQzvgR6PtCn2eNU4GAb43yktXZprfcCWZiA7zbmmujShy6EEI18CfT1QJpSaoBSKhiYBSxpNc6HwHQApVQ8pgsmtysLbS08OIhap6c730IIISzlhIGutW4AHgSWAzuBRVrr7UqpZ5VS13lHWw6UKKV2ACuAx7XWJd1VNJgjXWpd0kIXQohGPp1YpLVeCixtNezpZv/XwA+8f6eF3IZOCCFasuS1XMAc6SKBLoQQx1g20MOD7XLqvxBCNGPhQA+So1yEEKIZywZ6mLTQhRCiBesGusMut6ATQohmLBvo0ocuhBAtWTbQw4Lt1Dd4cHvkridCCAEWDvRwuWuREEK0YNlADwtuvMmFHOkihBBg4UAPd8ht6IQQojnrBrrc5EIIIVqwbKCHSaALIUQL1g106XIRQogWLBvo4d6donKUixBCGJYN9GNdLnKUixBCgIUDvek4dOlyEUIIIAACXXaKCiGEYdlAD5MzRYUQogXLBnqw3YbdpqQPXQghvCwb6EopwuU2dEII0cSygQ4QGmynTrpchBACsHighwdLC10IIRpZOtDDpMtFCCGaWDrQ5a5FQghxjKUDPSrUQXmty99lCCHEGcHSgZ4YFUJRZb2/yxBCiDOCtQM9OoTiqno8cl9RIYSweKBHhdLg0ZTWOP1dihBC+J3FAz0EgMIK6XYRQghrB3q0N9Ar6/xciRBC+J9Pga6UmqGUylJK5Silnmjj+TuVUkVKqUzv3/e6vlSvTX+Hl6dCg5PEqFAACmXHqBBCEHSiEZRSdmAecBmQD6xXSi3RWu9oNepCrfWD3VBjS/VVULgDnFUkRMUAyJEuQgiBby30SUCO1jpXa+0EFgAzu7esDoREmX/rygl12IkODaKwQrpchBDCl0BPAQ40e5zvHdbajUqpLUqp95VSfdqakFJqrlIqQymVUVRU1IlygdBo8299JQCJ0aHS5SKEEPgW6KqNYa0P/P4Y6K+1HgP8F3izrQlpredrrdO11ukJCQknV2mjxhZ6fQVgjnSRQBdCCN8CPR9o3uJOBQ42H0FrXaK1bkzVvwLndE15bQhp1UKPCpGjXIQQAt8CfT2QppQaoJQKBmYBS5qPoJTq1ezhdcDOriuxlcZAr/O20KNDKayoR2s5W1QIcXY74VEuWusGpdSDwHLADrymtd6ulHoWyNBaLwEeVkpdBzQApcCd3VZxUx/6sS6X+gYPFXUNxIQ5uu1thRDiTHfCQAfQWi8FlrYa9nSz/z8JPNm1pbWjVR96gvds0aLKOgl0IcRZzXpnigaFgs1xrMul8eQiOf1fCHGWs16gK2Va6U2HLTae/i+BLoQ4u1kv0MH0ozfrQwe5nosQQlgz0Ju10CNDgghz2KXLRQhx1rNooMc09aErpUiMlpOLhBDCooF+rIUOcnKREEKAVQM9NBrqy5seJkbJ9VyEEMKagR4S3aKFnhAVQpH0oQshznIWDfQo04fuPd0/MTqEyvoGap1uPxcmhBD+Y81AD40G7QZXLdDs5CLpRxdCnMWsGehtXEIX4Ih0uwghzmIWDXRz67njzxaVFroQ4uxl0UBvvA2dXM9FCCEaWTPQW11Ct0e4g5gwB7sLKzt4kRBCBDZrBnqrPnSlFGP7xLIpr8yPRQkhhH9ZNNBb3oYOYHyfWLKPVFJd3+CnooQQwr8sGugt+9ABxvWNxaNhS355Oy8SQojAZtFAP76FPi41FoDMA9LtIoQ4O1kz0O1B4Ahv6kMH6BERTP+4cDIPHPVjYUII4T/WDHTwXs+losWgcd4do9p7SQAhhDibWDjQo1r0oQOM79uDwsp6DpXLCUZCiLOPdQM9tOUVF8G00EH60YUQZyfrBnpI1HFdLsN7RRMcZJNAF0KclSwc6Me30IODbIzsHc2mPNkxKoQ4+1g70Fv1oYPpdtlaUI7L7fFDUUII4T/WDfQ2+tDB7Bitc3nYeej4sBdCiEBm3UAPiQZnJXha3qVoysCeAKzOKfZHVUII4TcWDnTv6f/OqhaDE6NCGd4rmi+zJdCFEGcX6wZ64yV02+hHvyAtng37j1LjlAt1CSHOHj4FulJqhlIqSymVo5R6ooPxblJKaaVUeteV2I6mS+ge349+QVo8TreHdXtLu70MIYQ4U5ww0JVSdmAecCUwApitlBrRxnhRwMPAuq4usk0hLW9y0dzE/j0JCbJJt4sQ4qziSwt9EpCjtc7VWjuBBcDMNsb7BfAccHrOu2/jiouNQh12Jg3oyZe7i05LKUIIcSbwJdBTgAPNHud7hzVRSo0H+mit/9XRhJRSc5VSGUqpjKKiUwzbpj70tq9/Pi0tgd2FVRwqrz219xFCCIvwJdBVG8OaLmeolLIBLwI/PNGEtNbztdbpWuv0hIQE36tsS6vb0LV2wZB4AL7cLd0uQoizgy+Bng/0afY4FTjY7HEUMAr4Qim1D5gCLOn2HaMddLkADE2KIiEqRAJdCHHW8CXQ1wNpSqkBSqlgYBawpPFJrXW51jpea91fa90fWAtcp7XO6JaKGwVHgLK1edgimBtHX5AWz5e7i+QyAEKIs8IJA11r3QA8CCwHdgKLtNbblVLPKqWu6+4C26WU94qLbbfQAa4a1YuyGherpZUuhDgLBPkyktZ6KbC01bCn2xn3olMvy0dt3LWouWlDEogJc/BRZgHThyWetrKEEMIfrHumKLR5Cd3mgoNsXDU6mf/sOEKt093ueEIIEQgsHuhR7R622Ojasb2pcbr5784jp6koIYTwD2sHekwKlOV1OMrkAXEkRYfwUebBDscTQgirs3agx6WZQHe1f/KQ3aa4ZkxvVmYXUl7jOo3FCSHE6WXtQI9PAzSU5nY42sxxvXG5Nf/eduj01CWEEH4QAIEOFGd3ONrolBj6x4Xzry0S6EKIwGXtQI8bbP4tzulwNKUUV4xKZm1uiXS7CCEClrUDPTgCYvqcsIUOMGNkMg0ezWe75GgXIURgsnagg2mll+w+4WhjU2NJig5h2bbDp6EoIYQ4/awf6PFDoHg3aN3haDab4oqRyazaXSS3phNCBKQACPQ0c6PoyhO3vGeMTKbO5WFVttz4QggReAIj0MGnfvRJA3oSG+5g+XbpRxdCBB7rB3qcN9B96EcPstu4dHgS/915BGeDXFJXCBFYrB/o0b3BEWH60X1wxchkKusaWJ0j3S5CiMBi/UBXCuIH+xzo04bEkxwdyssr9qBPsCNVCCGsxPqBDseOdPFBSJCdB6YPImP/Ubk9nRAioARGoMelQfkBcNb4NPrNE/vQOyaUF/+bLa10IUTACIxAb7pI1x6fRg8JsnP/9MFsyitjpRzCKIQIEAEU6Pjc7QJwc3ofUmLDePHTbLmJtBAiIARGoMcNBmWDI9t8fklwkI0fXDaEzfnl3PX6espr5aJdQghrC4xAd4RByjmwd9VJvezGc1J57sYxrNtbwvUvf8W+4upuKlAIIbpfYAQ6wMDpULABastO6mU3T+zD23dPprTayd1vrsftkZ2kQghrCqBAvwi0B/atPumXThkYx6+vH82eomo+2So3wRBCWFPgBHrqRHPGaO6KTr18xshk0hIj+dPnu/FIK10IYUGBE+hBwdD/PMj9olMvt9kUD148mOwjVSzfLtdMF0JYT+AEOph+9JIcKDvQqZdfM6Y3A+IjeOnzHDnhSAhhOQEW6BeZfzvZSrfbFA9MH8yOQxX8/OMdfL2nmDqXu6uqE0KIbhVYgZ44HCKTOt2PDjBzXG8uHpbIW2v2cetf1zHhF5+ybJvsKBVCnPkCK9CVMq303JXg6dzZnw67jdfunEjmzy7ntTvTGZocxf3vbGTBN3ldWqoQQnS1wAp0MP3oNcVwcNMpTSY61MHFw5J453uTmTYkgScWb+XlL6RvXQhx5vIp0JVSM5RSWUqpHKXUE208f69SaqtSKlMptVopNaLrS/XR0CshKAw2vtklkwsPDuKvd6Qzc1xvnluWxc8/3iEnHwkhzkgnDHSllB2YB1wJjABmtxHY72qtR2utxwHPAS90eaW+CouFUTfC1vehrqJLJumw23jx5nHMnTaQN77exwPvbJSdpUKIM44vLfRJQI7WOldr7QQWADObj6C1bp6cEYB/m7Dp3wVXNWxd1GWTtNkU/3PVcJ6+ZgTLdxzm7jfXU98goS6EOHP4EugpQPMDu/O9w1pQSj2glNqDaaE/3NaElFJzlVIZSqmMoqJuvA55ygRIHg0Zb0AX93l/9/wB/P6msXyVU8Ij72XSIJfeFUKcIXwJdNXGsONSUms9T2s9CPgx8NO2JqS1nq+1TtdapyckJJxcpSdDKTjnLjiy1Vywq4vdeE4qT18zgmXbD/M//9zaItS3FZTzyIJNzHl1LS9/kcO2gnK5lIAQ4rQI8mGcfKBPs8epwMEOxl8AvHIqRXWJMTfDp09DxmuQmt7lk//u+QMoq3Hyx89zWLL5IKN6x2C3KdbtLSUqJIjesWE8tyyL55ZlERvuYPKAnkwbksAt6X0Isre9HvV4NDZbW+tPIYQ4MV8CfT2QppQaABQAs4Bbm4+glErTWjfeLuhqwPdbB3WXkCizc3TLIrjyd+ZxF3vssiGMTIlhXW4pW/LLKKys58czhjFnSl+iQx0UVtTx5e5i1uSWsDa3hOXbj3C02smDF6c1TWPh+jw+2XqYPYVVHCqv5aXZE7h6TK8ur1UIEfiUL8dVK6WuAv4PsAOvaa1/pZR6FsjQWi9RSv0BuBRwAUeBB7XW2zuaZnp6us7IyDjlGehQ3lp47Qq4/i8wdlb3vpcPHnh3I59uP8LSRy5gcGIkn2w5xAPvbmRwYiQje0eTse8o8ZHBfPTg+f4uVQhxhlJKbdBat9nt4FOgd4fTEuhawx/GQFwa3L64e9/LB0WV9Vz6wkrSEiP5xbdGceMrXzMsOYoFc6cSHGTjza/38bMl2/nogfMY2yfW3+UKIc5AHQV64J0p2pxSMPpmc22XyiP+roaEqBCeumYEGfuPctMrXxMREsQrt51DcJBZDDdMSCEi2M5ba/b7uVIhhBUFdqCD2TmqPbDtA39XAsCNE1K4IC2e+gYPL8+ZQFJ0aNNzUaEOrp+QwsdbDlJa7TzutblFVVTVN7Q53Vqnm7+s3MPvl2fxzd5SXHI4pRBnncDucmn0l2mgbDD3i9PzfidQ63RzqLyWgQmRxz2XfaSSy19cxRNXDuPeCwc1Df/P9sPc/85G+sWF89bdk0mJDQNAa82/tx3mV5/spKCsFrtN4fZookKCGN+vBxP6xjK2Tyz94yJIiQ1r2ho4EWeDB5ui3SNyTobWGrdHd8m0hDjbddTl4stRLtY35hZY/j9QvBvi0048fjcLC7a3GeYAQ5KimDKwJ2+v2c+lwxMZnBjFiqxCHnh3I2lJUeSX1nDTK1/z+l0TySmsYv6qXLbklzO8VzQv3DyW4b2j+TqnhC93F7Fh/1H+8NnupnOrlIJhydHcNqUv149PITzYLH6PR3OwvJbcomq2HSzn65wS1u8rZUTvaN67ZwqhDjtgViq/XbaLK0YmM2dyX1J7hLeo/WBZLYs35jNrUl/iI0MAs2K4641vWLOnhOToUFJ7hPPDy4cweWBcN3264mTUudzUuzzEhDv8XYroAmdHC73yMLwwHC74IVzc5jlPZ5TVu4u5641vcLk1E/rGsv1gBYMTI3n3e1MoKKvljte+obiqHoAB8RF8f9pAvp3eB3sbx7BX1rnYeaiSvNIaDpTW8N+dR9h+sIKo0CB6x4RRVuvkaLULZ7MummHJUYxKieGDjflcNaoXL80ez4a8o8x5dR09wh0UVZr3viAtgctGJHFBWjwfbz7IvBV7qHW5GZsaw4K5UwkLtvPLf+3g1dV7mTO5L7UuN+tyS6mqb+DjB8+nb1z4cfWejKLKenpGBLc5383lldSwIa+Uy0ckExFy8m0Yl9vD//03m349I7h5Yp8Tv+AMV+dy8/KKHFbnFLOtoIKQIBsr/990ekYE+7s04YOz9yiX5v5+IxTuhEe3gs1++t63k4qr6lm8MZ8F6w8Q5rDz9t2Tm35weSU1vPT5bi4ZnsRlI5JOGGjNaa3ZmHeUd9cdoLq+gZgwB7ERDvr1jGBAfARpSZFNrev5q/bw66W7uDk9lWXbDhMfFcL7955LrcvNu+v28/HmQ+SV1jRNe8bIZKYNSeAnH27l8hFJ3DAhle+/vYE7pvbj2ZmjANhXXM11f1pNSo9wFt93LkWV9Tz7r+1kHihjaHIUI3vH8K1xKYzoHd3hfHy4qYAf/WMzUwfF8Zfbz2na2gATWOv3lbIyq4gVWYXsKaoG4NLhScy//Zx2T97SWlPrcpsuq1DTYi2vdXH/Oxv4KqcEgOduHNMU6h9uKuBfWw5xyfBErh3bm8gOVhaHymvZU1jN+WnxbT7/+a4jvPLFHuZM7se1Y3v7vEzX5pawYf9Rwhx2IkODuHJUclPtAFvzyzlSUcelI5Kahj2/fBfzVuxhQt9YRvaO4e21+/nBZUN4+BL/b722RWuNUt17wl2dy920JeqrBreHBo8+6dedKgl0gB0fwaI74NZ/wJDLT9/7doHT8YVu732fXLyVBesPkBAVwuL7zqVPz/AWz+cUVvHl7mKGJUdx7mATVq9/tZeff7wDm4LhvaL54L5zW3zpV2QV8t031jM6JYasw5UE2RSXjUhiT1E1WYcrAXj62hHMmdwXgC93F7O1oJzJA3oyrk8sr3+1j18t3cmw5Ciyj1QyoW8PXrtrIgVHa5m/Kpd/bztEnctDsN3G5IE9mT40kcq6Bl78b3aL4Nqw/yif7TzC1oJythWUc7TG1VTjwIQIzh0Ux7rcUvaVVPOLmaP4ZOshvsop5vmbxvLVnmIWbywgJsxBea2LiGA7V47uxaXDzRZL8y2BfcXVzJq/lsMVdfz2htHMmtS3xef89Z5i7nx9PTYFdS4PQ5IiuX58KhpNg1vTLy6cqQPjSGy2A73xdXf87Rsaml1aYkLfWN753hTCgu3sOlzBTa+sodrZwPzb07lsRBJZhyu5+o9fMnNcCv9781gA7nr9G7YWlLP6xxcT6rCjtWZFViElVU4iQ4LoERHMmNSYFivNznA2eFiRVUh4sJ1eMWH06RlGSFDLMKyocxHdbIVU53Jzz1sZVNS6+NOtE1p8/7pCea2LX3+yk39sOMCjlw7hoYsHo5SitNrJr5fuZESvaL57/oDjXrclv4yH3zP3XPjogfNPa5eVBDpAgxNeHAF9JsOsd07f+1qcy+3hLyv3cMXIZNKSfD/b9tdLd7J4Yz7v33su/eMjjnt+3oocnl+exTVjevHTq0eQHGPCqrTayWMLM1mZXcQlwxLZV1Ld1MIGiAwJoqq+gavH9OKFm8fy2c5CHlmwiehQByXVTsKD7Vw/PoVLhycxeWDPphDSWvODRZv5MLOAn10zgpXZRazIKiLIphiaHMWo3jEkRYcQFhyER2sy9pWyft9RguyKl+dM4NxB8dQ4G7jt1XVszCvDpuChi9N46OLBbCko5911eSzffpjKugaC7TYuG5nEd8/rT8+IEGbNX4OzwUNaUhQZ+0r5izdcATYfKOPWv64lpUcYC+ZO5es9xbzwaTa5zea50eDESO67cBDXj09hf2kN35r3FYlRISyYO4Ugm40vsgt5dGEmlwxL5NmZo7jpla9xa01CVAh7i6r54P5z+ck/t5FbVMVnP7yoaYvv65xibn11XdPK5q01+3j6o5bnBTrsivF9e9A/LpySKifFVfU0eDQOuw2HXeHR4PZoEr2H5rYVvE8u3sJ73xy7zl/PiGCeumY43xqXQo3TzfPLs3hzzT5unJDKL781CofdxoPvbuTf2w4TGYxUSV8AABQgSURBVBJEkF3xp9kT2t3KAcgprCI23NG0ldmoqr6BpVsO8cHGfMpqXIztE8OA+Eje+HovRZX1jOsTy8a8Mq4Z04sbJqTw5OKtHKmox6Zg0fenkt6/J2D2N83/MpffL88iPjKE4qp6LhqawPzb04/b8ss6XMk3e0uYPalv0wEB1fUN/OJfO7j3wkFt/i58IYHe6D9PwZp58IOdEJV04vHFKXF7dIddB4WVdSRGhR433OPRvPxFDi/+dzejUmK489x+nD84gW/2lrIyu5B+cRHcd+Ggph/QiqxCnl+WxdVjenHb5H7ttpZqnW5ufOVrdhyqICbMwb0XDuKOqf3a7VdvcHvQmOvhNyqvcfH7/5j3mtJqx67L7SFj31GWbz/MBxvzm8I9MjSId++ZTJ8e4dz617XsOlzJ7El92V1Yycb9ZcRHBfP+vec2HcKqtaaqvgGH3Ybdpth1qJI1ucV8vPkQWwvKGZsaQ0VdA+W1Lj68/7wW+yLeXrOPpz7aTkSwHY0Jo/jIEK7902pq6huodrr5/bfHctM5qU2v0VpzzUurqXO5eea6kdz5+nqmD03g6WtGUlXfwJGKOtbmlvDVnmKOVNQTHxlCfGQwwXYbTreHBrfGZgObUmQeKAPg+ZvGMmNUctN7fJRZwCMLMvnueQO4YmQSB8treWvNfjbllXHe4DjvPp5apg1JYFV2ESN7RzMsOZoPNubz06uHc+nwJOa+nUFOYRV3TO3PfRcNanHIb3FVPc8vy2LRhgP0ig7lvblT6BcXgdaaN7/ex++WZVHrcjMwPoI+PcPZkl/G0RoXw5KjeO6mMYxOieHPK3N5bvkutIZBCRH85oYx/PAfmSgUSx+5AAU8tjCT/+w4wpWjkvntDWP456Z8nvl4B49fMZQHpg9uqmdPURXf/vMaSqudTB7Qk5duHU9FbQP3/X0De4qq+O0NYzq9P0YCvVFxDvzpHLjkZ3DBD07ve4uT1uD2dPmhjofL6/jPjsPMHJdCTFj3bSZX1zeweFMBX+wq5PEZQxmWbPYJlFTVM+fVdeQWVzftfL7vwkE+dSV4PJoPMwv43bJdlFY7eed7U5g0oOdx473waTZ//mIPf7n9HKYPSwRgU95Rbpm/lvR+PXjne5OP68L7cFMBjy7MJNhuY0B8BB/cf26H+wTac6C0hgff3cjm/HJmjuvN7El9SYgK4bqXVjO8V7TZmvAuU7dH8/e1+3lu2S4SokJ47qaxTBrQk893HeHRBZlU1DXw/WkDefKq4U2f6S8/2cGijHzsNsV1Y3sTHGSjrMbJl7uLqXW6uWViH5ZuPUSow87bd0/ib6v38d43eUwfmsBDl6Qxvk8sSim01t6VU3CL79gXWYVszCvj3gsHEh4cxPp9pdz8lzVcNboXe4uq2XW4gp9ePYK7zuvfNJ2HF2TyyZaDPH3NCG48J5Uap5sbXv6aOpebey8cxP9+mkVkiINaZwMhDjt/nDW+w62ME5FAb+71q6DiIDy8yRzHJ8RpdqrH5dc63ZRU1x932GjrccKCW/ZP7y+pJiEqpM2+cJfbw4XPraDW5WbJg+efUl+1s8HD/36axTtr86iqbyDIpogMDWLpwxfQ23v+RHMVdS5Cg+wtzpHIK6nhm32l3DA+5biujLySGl5ZmcNHmQcJc9iJCXcwNCmKH14+lMGJkew8VMGcV9dRVuPEo+H+iwbxo8uHdvpKpr9btotXvthDZEgQL906nulDE1s8X13fwO1/M11xYQ47MWEOKutcLJg7ldGpZj/RA+9uJDbMwUu3jqdXzPGfwcmQQG9u80L451y4/Z8w6OLT//5CnKHySmqw2ehwRXEyap1ulm8/zL+3HeKOqf05b3DnW6UnK+twJU8u3sJtU/pxw4TUE7+gA84Gsx9pxqj29yNprdl0oIx/ZBzgq5wSfnPD6Bbz25izXXFwgwR6c646eOkciIiHe1aATc5eFEJYx9l7ca62OELhkqfgUCZse9/f1QghRJc5+wIdzBUYe42Fz54FV62/qxFCiC5xdga6zQaX/xLKD8C6P/u7GiGE6BJnZ6ADDJgGQ66EL1+A8gJ/VyOEEKfs7A10gCt+BZ4G+OgB8NPOYSGE6Cpnd6DHDYLLf2HuaLT+VX9XI4QQp+TsDnSA9Lth0CXmsgAle/xdjRBCdJoEulIw808QFAz//D543P6uSAghOkUCHSC6N1z1e8hfD2tf9nc1QgjRKRLojUZ/G4ZeBZ//0tyqTgghLEYCvZFScM2LEBRijnqRrhchhMVIoDcXlQxXPgcH1sHHD0N9pb8rEkIIn0mgtzbmFjjvUdj0DrxyLuSu9HdFQgjhEwn01pSCy34O310GNge8dR189Uc58UgIccaTQG9P3ylw72oYeT18+hQs/x/wePxdlRBCtOvUbuMd6ILD4cbXIKqXOZyxeDekf9fcGMNx/L0whRDCn3xqoSulZiilspRSOUqpJ9p4/gdKqR1KqS1Kqc+UUv26vlQ/sdngil/DFb8xx6kvmA3PD4IVv5ZuGCHEGeWEga6UsgPzgCuBEcBspdSIVqNtAtK11mOA94HnurpQv1IKpt4Pj+fAbYth8KWw8nfw4X3gdvm7OiGEAHxroU8CcrTWuVprJ7AAmNl8BK31Cq11jffhWuDUbuJ3prI7YPAl8O03YPpPYPN78N4sqCv3d2VCCOFToKcAB5o9zvcOa8/dwL/bekIpNVcplaGUyigqKvK9yjONUnDh/4Nr/wB7Pod5k2H7h9IFI4TwK192irZ1m+o2k0spdRuQDlzY1vNa6/nAfDA3ifaxxjPXOXdC8mj4+BH4x3cgdSI4wsFZZXakjr8NBl8Gdtn3LITofr4kTT7Qp9njVOBg65GUUpcCPwEu1FrXd015FpByDtzzhbmV3ZaFoGwQGmt2oO76F0Qmw7CroP/50O98iEryd8VCiACl9Am6CZRSQUA2cAlQAKwHbtVab282znjMztAZWmufrmyVnp6uMzIyOlv3mc/tgt3/MWec7l0FTu9lBJJGQ9plMPBCSBwBEQnQUAf5GXB4iwn+XmP9W7sQ4oyllNqgtU5v87kTBbp3AlcB/wfYgde01r9SSj0LZGitlyil/guMBg55X5Kntb6uo2kGfKA3526Aw5vNZQRyPoO8NaC9F/8KjQVnNXiaHS3T/wKYcp+58Yavx7t73LDtA/PvyG+BI6zr50MI4XenHOjd4awK9NZqy+DgRijKhuIsCImCfudB4nDY/k9Y9xeoKDD98f0vgD4TITwOwnqae6DWlZtWfVyaac0f3QdLf2Ra+GDGGz/H9OPXV4HNDmNnQ0wH+7KdNbB3JZTuhV5joNc4CIk8ufly1UFBBgSFmnojk8zJWY08HijOhspDUFtqLn5mc4A9GHoOhNRzTvqjFKJL1B6FmlLz23K7zG8yNNr8hmz2rn0vV505sCIopFMvl0C3GrcL9qyAnE9Nt83RfSd+TXSKuT9qRAJ8Mx92fQK62aUKbA4Yewv0PRcOrIUD35iVQ3i8uVvTgW/MSqKRspn9AyOvhxEzzcqlqhDqykwAO8LM66sKTUDvWQHZy8wO4WMTMfdt7TUWXLWw/2vz+vYMuRIuexYShjT7LBpgx4eQ+wVExJsfWFQvs3KKToXIRPPjAHOUUUkOVByEpFEQEXdseO1Rc4vB4izzox0wzYyjmu3z1xoKd8DhbZA00nSJgfm8diwxP/Axt5h5AqgugdJcsxXlCDc3SvHXlpHW5s/WwYFrWrec34pDkPGamYcZv4XIhO6v80Q8no7nwVkD9RXm++RpgNAY04CxB5n5a6gzy//oXigvgPg06D3eNDIKNsLOJVBdDD0HQGxfOLINdn9qlntbYvrCeQ+bAxxKckwXau4XZqu6oRaCwsy0eg6AiERTT3C4aazUHjXTSBplDp44ut9sRWcthav/F8bc3KmPSALd6lx1pkVbU2qOhQ+JNqFatAsObTbBfc6dLVvUdeVmeHCUae1//RJsett84UNjoM9kE0I1JebL12cyDL3SbCUc2mJa2ln/PtbqP5HwOBh2jZkGCmqKoewAHN5qpmF3QL9zzZZIj/4Q1sO0gjxucDth58fw5QvgqjH7ERJHmEDe9HezQguNNSsLT0PL9w3rCb3HQUwq7P3S/JAbRaeYz6nysPnxtRadYn5oymbqOLgJqguPPR8aawK68hDYQ0y3mPaYrZfaUijLazm90FiY+iBMnms+Yzi2ksn9Ag5mQtl+KD9gfvzDrjZ/0b29WyqOloHbEa1NOK34DRTtPDY8caRZWSUOMwG2b7X5/Bq7+MLjIG6wqW/P52a+7Q6z8372e5A8quP3rSv3bllmm8ZDv6lmOTZundUehSEzzDQ7Ul9lgll5w3vvl7B1kakpfigMvtjsbzq6F4qyzEqnLM987m0JCm3ZIGnO5oCwWKguAluQ+c40Lmebw8zDwIvM9yE0xgyrrzDvtXkh5H9jfiuuGvPcwItM4yIo1AT70b2mvppSWhwAaPN+Bs27U0NjYNi1MOl7ZkXTCRLowqguNl/q+KEdt4KaK84xLW9lM63hsFizBeGqNZuiEYlmeGy/Uz88s7oYVr8I+78yP2JXDaSkwwU/MK13MCugyoOm9VWWZ1pYjUHZdwoMuQJ6DIAj283KBG26fqJ6mZZ1/BAT0ns+N/N1dN+x32DiMPNj7TXWtNL3r4a6Chh+HQydYVZ8WxaZFV1UMqRMMJ+lx2We27EEsv9tfrQ9B5mAqSmBqiNm+hGJpiUXk2pC/tDmlvMfFAaDpps7Z0XEm30u+1eb4IgfYl7r8ZgV295VcCjTvP/wa01QeVxQsAHy1pr3DokxK9HEYSZclM2snEr2mH+HXgkT7zZdgAtuNfMw7lYT2jUlZj6iU0wD4sg2003YeiWm7Ka20lxwew9uixsMF/8URnzr2ArqyA7YsgCyl5tl52zjXgPRqeZzLs428+B2Agp69DOfZ2xfiO1zbEVrc5gtvpoSE6yOMPNZRSaaLryoXqbRk7fGtNoHXWKmH9bDrFDKD5hlERLV/ndSa/N93PweJI8xdzYL79n2uB6PmS9ntfnMgiPMb6U4y3wXw+Ng4HSzRXwKJNCF9Xg85ocaEe97q/VMcDAT1swzrTtHmNlC6jMRBlxoQqb5vJTlmZ3kdeUmjCsPQ9YyqMg3zweFmpWUx20uDFd1+NjwmD5wwQ/NZnvrPl5XHZTnmxWAr/2/FYfg/bvMSiYi3rRi68pMELqdZoWdMsGEWuJwE+Ll+cdWLAnDzJaVpwE+/5XZagiONGFpCzLhaQsyn0N8mlkhhsZ4u4o8Zpp9zz3W0HBWmy6KHv1b7ocREuhCWIbWpouqvtJsnTQ/yslVa7qQunonXUc8HtNdFRxxEq/xHnF1cJPZmnDWmJPuRt14ZvTTW1xHgS6nMApxJlGq/fMQ/LHD1WY7uTAHs8IZc3Ond/qJzpMbXAghRICQQBdCiAAhgS6EEAFCAl0IIQKEBLoQQgQICXQhhAgQEuhCCBEgJNCFECJA+O1MUaVUEbC/ky+PB4q7sJwzSaDOm8yX9QTqvFl9vvpprds85dZvgX4qlFIZ7Z36anWBOm8yX9YTqPMWqPMF0uUihBABQwJdCCEChFUDfb6/C+hGgTpvMl/WE6jzFqjzZc0+dCGEEMezagtdCCFEKxLoQggRICwX6EqpGUqpLKVUjlLqCX/X01lKqT5KqRVKqZ1Kqe1KqUe8w3sqpT5VSu32/tvD37V2hlLKrpTapJT6l/fxAKXUOu98LVRKndqNFf1EKRWrlHpfKbXLu+ymBsIyU0o95v0eblNKvaeUCrXqMlNKvaaUKlRKbWs2rM1lpIw/evNki1Jqgv8qP3WWCnSllB2YB1wJjABmK6VG+LeqTmsAfqi1Hg5MAR7wzssTwGda6zTgM+9jK3oEaHY7en4HvOidr6PA3X6p6tT9AVimtR4GjMXMo6WXmVIqBXgYSNdajwLswCysu8zeAGa0GtbeMroSSPP+zQVeOU01dgtLBTowCcjRWudqrZ3AAmCmn2vqFK31Ia31Ru//KzHBkIKZnze9o70JfMs/FXaeUioVuBp41ftYARcD73tHsep8RQPTgL8BaK2dWusyAmCZYW5HGaaUCgLCgUNYdJlprVcBpa0Gt7eMZgJvaWMtEKuU6nV6Ku16Vgv0FOBAs8f53mGWppTqD4wH1gFJWutDYEIfSPRfZZ32f8D/Azzex3FAmda6wfvYqsttIFAEvO7tTnpVKRWBxZeZ1roA+D2QhwnycmADgbHMGrW3jAIqU6wW6KqNYZY+7lIpFQl8ADyqta7wdz2nSil1DVCotd7QfHAbo1pxuQUBE4BXtNbjgWos1r3SFm9/8kxgANAbiMB0RbRmxWV2IoHy3QSsF+j5QJ9mj1OBg36q5ZQppRyYMH9Ha73YO/hI4yaf999Cf9XXSecB1yml9mG6xC7GtNhjvZvzYN3llg/ka63XeR+/jwl4qy+zS4G9WusirbULWAycS2Ass0btLaOAyhSrBfp6IM279z0Ys+NmiZ9r6hRvv/LfgJ1a6xeaPbUE+I73/98BPjrdtZ0KrfWTWutUrXV/zPL5XGs9B1gB3OQdzXLzBaC1PgwcUEoN9Q66BNiBxZcZpqtlilIq3Pu9bJwvyy+zZtpbRkuAO7xHu0wByhu7ZixJa22pP+AqIBvYA/zE3/Wcwnycj9m02wJkev+uwvQ3fwbs9v7b09+1nsI8XgT8y/v/gcA3QA7wDyDE3/V1cp7GARne5fYh0CMQlhnwc2AXsA14Gwix6jID3sPsC3BhWuB3t7eMMF0u87x5shVzpI/f56Gzf3LqvxBCBAirdbkIIYRohwS6EEIECAl0IYQIEBLoQggRICTQhRAiQEigCyFEgJBAF0KIAPH/ASBAv4GmUUkhAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2cdZI8Jz_BDG",
        "outputId": "b691e8f5-c0c4-4dfe-88f9-e807f1309684"
      },
      "source": [
        "predictions = model.predict_classes(X_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-83-bc83193b8b59>:1: Sequential.predict_classes (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n",
            "Instructions for updating:\n",
            "Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqH7oEbd_BDH",
        "outputId": "df244091-31cb-4845-eb15-0458acd27f20"
      },
      "source": [
        "print(classification_report(y_test,predictions))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.93      0.94      0.93      1577\n",
            "           1       0.94      0.93      0.93      1610\n",
            "\n",
            "    accuracy                           0.93      3187\n",
            "   macro avg       0.93      0.93      0.93      3187\n",
            "weighted avg       0.93      0.93      0.93      3187\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P91mRvqP_BDH"
      },
      "source": [
        "**we have better results from NN model compare to other two models but its very close to knearst classifier**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uSLJFz1Z_BDI",
        "outputId": "91e7e729-3c0c-42a8-d91d-90013c7a14c0"
      },
      "source": [
        "#probability for the ANN model\n",
        "model.predict_proba(x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.99996644],\n",
              "       [0.5396292 ],\n",
              "       [0.5007621 ],\n",
              "       ...,\n",
              "       [0.00301939],\n",
              "       [0.25202465],\n",
              "       [0.07568565]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 233
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cxQngzsP_BDI"
      },
      "source": [
        "# Probability calculation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JQZ-LHPP_BDI"
      },
      "source": [
        "#finding out the probability of lable=1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o95EMi9E_BDJ"
      },
      "source": [
        "dftest=pd.read_csv('exercise_02_test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JEtNZbWC_BDJ",
        "outputId": "a149dfed-aa72-4c02-9945-4c59952ead63"
      },
      "source": [
        "dftest"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x0</th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x4</th>\n",
              "      <th>x5</th>\n",
              "      <th>x6</th>\n",
              "      <th>x7</th>\n",
              "      <th>x8</th>\n",
              "      <th>x9</th>\n",
              "      <th>...</th>\n",
              "      <th>x90</th>\n",
              "      <th>x91</th>\n",
              "      <th>x92</th>\n",
              "      <th>x93</th>\n",
              "      <th>x94</th>\n",
              "      <th>x95</th>\n",
              "      <th>x96</th>\n",
              "      <th>x97</th>\n",
              "      <th>x98</th>\n",
              "      <th>x99</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6.625366</td>\n",
              "      <td>54.479467</td>\n",
              "      <td>15.285444</td>\n",
              "      <td>-0.794648</td>\n",
              "      <td>22.498346</td>\n",
              "      <td>-29.212209</td>\n",
              "      <td>1.435134</td>\n",
              "      <td>-4.551934</td>\n",
              "      <td>5.930404</td>\n",
              "      <td>-3.319388</td>\n",
              "      <td>...</td>\n",
              "      <td>-18.919238</td>\n",
              "      <td>1.774657</td>\n",
              "      <td>2.216406</td>\n",
              "      <td>america</td>\n",
              "      <td>0.987554</td>\n",
              "      <td>34.396477</td>\n",
              "      <td>-55.883515</td>\n",
              "      <td>13.739194</td>\n",
              "      <td>2.824056</td>\n",
              "      <td>1.141799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3.796927</td>\n",
              "      <td>-20.244923</td>\n",
              "      <td>-18.084196</td>\n",
              "      <td>-1.113454</td>\n",
              "      <td>-3.551728</td>\n",
              "      <td>-4.025589</td>\n",
              "      <td>1.971885</td>\n",
              "      <td>-1.965186</td>\n",
              "      <td>13.210722</td>\n",
              "      <td>-4.262240</td>\n",
              "      <td>...</td>\n",
              "      <td>33.878264</td>\n",
              "      <td>-1.027421</td>\n",
              "      <td>7.924785</td>\n",
              "      <td>asia</td>\n",
              "      <td>2.615088</td>\n",
              "      <td>15.038461</td>\n",
              "      <td>-13.110477</td>\n",
              "      <td>-13.018951</td>\n",
              "      <td>-5.069025</td>\n",
              "      <td>0.568757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>31.875080</td>\n",
              "      <td>-61.467354</td>\n",
              "      <td>14.943580</td>\n",
              "      <td>0.979055</td>\n",
              "      <td>6.796937</td>\n",
              "      <td>-29.708041</td>\n",
              "      <td>4.778812</td>\n",
              "      <td>-2.682217</td>\n",
              "      <td>-17.122526</td>\n",
              "      <td>0.903299</td>\n",
              "      <td>...</td>\n",
              "      <td>-28.553940</td>\n",
              "      <td>1.755786</td>\n",
              "      <td>-0.325669</td>\n",
              "      <td>asia</td>\n",
              "      <td>-0.986222</td>\n",
              "      <td>-1.769850</td>\n",
              "      <td>-7.140415</td>\n",
              "      <td>0.791425</td>\n",
              "      <td>-3.224037</td>\n",
              "      <td>-0.816682</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>15.266588</td>\n",
              "      <td>-18.454831</td>\n",
              "      <td>1.105534</td>\n",
              "      <td>-2.718771</td>\n",
              "      <td>-5.511702</td>\n",
              "      <td>2.252314</td>\n",
              "      <td>-8.017649</td>\n",
              "      <td>3.635776</td>\n",
              "      <td>-13.024884</td>\n",
              "      <td>-1.751722</td>\n",
              "      <td>...</td>\n",
              "      <td>14.380384</td>\n",
              "      <td>-0.756253</td>\n",
              "      <td>5.930171</td>\n",
              "      <td>asia</td>\n",
              "      <td>-2.199690</td>\n",
              "      <td>-16.389740</td>\n",
              "      <td>-2.564346</td>\n",
              "      <td>-16.716012</td>\n",
              "      <td>5.559949</td>\n",
              "      <td>0.603007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-17.616761</td>\n",
              "      <td>15.810515</td>\n",
              "      <td>-17.972025</td>\n",
              "      <td>-1.995724</td>\n",
              "      <td>-23.112552</td>\n",
              "      <td>-15.899861</td>\n",
              "      <td>-17.054154</td>\n",
              "      <td>4.097427</td>\n",
              "      <td>-7.755402</td>\n",
              "      <td>-2.553410</td>\n",
              "      <td>...</td>\n",
              "      <td>-22.999896</td>\n",
              "      <td>-1.364063</td>\n",
              "      <td>1.207973</td>\n",
              "      <td>asia</td>\n",
              "      <td>1.694508</td>\n",
              "      <td>-4.086831</td>\n",
              "      <td>-28.709156</td>\n",
              "      <td>23.003355</td>\n",
              "      <td>-4.358594</td>\n",
              "      <td>1.929231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>-11.222816</td>\n",
              "      <td>-30.626970</td>\n",
              "      <td>58.991811</td>\n",
              "      <td>0.401397</td>\n",
              "      <td>-6.735698</td>\n",
              "      <td>6.647318</td>\n",
              "      <td>-5.082310</td>\n",
              "      <td>4.345536</td>\n",
              "      <td>5.384552</td>\n",
              "      <td>-1.835131</td>\n",
              "      <td>...</td>\n",
              "      <td>13.127413</td>\n",
              "      <td>3.771388</td>\n",
              "      <td>-2.735344</td>\n",
              "      <td>asia</td>\n",
              "      <td>1.224840</td>\n",
              "      <td>6.624887</td>\n",
              "      <td>-34.474480</td>\n",
              "      <td>8.775300</td>\n",
              "      <td>5.180422</td>\n",
              "      <td>-1.393041</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>-11.499750</td>\n",
              "      <td>-15.208325</td>\n",
              "      <td>9.735200</td>\n",
              "      <td>0.705614</td>\n",
              "      <td>-1.683272</td>\n",
              "      <td>-21.036041</td>\n",
              "      <td>13.915810</td>\n",
              "      <td>-8.038178</td>\n",
              "      <td>-1.497275</td>\n",
              "      <td>-3.215643</td>\n",
              "      <td>...</td>\n",
              "      <td>135.662872</td>\n",
              "      <td>5.285597</td>\n",
              "      <td>-0.194734</td>\n",
              "      <td>euorpe</td>\n",
              "      <td>1.185143</td>\n",
              "      <td>38.364908</td>\n",
              "      <td>9.031730</td>\n",
              "      <td>-18.614015</td>\n",
              "      <td>2.463580</td>\n",
              "      <td>2.555154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>30.593345</td>\n",
              "      <td>16.310727</td>\n",
              "      <td>7.145467</td>\n",
              "      <td>0.320570</td>\n",
              "      <td>-24.368449</td>\n",
              "      <td>10.918366</td>\n",
              "      <td>-1.102491</td>\n",
              "      <td>0.761505</td>\n",
              "      <td>4.984238</td>\n",
              "      <td>0.864588</td>\n",
              "      <td>...</td>\n",
              "      <td>-7.309901</td>\n",
              "      <td>-1.755134</td>\n",
              "      <td>-7.582438</td>\n",
              "      <td>asia</td>\n",
              "      <td>1.784907</td>\n",
              "      <td>-11.820404</td>\n",
              "      <td>19.663238</td>\n",
              "      <td>16.450237</td>\n",
              "      <td>18.993477</td>\n",
              "      <td>0.508249</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>22.669325</td>\n",
              "      <td>4.404143</td>\n",
              "      <td>-41.708040</td>\n",
              "      <td>-0.739057</td>\n",
              "      <td>-12.889045</td>\n",
              "      <td>-21.016407</td>\n",
              "      <td>-4.315463</td>\n",
              "      <td>7.069085</td>\n",
              "      <td>4.114309</td>\n",
              "      <td>0.271119</td>\n",
              "      <td>...</td>\n",
              "      <td>-79.950778</td>\n",
              "      <td>1.558892</td>\n",
              "      <td>2.806693</td>\n",
              "      <td>asia</td>\n",
              "      <td>0.870775</td>\n",
              "      <td>-2.363894</td>\n",
              "      <td>5.375669</td>\n",
              "      <td>6.324780</td>\n",
              "      <td>0.024320</td>\n",
              "      <td>-0.971681</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>16.665983</td>\n",
              "      <td>16.531172</td>\n",
              "      <td>-34.819552</td>\n",
              "      <td>-0.818345</td>\n",
              "      <td>-15.859801</td>\n",
              "      <td>-18.004868</td>\n",
              "      <td>1.661508</td>\n",
              "      <td>-7.131644</td>\n",
              "      <td>-13.612265</td>\n",
              "      <td>1.749355</td>\n",
              "      <td>...</td>\n",
              "      <td>-33.533262</td>\n",
              "      <td>-0.471731</td>\n",
              "      <td>-1.226588</td>\n",
              "      <td>asia</td>\n",
              "      <td>0.322634</td>\n",
              "      <td>-5.825642</td>\n",
              "      <td>-7.091229</td>\n",
              "      <td>-19.612922</td>\n",
              "      <td>-5.280292</td>\n",
              "      <td>-1.466352</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 100 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             x0         x1         x2        x3         x4         x5  \\\n",
              "0      6.625366  54.479467  15.285444 -0.794648  22.498346 -29.212209   \n",
              "1      3.796927 -20.244923 -18.084196 -1.113454  -3.551728  -4.025589   \n",
              "2     31.875080 -61.467354  14.943580  0.979055   6.796937 -29.708041   \n",
              "3     15.266588 -18.454831   1.105534 -2.718771  -5.511702   2.252314   \n",
              "4    -17.616761  15.810515 -17.972025 -1.995724 -23.112552 -15.899861   \n",
              "...         ...        ...        ...       ...        ...        ...   \n",
              "9995 -11.222816 -30.626970  58.991811  0.401397  -6.735698   6.647318   \n",
              "9996 -11.499750 -15.208325   9.735200  0.705614  -1.683272 -21.036041   \n",
              "9997  30.593345  16.310727   7.145467  0.320570 -24.368449  10.918366   \n",
              "9998  22.669325   4.404143 -41.708040 -0.739057 -12.889045 -21.016407   \n",
              "9999  16.665983  16.531172 -34.819552 -0.818345 -15.859801 -18.004868   \n",
              "\n",
              "             x6        x7         x8        x9  ...         x90       x91  \\\n",
              "0      1.435134 -4.551934   5.930404 -3.319388  ...  -18.919238  1.774657   \n",
              "1      1.971885 -1.965186  13.210722 -4.262240  ...   33.878264 -1.027421   \n",
              "2      4.778812 -2.682217 -17.122526  0.903299  ...  -28.553940  1.755786   \n",
              "3     -8.017649  3.635776 -13.024884 -1.751722  ...   14.380384 -0.756253   \n",
              "4    -17.054154  4.097427  -7.755402 -2.553410  ...  -22.999896 -1.364063   \n",
              "...         ...       ...        ...       ...  ...         ...       ...   \n",
              "9995  -5.082310  4.345536   5.384552 -1.835131  ...   13.127413  3.771388   \n",
              "9996  13.915810 -8.038178  -1.497275 -3.215643  ...  135.662872  5.285597   \n",
              "9997  -1.102491  0.761505   4.984238  0.864588  ...   -7.309901 -1.755134   \n",
              "9998  -4.315463  7.069085   4.114309  0.271119  ...  -79.950778  1.558892   \n",
              "9999   1.661508 -7.131644 -13.612265  1.749355  ...  -33.533262 -0.471731   \n",
              "\n",
              "           x92      x93       x94        x95        x96        x97        x98  \\\n",
              "0     2.216406  america  0.987554  34.396477 -55.883515  13.739194   2.824056   \n",
              "1     7.924785     asia  2.615088  15.038461 -13.110477 -13.018951  -5.069025   \n",
              "2    -0.325669     asia -0.986222  -1.769850  -7.140415   0.791425  -3.224037   \n",
              "3     5.930171     asia -2.199690 -16.389740  -2.564346 -16.716012   5.559949   \n",
              "4     1.207973     asia  1.694508  -4.086831 -28.709156  23.003355  -4.358594   \n",
              "...        ...      ...       ...        ...        ...        ...        ...   \n",
              "9995 -2.735344     asia  1.224840   6.624887 -34.474480   8.775300   5.180422   \n",
              "9996 -0.194734   euorpe  1.185143  38.364908   9.031730 -18.614015   2.463580   \n",
              "9997 -7.582438     asia  1.784907 -11.820404  19.663238  16.450237  18.993477   \n",
              "9998  2.806693     asia  0.870775  -2.363894   5.375669   6.324780   0.024320   \n",
              "9999 -1.226588     asia  0.322634  -5.825642  -7.091229 -19.612922  -5.280292   \n",
              "\n",
              "           x99  \n",
              "0     1.141799  \n",
              "1     0.568757  \n",
              "2    -0.816682  \n",
              "3     0.603007  \n",
              "4     1.929231  \n",
              "...        ...  \n",
              "9995 -1.393041  \n",
              "9996  2.555154  \n",
              "9997  0.508249  \n",
              "9998 -0.971681  \n",
              "9999 -1.466352  \n",
              "\n",
              "[10000 rows x 100 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 236
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZXnFcgp_BDJ",
        "outputId": "1378d241-2d40-4ccd-bd4d-b761a1831dd4"
      },
      "source": [
        "#check for any categorical columns in df1\n",
        "cols = dftest.columns\n",
        "num_cols = dftest._get_numeric_data().columns\n",
        "list(set(cols) - set(num_cols))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['x45', 'x34', 'x68', 'x93', 'x35', 'x41']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 239
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oXbkWan-_BDK",
        "outputId": "31f5a67b-851c-441e-af90-0051ff60e9b0"
      },
      "source": [
        "#Cleaning the X34,35,68,93\n",
        "print(dftest['x34'].unique())\n",
        "print(dftest['x35'].unique())\n",
        "print(dftest['x68'].unique())\n",
        "print(dftest['x93'].unique())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['volkswagon' 'bmw' 'Toyota' 'tesla' 'Honda' 'chrystler' 'ford' 'nissan'\n",
            " nan 'chevrolet' 'mercades']\n",
            "['wed' 'thurday' 'wednesday' 'thur' 'tuesday' 'friday' 'monday' 'fri']\n",
            "['Aug' 'Jun' 'sept.' 'July' 'Apr' 'May' 'Oct' 'Nov' 'Mar' 'January' 'Dev'\n",
            " 'Feb' nan]\n",
            "['america' 'asia' 'euorpe' nan]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVpH8wIw_BDK"
      },
      "source": [
        "dftest[\"x35\"].replace({\"wed\": \"wednesday\", \"thur\": \"thurday\", \"fri\": \"friday\"}, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PMv13Y4v_BDK"
      },
      "source": [
        "dftest = pd.get_dummies(data = dftest, columns = ['x34', 'x35', 'x68', 'x93'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1qVb-e7x_BDR"
      },
      "source": [
        "#removing dollar sign from column x41\n",
        "dftest['x41'] = dftest['x41'].str.replace('$', '').astype(float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UOL9o1wA_BDR"
      },
      "source": [
        "dftest['x45'] = dftest['x45'].str.replace('%', '').astype(float)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xMeKTsWN_BDS",
        "outputId": "e967b68b-f497-42c7-8fd0-98346a855ea3"
      },
      "source": [
        "# Check at first if we have any missing values\n",
        "dftest.isnull().sum().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "176"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 246
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EfgsF-2Q_BDS"
      },
      "source": [
        "dftest.fillna(dftest.mean(), inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_8otckx_BDS",
        "outputId": "51c69a43-6988-41dc-a7a9-64bad0488964"
      },
      "source": [
        "dftest.isnull().sum().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 255
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WeDHa4D-_BDT",
        "outputId": "0464f03d-b56b-4f7a-e684-853a4c11846d"
      },
      "source": [
        "relevant_features.index.values"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['x1', 'x2', 'x3', 'x5', 'x10', 'x20', 'x21', 'x22', 'x33', 'x37',\n",
              "       'x40', 'x41', 'x45', 'x50', 'x51', 'x56', 'x58', 'x63', 'x66',\n",
              "       'x69', 'x70', 'x72', 'x73', 'x75', 'x78', 'x79', 'x83', 'x85',\n",
              "       'x96', 'x97', 'x99', 'dependent'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 256
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BN58NRRh_BDT"
      },
      "source": [
        "df2test = dftest[['x1', 'x2', 'x3', 'x5', 'x10', 'x20', 'x21', 'x22', 'x33', 'x37',\n",
        "       'x40', 'x41', 'x45', 'x50', 'x51', 'x56', 'x58', 'x63', 'x66',\n",
        "       'x69', 'x70', 'x72', 'x73', 'x75', 'x78', 'x79', 'x83', 'x85',\n",
        "       'x96', 'x97', 'x99']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1EUjMBQc_BDT",
        "outputId": "37e51a68-4abc-441c-bbde-9c8605b86156"
      },
      "source": [
        "df2test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>x1</th>\n",
              "      <th>x2</th>\n",
              "      <th>x3</th>\n",
              "      <th>x5</th>\n",
              "      <th>x10</th>\n",
              "      <th>x20</th>\n",
              "      <th>x21</th>\n",
              "      <th>x22</th>\n",
              "      <th>x33</th>\n",
              "      <th>x37</th>\n",
              "      <th>...</th>\n",
              "      <th>x72</th>\n",
              "      <th>x73</th>\n",
              "      <th>x75</th>\n",
              "      <th>x78</th>\n",
              "      <th>x79</th>\n",
              "      <th>x83</th>\n",
              "      <th>x85</th>\n",
              "      <th>x96</th>\n",
              "      <th>x97</th>\n",
              "      <th>x99</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>54.479467</td>\n",
              "      <td>15.285444</td>\n",
              "      <td>-0.794648</td>\n",
              "      <td>-29.212209</td>\n",
              "      <td>72.390215</td>\n",
              "      <td>2.975753</td>\n",
              "      <td>5.103209</td>\n",
              "      <td>-59.107388</td>\n",
              "      <td>6.153179</td>\n",
              "      <td>-32.086998</td>\n",
              "      <td>...</td>\n",
              "      <td>2.317579</td>\n",
              "      <td>36.138153</td>\n",
              "      <td>80.851384</td>\n",
              "      <td>5.743085</td>\n",
              "      <td>12.317578</td>\n",
              "      <td>3.358217</td>\n",
              "      <td>10.211297</td>\n",
              "      <td>-55.883515</td>\n",
              "      <td>13.739194</td>\n",
              "      <td>1.141799</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-20.244923</td>\n",
              "      <td>-18.084196</td>\n",
              "      <td>-1.113454</td>\n",
              "      <td>-4.025589</td>\n",
              "      <td>65.711033</td>\n",
              "      <td>4.486136</td>\n",
              "      <td>23.995539</td>\n",
              "      <td>24.797640</td>\n",
              "      <td>-5.280295</td>\n",
              "      <td>29.391786</td>\n",
              "      <td>...</td>\n",
              "      <td>5.297850</td>\n",
              "      <td>8.353935</td>\n",
              "      <td>-21.295879</td>\n",
              "      <td>-0.544251</td>\n",
              "      <td>2.399873</td>\n",
              "      <td>5.950392</td>\n",
              "      <td>20.238130</td>\n",
              "      <td>-13.110477</td>\n",
              "      <td>-13.018951</td>\n",
              "      <td>0.568757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-61.467354</td>\n",
              "      <td>14.943580</td>\n",
              "      <td>0.979055</td>\n",
              "      <td>-29.708041</td>\n",
              "      <td>90.438310</td>\n",
              "      <td>9.143171</td>\n",
              "      <td>-57.616974</td>\n",
              "      <td>-43.275370</td>\n",
              "      <td>3.526041</td>\n",
              "      <td>-30.329997</td>\n",
              "      <td>...</td>\n",
              "      <td>-7.591016</td>\n",
              "      <td>-4.399284</td>\n",
              "      <td>27.719905</td>\n",
              "      <td>-6.045367</td>\n",
              "      <td>4.351231</td>\n",
              "      <td>-1.105144</td>\n",
              "      <td>16.438294</td>\n",
              "      <td>-7.140415</td>\n",
              "      <td>0.791425</td>\n",
              "      <td>-0.816682</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-18.454831</td>\n",
              "      <td>1.105534</td>\n",
              "      <td>-2.718771</td>\n",
              "      <td>2.252314</td>\n",
              "      <td>-43.284428</td>\n",
              "      <td>8.747259</td>\n",
              "      <td>-6.334656</td>\n",
              "      <td>48.633180</td>\n",
              "      <td>-3.051543</td>\n",
              "      <td>11.088216</td>\n",
              "      <td>...</td>\n",
              "      <td>3.419525</td>\n",
              "      <td>16.864636</td>\n",
              "      <td>-4.053955</td>\n",
              "      <td>2.595559</td>\n",
              "      <td>-9.223152</td>\n",
              "      <td>-5.203346</td>\n",
              "      <td>-12.050183</td>\n",
              "      <td>-2.564346</td>\n",
              "      <td>-16.716012</td>\n",
              "      <td>0.603007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>15.810515</td>\n",
              "      <td>-17.972025</td>\n",
              "      <td>-1.995724</td>\n",
              "      <td>-15.899861</td>\n",
              "      <td>0.317012</td>\n",
              "      <td>47.448220</td>\n",
              "      <td>-9.378142</td>\n",
              "      <td>22.100129</td>\n",
              "      <td>9.049640</td>\n",
              "      <td>-21.955105</td>\n",
              "      <td>...</td>\n",
              "      <td>-4.029329</td>\n",
              "      <td>-31.872595</td>\n",
              "      <td>21.743536</td>\n",
              "      <td>-3.722035</td>\n",
              "      <td>4.915405</td>\n",
              "      <td>3.971766</td>\n",
              "      <td>17.245945</td>\n",
              "      <td>-28.709156</td>\n",
              "      <td>23.003355</td>\n",
              "      <td>1.929231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>-30.626970</td>\n",
              "      <td>58.991811</td>\n",
              "      <td>0.401397</td>\n",
              "      <td>6.647318</td>\n",
              "      <td>9.943610</td>\n",
              "      <td>13.363441</td>\n",
              "      <td>-4.703320</td>\n",
              "      <td>27.014896</td>\n",
              "      <td>-1.410838</td>\n",
              "      <td>32.571079</td>\n",
              "      <td>...</td>\n",
              "      <td>-10.965044</td>\n",
              "      <td>30.324111</td>\n",
              "      <td>12.957945</td>\n",
              "      <td>-6.417189</td>\n",
              "      <td>-1.376996</td>\n",
              "      <td>-0.641675</td>\n",
              "      <td>1.853951</td>\n",
              "      <td>-34.474480</td>\n",
              "      <td>8.775300</td>\n",
              "      <td>-1.393041</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>-15.208325</td>\n",
              "      <td>9.735200</td>\n",
              "      <td>0.705614</td>\n",
              "      <td>-21.036041</td>\n",
              "      <td>-10.303662</td>\n",
              "      <td>-3.407283</td>\n",
              "      <td>-23.849387</td>\n",
              "      <td>-24.176855</td>\n",
              "      <td>-0.031866</td>\n",
              "      <td>65.228156</td>\n",
              "      <td>...</td>\n",
              "      <td>-8.392947</td>\n",
              "      <td>-35.426111</td>\n",
              "      <td>33.440937</td>\n",
              "      <td>-2.299726</td>\n",
              "      <td>23.301768</td>\n",
              "      <td>-1.847732</td>\n",
              "      <td>0.883073</td>\n",
              "      <td>9.031730</td>\n",
              "      <td>-18.614015</td>\n",
              "      <td>2.555154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>16.310727</td>\n",
              "      <td>7.145467</td>\n",
              "      <td>0.320570</td>\n",
              "      <td>10.918366</td>\n",
              "      <td>-9.197691</td>\n",
              "      <td>-3.323142</td>\n",
              "      <td>40.422099</td>\n",
              "      <td>-40.071638</td>\n",
              "      <td>-8.943171</td>\n",
              "      <td>-2.316092</td>\n",
              "      <td>...</td>\n",
              "      <td>-6.879382</td>\n",
              "      <td>9.161562</td>\n",
              "      <td>6.073351</td>\n",
              "      <td>-1.784711</td>\n",
              "      <td>-58.597495</td>\n",
              "      <td>8.598494</td>\n",
              "      <td>-31.900233</td>\n",
              "      <td>19.663238</td>\n",
              "      <td>16.450237</td>\n",
              "      <td>0.508249</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>4.404143</td>\n",
              "      <td>-41.708040</td>\n",
              "      <td>-0.739057</td>\n",
              "      <td>-21.016407</td>\n",
              "      <td>64.626297</td>\n",
              "      <td>16.078593</td>\n",
              "      <td>19.669888</td>\n",
              "      <td>28.352634</td>\n",
              "      <td>-1.842090</td>\n",
              "      <td>-42.531140</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.102775</td>\n",
              "      <td>35.508177</td>\n",
              "      <td>26.801064</td>\n",
              "      <td>-4.229037</td>\n",
              "      <td>0.915377</td>\n",
              "      <td>0.865563</td>\n",
              "      <td>4.338038</td>\n",
              "      <td>5.375669</td>\n",
              "      <td>6.324780</td>\n",
              "      <td>-0.971681</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>16.531172</td>\n",
              "      <td>-34.819552</td>\n",
              "      <td>-0.818345</td>\n",
              "      <td>-18.004868</td>\n",
              "      <td>102.287759</td>\n",
              "      <td>17.047219</td>\n",
              "      <td>-8.146332</td>\n",
              "      <td>-2.756645</td>\n",
              "      <td>1.886186</td>\n",
              "      <td>-17.550785</td>\n",
              "      <td>...</td>\n",
              "      <td>7.679574</td>\n",
              "      <td>1.732969</td>\n",
              "      <td>32.643389</td>\n",
              "      <td>-2.001553</td>\n",
              "      <td>21.913632</td>\n",
              "      <td>6.518906</td>\n",
              "      <td>21.309646</td>\n",
              "      <td>-7.091229</td>\n",
              "      <td>-19.612922</td>\n",
              "      <td>-1.466352</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 31 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "             x1         x2        x3         x5         x10        x20  \\\n",
              "0     54.479467  15.285444 -0.794648 -29.212209   72.390215   2.975753   \n",
              "1    -20.244923 -18.084196 -1.113454  -4.025589   65.711033   4.486136   \n",
              "2    -61.467354  14.943580  0.979055 -29.708041   90.438310   9.143171   \n",
              "3    -18.454831   1.105534 -2.718771   2.252314  -43.284428   8.747259   \n",
              "4     15.810515 -17.972025 -1.995724 -15.899861    0.317012  47.448220   \n",
              "...         ...        ...       ...        ...         ...        ...   \n",
              "9995 -30.626970  58.991811  0.401397   6.647318    9.943610  13.363441   \n",
              "9996 -15.208325   9.735200  0.705614 -21.036041  -10.303662  -3.407283   \n",
              "9997  16.310727   7.145467  0.320570  10.918366   -9.197691  -3.323142   \n",
              "9998   4.404143 -41.708040 -0.739057 -21.016407   64.626297  16.078593   \n",
              "9999  16.531172 -34.819552 -0.818345 -18.004868  102.287759  17.047219   \n",
              "\n",
              "            x21        x22       x33        x37  ...        x72        x73  \\\n",
              "0      5.103209 -59.107388  6.153179 -32.086998  ...   2.317579  36.138153   \n",
              "1     23.995539  24.797640 -5.280295  29.391786  ...   5.297850   8.353935   \n",
              "2    -57.616974 -43.275370  3.526041 -30.329997  ...  -7.591016  -4.399284   \n",
              "3     -6.334656  48.633180 -3.051543  11.088216  ...   3.419525  16.864636   \n",
              "4     -9.378142  22.100129  9.049640 -21.955105  ...  -4.029329 -31.872595   \n",
              "...         ...        ...       ...        ...  ...        ...        ...   \n",
              "9995  -4.703320  27.014896 -1.410838  32.571079  ... -10.965044  30.324111   \n",
              "9996 -23.849387 -24.176855 -0.031866  65.228156  ...  -8.392947 -35.426111   \n",
              "9997  40.422099 -40.071638 -8.943171  -2.316092  ...  -6.879382   9.161562   \n",
              "9998  19.669888  28.352634 -1.842090 -42.531140  ...  -0.102775  35.508177   \n",
              "9999  -8.146332  -2.756645  1.886186 -17.550785  ...   7.679574   1.732969   \n",
              "\n",
              "            x75       x78        x79       x83        x85        x96  \\\n",
              "0     80.851384  5.743085  12.317578  3.358217  10.211297 -55.883515   \n",
              "1    -21.295879 -0.544251   2.399873  5.950392  20.238130 -13.110477   \n",
              "2     27.719905 -6.045367   4.351231 -1.105144  16.438294  -7.140415   \n",
              "3     -4.053955  2.595559  -9.223152 -5.203346 -12.050183  -2.564346   \n",
              "4     21.743536 -3.722035   4.915405  3.971766  17.245945 -28.709156   \n",
              "...         ...       ...        ...       ...        ...        ...   \n",
              "9995  12.957945 -6.417189  -1.376996 -0.641675   1.853951 -34.474480   \n",
              "9996  33.440937 -2.299726  23.301768 -1.847732   0.883073   9.031730   \n",
              "9997   6.073351 -1.784711 -58.597495  8.598494 -31.900233  19.663238   \n",
              "9998  26.801064 -4.229037   0.915377  0.865563   4.338038   5.375669   \n",
              "9999  32.643389 -2.001553  21.913632  6.518906  21.309646  -7.091229   \n",
              "\n",
              "            x97       x99  \n",
              "0     13.739194  1.141799  \n",
              "1    -13.018951  0.568757  \n",
              "2      0.791425 -0.816682  \n",
              "3    -16.716012  0.603007  \n",
              "4     23.003355  1.929231  \n",
              "...         ...       ...  \n",
              "9995   8.775300 -1.393041  \n",
              "9996 -18.614015  2.555154  \n",
              "9997  16.450237  0.508249  \n",
              "9998   6.324780 -0.971681  \n",
              "9999 -19.612922 -1.466352  \n",
              "\n",
              "[10000 rows x 31 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 258
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QygROnIq_BDU",
        "outputId": "2aeb1f8b-6930-4006-d3e7-70d9cd70aeda"
      },
      "source": [
        "log_reg.classes_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 262
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hsKNYb-m_BDV",
        "outputId": "f939b3fe-9c53-4e63-c660-79c13e211fb0"
      },
      "source": [
        "knears_neighbors.classes_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1], dtype=int64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 263
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYKvp879_BDV"
      },
      "source": [
        "#Normalization\n",
        "sc=StandardScaler()\n",
        "x=sc.fit_transform(df2test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "8mLwACHw_BDV"
      },
      "source": [
        "l = []\n",
        "\n",
        "for i in range(x.shape[0]):\n",
        "    l.append(knears_neighbors.predict_proba(x[i].reshape(1, -1)).reshape(-1,1)[1][0])        \n",
        "#     break\n",
        "prob_df = pd.DataFrame({\"prob\": l})\n",
        "prob_df.to_csv(\"results2.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yw7IurCL_BDW"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}