H1N1_Comp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import pickle\n",
    "import statistics\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "from sklearn.decomposition import PCA\n",
    "from mlxtend.plotting import plot_decision_regions\n",
    "from mlxtend.classifier import StackingClassifier \n",
    "\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.preprocessing import StandardScaler, label_binarize, LabelEncoder, OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score, log_loss, make_scorer\n",
    "from sklearn.metrics import precision_recall_fscore_support as score\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, classification_report, confusion_matrix, auc\n",
    "from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB\n",
    "from imblearn.over_sampling import SMOTE, ADASYN, RandomOverSampler\n",
    "from xgboost import XGBClassifier\n",
    "from catboost import CatBoostRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_drop=pd.read_pickle(\"Data/df_drop.pkl\")\n",
    "test = pd.read_csv(\"Data/test_set_features.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_pickle(\"Data/df.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test=test.iloc[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_id=test.iloc[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=df.values\n",
    "\n",
    "X = df_train[:, 3:].astype(str)\n",
    "y = df_train[:, 1].astype(str) #1=H1N1 2=Seasonal\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=4444)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encode input variables\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "onehot_encoder.fit(X)\n",
    "X = onehot_encoder.transform(X)\n",
    "# X_test = onehot_encoder.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fancyimpute import IterativeImputer \n",
    "mice_imputer = IterativeImputer() \n",
    "X = mice_imputer.fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=4444)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encode input variables\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "onehot_encoder.fit(X_train)\n",
    "X_train = onehot_encoder.transform(X_train)\n",
    "X_test = onehot_encoder.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ordinal encode target variable\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(y_train)\n",
    "y_train = label_encoder.transform(y_train)\n",
    "y_test = label_encoder.transform(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import RandomizedSearchCV\n",
    "# # Number of trees in random forest\n",
    "# n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# # Number of features to consider at every split\n",
    "# max_features = ['auto', 'sqrt']\n",
    "# # Maximum number of levels in tree\n",
    "# max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "# max_depth.append(None)\n",
    "# # Minimum number of samples required to split a node\n",
    "# min_samples_split = [2, 5, 10]\n",
    "# # Minimum number of samples required at each leaf node\n",
    "# min_samples_leaf = [1, 2, 4]\n",
    "# # Method of selecting samples for training each tree\n",
    "# bootstrap = [True, False]\n",
    "# # Create the random grid\n",
    "# random_grid = {'n_estimators': n_estimators,\n",
    "#                'max_features': max_features,\n",
    "#                'max_depth': max_depth,\n",
    "#                'min_samples_split': min_samples_split,\n",
    "#                'min_samples_leaf': min_samples_leaf,\n",
    "#                'bootstrap': bootstrap}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import RandomForestRegressor\n",
    "# rf = RandomForestRegressor(random_state = 42)\n",
    "\n",
    "# # Use the random grid to search for best hyperparameters\n",
    "# # First create the base model to tune\n",
    "# rf = RandomForestRegressor()\n",
    "# # Random search of parameters, using 3 fold cross validation, \n",
    "# # search across 100 different combinations, and use all available cores\n",
    "# rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# # Fit the random search model\n",
    "# rf_random.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "model = RandomForestClassifier(n_estimators=17, n_jobs=2)\n",
    "# fit on the training set\n",
    "model.fit(X_train, y_train)\n",
    "# predict on test set\n",
    "yhat = model.predict(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training:  99.61%\n",
      "Test set:  86.07%\n",
      "Accuracy_: 86.07\n",
      "roc_auc_score: 74.32\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.95      0.91      4203\n",
      "           1       0.74      0.54      0.62      1139\n",
      "\n",
      "    accuracy                           0.86      5342\n",
      "   macro avg       0.81      0.74      0.77      5342\n",
      "weighted avg       0.85      0.86      0.85      5342\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# predict on test set\n",
    "# evaluate predictions\n",
    "print(\"Training: {:6.2f}%\".format(100*model.score(X_train, y_train)))\n",
    "print(\"Test set: {:6.2f}%\".format(100*model.score(X_test, y_test)))\n",
    "accuracy = accuracy_score(y_test, yhat)\n",
    "roc_auc = roc_auc_score(y_test, yhat)\n",
    "print('Accuracy_: %.2f' % (accuracy*100))\n",
    "print('roc_auc_score: %.2f' % (roc_auc*100))\n",
    "print(classification_report(y_test, yhat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_test=df_test.values.astype(str)\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "onehot_encoder.fit(test_test)\n",
    "test_test = onehot_encoder.transform(test_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_s1 = model.predict_proba(test_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_s2 = model.predict_proba(test_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.18, 0.04, 0.18, ..., 0.16, 0.04, 0.5 ])"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_s1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.42857143, 0.14285714, 0.85714286, ..., 0.42857143, 0.28571429,\n",
       "       0.85714286])"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame(\n",
    "    {\n",
    "    \"respondent_id\": test_id,\n",
    "    \"h1n1_vaccine\": y_s1[:,1],\n",
    "    \"seasonal_vaccine\": y_s2[:,1],\n",
    "    } \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv(\"Data/sub_rf.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.to_csv(\"Data/df_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sea = df.iloc[:,2:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_h1n1 = pd.DataFrame(df.iloc[:,1:]).drop(['seasonal_vaccine'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_h1n1.to_csv(\"Data/df_h1n1.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sea.to_csv(\"Data/df_sea.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_h1n1 = pd.read_csv(\"h1n1_res.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_h1n1 = res_h1n1[\"Prediction\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_sea = pd.read_csv(\"sea_res.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_sea = res_sea[\"Prediction\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_2 = pd.DataFrame(\n",
    "    {\n",
    "    \"respondent_id\": test_id,\n",
    "    \"h1n1_vaccine\": res_h1n1,\n",
    "    \"seasonal_vaccine\": res_sea,\n",
    "    } \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_2.to_csv(\"Data/sub_2.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMClassifier\n",
    "from hyperopt import STATUS_OK, Trials, fmin, hp, tpe, space_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.854201762595539                                     \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.9400000000000001, 'learning_rate': 0.0397, 'max_depth': 5, 'min_child_weight': 2.6750000000000003, 'n_estimators': 613, 'num_leaves': 1426, 'objective': 'binary'}\n",
      "0.8503640162010869                                                             \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.68, 'learning_rate': 0.0436, 'max_depth': 8, 'min_child_weight': 3.4250000000000003, 'n_estimators': 947, 'num_leaves': 2764, 'objective': 'binary'}\n",
      "0.8113272000897334                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.725, 'learning_rate': 0.0015, 'max_depth': 10, 'min_child_weight': 5.375, 'n_estimators': 419, 'num_leaves': 182, 'objective': 'binary'}\n",
      "0.7877369961039575                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.53, 'learning_rate': 0.0009000000000000001, 'max_depth': 4, 'min_child_weight': 4.45, 'n_estimators': 603, 'num_leaves': 3782, 'objective': 'binary'}\n",
      "0.8540612036194812                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.585, 'learning_rate': 0.0347, 'max_depth': 8, 'min_child_weight': 4.75, 'n_estimators': 522, 'num_leaves': 3808, 'objective': 'binary'}\n",
      "0.8559331398456654                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.66, 'learning_rate': 0.0056, 'max_depth': 8, 'min_child_weight': 2.4250000000000003, 'n_estimators': 683, 'num_leaves': 720, 'objective': 'binary'}\n",
      "0.8514860781767293                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.635, 'learning_rate': 0.0056, 'max_depth': 3, 'min_child_weight': 8.225, 'n_estimators': 925, 'num_leaves': 2618, 'objective': 'binary'}\n",
      "0.8569163078206523                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.84, 'learning_rate': 0.046900000000000004, 'max_depth': 6, 'min_child_weight': 5.825, 'n_estimators': 260, 'num_leaves': 1184, 'objective': 'binary'}\n",
      "0.8550906622921628                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.525, 'learning_rate': 0.021500000000000002, 'max_depth': 9, 'min_child_weight': 8.25, 'n_estimators': 161, 'num_leaves': 2558, 'objective': 'binary'}\n",
      "0.8576651086002114                                                               \n",
      "{'boosting_type': 'gbdt', 'colsample_bytree': 0.8150000000000001, 'learning_rate': 0.013000000000000001, 'max_depth': 4, 'min_child_weight': 7.9750000000000005, 'n_estimators': 809, 'num_leaves': 3400, 'objective': 'binary'}\n",
      "100%|██████████| 10/10 [57:15<00:00, 343.57s/trial, best loss: 0.7877369961039575]\n",
      "best:\n",
      "{'colsample_bytree': 0.53, 'learning_rate': 0.0009000000000000001, 'max_depth': 2, 'min_child_weight': 4.45, 'n_estimators': 503, 'num_leaves': 1889}\n"
     ]
    }
   ],
   "source": [
    "def hyperopt_lgb_score(params):\n",
    "    clf = LGBMClassifier(**params)\n",
    "    current_score = cross_val_score(clf, X_train, y_train, cv=10).mean()\n",
    "    print(current_score, params)\n",
    "    return current_score \n",
    " \n",
    "space_lgb = {\n",
    "            'learning_rate': hp.quniform('learning_rate', 0, 0.05, 0.0001),\n",
    "            'n_estimators': hp.choice('n_estimators', range(100, 1000)),\n",
    "            'max_depth':  hp.choice('max_depth', np.arange(2, 12, dtype=int)),\n",
    "            'num_leaves': hp.choice('num_leaves', 2*np.arange(2, 2**11, dtype=int)),\n",
    "            'min_child_weight': hp.quniform('min_child_weight', 1, 9, 0.025),\n",
    "            'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.005),\n",
    "            'objective': 'binary',\n",
    "            'boosting_type': 'gbdt',\n",
    "            }\n",
    " \n",
    "best = fmin(fn=hyperopt_lgb_score, space=space_lgb, algo=tpe.suggest, max_evals=10)\n",
    "print('best:')\n",
    "print(best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'boosting_type': 'gbdt',\n",
       " 'colsample_bytree': 0.53,\n",
       " 'learning_rate': 0.0009000000000000001,\n",
       " 'max_depth': 4,\n",
       " 'min_child_weight': 4.45,\n",
       " 'n_estimators': 603,\n",
       " 'num_leaves': 3782,\n",
       " 'objective': 'binary'}"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = space_eval(space_lgb, best)\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78.77"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LGB_Classifier = LGBMClassifier(**params)\n",
    "LGB_Classifier.fit(X_train, y_train)\n",
    "Y_pred = LGB_Classifier.predict_proba(test_test).astype(int)\n",
    "LGB_Classifier.score(X_train, y_train)\n",
    "acc_LGB_Classifier = round(LGB_Classifier.score(X_train, y_train) * 100, 2)\n",
    "acc_LGB_Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0],\n",
       "       [0, 0],\n",
       "       [0, 0],\n",
       "       ...,\n",
       "       [0, 0],\n",
       "       [0, 0],\n",
       "       [0, 0]])"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:metis] *",
   "language": "python",
   "name": "conda-env-metis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}