intial-submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Loading and Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%store -r __imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The libraries have been loaded!\n"
     ]
    }
   ],
   "source": [
    "__imp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "from tqdm.auto import tqdm\n",
    "import featuretools as ft\n",
    "from joblib import dump,load\n",
    "\n",
    "from sklearn.preprocessing import minmax_scale, StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression,LogisticRegressionCV, RidgeClassifierCV, RidgeClassifier\n",
    "from sklearn.metrics import make_scorer, log_loss\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from  sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "\n",
    "RANDOM_SEED = 777\n",
    "np.random.seed(RANDOM_SEED)\n",
    "import warnings\n",
    "warnings.filterwarnings(\n",
    "    action='ignore',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_csv('data/metadata.csv')\n",
    "train_labels = pd.read_csv('data/train_labels.csv', index_col=\"sample_id\")\n",
    "submission_format = pd.read_csv('data/submission_format.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>m/z</th>\n",
       "      <th colspan=\"10\" halign=\"left\">0</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">99</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_bin</th>\n",
       "      <th>(-100, 0]</th>\n",
       "      <th>(0, 100]</th>\n",
       "      <th>(100, 200]</th>\n",
       "      <th>(1000, 1100]</th>\n",
       "      <th>(1100, 1200]</th>\n",
       "      <th>(1200, 1300]</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>...</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>(400, 500]</th>\n",
       "      <th>(500, 600]</th>\n",
       "      <th>(600, 700]</th>\n",
       "      <th>(700, 800]</th>\n",
       "      <th>(800, 900]</th>\n",
       "      <th>(900, 1000]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0000</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004085</td>\n",
       "      <td>0.004641</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001394</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.898509e-06</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0001</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0002</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002270</td>\n",
       "      <td>0.002495</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001688</td>\n",
       "      <td>0.000636</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.257968e-07</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>9.397170e-07</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0003</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.002464</td>\n",
       "      <td>0.000306</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001099</td>\n",
       "      <td>0.000992</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.379099e-06</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>4.693477e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0004</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005993</td>\n",
       "      <td>0.012429</td>\n",
       "      <td>0.000593</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009380</td>\n",
       "      <td>0.005099</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.904920e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>3.174421e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1600 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "m/z               0                                                 \\\n",
       "temp_bin  (-100, 0]  (0, 100] (100, 200] (1000, 1100] (1100, 1200]   \n",
       "sample_id                                                            \n",
       "S0000           0.0  0.004085   0.004641     0.000000          0.0   \n",
       "S0001           0.0  0.000000   0.000000     0.000000          0.0   \n",
       "S0002           0.0  0.002270   0.002495     0.000000          0.0   \n",
       "S0003           0.0  0.001336   0.002464     0.000306          0.0   \n",
       "S0004           0.0  0.005993   0.012429     0.000593          0.0   \n",
       "\n",
       "m/z                                                                     ...  \\\n",
       "temp_bin  (1200, 1300] (1300, 1400] (1400, 1500] (200, 300] (300, 400]  ...   \n",
       "sample_id                                                               ...   \n",
       "S0000              0.0          0.0          0.0   0.001394   0.000188  ...   \n",
       "S0001              0.0          0.0          0.0   0.000000   0.000000  ...   \n",
       "S0002              0.0          0.0          0.0   0.001688   0.000636  ...   \n",
       "S0003              0.0          0.0          0.0   0.001099   0.000992  ...   \n",
       "S0004              0.0          0.0          0.0   0.009380   0.005099  ...   \n",
       "\n",
       "m/z                 99                                                   \\\n",
       "temp_bin  (1300, 1400] (1400, 1500]    (200, 300] (300, 400] (400, 500]   \n",
       "sample_id                                                                 \n",
       "S0000              0.0          0.0  2.898509e-06   0.000003   0.000000   \n",
       "S0001              0.0          0.0  0.000000e+00   0.000000   0.000000   \n",
       "S0002              0.0          0.0  9.257968e-07   0.000001   0.000001   \n",
       "S0003              0.0          0.0  4.379099e-06   0.000006   0.000005   \n",
       "S0004              0.0          0.0  3.904920e-06   0.000004   0.000004   \n",
       "\n",
       "m/z                                                                   \n",
       "temp_bin  (500, 600]    (600, 700] (700, 800] (800, 900] (900, 1000]  \n",
       "sample_id                                                             \n",
       "S0000       0.000000  0.000000e+00   0.000000   0.000000    0.000000  \n",
       "S0001       0.000000  0.000000e+00   0.000000   0.000000    0.000000  \n",
       "S0002       0.000001  9.397170e-07   0.000001   0.000001    0.000001  \n",
       "S0003       0.000005  4.693477e-06   0.000004   0.000004    0.000004  \n",
       "S0004       0.000005  3.174421e-06   0.000004   0.000004    0.000003  \n",
       "\n",
       "[5 rows x 1600 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('data/train_df.csv',index_col=0,header=[0,1])\n",
    "train_df.sort_index(inplace=True,axis=1)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>m/z</th>\n",
       "      <th colspan=\"10\" halign=\"left\">0</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">99</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_bin</th>\n",
       "      <th>(-100, 0]</th>\n",
       "      <th>(0, 100]</th>\n",
       "      <th>(100, 200]</th>\n",
       "      <th>(1000, 1100]</th>\n",
       "      <th>(1100, 1200]</th>\n",
       "      <th>(1200, 1300]</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>...</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>(400, 500]</th>\n",
       "      <th>(500, 600]</th>\n",
       "      <th>(600, 700]</th>\n",
       "      <th>(700, 800]</th>\n",
       "      <th>(800, 900]</th>\n",
       "      <th>(900, 1000]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0766</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.410100</td>\n",
       "      <td>0.593334</td>\n",
       "      <td>0.206508</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.606578</td>\n",
       "      <td>0.678006</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001801</td>\n",
       "      <td>0.001626</td>\n",
       "      <td>0.001498</td>\n",
       "      <td>0.001380</td>\n",
       "      <td>0.001495</td>\n",
       "      <td>0.001755</td>\n",
       "      <td>0.001762</td>\n",
       "      <td>0.001607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0767</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058563</td>\n",
       "      <td>0.073878</td>\n",
       "      <td>0.088571</td>\n",
       "      <td>0.097939</td>\n",
       "      <td>0.063057</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.081069</td>\n",
       "      <td>0.083869</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000187</td>\n",
       "      <td>0.000173</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>0.000183</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.000133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0768</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029256</td>\n",
       "      <td>0.040753</td>\n",
       "      <td>0.013179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.037022</td>\n",
       "      <td>0.041347</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.000248</td>\n",
       "      <td>0.000144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0769</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001313</td>\n",
       "      <td>0.001544</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0770</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003460</td>\n",
       "      <td>0.003929</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>0.002893</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1600 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "m/z               0                                                 \\\n",
       "temp_bin  (-100, 0]  (0, 100] (100, 200] (1000, 1100] (1100, 1200]   \n",
       "sample_id                                                            \n",
       "S0766           0.0  0.410100   0.593334     0.206508     0.000000   \n",
       "S0767           0.0  0.058563   0.073878     0.088571     0.097939   \n",
       "S0768           0.0  0.029256   0.040753     0.013179     0.000000   \n",
       "S0769           0.0  0.000261   0.000842     0.000000     0.000000   \n",
       "S0770           0.0  0.003460   0.003929     0.000467     0.000000   \n",
       "\n",
       "m/z                                                                     ...  \\\n",
       "temp_bin  (1200, 1300] (1300, 1400] (1400, 1500] (200, 300] (300, 400]  ...   \n",
       "sample_id                                                               ...   \n",
       "S0766         0.000000          0.0          0.0   0.606578   0.678006  ...   \n",
       "S0767         0.063057          0.0          0.0   0.081069   0.083869  ...   \n",
       "S0768         0.000000          0.0          0.0   0.037022   0.041347  ...   \n",
       "S0769         0.000000          0.0          0.0   0.001313   0.001544  ...   \n",
       "S0770         0.000000          0.0          0.0   0.003286   0.002893  ...   \n",
       "\n",
       "m/z                 99                                                \\\n",
       "temp_bin  (1300, 1400] (1400, 1500] (200, 300] (300, 400] (400, 500]   \n",
       "sample_id                                                              \n",
       "S0766              0.0          0.0   0.001801   0.001626   0.001498   \n",
       "S0767              0.0          0.0   0.000153   0.000187   0.000173   \n",
       "S0768              0.0          0.0   0.000205   0.000122   0.000152   \n",
       "S0769              0.0          0.0   0.000010   0.000010   0.000008   \n",
       "S0770              0.0          0.0   0.000002   0.000002   0.000002   \n",
       "\n",
       "m/z                                                                \n",
       "temp_bin  (500, 600] (600, 700] (700, 800] (800, 900] (900, 1000]  \n",
       "sample_id                                                          \n",
       "S0766       0.001380   0.001495   0.001755   0.001762    0.001607  \n",
       "S0767       0.000150   0.000183   0.000162   0.000146    0.000133  \n",
       "S0768       0.000161   0.000158   0.000182   0.000248    0.000144  \n",
       "S0769       0.000009   0.000012   0.000011   0.000012    0.000000  \n",
       "S0770       0.000002   0.000001   0.000003   0.000002    0.000002  \n",
       "\n",
       "[5 rows x 1600 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('test_df.csv',index_col=0,header=[0,1])\n",
    "test_df.sort_index(inplace=True,axis=1)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training samples:  766\n",
      "Number of validation samples:  293\n",
      "Number of testing samples:  511\n"
     ]
    }
   ],
   "source": [
    "train_files = metadata[metadata[\"split\"] == \"train\"][\"features_path\"].to_dict()\n",
    "val_files = metadata[metadata[\"split\"] == \"val\"][\"features_path\"].to_dict()\n",
    "test_files = metadata[metadata[\"split\"] == \"test\"][\"features_path\"].to_dict()\n",
    "\n",
    "print(\"Number of training samples: \", len(train_files))\n",
    "print(\"Number of validation samples: \", len(val_files))\n",
    "print(\"Number of testing samples: \", len(test_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total test files:  804\n"
     ]
    }
   ],
   "source": [
    "all_test_files = val_files.copy()\n",
    "all_test_files.update(test_files)\n",
    "print(\"Total test files: \", len(all_test_files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import submission format\n",
    "submission_template_df = submission_format.copy(deep=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_frac_and_He(df):\n",
    "    \"\"\"\n",
    "    Drops fractional m/z values, m/z values > 100, and carrier gas m/z\n",
    "\n",
    "    Args:\n",
    "        df: a dataframe representing a single sample, containing m/z values\n",
    "\n",
    "    Returns:\n",
    "        The dataframe without fractional an carrier gas m/z\n",
    "    \"\"\"\n",
    "\n",
    "    # drop fractional m/z values\n",
    "    df = df[df[\"m/z\"].transform(round) == df[\"m/z\"]]\n",
    "    assert df[\"m/z\"].apply(float.is_integer).all(), \"not all m/z are integers\"\n",
    "\n",
    "    # drop m/z values greater than 99\n",
    "    df = df[df[\"m/z\"] < 100]\n",
    "\n",
    "    # drop carrier gas\n",
    "    df = df[df[\"m/z\"] != 4]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_background_abundance(df):\n",
    "    \"\"\"\n",
    "    Subtracts minimum abundance value\n",
    "\n",
    "    Args:\n",
    "        df: dataframe with 'm/z' and 'abundance' columns\n",
    "\n",
    "    Returns:\n",
    "        dataframe with minimum abundance subtracted for all observations\n",
    "    \"\"\"\n",
    "\n",
    "    df[\"abundance_minsub\"] = df.groupby([\"m/z\"])[\"abundance\"].transform(\n",
    "        lambda x: (x - x.min())\n",
    "    )\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_abun(df):\n",
    "    \"\"\"\n",
    "    Scale abundance from 0-100 according to the min and max values across entire sample\n",
    "\n",
    "    Args:\n",
    "        df: dataframe containing abundances and m/z\n",
    "\n",
    "    Returns:\n",
    "        dataframe with additional column of scaled abundances\n",
    "    \"\"\"\n",
    "\n",
    "    df[\"abun_minsub_scaled\"] = minmax_scale(df[\"abundance_minsub\"].astype(float))\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess function\n",
    "def preprocess_sample(df):\n",
    "    df = drop_frac_and_He(df)\n",
    "    df = remove_background_abundance(df)\n",
    "    df = scale_abun(df)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1600, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>temp_bin</th>\n",
       "      <th>m/z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    temp_bin  m/z\n",
       "0  (-100, 0]    0\n",
       "1  (-100, 0]    1\n",
       "2  (-100, 0]    2\n",
       "3  (-100, 0]    3\n",
       "4  (-100, 0]    4"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a series of temperature bins\n",
    "temprange = pd.interval_range(start=-100, end=1500, freq=100)\n",
    "temprange\n",
    "\n",
    "# Make dataframe with rows that are combinations of all temperature bins\n",
    "# and all m/z values\n",
    "allcombs = list(itertools.product(temprange, [*range(0, 100)]))\n",
    "\n",
    "allcombs_df = pd.DataFrame(allcombs, columns=[\"temp_bin\", \"m/z\"])\n",
    "print(allcombs_df.shape)\n",
    "allcombs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(allcombs_df.columns[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def abun_per_tempbin(df):\n",
    "\n",
    "    \"\"\"\n",
    "    Transforms dataset to take the preprocessed max abundance for each\n",
    "    temperature range for each m/z value\n",
    "\n",
    "    Args:\n",
    "        df: dataframe to transform\n",
    "\n",
    "    Returns:\n",
    "        transformed dataframe\n",
    "    \"\"\"\n",
    "\n",
    "    # Bin temperatures\n",
    "    df[\"temp_bin\"] = pd.cut(df[\"temp\"], bins=temprange)\n",
    "\n",
    "    # Combine with a list of all temp bin-m/z value combinations\n",
    "    df = pd.merge(allcombs_df, df, on=[\"temp_bin\", \"m/z\"], how=\"left\")\n",
    "\n",
    "    # Aggregate to temperature bin level to find max\n",
    "    df = df.groupby([\"temp_bin\", \"m/z\"]).max(\"abun_minsub_scaled\").reset_index()\n",
    "\n",
    "    # Fill in 0 for abundance values without information\n",
    "    df = df.replace(np.nan, 0)\n",
    "\n",
    "    # Reshape so each row is a single sample\n",
    "    df = df.pivot_table(columns=[\"m/z\", \"temp_bin\"], values=[\"abun_minsub_scaled\"])\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/766 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of train files:  766\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 766/766 [04:38<00:00,  2.75it/s]\n"
     ]
    }
   ],
   "source": [
    "# Assembling preprocessed and transformed training set\n",
    "\n",
    "train_features_dict = {}\n",
    "print(\"Total number of train files: \", len(train_files))\n",
    "\n",
    "for i, (sample_id, filepath) in enumerate(tqdm(train_files.items())):\n",
    "\n",
    "    # Load training sample\n",
    "    temp = pd.read_csv('data/' + filepath)\n",
    "\n",
    "    # Preprocessing training sample\n",
    "    train_sample_pp = preprocess_sample(temp)\n",
    "\n",
    "    # Feature engineering\n",
    "    train_sample_fe = abun_per_tempbin(train_sample_pp).reset_index(drop=True)\n",
    "    train_features_dict[sample_id] = train_sample_fe\n",
    "\n",
    "train_features = pd.concat(\n",
    "    train_features_dict, names=[\"sample_id\", \"dummy_index\"]\n",
    ").reset_index(level=\"dummy_index\", drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_frac_and_He_test(df):\n",
    "    \"\"\"\n",
    "    Drops fractional m/z values, m/z values > 100, and carrier gas m/z\n",
    "\n",
    "    Args:\n",
    "        df: a dataframe representing a single sample, containing m/z values\n",
    "\n",
    "    Returns:\n",
    "        The dataframe without fractional an carrier gas m/z\n",
    "    \"\"\"\n",
    "\n",
    "    # drop fractional m/z values\n",
    "    df = df[df[\"m/z\"].transform(round) == df[\"m/z\"]]\n",
    "    assert df[\"m/z\"].apply(float.is_integer).all(), \"not all m/z are integers\"\n",
    "\n",
    "    # drop m/z values greater than 99\n",
    "    df = df[df[\"m/z\"] < 100]\n",
    "\n",
    "    # drop carrier gas\n",
    "    df = df[df[\"m/z\"] != 4]\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_background_abundance_test(df):\n",
    "    \"\"\"\n",
    "    Subtracts minimum abundance value\n",
    "\n",
    "    Args:\n",
    "        df: dataframe with 'm/z' and 'abundance' columns\n",
    "\n",
    "    Returns:\n",
    "        dataframe with minimum abundance subtracted for all observations\n",
    "    \"\"\"\n",
    "\n",
    "    df[\"abundance_minsub\"] = df.groupby([\"m/z\"])[\"abundance\"].transform(\n",
    "        lambda x: (x - x.min())\n",
    "    )\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale_abun_test(df):\n",
    "    \"\"\"\n",
    "    Scale abundance from 0-100 according to the min and max values across entire sample\n",
    "\n",
    "    Args:\n",
    "        df: dataframe containing abundances and m/z\n",
    "\n",
    "    Returns:\n",
    "        dataframe with additional column of scaled abundances\n",
    "    \"\"\"\n",
    "\n",
    "    df[\"abun_minsub_scaled\"] = minmax_scale(df[\"abundance_minsub\"].astype(float))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocess function\n",
    "def preprocess_sample_test(df):\n",
    "    df = drop_frac_and_He_test(df)\n",
    "    df = remove_background_abundance_test(df)\n",
    "    df = scale_abun_test(df)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1600, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>temp_bin</th>\n",
       "      <th>m/z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(-100, 0]</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    temp_bin  m/z\n",
       "0  (-100, 0]    0\n",
       "1  (-100, 0]    1\n",
       "2  (-100, 0]    2\n",
       "3  (-100, 0]    3\n",
       "4  (-100, 0]    4"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a series of temperature bins\n",
    "temprange_test = pd.interval_range(start=-100, end=1500, freq=100)\n",
    "temprange_test\n",
    "\n",
    "# Make dataframe with rows that are combinations of all temperature bins\n",
    "# and all m/z values\n",
    "allcombs_test = list(itertools.product(temprange_test, [*range(0, 100)]))\n",
    "\n",
    "allcombs_df_test = pd.DataFrame(allcombs_test, columns=[\"temp_bin\", \"m/z\"])\n",
    "print(allcombs_df_test.shape)\n",
    "allcombs_df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a series of temperature bins\n",
    "temprange = pd.interval_range(start=-100, end=2000, freq=200)\n",
    "temprange\n",
    "\n",
    "# Make dataframe with rows that are combinations of all temperature bins\n",
    "# and all m/z values\n",
    "allcombs = list(itertools.product(temprange, [*range(0, 500)]))\n",
    "\n",
    "allcombs_df = pd.DataFrame(allcombs, columns=[\"temp_bin\", \"m/z\"])\n",
    "display(allcombs_df.head())\n",
    "print(allcombs_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def abun_per_tempbin_test(df):\n",
    "\n",
    "    \"\"\"\n",
    "    Transforms dataset to take the preprocessed max abundance for each\n",
    "    temperature range for each m/z value\n",
    "\n",
    "    Args:\n",
    "        df: dataframe to transform\n",
    "\n",
    "    Returns:\n",
    "        transformed dataframe\n",
    "    \"\"\"\n",
    "\n",
    "    # Bin temperatures\n",
    "    df[\"temp_bin\"] = pd.cut(df[\"temp\"], bins=temprange_test)\n",
    "\n",
    "    # Combine with a list of all temp bin-m/z value combinations\n",
    "    df = pd.merge(allcombs_df_test, df, on=[\"temp_bin\", \"m/z\"], how=\"left\")\n",
    "\n",
    "    # Aggregate to temperature bin level to find max\n",
    "    df = df.groupby([\"temp_bin\", \"m/z\"]).max(\"abun_minsub_scaled\").reset_index()\n",
    "\n",
    "    # Fill in 0 for abundance values without information\n",
    "    df = df.replace(np.nan, 0)\n",
    "\n",
    "    # Reshape so each row is a single sample\n",
    "    df = df.pivot_table(columns=[\"m/z\", \"temp_bin\"], values=[\"abun_minsub_scaled\"])\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/804 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of test files:  804\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 804/804 [03:02<00:00,  4.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3min 4s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Assembling preprocessed and transformed testing set\n",
    "\n",
    "test_feat_dict = {}\n",
    "print(\"Total number of test files: \", len(all_test_files))\n",
    "\n",
    "for i, (sample_id, filepath) in enumerate(tqdm(all_test_files.items())):\n",
    "\n",
    "    # Load testing sample\n",
    "    temp = pd.read_csv('data/' + filepath)\n",
    "\n",
    "    # Preprocessing testing sample\n",
    "    test_sample_pp = preprocess_sample_test(temp)\n",
    "\n",
    "    # Feature engineering\n",
    "    test_sample_fe = abun_per_tempbin_test(test_sample_pp).reset_index(drop=True)\n",
    "    test_feat_dict[sample_id] = test_sample_fe\n",
    "\n",
    "\n",
    "test_features = pd.concat(\n",
    "    test_feat_dict, names=[\"sample_id\", \"dummy_index\"]\n",
    ").reset_index(level=\"dummy_index\", drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross validate\n",
    "- cv models about to make preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logloss_cross_val(clf, X, y,nsplits=5):\n",
    "    # Define stratified k-fold validation\n",
    "    skf = StratifiedKFold(n_splits=nsplits, random_state=RANDOM_SEED, shuffle=True)\n",
    "    # Define log loss\n",
    "    log_loss_scorer = make_scorer(log_loss,needs_proba=True)\n",
    "\n",
    "    # Generate a score for each label class\n",
    "    log_loss_cv = {}\n",
    "    for col in y.columns:\n",
    "        y_col = y[col]  # take one label at a time\n",
    "        log_loss_cv[col] = np.mean(cross_val_score(clf, X.values, y_col, cv=skf, scoring=log_loss_scorer))\n",
    "    avg_log_loss = np.mean(list(log_loss_cv.values()))\n",
    "\n",
    "    return log_loss_cv, avg_log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.26688889041325586 0.050806806573264605\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x16d07971e80>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAatUlEQVR4nO3de5ScdX3H8fc32VxICLmYK7mwQQIaETVdMF5oCaRA0IqtlkKpRbTNEdHWlqMlpsdTTw+VamtrjxaMt2qlWkTU1GIRKCJqA24oBAIJWSE0CSTZIIRISMjl2z+eZ3Znw2x2nnmu89vP65yczDzP7Py+89udz/zm99zM3RERkTCNKLsAERHJj0JeRCRgCnkRkYAp5EVEAqaQFxEJWEfZBdSbOnWqd3Z2ll2GiEhbWbt27S53n9ZoXaVCvrOzk+7u7rLLEBFpK2b2xGDrNF0jIhIwhbyISMAU8iIiAVPIi4gETCEvIhIwhbyISMAU8iIiAVPIS2n2HzzEt7q3oNNdi+RHIR+Yrc/s5b8eeqrsMpry6dse5cM3reOHD+8ouxSRYCnkA/O2z/6U9339vrLLaErvnv0A7Nl3sORKRMKlkA/ML59/sewSRKRCgg35B7fuZv2Tu8suQ0SkVJU6QVmWfuuzPwFg87VvKbkSEZHyBDuSFxGRDELezOaa2Z1m9rCZrTezP42XTzGz28xsU/z/5PTlSrO0W6KIQDYj+YPAVe6+EFgMXGlmC4GrgTvcfQFwR3xfCqKMFxHIIOTd/Sl3vy++vQd4BJgNXAh8NX7YV4G3p21LmqeMFxHIeE7ezDqB1wH3ADPcvXZUznZgxiA/s9zMus2su7e3N8tyhjVN14gIZBjyZnYs8G3gQ+7+XP06jxKnYeq4+yp373L3rmnTGl6iUFrQDhFvWPy/iOQlk5A3s1FEAX+Du98cL95hZrPi9bOAnVm0Jc1pp4F8G5Uq0nay2LvGgC8Bj7j7p+tWrQYui29fBnwvbVvNenTHnqKaqixvg+hshxqr6rHeX/Hwk88N/UAZ9rI4GOpNwLuAB83s/njZR4FrgRvN7L3AE8BFGbTVlPP/8cdFNVVZ7TSSl+TO/vu7AB3sJ0NLHfLu/hMGn1Y9J+3zt+KwAk5EBNARr8HSSF5EQCEfLM13iwgEdoKya3+wgWkTxpRdRiWENpLfvnsfP+3ZxTt+bU7ZpYi0laBC/vq7flF2CZURWMZz2ZfvZeOOPSx95QwmjhtVdjkibUPTNYEK7YjXnXv2AXAosNclkjeFfKBCjcLQPrxE8qaQD1TSLHzhxUN0Xv2frPpxNae8omPuRCQphXyoEob8M3uja8P+zS0bcigmOxrHiySjkA9U0l0oqz5Qrnh5IpWlkA9UqFPXob4ukbwo5AMVWhbWvmnoIC+RZBTygQpvL5S+lBeRBBTygUqShb179rN+W3uctlYnnxNJJqgjXqVfkoH8ss/8mF2/ejG/YjKk6RqRZDSSD1SSMGyHgK/NyWskL5KMQj4D//XQdv7kG/9bdhkDhDYlX9uF8rBSXiQRhXwG3vf1tax+4MmyyxgguJCvbXcN7HWJ5E0hH6hQ564PK+VFElHIByrULAz0ZYnkRiE/iN0vHODSL67hyWdfKLuUloQWhhbPymskL5KMQn4Qq+/fxk97nuazd/Y0/TNlbxT80cadfbdDOxiqf04+rNclkjeFfIZ++7qfsa3Ekf/7b7iv73ZoWdi3d01gr0skbwr5wcRDxyRnP3xgy7N84ceP5VNPE4bDVMZweI0iWVLIZ2zkiPJOils/yg0tC5/cHV3+7/DhkgsRaTMK+YyVGfL189Wh7kIZ6utKQtslJAmFfMbKCvk7N+zkwKG6kA80B0J9XUkc0oYJSUAhn7GOkkL+8n/5+YD7ocaA5uThkPpAElDIZ2xERa6j1+xX+nYbFbZZubnQdglJQiGf0s49+wbcL2skf6Rms/DAofZKjOdeOFB2CaXTSF6SUMintPWZgfvFjygh5PcdOPSSZc3mwP6D7RXy/71h59APCly7ffuScuUe8mZ2vpltNLMeM7s67/aKtmnHngH3yxjJ3/LgUw2WNhcEVR/JuzudV/9n3/3Zk44psZpqKPvIamkvuYa8mY0EPgcsAxYCl5jZwjzbLNpffPvBAffL2Lum0ciu2ZH8ixUfye/Zf3DA/Rcr/qFUhL0NvrmJDMby3OfWzN4A/JW7nxffXwHg7p9o9Piuri7v7u5O3M49j0XnmLl7066G689cMDXxc2755V42P72X0R0jeP38KX3L7960i+kTxnDKzAk8umMPO57b37C9PfsOcv+WZ1l84hTWPPbLlmpo1rZnXuCxXc8PWLZo3iTGj+ng7k27mHncWBbMOLbhz+47cIifb37mJfU34+5NuzhtzkQmHjOqpbp/0rMLd1gw/VhmThzb8DH7Dxzm3s2/TFTf3Zt2MWFMB3v2H+TNJ02lItvCM9Gz81c8tbt/O1Cef1dSrLeeNovfO31eSz9rZmvdvavRuryv8Tob2FJ3fyvw+voHmNlyYDnAvHmtvcBDh53n60Z80yaMoXdPf/hu372PCWOTvdTJ40ez+em9vGLmhL7nro2Yd+7Zz5zJxzQMeIDn90cBD7DmsSigHt/1PNMnjElUQ7MmjWscsrW6tz+3j+MnNQ7RRp54ei9Tjx191Mfs2Rc997qtu1k0b1LTz13v1OMn8uC23UwY2zHg9zeUZh5b+wawbuuznDS98QdcO6oF/MgRxvyp4xP1m1RbXt+qS7+Qt7uvAlZBNJJv5TneeNJUbj5p4IjmW91b+PBN6wD4xO+8mq7OKY1+NJHdLxzgNR//IQA3v/9NA+aK6938/jex7DN388hTzzHjuDHseG4/f3LOAi7qmpu6hsGc9ak72fz0XgCWvnIGX7ws+lD/3et/xtmvmMEVZ7180J898nV8YMlJXHT60Wv9n188zSVfWANErzdPa594hndc97O++0O1V/96Lnn9PFYse2VutRWt9tq6Vy5l8vijfxCLQP4hvw2oT4s58bLc1c+Nd4zMd/vyuNEj2fti43nSkS2c6KwVVjcnUT898a33vTHxc5Wxh9DRpJluOXZ06eOYXFTleAypvrz3rvk5sMDM5pvZaOBiYHXObQID3wR57/FyxW8MPkoeObL4N2PaFnP+TEwszeup2gdWVqxivyOprlz/VNz9IPAB4FbgEeBGd1+fZ5s19QOdURml1mCDJzP4tz+ONjWMjtuqbdAeWcKIK2mTn/v9RZw4bXzf/WZGiUW+LEvRWKgDXo3kpVm5jwfc/RZ3P9ndX+7u1+TdXs2AkXxOo+n6DZ5nxHP+7z1z/qB1FCVpm285bRbLTp3Zd7/MM2k2kmokH2gYljF4kPYU7Je++jd3Xm+IS1/fvzdQx8gRPP6JC/jIeacMeExthDwtpz1rGmnl5RbRX61KU061Xkl2KvYrkgoLc6sUUD8YzeoNMdTTDNz4Gd2+cslJXLr4BM46ZXo2ReSkPuSbmccuMmMsRWuhhmGo31Ake8GO5NPM4zbrXYs7efXsiUfdNXLUyBEsKTjgW3nt73lz/zRTVU6yVpNuJF+t15KViv2KpMKCDfn6N8FxY1s7IvNIR4bnzIlj+Y8Pvpnpx730QKPahtcyBlytNDnxmFF0nTAZCGuPlFAHvBrJS7MCDvnoTTCmY0SpB42UMZJs9VvM4RL3CDqaVCP5ir2WrAT6siQH4YZ8/MqK3OBZFa2+/2vnOWtm75oiw7PVD8qxo0Zw3qtmZFxNNYT64SXZC3bDa+1NkOX519rlbdXq+782xVS1qYBWy9nw18uyLUSkDYU7kq9IUHkJV1stYiRfpIr8KkXaUsAhH/0/HC/8nHpOvom/ikKPeG2b71Ai1RNwyOcwXdNC1pSy4bXFn6uN5KvyLaimYuWItJVgQ940kk+sdlm5yk3XlF2ASBsLNuRruwFmeTnMJKPyv1j2CiaNG0Xn1HHZFdCkVjP6UPyB2DFi6D+LWhNFjLI1khdpXbB719QO6Mnz8oZHs+SU6dz/sXNLaXvpwtZ2G6xd+WpUghO6FZO/SnmRVoUb8nEuDLfJmq+8+3SWvKK10ygcPBxdfizJRVaK2F9bI3mR1gU7XWN90zXZxXw7hM3cKa1PDx06VJuuaf588kV0SRt0u0hlBRvyeexdU2VZzJEf7JuuSTKSb7295ttQzIu0KuCQj/4va06+aFm8ylrIJ7nIShG7iCriRVoX8Jz88BrJZ+HgoWhOflQTe9f0Ra/2rinUXR8+i23PvlB2GdJGgg15Sa62d02Si48XMyevlK854WXjOeFl44d+oEgs2OmaPIQ+oqxN1yQ51bD2kxeptmBD3obZLpRZ5OBfv/1UJo0bxZiOJg6Gihus2ikQRGSgYKdrhttX/Cw+zC7qmnvUSxk2Ush0zfD6VYpkKviRfKbPOcw+OKpCu1CKtC7YkB9uio7BWntFTNco4kVaF2zI2zDbT7402vAqUmnhhnwO6aOwKYemyURaF2zI12g+Nx+1fi1kuka/QpGWBRvyeUzXKGteqpD95PNvQiRY4YZ82QUME4X0cwuNjG5iX3+R4SDYd8Jw+4r/gbNPAmDmcWMLaa//rJdF7F2TrI0HPnYua/9yaU7ViLSXVCFvZp8ysw1mts7MvmNmk+rWrTCzHjPbaGbnpa60Aqo8v/87i+aw+dq3MH5MeMe3Je32ieNGMWHsqHyKEWkzaUfytwGnuvtpwKPACgAzWwhcDLwKOB/4ZzMbmbKthOKzUBbbqOSguh+tItWXKuTd/YfufjC+uwaYE9++EPimu+9398eBHuCMNG0lVeFBd1CKOa2BfpkircpyTv49wA/i27OBLXXrtsbLXsLMlptZt5l19/b2ZlZMHrGgqOlX5Ang1O8irRtyAtfMbgdmNli10t2/Fz9mJXAQuCFpAe6+ClgF0NXVpdkVEZEMDRny7n7U3RTM7N3AW4FzvH+n9G1A/ekM58TLCmM5XBlKswYvpS4Rqba0e9ecD3wEeJu7761btRq42MzGmNl8YAFwb5q2EtdWZGPDUJGnGtCHq0jr0u5v91lgDHBbPHJe4+7vc/f1ZnYj8DDRNM6V7n4oZVstUUCIyHCWKuTd/aSjrLsGuCbN86fRf1qDLJ9Tnxgi0l7CPeJVEza50uedSHsIN+QVQiIi4Ya8iIgMg5B3ndig7WnqTaR1wYa8pmvCcczogk97JBKQgENeKV+EIrp5dMcINl/7lvwbEglQsCEv+dJnqEh7CDbkaxmU5X7yIiLtJtyQ10hTRCTckK9R2OdDe7yItIdgQ74WQpquEZHhLNyQ10BTRCTgkC+7gMD1f4iqp0WqLO2phkUK86l3nsbuFw6UXYZIWwk35Au8BqkU43e75g79IBEZIODpGk0j5EnbPETaQ7ghrxASEQk35EVEJOCQ7xvIa1I+F5oOE2kP4Ya85mtERMIN+T7KehEZxoINeU3XFENfmESqLdyQV/jkqta/OjeQSLWFG/KapxERCTfkpRj6xiRSbeGGfN9pDTSfkAdlu0h7CDbkNcIUEQk55MsuQESkAoINecmXvimJtIdMQt7MrjIzN7Op8X0zs38ysx4zW2dmi7JoJ2FNgHbxE5HhLXXIm9lc4Fzg/+oWLwMWxP+WA9elbSdxXUU3KCJSQVmM5P8B+AgDjy29EPiaR9YAk8xsVgZtNU3TCXlTB4u0g1Qhb2YXAtvc/YEjVs0GttTd3xova/Qcy82s28y6e3t705QjIiJHGPLyf2Z2OzCzwaqVwEeJpmpa5u6rgFUAXV1dmc2g64jXYqiXRaptyJB396WNlpvZq4H5wAPxRs45wH1mdgawDai/IOeceFlhNF2TL/WvSHtoebrG3R909+nu3ununURTMovcfTuwGvjDeC+bxcBud38qm5JFRKRZQ47kW3QLcAHQA+wFLs+pnSFpD0oRGc4yC/l4NF+77cCVWT13KzSdkC91r0h7CPaIV214FREJOORFRCTgkO+/cpFm5fOgC6WLtIdwQ77sAkREKiDYkJdiaEAvUm0KeWlJLds1GyZSbcGG/Ih4iDl/6viSKxERKU9eB0OVbsQI4yuXn86px08su5SgabpGpNqCDXmAJadML7uEYCncRdpDsNM1IiKikBcRCZpCXlqi00aItAeFvIhIwBTyIiIBU8iLiARMIS8t0S6UIu1BIS+paAOsSLUp5EVEAqaQFxEJmEJeRCRgCnkRkYAp5KUl2rtGpD0o5EVEAqaQl1Q0ohepNoW8tMSU7iJtQSEvIhIwhbyISMAU8tISTdaItAeFvIhIwBTyIiIBSx3yZvZBM9tgZuvN7JN1y1eYWY+ZbTSz89K2IyIiyXWk+WEzWwJcCLzG3feb2fR4+ULgYuBVwPHA7WZ2srsfSluwVIOXXYCINCXtSP4K4Fp33w/g7jvj5RcC33T3/e7+ONADnJGyLakgbYAVqba0IX8ycKaZ3WNmd5nZ6fHy2cCWusdtjZe9hJktN7NuM+vu7e1NWY4UpRbuGtGLVNuQ0zVmdjsws8GqlfHPTwEWA6cDN5rZiUkKcPdVwCqArq4uZUab0C9KpD0MGfLuvnSwdWZ2BXCzuztwr5kdBqYC24C5dQ+dEy+TwGi6RqTa0k7XfBdYAmBmJwOjgV3AauBiMxtjZvOBBcC9KdsSEZGEUu1dA3wZ+LKZPQS8CFwWj+rXm9mNwMPAQeBK7VkjIlK8VCHv7i8CfzDIumuAa9I8v4iIpKMjXkVEAqaQFxEJmEJeWhJtehGRqlPIi4gETCEvIhIwhbykomu9ilSbQl5EJGBpD4Yadl4+bTxnLphWdhkiIk1RyCd0x1VnlV2CiEjTNF0jIhIwhbyISMAU8tISHQsl0h4U8iIiAVPIi4gETCEvIhIwhbyISMAU8iIiAVPIi4gETCEvIhIwhbyISMAU8iIiAVPIi4gETCEvIhIwhbyISMAU8iIiAVPIi4gETCEvqeg63iLVppCXltTCfeyokeUWIiJHpWu8SktmTzqGq37zZN7+utlllyIiR5FqJG9mrzWzNWZ2v5l1m9kZ8XIzs38ysx4zW2dmi7IpV6rCzPjgOQuYO2Vc2aWIyFGkna75JPBxd38t8LH4PsAyYEH8bzlwXcp2RESkBWlD3oHj4tsTgSfj2xcCX/PIGmCSmc1K2ZaIiCSUdk7+Q8CtZvZ3RB8Yb4yXzwa21D1ua7zsqZTtiYhIAkOGvJndDsxssGolcA7wZ+7+bTO7CPgSsDRJAWa2nGhKh3nz5iX5URERGYK5e+s/bLYbmOTubmYG7Hb348zs88CP3P0b8eM2Ame5+1FH8l1dXd7d3d1yPSIiw5GZrXX3rkbr0s7JPwn8Rnz7bGBTfHs18IfxXjaLicJfUzUiIgVLOyf/x8BnzKwD2Ec87QLcAlwA9AB7gctTtiMiIi1IFfLu/hPg1xosd+DKNM8tIiLppZqTz5qZ9QJPtPjjU4FdGZaTlarWBdWtTXUlo7qSCbGuE9x9WqMVlQr5NMyse7AND2Wqal1Q3dpUVzKqK5nhVpdOUCYiEjCFvIhIwEIK+VVlFzCIqtYF1a1NdSWjupIZVnUFMycvIiIvFdJIXkREjqCQFxEJWBAhb2bnm9nG+CIlVxfc9lwzu9PMHjaz9Wb2p/HyKWZ2m5ltiv+fHC8v9IIqZjbSzP7XzL4f359vZvfE7f+7mY2Ol4+J7/fE6ztzrGmSmd1kZhvM7BEze0MV+svM/iz+HT5kZt8ws7Fl9JeZfdnMdprZQ3XLEvePmV0WP36TmV2WU12fin+P68zsO2Y2qW7diriujWZ2Xt3yTN+vjeqqW3eVmbmZTY3vl9pf8fIPxn223sw+Wbc8n/5y97b+B4wEfgGcCIwGHgAWFtj+LGBRfHsC8CiwkOgCKlfHy68G/ja+fQHwA8CAxcA9Odf358C/Ad+P798IXBzfvh64Ir79fuD6+PbFwL/nWNNXgT+Kb48GJpXdX0Snwn4cOKaun95dRn8Bvw4sAh6qW5aof4ApwGPx/5Pj25NzqOtcoCO+/bd1dS2M34tjgPnxe3RkHu/XRnXFy+cCtxIdYDm1Iv21BLgdGBPfn553f+XyJi7yH/AG4Na6+yuAFSXW8z3gN4GNwKx42SxgY3z788AldY/ve1wOtcwB7iA6edz34z/sXXVvyr6+i98Mb4hvd8SPsxxqmkgUpnbE8lL7i/5rIEyJX//3gfPK6i+g84hwSNQ/wCXA5+uWD3hcVnUdse63gRvi2wPeh7X+yuv92qgu4CbgNcBm+kO+1P4iGjQsbfC43PorhOmawS5QUrj4K/vrgHuAGd5/5s3twIz4dpH1/iPwEeBwfP9lwLPufrBB2311xet3x4/P2nygF/hKPI30RTMbT8n95e7bgL8D/o/o4ja7gbWU3181SfunjPfFe4hGyaXXZWYXAtvc/YEjVpXdXycDZ8ZTfHeZ2el51xVCyFeCmR0LfBv4kLs/V7/Oo4/gQvdVNbO3AjvdfW2R7Tahg+gr7HXu/jrgeaLphz4l9ddkostWzgeOB8YD5xdZQ7PK6J+hmNlK4CBwQwVqGQd8lOi601XTQfRtcTHwYeBGM7M8Gwwh5LcRzb3VzImXFcbMRhEF/A3ufnO8eIfF17WN/98ZLy+q3jcBbzOzzcA3iaZsPkN0vd3a2Ufr2+6rK14/EXg6h7q2Alvd/Z74/k1EoV92fy0FHnf3Xnc/ANxM1Idl91dN0v4p7H1hZu8G3gpcGn8AlV3Xy4k+rB+I//7nAPeZ2cyS64Lo7/9mj9xL9C17ap51hRDyPwcWxHtBjCbaCLa6qMbjT+EvAY+4+6frVq0GalvoLyOaq68tz/2CKu6+wt3nuHsnUZ/8t7tfCtwJvHOQumr1vjN+fOajRXffDmwxs1PiRecAD1NyfxFN0yw2s3Hx77RWV6n9VSdp/9wKnGtmk+NvKefGyzJlZucTTQm+zd33HlHvxRbthTQfWADcSwHvV3d/0N2nu3tn/Pe/lWjniO2U3F/Ad4k2vmJmJxNtTN1Fnv2VdsNCFf4RbTF/lGgr9MqC234z0VfndcD98b8LiOZn7yC6WtbtwJT48QZ8Lq71QaCrgBrPon/vmhPjP54e4Fv0b+UfG9/videfmGM9rwW64z77LtHeDKX3F/BxYAPwEPCvRHs6FN5fwDeItgscIAqo97bSP0Rz5D3xv8tzqquHaM649rd/fd3jV8Z1bQSW1S3P9P3aqK4j1m+mf8Nr2f01Gvh6/Dd2H3B23v2l0xqIiAQshOkaEREZhEJeRCRgCnkRkYAp5EVEAqaQFxEJmEJeRCRgCnkRkYD9P/53POchAhyUAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "logreg_basalt = LogisticRegression(\n",
    "    penalty=\"l1\", solver=\"liblinear\", C=20, random_state=RANDOM_SEED, max_iter = 100, verbose = 0\n",
    ")\n",
    "basalt_cv = cross_val_score(logreg_basalt, train_df, train_labels['basalt'], cv=skf,scoring=log_loss_scorer,)\n",
    "print(basalt_cv.mean(),basalt_cv.std())\n",
    "logreg_basalt.fit(train_df, train_labels['basalt'])\n",
    "plt.plot(logreg_basalt.coef_.ravel(),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([578, 351,  24,  20, 991,   8, 635, 427, 429, 419, 990, 167,  19,\n",
       "            111, 346, 174, 420, 633, 417, 339, 171, 424, 425, 169, 164, 637,\n",
       "            636, 173, 337, 627, 989, 161, 162, 168, 163, 170, 370, 345, 428,\n",
       "            166, 700, 172, 418, 638, 156, 175, 344, 205, 426, 158,  31, 634,\n",
       "            699, 431, 194, 348, 347, 981,  27, 430, 691, 341, 491, 632, 639,\n",
       "             28, 139, 490],\n",
       "           dtype='int64')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imps = logreg_basalt.coef_\n",
    "imps = pd.Series(imps.ravel())\n",
    "imps = imps.where(imps != 0.0).dropna().sort_values(ascending=False).index\n",
    "imps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cb_clf = CatBoostClassifier(random_seed=RANDOM_SEED,verbose=999,eval_metric='Logloss')\n",
    "cb_cvscore = logloss_cross_val(cb_clf,train_df.iloc[:,imps],train_labels,nsplits=3)\n",
    "#  eta 0.008354"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'basalt': 0.18828937417066896,\n",
       "  'carbonate': 0.12024706110486756,\n",
       "  'chloride': 0.19061186090833537,\n",
       "  'iron_oxide': 0.21487846278805361,\n",
       "  'oxalate': 0.007846615665656557,\n",
       "  'oxychlorine': 0.1699206932772332,\n",
       "  'phyllosilicate': 0.27102300942991164,\n",
       "  'silicate': 0.2095405184414981,\n",
       "  'sulfate': 0.18617458310250823,\n",
       "  'sulfide': 0.08407230957793475},\n",
       " 0.1642604488466668)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cb_cvscore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost on different imp features for different targets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target : basalt\n",
      "[0.28145629 0.20222666 0.36910234 0.22798772 0.21757914 0.29544072\n",
      " 0.20097045 0.29409363 0.28466385 0.2953681 ]\n",
      "cv mean:  0.26688889041325586 cv std:  0.050806806573264605\n",
      "Target : carbonate\n",
      "[0.19482698 0.41037536 0.19710714 0.17112708 0.2341675  0.64381442\n",
      " 0.27073409 0.23091639 0.16539812 0.57081833]\n",
      "cv mean:  0.30892854070010684 cv std:  0.16409558473395217\n",
      "Target : chloride\n",
      "[0.82290436 0.40821751 0.20691256 0.25194617 0.42625937 0.26595152\n",
      " 0.15623535 0.23460981 0.46128416 0.27724636]\n",
      "cv mean:  0.3511567163236281 cv std:  0.18398900294459541\n",
      "Target : iron_oxide\n",
      "[0.3648099  0.30963721 0.53064393 0.26110411 0.35804746 0.58574187\n",
      " 0.39385339 0.47832248 0.44775951 0.25645293]\n",
      "cv mean:  0.3986372799761864 cv std:  0.10538695097778113\n",
      "Target : oxalate\n",
      "[0.00043246 0.00117575 0.00161927 0.00470163 0.00081504 0.00119717\n",
      " 0.0015451  0.00128765 0.00474454 0.00154794]\n",
      "cv mean:  0.001906653191792414 cv std:  0.001448900402760595\n",
      "Target : oxychlorine\n",
      "[0.14260448 0.36150856 0.23878873 0.3143201  0.23947054 0.2990744\n",
      " 0.47461102 0.40202199 0.4412648  0.28943223]\n",
      "cv mean:  0.32030968431274254 cv std:  0.09644340002869166\n",
      "Target : phyllosilicate\n",
      "[0.52229731 0.52156728 0.36410197 0.38315554 0.50902175 0.27909754\n",
      " 0.34029791 0.40327693 0.4489649  0.38559995]\n",
      "cv mean:  0.41573810920943516 cv std:  0.07843080173301654\n",
      "Target : silicate\n",
      "[0.67607275 0.22380803 0.25072161 0.4303042  0.21489853 0.53311494\n",
      " 0.22146998 0.41230126 0.42965277 0.58780669]\n",
      "cv mean:  0.3980150767810053 cv std:  0.15829713419837416\n",
      "Target : sulfate\n",
      "[0.25428857 0.51785727 0.31619247 0.27004178 0.30694919 0.39254191\n",
      " 0.22536172 0.45988377 0.54257683 0.41660285]\n",
      "cv mean:  0.37022963574533263 cv std:  0.10657144757378245\n",
      "Target : sulfide\n",
      "[0.06695149 0.29938002 0.09962825 0.29808231 0.10090687 0.1206861\n",
      " 0.08377121 0.17494039 0.28113219 0.46041315]\n",
      "cv mean:  0.1985891980362468 cv std:  0.12324679164771461\n"
     ]
    }
   ],
   "source": [
    "imp_fts_dict = {}\n",
    "for col in train_labels.columns:\n",
    "    print(\"Target :\",col)\n",
    "    # Define stratified k-fold validation\n",
    "    skf = StratifiedKFold(n_splits=10, random_state=RANDOM_SEED, shuffle=True)\n",
    "    # Define log loss\n",
    "    log_loss_scorer = make_scorer(log_loss,needs_proba=True)\n",
    "\n",
    "    logreg_clf = LogisticRegression(penalty='l1',random_state=RANDOM_SEED,solver='liblinear',C=20)\n",
    "    # check cv score for each target\n",
    "    cv = cross_val_score(logreg_clf, train_df, train_labels[col], cv=skf,scoring=log_loss_scorer)\n",
    "    print(cv)\n",
    "    print(\"cv mean: \", cv.mean(), \"cv std: \", cv.std())\n",
    "\n",
    "    # collect non zero coeffs and store in imp_fts_dict\n",
    "    logreg_clf.fit(train_df,train_labels[col])\n",
    "    coefs = logreg_clf.coef_\n",
    "    coefs = pd.Series(coefs.ravel())\n",
    "    coefs = coefs.where(coefs != 0.0).dropna().sort_values(ascending=False).index\n",
    "    imp_fts_dict[col] = coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[68, 68, 70, 88, 11, 67, 82, 87, 74, 56]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(imp_fts_dict[col]) for col in imp_fts_dict.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target : basalt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Custom logger is already specified. Specify more than one logger at same time is not thread safe."
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.15052713 0.15040269 0.3350209  0.12793076 0.15019269 0.20523295\n",
      " 0.19710424 0.21792255 0.1862148  0.157915  ]\n",
      "cv mean:  0.18784637155072298 cv std:  0.05626180235471972\n",
      "Target : carbonate\n",
      "[0.0680218  0.06599585 0.08776372 0.05272954 0.10583692 0.2447081\n",
      " 0.13912305 0.08835207 0.13469994 0.14814648]\n",
      "cv mean:  0.11353774590107993 cv std:  0.05375255042012596\n",
      "Target : chloride\n",
      "[0.17517862 0.16311785 0.12559912 0.19640469 0.28909492 0.2042061\n",
      " 0.09385    0.12773808 0.21785161 0.17339423]\n",
      "cv mean:  0.17664352081186974 cv std:  0.05247981458318296\n",
      "Target : iron_oxide\n",
      "[0.29324072 0.1998568  0.24477163 0.17379342 0.14257655 0.1473208\n",
      " 0.24704426 0.28284446 0.19559076 0.1790757 ]\n",
      "cv mean:  0.2106115093533889 cv std:  0.05085792083698198\n",
      "Target : oxalate\n",
      "[0.0009267  0.01764039 0.00162672 0.00706897 0.01897874 0.01061441\n",
      " 0.00347737 0.0010289  0.00731543 0.00672949]\n",
      "cv mean:  0.0075407113695742465 cv std:  0.00617127023663031\n",
      "Target : oxychlorine\n",
      "[0.10779347 0.1406152  0.13452193 0.22934622 0.13181356 0.27907312\n",
      " 0.19889586 0.15622008 0.17299448 0.12147225]\n",
      "cv mean:  0.16727461773576674 cv std:  0.051169740048721005\n",
      "Target : phyllosilicate\n",
      "[0.3042465  0.26820882 0.30710333 0.22463674 0.24410289 0.1776356\n",
      " 0.23068109 0.19346851 0.26574525 0.33960599]\n",
      "cv mean:  0.2555434721846475 cv std:  0.048978145978670236\n",
      "Target : silicate\n",
      "[0.15471218 0.19525064 0.21631273 0.21275209 0.15109614 0.32917337\n",
      " 0.15907699 0.27125853 0.24192169 0.20280191]\n",
      "cv mean:  0.2134356266280073 cv std:  0.05319041452429881\n",
      "Target : sulfate\n",
      "[0.11814955 0.18755453 0.11004119 0.12252881 0.201986   0.12208406\n",
      " 0.16850809 0.19153356 0.26235881 0.22975016]\n",
      "cv mean:  0.17144947622728904 cv std:  0.04970853516608093\n",
      "Target : sulfide\n",
      "[0.01736265 0.1744997  0.04779251 0.10693872 0.0279274  0.05931241\n",
      " 0.03976703 0.10004632 0.06395543 0.09332509]\n",
      "cv mean:  0.07309272619430875 cv std:  0.044423651488893344\n"
     ]
    }
   ],
   "source": [
    "for col in train_labels.columns:\n",
    "    print(\"Target :\",col)\n",
    "    # Define stratified k-fold validation\n",
    "    skf = StratifiedKFold(n_splits=10, random_state=RANDOM_SEED, shuffle=True)\n",
    "    # Define log loss\n",
    "    log_loss_scorer = make_scorer(log_loss,needs_proba=True)\n",
    "\n",
    "    cb_clf = CatBoostClassifier(random_seed=RANDOM_SEED,verbose=False,eval_metric='Logloss')\n",
    "    cv =  cross_val_score(cb_clf,train_df.iloc[:,imp_fts_dict[col]],train_labels[col],cv=skf,scoring=log_loss_scorer)\n",
    "    print(cv)\n",
    "    print(\"cv mean: \", cv.mean(), \"cv std: \", cv.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvs = [0.18784637155072298,0.11353774590107993,0.17664352081186974 , 0.2106115093533889,0.007540711369574246,0.16727461773576674, 0.2555434721846475,0.2134356266280073,0.1714494762272890,0.07309272619430875]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15769757779566554"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(cvs).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What about nns on selected fts??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import layers,models,metrics,callbacks,losses\n",
    "from tensorflow.keras.activations import relu,sigmoid,swish,tanh,selu\n",
    "from tensorflow.keras.optimizers import SGD,Adam,RMSprop\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_11\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_21 (Dense)             (None, 1)                 69        \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 800)               1600      \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 400)               320400    \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 10)                4010      \n",
      "_________________________________________________________________\n",
      "lambda_10 (Lambda)           (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 326,079\n",
      "Trainable params: 326,079\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "nn = models.Sequential()\n",
    "nn.add(layers.Dense(1, input_shape=(train_df.iloc[:,imp_fts_dict['basalt']].shape[1],)))\n",
    "nn.add(layers.Dense(800, activation=tanh))\n",
    "nn.add(layers.Dense(400, activation=tanh))\n",
    "nn.add(layers.Dense(10))\n",
    "nn.add(layers.Lambda(lambda x: tf.keras.backend.clip(x, 0.0001, 0.9999)))\n",
    "\n",
    "nn.compile(optimizer=Adam(learning_rate=0.001), loss=losses.CategoricalCrossentropy(), metrics=[metrics.CategoricalCrossentropy()])\n",
    "print(nn.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = callbacks.EarlyStopping(monitor='val_loss', patience=10,restore_best_weights=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "1/1 - 1s - loss: 4.7251 - categorical_crossentropy: 4.7251 - val_loss: 6.0336 - val_categorical_crossentropy: 6.0336\n",
      "Epoch 2/150\n",
      "1/1 - 0s - loss: 6.6787 - categorical_crossentropy: 6.6787 - val_loss: 5.6509 - val_categorical_crossentropy: 5.6509\n",
      "Epoch 3/150\n",
      "1/1 - 0s - loss: 6.0273 - categorical_crossentropy: 6.0273 - val_loss: 5.6162 - val_categorical_crossentropy: 5.6162\n",
      "Epoch 4/150\n",
      "1/1 - 0s - loss: 5.9938 - categorical_crossentropy: 5.9938 - val_loss: 5.6236 - val_categorical_crossentropy: 5.6236\n",
      "Epoch 5/150\n",
      "1/1 - 0s - loss: 5.9916 - categorical_crossentropy: 5.9916 - val_loss: 5.5738 - val_categorical_crossentropy: 5.5738\n",
      "Epoch 6/150\n",
      "1/1 - 0s - loss: 5.9839 - categorical_crossentropy: 5.9839 - val_loss: 5.5476 - val_categorical_crossentropy: 5.5476\n",
      "Epoch 7/150\n",
      "1/1 - 0s - loss: 5.9522 - categorical_crossentropy: 5.9522 - val_loss: 5.5701 - val_categorical_crossentropy: 5.5701\n",
      "Epoch 8/150\n",
      "1/1 - 0s - loss: 5.9118 - categorical_crossentropy: 5.9118 - val_loss: 5.5108 - val_categorical_crossentropy: 5.5108\n",
      "Epoch 9/150\n",
      "1/1 - 0s - loss: 5.8710 - categorical_crossentropy: 5.8710 - val_loss: 5.3593 - val_categorical_crossentropy: 5.3593\n",
      "Epoch 10/150\n",
      "1/1 - 0s - loss: 5.7662 - categorical_crossentropy: 5.7662 - val_loss: 5.2467 - val_categorical_crossentropy: 5.2467\n",
      "Epoch 11/150\n",
      "1/1 - 0s - loss: 5.6567 - categorical_crossentropy: 5.6567 - val_loss: 5.3337 - val_categorical_crossentropy: 5.3337\n",
      "Epoch 12/150\n",
      "1/1 - 0s - loss: 5.6828 - categorical_crossentropy: 5.6828 - val_loss: 5.3548 - val_categorical_crossentropy: 5.3548\n",
      "Epoch 13/150\n",
      "1/1 - 0s - loss: 5.6884 - categorical_crossentropy: 5.6884 - val_loss: 5.3357 - val_categorical_crossentropy: 5.3357\n",
      "Epoch 14/150\n",
      "1/1 - 0s - loss: 5.5836 - categorical_crossentropy: 5.5836 - val_loss: 5.1247 - val_categorical_crossentropy: 5.1247\n",
      "Epoch 15/150\n",
      "1/1 - 0s - loss: 5.3208 - categorical_crossentropy: 5.3208 - val_loss: 4.8192 - val_categorical_crossentropy: 4.8192\n",
      "Epoch 16/150\n",
      "1/1 - 0s - loss: 5.0157 - categorical_crossentropy: 5.0157 - val_loss: 4.6706 - val_categorical_crossentropy: 4.6706\n",
      "Epoch 17/150\n",
      "1/1 - 0s - loss: 4.9237 - categorical_crossentropy: 4.9237 - val_loss: 4.6110 - val_categorical_crossentropy: 4.6110\n",
      "Epoch 18/150\n",
      "1/1 - 0s - loss: 4.8786 - categorical_crossentropy: 4.8786 - val_loss: 4.5248 - val_categorical_crossentropy: 4.5248\n",
      "Epoch 19/150\n",
      "1/1 - 0s - loss: 4.8379 - categorical_crossentropy: 4.8379 - val_loss: 4.5051 - val_categorical_crossentropy: 4.5051\n",
      "Epoch 20/150\n",
      "1/1 - 0s - loss: 4.8122 - categorical_crossentropy: 4.8122 - val_loss: 4.4670 - val_categorical_crossentropy: 4.4670\n",
      "Epoch 21/150\n",
      "1/1 - 0s - loss: 4.7951 - categorical_crossentropy: 4.7951 - val_loss: 4.4929 - val_categorical_crossentropy: 4.4929\n",
      "Epoch 22/150\n",
      "1/1 - 0s - loss: 4.7801 - categorical_crossentropy: 4.7801 - val_loss: 4.5452 - val_categorical_crossentropy: 4.5452\n",
      "Epoch 23/150\n",
      "1/1 - 0s - loss: 4.7589 - categorical_crossentropy: 4.7589 - val_loss: 4.5095 - val_categorical_crossentropy: 4.5095\n",
      "Epoch 24/150\n",
      "1/1 - 0s - loss: 4.7461 - categorical_crossentropy: 4.7461 - val_loss: 4.4277 - val_categorical_crossentropy: 4.4277\n",
      "Epoch 25/150\n",
      "1/1 - 0s - loss: 4.7416 - categorical_crossentropy: 4.7416 - val_loss: 4.4367 - val_categorical_crossentropy: 4.4367\n",
      "Epoch 26/150\n",
      "1/1 - 0s - loss: 4.7456 - categorical_crossentropy: 4.7456 - val_loss: 4.4536 - val_categorical_crossentropy: 4.4536\n",
      "Epoch 27/150\n",
      "1/1 - 0s - loss: 4.7429 - categorical_crossentropy: 4.7429 - val_loss: 4.4516 - val_categorical_crossentropy: 4.4516\n",
      "Epoch 28/150\n",
      "1/1 - 0s - loss: 4.7426 - categorical_crossentropy: 4.7426 - val_loss: 4.4172 - val_categorical_crossentropy: 4.4172\n",
      "Epoch 29/150\n",
      "1/1 - 0s - loss: 4.7365 - categorical_crossentropy: 4.7365 - val_loss: 4.3697 - val_categorical_crossentropy: 4.3697\n",
      "Epoch 30/150\n",
      "1/1 - 0s - loss: 4.7172 - categorical_crossentropy: 4.7172 - val_loss: 4.3603 - val_categorical_crossentropy: 4.3603\n",
      "Epoch 31/150\n",
      "1/1 - 0s - loss: 4.7081 - categorical_crossentropy: 4.7081 - val_loss: 4.3482 - val_categorical_crossentropy: 4.3482\n",
      "Epoch 32/150\n",
      "1/1 - 0s - loss: 4.7231 - categorical_crossentropy: 4.7231 - val_loss: 4.3353 - val_categorical_crossentropy: 4.3353\n",
      "Epoch 33/150\n",
      "1/1 - 0s - loss: 4.7275 - categorical_crossentropy: 4.7275 - val_loss: 4.3394 - val_categorical_crossentropy: 4.3394\n",
      "Epoch 34/150\n",
      "1/1 - 0s - loss: 4.6924 - categorical_crossentropy: 4.6924 - val_loss: 4.3439 - val_categorical_crossentropy: 4.3439\n",
      "Epoch 35/150\n",
      "1/1 - 0s - loss: 4.6837 - categorical_crossentropy: 4.6837 - val_loss: 4.3527 - val_categorical_crossentropy: 4.3527\n",
      "Epoch 36/150\n",
      "1/1 - 0s - loss: 4.6865 - categorical_crossentropy: 4.6865 - val_loss: 4.3717 - val_categorical_crossentropy: 4.3717\n",
      "Epoch 37/150\n",
      "1/1 - 0s - loss: 4.6852 - categorical_crossentropy: 4.6852 - val_loss: 4.3679 - val_categorical_crossentropy: 4.3679\n",
      "Epoch 38/150\n",
      "1/1 - 0s - loss: 4.6890 - categorical_crossentropy: 4.6890 - val_loss: 4.3596 - val_categorical_crossentropy: 4.3596\n",
      "Epoch 39/150\n",
      "1/1 - 0s - loss: 4.6836 - categorical_crossentropy: 4.6836 - val_loss: 4.3503 - val_categorical_crossentropy: 4.3503\n",
      "Epoch 40/150\n",
      "1/1 - 0s - loss: 4.6830 - categorical_crossentropy: 4.6830 - val_loss: 4.3092 - val_categorical_crossentropy: 4.3092\n",
      "Epoch 41/150\n",
      "1/1 - 0s - loss: 4.6790 - categorical_crossentropy: 4.6790 - val_loss: 4.2666 - val_categorical_crossentropy: 4.2666\n",
      "Epoch 42/150\n",
      "1/1 - 0s - loss: 4.6645 - categorical_crossentropy: 4.6645 - val_loss: 4.2269 - val_categorical_crossentropy: 4.2269\n",
      "Epoch 43/150\n",
      "1/1 - 0s - loss: 4.6574 - categorical_crossentropy: 4.6574 - val_loss: 4.1929 - val_categorical_crossentropy: 4.1929\n",
      "Epoch 44/150\n",
      "1/1 - 0s - loss: 4.6510 - categorical_crossentropy: 4.6510 - val_loss: 4.2218 - val_categorical_crossentropy: 4.2218\n",
      "Epoch 45/150\n",
      "1/1 - 0s - loss: 4.6442 - categorical_crossentropy: 4.6442 - val_loss: 4.1968 - val_categorical_crossentropy: 4.1968\n",
      "Epoch 46/150\n",
      "1/1 - 0s - loss: 4.6403 - categorical_crossentropy: 4.6403 - val_loss: 4.1845 - val_categorical_crossentropy: 4.1845\n",
      "Epoch 47/150\n",
      "1/1 - 0s - loss: 4.6237 - categorical_crossentropy: 4.6237 - val_loss: 4.2035 - val_categorical_crossentropy: 4.2035\n",
      "Epoch 48/150\n",
      "1/1 - 0s - loss: 4.6337 - categorical_crossentropy: 4.6337 - val_loss: 4.1917 - val_categorical_crossentropy: 4.1917\n",
      "Epoch 49/150\n",
      "1/1 - 0s - loss: 4.6630 - categorical_crossentropy: 4.6630 - val_loss: 4.1750 - val_categorical_crossentropy: 4.1750\n",
      "Epoch 50/150\n",
      "1/1 - 0s - loss: 4.6556 - categorical_crossentropy: 4.6556 - val_loss: 4.1308 - val_categorical_crossentropy: 4.1308\n",
      "Epoch 51/150\n",
      "1/1 - 0s - loss: 4.6316 - categorical_crossentropy: 4.6316 - val_loss: 4.1286 - val_categorical_crossentropy: 4.1286\n",
      "Epoch 52/150\n",
      "1/1 - 0s - loss: 4.6071 - categorical_crossentropy: 4.6071 - val_loss: 4.1350 - val_categorical_crossentropy: 4.1350\n",
      "Epoch 53/150\n",
      "1/1 - 0s - loss: 4.5927 - categorical_crossentropy: 4.5927 - val_loss: 4.1154 - val_categorical_crossentropy: 4.1154\n",
      "Epoch 54/150\n",
      "1/1 - 0s - loss: 4.5815 - categorical_crossentropy: 4.5815 - val_loss: 4.0878 - val_categorical_crossentropy: 4.0878\n",
      "Epoch 55/150\n",
      "1/1 - 0s - loss: 4.5735 - categorical_crossentropy: 4.5735 - val_loss: 4.0377 - val_categorical_crossentropy: 4.0377\n",
      "Epoch 56/150\n",
      "1/1 - 0s - loss: 4.5674 - categorical_crossentropy: 4.5674 - val_loss: 4.0226 - val_categorical_crossentropy: 4.0226\n",
      "Epoch 57/150\n",
      "1/1 - 0s - loss: 4.5521 - categorical_crossentropy: 4.5521 - val_loss: 4.0160 - val_categorical_crossentropy: 4.0160\n",
      "Epoch 58/150\n",
      "1/1 - 0s - loss: 4.5577 - categorical_crossentropy: 4.5577 - val_loss: 4.0074 - val_categorical_crossentropy: 4.0074\n",
      "Epoch 59/150\n",
      "1/1 - 0s - loss: 4.5733 - categorical_crossentropy: 4.5733 - val_loss: 4.0175 - val_categorical_crossentropy: 4.0175\n",
      "Epoch 60/150\n",
      "1/1 - 0s - loss: 4.5758 - categorical_crossentropy: 4.5758 - val_loss: 4.0081 - val_categorical_crossentropy: 4.0081\n",
      "Epoch 61/150\n",
      "1/1 - 0s - loss: 4.5764 - categorical_crossentropy: 4.5764 - val_loss: 3.9942 - val_categorical_crossentropy: 3.9942\n",
      "Epoch 62/150\n",
      "1/1 - 0s - loss: 4.5505 - categorical_crossentropy: 4.5505 - val_loss: 3.9658 - val_categorical_crossentropy: 3.9658\n",
      "Epoch 63/150\n",
      "1/1 - 0s - loss: 4.5524 - categorical_crossentropy: 4.5524 - val_loss: 3.9530 - val_categorical_crossentropy: 3.9530\n",
      "Epoch 64/150\n",
      "1/1 - 0s - loss: 4.5286 - categorical_crossentropy: 4.5286 - val_loss: 3.9507 - val_categorical_crossentropy: 3.9507\n",
      "Epoch 65/150\n",
      "1/1 - 0s - loss: 4.5163 - categorical_crossentropy: 4.5163 - val_loss: 3.9424 - val_categorical_crossentropy: 3.9424\n",
      "Epoch 66/150\n",
      "1/1 - 0s - loss: 4.5133 - categorical_crossentropy: 4.5133 - val_loss: 3.9491 - val_categorical_crossentropy: 3.9491\n",
      "Epoch 67/150\n",
      "1/1 - 0s - loss: 4.5011 - categorical_crossentropy: 4.5011 - val_loss: 3.9619 - val_categorical_crossentropy: 3.9619\n",
      "Epoch 68/150\n",
      "1/1 - 0s - loss: 4.4860 - categorical_crossentropy: 4.4860 - val_loss: 3.9652 - val_categorical_crossentropy: 3.9652\n",
      "Epoch 69/150\n",
      "1/1 - 0s - loss: 4.4850 - categorical_crossentropy: 4.4850 - val_loss: 3.9745 - val_categorical_crossentropy: 3.9745\n",
      "Epoch 70/150\n",
      "1/1 - 0s - loss: 4.4815 - categorical_crossentropy: 4.4815 - val_loss: 3.9711 - val_categorical_crossentropy: 3.9711\n",
      "Epoch 71/150\n",
      "1/1 - 0s - loss: 4.4822 - categorical_crossentropy: 4.4822 - val_loss: 3.9842 - val_categorical_crossentropy: 3.9842\n",
      "Epoch 72/150\n",
      "1/1 - 0s - loss: 4.4859 - categorical_crossentropy: 4.4859 - val_loss: 3.9737 - val_categorical_crossentropy: 3.9737\n",
      "Epoch 73/150\n",
      "1/1 - 0s - loss: 4.4827 - categorical_crossentropy: 4.4827 - val_loss: 3.9616 - val_categorical_crossentropy: 3.9616\n",
      "Epoch 74/150\n",
      "1/1 - 0s - loss: 4.4784 - categorical_crossentropy: 4.4784 - val_loss: 3.9611 - val_categorical_crossentropy: 3.9611\n",
      "Epoch 75/150\n",
      "1/1 - 0s - loss: 4.4755 - categorical_crossentropy: 4.4755 - val_loss: 3.9523 - val_categorical_crossentropy: 3.9523\n"
     ]
    }
   ],
   "source": [
    "hist = nn.fit(train_df.iloc[:,imp_fts_dict['basalt']],train_labels,batch_size=612,epochs=150,validation_split=0.2,verbose=2,callbacks=[callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  i think nns are absolutely a waste of time in this case"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tried svm and logreg on train imp features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'basalt': 0.2683067334157245,\n",
       "  'carbonate': 0.2824961882865381,\n",
       "  'chloride': 0.2718993559266498,\n",
       "  'iron_oxide': 0.3804138783590792,\n",
       "  'oxalate': 0.003494474156954956,\n",
       "  'oxychlorine': 0.3311883782764481,\n",
       "  'phyllosilicate': 0.4645399716330522,\n",
       "  'silicate': 0.3336856568198504,\n",
       "  'sulfate': 0.328319778109153,\n",
       "  'sulfide': 0.14695751219567454},\n",
       " 0.2811301927179125)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_clf = LogisticRegression(random_state=RANDOM_SEED,verbose=0,penalty='l1',C=10,solver='liblinear')\n",
    "log_cvscore = logloss_cross_val(log_clf,train_df.iloc[:,imps],train_labels,nsplits=5)\n",
    "log_cvscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM][LibSVM]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'basalt': 0.2602430128289928,\n",
       "  'carbonate': 0.18745751504389904,\n",
       "  'chloride': 0.2869690909407553,\n",
       "  'iron_oxide': 0.352568430674568,\n",
       "  'oxalate': 0.012864080811357781,\n",
       "  'oxychlorine': 0.25436904080238104,\n",
       "  'phyllosilicate': 0.37366108764461786,\n",
       "  'silicate': 0.2983867983800075,\n",
       "  'sulfate': 0.27253603759571926,\n",
       "  'sulfide': 0.10621789869385134},\n",
       " 0.24052729934161493)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_clf = SVC(random_state=RANDOM_SEED,verbose=1,probability=True)\n",
    "svm_cvscore = logloss_cross_val(svm_clf,train_df.iloc[:,imps],train_labels,nsplits=3)\n",
    "svm_cvscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost on l1_tot_abund_imp_fts \n",
    "- upsampled cv loss is 0.12\n",
    "- without upsampling cv loss is 0.19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([55, 69, 30, 72, 39,  1, 14, 42, 34, 17, 31,  2, 13, 43, 29, 27, 99,\n",
       "            18, 35, 16, 15,  4, 26, 40,  0, 47, 38,  3, 33, 19],\n",
       "           dtype='int64')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot_abund_imp_fts = load('l1_tot_abund_imp_fts_dict.joblib')\n",
    "tot_abund_imp_fts['phyllosilicate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_clf = LogisticRegression(random_state=RANDOM_SEED,verbose=0,penalty='l1',C=10,solver='liblinear')\n",
    "logreg_cvscore = logloss_cross_val(logreg_clf,train_df,train_labels,nsplits=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'basalt': 0.2908690428327613,\n",
       "  'carbonate': 0.3066242555497567,\n",
       "  'chloride': 0.3391700401132475,\n",
       "  'iron_oxide': 0.37640322140497917,\n",
       "  'oxalate': 0.004054584453948384,\n",
       "  'oxychlorine': 0.2866645348619691,\n",
       "  'phyllosilicate': 0.4154358926880845,\n",
       "  'silicate': 0.37181695938703097,\n",
       "  'sulfate': 0.409961917558196,\n",
       "  'sulfide': 0.18903867440456762},\n",
       " 0.29900391232545415)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg_cvscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target : basalt\n",
      "[0.24061223 0.18193615 0.39784157 0.20591818 0.48632038 0.3646586\n",
      " 0.2522026  0.25311382 0.31064896 0.21543795]\n",
      "cv mean:  0.2908690428327613 cv std:  0.09257411236049491\n",
      "Target : carbonate\n",
      "[0.20083117 0.31059588 0.17571363 0.16892331 0.24455976 0.80774629\n",
      " 0.21758708 0.24215335 0.26802696 0.43010512]\n",
      "cv mean:  0.30662425554975664 cv std:  0.18196257336726052\n",
      "Target : chloride\n",
      "[0.84596205 0.36553713 0.15532271 0.27963954 0.41581153 0.28230725\n",
      " 0.17545554 0.2479177  0.36310199 0.26064496]\n",
      "cv mean:  0.3391700401132475 cv std:  0.1859711640548894\n",
      "Target : iron_oxide\n",
      "[0.38733524 0.37812592 0.40641022 0.2566563  0.37429869 0.53253654\n",
      " 0.37542888 0.42459025 0.40248556 0.22616461]\n",
      "cv mean:  0.3764032214049792 cv std:  0.0808202524475268\n",
      "Target : oxalate\n",
      "[0.00123825 0.000865   0.00246681 0.00403684 0.00053017 0.01267582\n",
      " 0.00204895 0.0017305  0.01077965 0.00417386]\n",
      "cv mean:  0.004054584453948381 cv std:  0.004025401286313474\n",
      "Target : oxychlorine\n",
      "[0.1832309  0.35475715 0.22729311 0.4027843  0.19575835 0.25653885\n",
      " 0.55039336 0.20819278 0.26587662 0.22181991]\n",
      "cv mean:  0.2866645348619691 cv std:  0.11039159850754554\n",
      "Target : phyllosilicate\n",
      "[0.52543651 0.58128091 0.37678837 0.40664755 0.36507033 0.29242367\n",
      " 0.37735725 0.38188794 0.46655206 0.38091435]\n",
      "cv mean:  0.41543589268808445 cv std:  0.08076157249225131\n",
      "Target : silicate\n",
      "[0.57607627 0.24957337 0.18353161 0.38502824 0.21246849 0.5039077\n",
      " 0.23935286 0.39543727 0.55010682 0.42268695]\n",
      "cv mean:  0.37181695938703097 cv std:  0.1369711114211719\n",
      "Target : sulfate\n",
      "[0.26165995 0.64558496 0.31198489 0.27353653 0.68826845 0.3451048\n",
      " 0.27122075 0.37541646 0.50615998 0.42068243]\n",
      "cv mean:  0.409961917558196 cv std:  0.14737555600976607\n",
      "Target : sulfide\n",
      "[0.06053367 0.24343603 0.08353548 0.30419356 0.08198294 0.12346715\n",
      " 0.07810786 0.14029474 0.33250756 0.44232775]\n",
      "cv mean:  0.18903867440456762 cv std:  0.12611305675405096\n"
     ]
    }
   ],
   "source": [
    "tot_abund_imp_fts_dict_mixed = {}\n",
    "cv_list = []\n",
    "for col in train_labels.columns:\n",
    "    print(\"Target :\",col)\n",
    "    # Define stratified k-fold validation\n",
    "    skf = StratifiedKFold(n_splits=10, random_state=RANDOM_SEED, shuffle=True)\n",
    "    # Define log loss\n",
    "    log_loss_scorer = make_scorer(log_loss,needs_proba=True)\n",
    "\n",
    "    logreg_clf = LogisticRegression(penalty='l1',random_state=RANDOM_SEED,solver='liblinear',C=10)\n",
    "    # check cv score for each target\n",
    "    cv = cross_val_score(logreg_clf, train_df, train_labels[col], cv=skf,scoring=log_loss_scorer)\n",
    "    print(cv)\n",
    "    print(\"cv mean: \", cv.mean(), \"cv std: \", cv.std())\n",
    "    cv_list.append(cv.mean())\n",
    "\n",
    "    # collect non zero coeffs and store in tot_abund_imp_fts_dict_mixed\n",
    "    logreg_clf.fit(train_df,train_labels[col])\n",
    "    coefs = logreg_clf.coef_\n",
    "    coefs = pd.Series(coefs.ravel())\n",
    "    coefs = coefs.where(coefs != 0.0).dropna().sort_values(ascending=False).index\n",
    "    tot_abund_imp_fts_dict_mixed[col] = coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.29900391232545415"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_list = [0.2908690428327613 ,0.30662425554975664 ,0.3391700401132475 ,0.3764032214049792 ,0.004054584453948381,0.2866645348619691 ,0.41543589268808445,0.37181695938703097,0.409961917558196 ,0.18903867440456762    ]\n",
    "np.array(cv_list).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['basalt', 'carbonate', 'chloride', 'iron_oxide', 'oxalate', 'oxychlorine', 'phyllosilicate', 'silicate', 'sulfate', 'sulfide'])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot_abund_imp_fts_dict_mixed.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target : basalt\n",
      "catboost model log-loss:\n",
      "[0.14591509 0.16290024 0.30852966 0.12917836 0.16941933 0.19605544\n",
      " 0.19931107 0.22922979 0.19879662 0.159036  ]\n",
      "mean logloss -->  0.1898371595962705  loss std -->  0.0485265075804131\n",
      "------------------------------------------------------------------------\n",
      "Target : carbonate\n",
      "catboost model log-loss:\n",
      "[0.0611044  0.06055104 0.10049762 0.05437403 0.10405143 0.22489677\n",
      " 0.1518459  0.08090222 0.11978311 0.15071867]\n",
      "mean logloss -->  0.11087251729595364  loss std -->  0.050597473146514665\n",
      "------------------------------------------------------------------------\n",
      "Target : chloride\n",
      "catboost model log-loss:\n",
      "[0.16920064 0.17976555 0.12714085 0.19307841 0.26745554 0.19358702\n",
      " 0.0953205  0.12497579 0.25133394 0.17848081]\n",
      "mean logloss -->  0.17803390585811468  loss std -->  0.051152087599242536\n",
      "------------------------------------------------------------------------\n",
      "Target : iron_oxide\n",
      "catboost model log-loss:\n",
      "[0.2911899  0.20040303 0.23526644 0.14811042 0.1385408  0.14902611\n",
      " 0.23516141 0.28010571 0.21576338 0.19585843]\n",
      "mean logloss -->  0.2089425629423284  loss std -->  0.05074127919319486\n",
      "------------------------------------------------------------------------\n",
      "Target : oxalate\n",
      "catboost model log-loss:\n",
      "[0.00103809 0.00434786 0.00205829 0.00546733 0.01036615 0.01082481\n",
      " 0.00287106 0.00108464 0.00442027 0.01259665]\n",
      "mean logloss -->  0.0055075149349432225  loss std -->  0.004037944162466829\n",
      "------------------------------------------------------------------------\n",
      "Target : oxychlorine\n",
      "catboost model log-loss:\n",
      "[0.10022086 0.13752546 0.14728284 0.23494679 0.12592965 0.29263253\n",
      " 0.21573163 0.14798523 0.16720963 0.11598144]\n",
      "mean logloss -->  0.16854460648696984  loss std -->  0.05754950474064574\n",
      "------------------------------------------------------------------------\n",
      "Target : phyllosilicate\n",
      "catboost model log-loss:\n",
      "[0.29000732 0.28587062 0.27646972 0.25162684 0.23794435 0.17583879\n",
      " 0.19237445 0.20155602 0.25002862 0.34586195]\n",
      "mean logloss -->  0.25075786923529536  loss std -->  0.04907860724049313\n",
      "------------------------------------------------------------------------\n",
      "Target : silicate\n",
      "catboost model log-loss:\n",
      "[0.15921283 0.17733137 0.2085064  0.2243665  0.14823882 0.32510369\n",
      " 0.15943986 0.26854527 0.24591434 0.19237411]\n",
      "mean logloss -->  0.21090332045409949  loss std -->  0.05338046011378124\n",
      "------------------------------------------------------------------------\n",
      "Target : sulfate\n",
      "catboost model log-loss:\n",
      "[0.12295766 0.18272964 0.09903211 0.1262956  0.1940325  0.11459198\n",
      " 0.16998794 0.1780163  0.25366768 0.21779481]\n",
      "mean logloss -->  0.1659106227413552  loss std -->  0.04706464467196016\n",
      "------------------------------------------------------------------------\n",
      "Target : sulfide\n",
      "catboost model log-loss:\n",
      "[0.0128014  0.18010083 0.05151428 0.10432472 0.02562858 0.06475225\n",
      " 0.04065614 0.09914392 0.06075147 0.08152127]\n",
      "mean logloss -->  0.07211948696322569  loss std -->  0.04560035707243781\n",
      "------------------------------------------------------------------------\n",
      "[0.1898371595962705, 0.11087251729595364, 0.17803390585811468, 0.2089425629423284, 0.0055075149349432225, 0.16854460648696984, 0.25075786923529536, 0.21090332045409949, 0.1659106227413552, 0.07211948696322569]\n",
      "0.15614295665085562\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(n_splits=10, random_state=RANDOM_SEED, shuffle=True)\n",
    "cv = []\n",
    "for col in train_labels.columns:\n",
    "    print(\"Target :\",col)\n",
    "    # sampler = SMOTETomek(random_state=RANDOM_SEED)\n",
    "    # sampled_train_col = sampler.fit_resample(tot_abund_df.iloc[:,tot_abund_imp_fts_dict[col]], train_labels[col])\n",
    "    cb_clf = CatBoostClassifier(random_state=RANDOM_SEED,verbose=False,eval_metric='Logloss')\n",
    "    cb_logloss = cross_val_score(cb_clf,train_df.iloc[:,tot_abund_imp_fts_dict_mixed[col]],train_labels[col],scoring=log_loss_scorer,cv=skf)\n",
    "    print(\"catboost model log-loss:\")\n",
    "    print(cb_logloss)\n",
    "    print(\"mean logloss --> \",cb_logloss.mean(),\" loss std --> \",cb_logloss.std())\n",
    "    print(\"------------------------------------------------------------------------\")\n",
    "    cv.append(cb_logloss.mean())\n",
    "print(cv)\n",
    "print(np.array(cv).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15614295665085562"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(cv).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coefs == imp_fts_index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>m/z</th>\n",
       "      <th colspan=\"10\" halign=\"left\">0</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"9\" halign=\"left\">99</th>\n",
       "      <th>tot_abund</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_bin</th>\n",
       "      <th>(-100, 0]</th>\n",
       "      <th>(0, 100]</th>\n",
       "      <th>(100, 200]</th>\n",
       "      <th>(1000, 1100]</th>\n",
       "      <th>(1100, 1200]</th>\n",
       "      <th>(1200, 1300]</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>...</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>(400, 500]</th>\n",
       "      <th>(500, 600]</th>\n",
       "      <th>(600, 700]</th>\n",
       "      <th>(700, 800]</th>\n",
       "      <th>(800, 900]</th>\n",
       "      <th>(900, 1000]</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0000</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004085</td>\n",
       "      <td>0.004641</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001394</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.898509e-06</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.014152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0001</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.760583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0002</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002270</td>\n",
       "      <td>0.002495</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001688</td>\n",
       "      <td>0.000636</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.257968e-07</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>9.397170e-07</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>4.175911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0003</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.002464</td>\n",
       "      <td>0.000306</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001099</td>\n",
       "      <td>0.000992</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.379099e-06</td>\n",
       "      <td>0.000006</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>4.693477e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>4.712937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0004</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005993</td>\n",
       "      <td>0.012429</td>\n",
       "      <td>0.000593</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009380</td>\n",
       "      <td>0.005099</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.904920e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>3.174421e-06</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>4.288637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1601 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "m/z               0                                                 \\\n",
       "temp_bin  (-100, 0]  (0, 100] (100, 200] (1000, 1100] (1100, 1200]   \n",
       "sample_id                                                            \n",
       "S0000           0.0  0.004085   0.004641     0.000000          0.0   \n",
       "S0001           0.0  0.000000   0.000000     0.000000          0.0   \n",
       "S0002           0.0  0.002270   0.002495     0.000000          0.0   \n",
       "S0003           0.0  0.001336   0.002464     0.000306          0.0   \n",
       "S0004           0.0  0.005993   0.012429     0.000593          0.0   \n",
       "\n",
       "m/z                                                                     ...  \\\n",
       "temp_bin  (1200, 1300] (1300, 1400] (1400, 1500] (200, 300] (300, 400]  ...   \n",
       "sample_id                                                               ...   \n",
       "S0000              0.0          0.0          0.0   0.001394   0.000188  ...   \n",
       "S0001              0.0          0.0          0.0   0.000000   0.000000  ...   \n",
       "S0002              0.0          0.0          0.0   0.001688   0.000636  ...   \n",
       "S0003              0.0          0.0          0.0   0.001099   0.000992  ...   \n",
       "S0004              0.0          0.0          0.0   0.009380   0.005099  ...   \n",
       "\n",
       "m/z                 99                                                 \\\n",
       "temp_bin  (1400, 1500]    (200, 300] (300, 400] (400, 500] (500, 600]   \n",
       "sample_id                                                               \n",
       "S0000              0.0  2.898509e-06   0.000003   0.000000   0.000000   \n",
       "S0001              0.0  0.000000e+00   0.000000   0.000000   0.000000   \n",
       "S0002              0.0  9.257968e-07   0.000001   0.000001   0.000001   \n",
       "S0003              0.0  4.379099e-06   0.000006   0.000005   0.000005   \n",
       "S0004              0.0  3.904920e-06   0.000004   0.000004   0.000005   \n",
       "\n",
       "m/z                                                       tot_abund  \n",
       "temp_bin     (600, 700] (700, 800] (800, 900] (900, 1000]            \n",
       "sample_id                                                            \n",
       "S0000      0.000000e+00   0.000000   0.000000    0.000000  3.014152  \n",
       "S0001      0.000000e+00   0.000000   0.000000    0.000000  1.760583  \n",
       "S0002      9.397170e-07   0.000001   0.000001    0.000001  4.175911  \n",
       "S0003      4.693477e-06   0.000004   0.000004    0.000004  4.712937  \n",
       "S0004      3.174421e-06   0.000004   0.000004    0.000003  4.288637  \n",
       "\n",
       "[5 rows x 1601 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['tot_abund'] = train_df.sum(axis=1)   # do for test\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>m/z</th>\n",
       "      <th colspan=\"10\" halign=\"left\">0</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"9\" halign=\"left\">99</th>\n",
       "      <th>tot_abund</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_bin</th>\n",
       "      <th>(-100, 0]</th>\n",
       "      <th>(0, 100]</th>\n",
       "      <th>(100, 200]</th>\n",
       "      <th>(1000, 1100]</th>\n",
       "      <th>(1100, 1200]</th>\n",
       "      <th>(1200, 1300]</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>...</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>(400, 500]</th>\n",
       "      <th>(500, 600]</th>\n",
       "      <th>(600, 700]</th>\n",
       "      <th>(700, 800]</th>\n",
       "      <th>(800, 900]</th>\n",
       "      <th>(900, 1000]</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0766</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.410100</td>\n",
       "      <td>0.593334</td>\n",
       "      <td>0.206508</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.606578</td>\n",
       "      <td>0.678006</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001801</td>\n",
       "      <td>0.001626</td>\n",
       "      <td>0.001498</td>\n",
       "      <td>0.001380</td>\n",
       "      <td>0.001495</td>\n",
       "      <td>0.001755</td>\n",
       "      <td>0.001762</td>\n",
       "      <td>0.001607</td>\n",
       "      <td>49.135066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0767</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058563</td>\n",
       "      <td>0.073878</td>\n",
       "      <td>0.088571</td>\n",
       "      <td>0.097939</td>\n",
       "      <td>0.063057</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.081069</td>\n",
       "      <td>0.083869</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000187</td>\n",
       "      <td>0.000173</td>\n",
       "      <td>0.000150</td>\n",
       "      <td>0.000183</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.000133</td>\n",
       "      <td>16.354304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0768</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029256</td>\n",
       "      <td>0.040753</td>\n",
       "      <td>0.013179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.037022</td>\n",
       "      <td>0.041347</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.000248</td>\n",
       "      <td>0.000144</td>\n",
       "      <td>14.889806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0769</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001313</td>\n",
       "      <td>0.001544</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.083412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0770</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003460</td>\n",
       "      <td>0.003929</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>0.002893</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>2.462505</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1601 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "m/z               0                                                 \\\n",
       "temp_bin  (-100, 0]  (0, 100] (100, 200] (1000, 1100] (1100, 1200]   \n",
       "sample_id                                                            \n",
       "S0766           0.0  0.410100   0.593334     0.206508     0.000000   \n",
       "S0767           0.0  0.058563   0.073878     0.088571     0.097939   \n",
       "S0768           0.0  0.029256   0.040753     0.013179     0.000000   \n",
       "S0769           0.0  0.000261   0.000842     0.000000     0.000000   \n",
       "S0770           0.0  0.003460   0.003929     0.000467     0.000000   \n",
       "\n",
       "m/z                                                                     ...  \\\n",
       "temp_bin  (1200, 1300] (1300, 1400] (1400, 1500] (200, 300] (300, 400]  ...   \n",
       "sample_id                                                               ...   \n",
       "S0766         0.000000          0.0          0.0   0.606578   0.678006  ...   \n",
       "S0767         0.063057          0.0          0.0   0.081069   0.083869  ...   \n",
       "S0768         0.000000          0.0          0.0   0.037022   0.041347  ...   \n",
       "S0769         0.000000          0.0          0.0   0.001313   0.001544  ...   \n",
       "S0770         0.000000          0.0          0.0   0.003286   0.002893  ...   \n",
       "\n",
       "m/z                 99                                                         \\\n",
       "temp_bin  (1400, 1500] (200, 300] (300, 400] (400, 500] (500, 600] (600, 700]   \n",
       "sample_id                                                                       \n",
       "S0766              0.0   0.001801   0.001626   0.001498   0.001380   0.001495   \n",
       "S0767              0.0   0.000153   0.000187   0.000173   0.000150   0.000183   \n",
       "S0768              0.0   0.000205   0.000122   0.000152   0.000161   0.000158   \n",
       "S0769              0.0   0.000010   0.000010   0.000008   0.000009   0.000012   \n",
       "S0770              0.0   0.000002   0.000002   0.000002   0.000002   0.000001   \n",
       "\n",
       "m/z                                          tot_abund  \n",
       "temp_bin  (700, 800] (800, 900] (900, 1000]             \n",
       "sample_id                                               \n",
       "S0766       0.001755   0.001762    0.001607  49.135066  \n",
       "S0767       0.000162   0.000146    0.000133  16.354304  \n",
       "S0768       0.000182   0.000248    0.000144  14.889806  \n",
       "S0769       0.000011   0.000012    0.000000   5.083412  \n",
       "S0770       0.000003   0.000002    0.000002   2.462505  \n",
       "\n",
       "[5 rows x 1601 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['tot_abund'] = test_df.sum(axis=1)   # do for test\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mz(mz):\n",
    "    cols_list = []\n",
    "    for col in train_df.columns:\n",
    "        if col[0] == str(mz):\n",
    "            cols_list.append(col)\n",
    "    return cols_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:01<00:00, 51.65it/s]\n",
      "100%|██████████| 100/100 [00:01<00:00, 54.70it/s]\n"
     ]
    }
   ],
   "source": [
    "for mz in tqdm(range(0,100)):\n",
    "    train_df[f\"tot_abund_{mz}\"] = train_df[get_mz(mz)].sum(axis=1)\n",
    "for mz in tqdm(range(0,100)):\n",
    "    test_df[f\"tot_abund_{mz}\"] = test_df[get_mz(mz)].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>m/z</th>\n",
       "      <th colspan=\"10\" halign=\"left\">0</th>\n",
       "      <th>...</th>\n",
       "      <th>tot_abund_90</th>\n",
       "      <th>tot_abund_91</th>\n",
       "      <th>tot_abund_92</th>\n",
       "      <th>tot_abund_93</th>\n",
       "      <th>tot_abund_94</th>\n",
       "      <th>tot_abund_95</th>\n",
       "      <th>tot_abund_96</th>\n",
       "      <th>tot_abund_97</th>\n",
       "      <th>tot_abund_98</th>\n",
       "      <th>tot_abund_99</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temp_bin</th>\n",
       "      <th>(-100, 0]</th>\n",
       "      <th>(0, 100]</th>\n",
       "      <th>(100, 200]</th>\n",
       "      <th>(1000, 1100]</th>\n",
       "      <th>(1100, 1200]</th>\n",
       "      <th>(1200, 1300]</th>\n",
       "      <th>(1300, 1400]</th>\n",
       "      <th>(1400, 1500]</th>\n",
       "      <th>(200, 300]</th>\n",
       "      <th>(300, 400]</th>\n",
       "      <th>...</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0766</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.410100</td>\n",
       "      <td>0.593334</td>\n",
       "      <td>0.206508</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.606578</td>\n",
       "      <td>0.678006</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017303</td>\n",
       "      <td>0.014516</td>\n",
       "      <td>0.016294</td>\n",
       "      <td>0.016206</td>\n",
       "      <td>0.016218</td>\n",
       "      <td>0.016532</td>\n",
       "      <td>0.014431</td>\n",
       "      <td>0.012585</td>\n",
       "      <td>0.014749</td>\n",
       "      <td>0.016790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0767</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.058563</td>\n",
       "      <td>0.073878</td>\n",
       "      <td>0.088571</td>\n",
       "      <td>0.097939</td>\n",
       "      <td>0.063057</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.081069</td>\n",
       "      <td>0.083869</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001840</td>\n",
       "      <td>0.002187</td>\n",
       "      <td>0.002017</td>\n",
       "      <td>0.001949</td>\n",
       "      <td>0.001945</td>\n",
       "      <td>0.002117</td>\n",
       "      <td>0.002137</td>\n",
       "      <td>0.002009</td>\n",
       "      <td>0.002131</td>\n",
       "      <td>0.002034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0768</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029256</td>\n",
       "      <td>0.040753</td>\n",
       "      <td>0.013179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.037022</td>\n",
       "      <td>0.041347</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001564</td>\n",
       "      <td>0.001702</td>\n",
       "      <td>0.001728</td>\n",
       "      <td>0.001620</td>\n",
       "      <td>0.001876</td>\n",
       "      <td>0.001640</td>\n",
       "      <td>0.001640</td>\n",
       "      <td>0.001637</td>\n",
       "      <td>0.001555</td>\n",
       "      <td>0.001742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0769</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001313</td>\n",
       "      <td>0.001544</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000116</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S0770</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003460</td>\n",
       "      <td>0.003929</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003286</td>\n",
       "      <td>0.002893</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S1565</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.086313</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S1566</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000492</td>\n",
       "      <td>0.002167</td>\n",
       "      <td>0.001428</td>\n",
       "      <td>0.000886</td>\n",
       "      <td>0.001625</td>\n",
       "      <td>0.001132</td>\n",
       "      <td>0.001428</td>\n",
       "      <td>0.001280</td>\n",
       "      <td>0.000985</td>\n",
       "      <td>0.000541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S1567</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001261</td>\n",
       "      <td>0.009706</td>\n",
       "      <td>0.004162</td>\n",
       "      <td>0.003262</td>\n",
       "      <td>0.004903</td>\n",
       "      <td>0.002782</td>\n",
       "      <td>0.002662</td>\n",
       "      <td>0.004102</td>\n",
       "      <td>0.002021</td>\n",
       "      <td>0.001141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S1568</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002863</td>\n",
       "      <td>0.012187</td>\n",
       "      <td>0.007543</td>\n",
       "      <td>0.006600</td>\n",
       "      <td>0.008276</td>\n",
       "      <td>0.006111</td>\n",
       "      <td>0.007578</td>\n",
       "      <td>0.007089</td>\n",
       "      <td>0.005029</td>\n",
       "      <td>0.002794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S1569</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.117340</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.069365</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>804 rows × 1701 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "m/z               0                                                 \\\n",
       "temp_bin  (-100, 0]  (0, 100] (100, 200] (1000, 1100] (1100, 1200]   \n",
       "sample_id                                                            \n",
       "S0766           0.0  0.410100   0.593334     0.206508     0.000000   \n",
       "S0767           0.0  0.058563   0.073878     0.088571     0.097939   \n",
       "S0768           0.0  0.029256   0.040753     0.013179     0.000000   \n",
       "S0769           0.0  0.000261   0.000842     0.000000     0.000000   \n",
       "S0770           0.0  0.003460   0.003929     0.000467     0.000000   \n",
       "...             ...       ...        ...          ...          ...   \n",
       "S1565           0.0  0.000000   0.000000     0.000000     0.000000   \n",
       "S1566           0.0  0.000049   0.000000     0.000000     0.000000   \n",
       "S1567           0.0  0.000000   0.000000     0.000000     0.000000   \n",
       "S1568           0.0  0.000000   0.000000     0.000000     0.000000   \n",
       "S1569           0.0  0.000000   0.000000     0.000000     0.000000   \n",
       "\n",
       "m/z                                                                     ...  \\\n",
       "temp_bin  (1200, 1300] (1300, 1400] (1400, 1500] (200, 300] (300, 400]  ...   \n",
       "sample_id                                                               ...   \n",
       "S0766         0.000000          0.0          0.0   0.606578   0.678006  ...   \n",
       "S0767         0.063057          0.0          0.0   0.081069   0.083869  ...   \n",
       "S0768         0.000000          0.0          0.0   0.037022   0.041347  ...   \n",
       "S0769         0.000000          0.0          0.0   0.001313   0.001544  ...   \n",
       "S0770         0.000000          0.0          0.0   0.003286   0.002893  ...   \n",
       "...                ...          ...          ...        ...        ...  ...   \n",
       "S1565         0.000000          0.0          0.0   0.000000   0.000000  ...   \n",
       "S1566         0.000000          0.0          0.0   0.000000   0.000000  ...   \n",
       "S1567         0.000000          0.0          0.0   0.000000   0.000000  ...   \n",
       "S1568         0.000000          0.0          0.0   0.000000   0.000000  ...   \n",
       "S1569         0.000000          0.0          0.0   0.000000   0.000000  ...   \n",
       "\n",
       "m/z       tot_abund_90 tot_abund_91 tot_abund_92 tot_abund_93 tot_abund_94  \\\n",
       "temp_bin                                                                     \n",
       "sample_id                                                                    \n",
       "S0766         0.017303     0.014516     0.016294     0.016206     0.016218   \n",
       "S0767         0.001840     0.002187     0.002017     0.001949     0.001945   \n",
       "S0768         0.001564     0.001702     0.001728     0.001620     0.001876   \n",
       "S0769         0.000112     0.000109     0.000088     0.000108     0.000098   \n",
       "S0770         0.000022     0.000025     0.000022     0.000022     0.000021   \n",
       "...                ...          ...          ...          ...          ...   \n",
       "S1565         0.000000     0.086313     0.000000     0.000000     0.000000   \n",
       "S1566         0.000492     0.002167     0.001428     0.000886     0.001625   \n",
       "S1567         0.001261     0.009706     0.004162     0.003262     0.004903   \n",
       "S1568         0.002863     0.012187     0.007543     0.006600     0.008276   \n",
       "S1569         0.000000     0.117340     0.000000     0.000000     0.069365   \n",
       "\n",
       "m/z       tot_abund_95 tot_abund_96 tot_abund_97 tot_abund_98 tot_abund_99  \n",
       "temp_bin                                                                    \n",
       "sample_id                                                                   \n",
       "S0766         0.016532     0.014431     0.012585     0.014749     0.016790  \n",
       "S0767         0.002117     0.002137     0.002009     0.002131     0.002034  \n",
       "S0768         0.001640     0.001640     0.001637     0.001555     0.001742  \n",
       "S0769         0.000098     0.000116     0.000125     0.000098     0.000093  \n",
       "S0770         0.000022     0.000020     0.000020     0.000022     0.000022  \n",
       "...                ...          ...          ...          ...          ...  \n",
       "S1565         0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "S1566         0.001132     0.001428     0.001280     0.000985     0.000541  \n",
       "S1567         0.002782     0.002662     0.004102     0.002021     0.001141  \n",
       "S1568         0.006111     0.007578     0.007089     0.005029     0.002794  \n",
       "S1569         0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "\n",
       "[804 rows x 1701 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training with imp_fts_index\n",
    "def train_and_predict(col):\n",
    "    train_imp_fts = train_df.iloc[:,imp_fts_dict[col]]\n",
    "    test_imp_fts = test_df.iloc[:,imp_fts_dict[col]]\n",
    "    clf = CatBoostClassifier(random_state=RANDOM_SEED,verbose=False,eval_metric='Logloss',task_type='CPU')\n",
    "    clf.fit(train_imp_fts,train_labels[col])\n",
    "    preds = clf.predict_proba(test_imp_fts)\n",
    "    preds = preds[:,1]\n",
    "    submission_template_df[col] = preds\n",
    "    return clf, preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = {}\n",
    "for col in tqdm(train_labels.columns):\n",
    "    results[col] = train_and_predict(col)\n",
    "print('all good :)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>basalt</th>\n",
       "      <th>carbonate</th>\n",
       "      <th>chloride</th>\n",
       "      <th>iron_oxide</th>\n",
       "      <th>oxalate</th>\n",
       "      <th>oxychlorine</th>\n",
       "      <th>phyllosilicate</th>\n",
       "      <th>silicate</th>\n",
       "      <th>sulfate</th>\n",
       "      <th>sulfide</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S0766</td>\n",
       "      <td>0.008515</td>\n",
       "      <td>0.004105</td>\n",
       "      <td>0.017981</td>\n",
       "      <td>0.009874</td>\n",
       "      <td>0.001412</td>\n",
       "      <td>0.008454</td>\n",
       "      <td>0.019390</td>\n",
       "      <td>0.271978</td>\n",
       "      <td>0.005496</td>\n",
       "      <td>0.002052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S0767</td>\n",
       "      <td>0.077800</td>\n",
       "      <td>0.035944</td>\n",
       "      <td>0.004325</td>\n",
       "      <td>0.036534</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>0.034601</td>\n",
       "      <td>0.045297</td>\n",
       "      <td>0.609227</td>\n",
       "      <td>0.091435</td>\n",
       "      <td>0.001141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S0768</td>\n",
       "      <td>0.484653</td>\n",
       "      <td>0.146311</td>\n",
       "      <td>0.002644</td>\n",
       "      <td>0.081566</td>\n",
       "      <td>0.002529</td>\n",
       "      <td>0.017212</td>\n",
       "      <td>0.394838</td>\n",
       "      <td>0.623719</td>\n",
       "      <td>0.043436</td>\n",
       "      <td>0.000775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>S0769</td>\n",
       "      <td>0.020934</td>\n",
       "      <td>0.012400</td>\n",
       "      <td>0.068250</td>\n",
       "      <td>0.019692</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.879791</td>\n",
       "      <td>0.100869</td>\n",
       "      <td>0.010727</td>\n",
       "      <td>0.871765</td>\n",
       "      <td>0.002046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S0770</td>\n",
       "      <td>0.022837</td>\n",
       "      <td>0.010769</td>\n",
       "      <td>0.014176</td>\n",
       "      <td>0.226716</td>\n",
       "      <td>0.000148</td>\n",
       "      <td>0.980885</td>\n",
       "      <td>0.899091</td>\n",
       "      <td>0.009366</td>\n",
       "      <td>0.014347</td>\n",
       "      <td>0.000715</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  sample_id    basalt  carbonate  chloride  iron_oxide   oxalate  oxychlorine  \\\n",
       "0     S0766  0.008515   0.004105  0.017981    0.009874  0.001412     0.008454   \n",
       "1     S0767  0.077800   0.035944  0.004325    0.036534  0.000102     0.034601   \n",
       "2     S0768  0.484653   0.146311  0.002644    0.081566  0.002529     0.017212   \n",
       "3     S0769  0.020934   0.012400  0.068250    0.019692  0.000095     0.879791   \n",
       "4     S0770  0.022837   0.010769  0.014176    0.226716  0.000148     0.980885   \n",
       "\n",
       "   phyllosilicate  silicate   sulfate   sulfide  \n",
       "0        0.019390  0.271978  0.005496  0.002052  \n",
       "1        0.045297  0.609227  0.091435  0.001141  \n",
       "2        0.394838  0.623719  0.043436  0.000775  \n",
       "3        0.100869  0.010727  0.871765  0.002046  \n",
       "4        0.899091  0.009366  0.014347  0.000715  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_template_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>basalt</th>\n",
       "      <th>carbonate</th>\n",
       "      <th>chloride</th>\n",
       "      <th>iron_oxide</th>\n",
       "      <th>oxalate</th>\n",
       "      <th>oxychlorine</th>\n",
       "      <th>phyllosilicate</th>\n",
       "      <th>silicate</th>\n",
       "      <th>sulfate</th>\n",
       "      <th>sulfide</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "      <td>804.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.078483</td>\n",
       "      <td>0.118999</td>\n",
       "      <td>0.097082</td>\n",
       "      <td>0.169565</td>\n",
       "      <td>0.035201</td>\n",
       "      <td>0.188936</td>\n",
       "      <td>0.296443</td>\n",
       "      <td>0.103878</td>\n",
       "      <td>0.162384</td>\n",
       "      <td>0.037270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.181363</td>\n",
       "      <td>0.268019</td>\n",
       "      <td>0.199739</td>\n",
       "      <td>0.262904</td>\n",
       "      <td>0.174971</td>\n",
       "      <td>0.328374</td>\n",
       "      <td>0.350624</td>\n",
       "      <td>0.206015</td>\n",
       "      <td>0.293087</td>\n",
       "      <td>0.146485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.002834</td>\n",
       "      <td>0.001488</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.003239</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>0.002775</td>\n",
       "      <td>0.002868</td>\n",
       "      <td>0.002762</td>\n",
       "      <td>0.001790</td>\n",
       "      <td>0.000368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.009633</td>\n",
       "      <td>0.006857</td>\n",
       "      <td>0.007400</td>\n",
       "      <td>0.017270</td>\n",
       "      <td>0.000134</td>\n",
       "      <td>0.010645</td>\n",
       "      <td>0.031280</td>\n",
       "      <td>0.010037</td>\n",
       "      <td>0.008597</td>\n",
       "      <td>0.000691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.019441</td>\n",
       "      <td>0.014295</td>\n",
       "      <td>0.017098</td>\n",
       "      <td>0.041274</td>\n",
       "      <td>0.000231</td>\n",
       "      <td>0.025661</td>\n",
       "      <td>0.099027</td>\n",
       "      <td>0.021206</td>\n",
       "      <td>0.018716</td>\n",
       "      <td>0.001284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.047220</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>0.055102</td>\n",
       "      <td>0.159096</td>\n",
       "      <td>0.000868</td>\n",
       "      <td>0.094257</td>\n",
       "      <td>0.582143</td>\n",
       "      <td>0.064119</td>\n",
       "      <td>0.098102</td>\n",
       "      <td>0.004943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.977683</td>\n",
       "      <td>0.982626</td>\n",
       "      <td>0.982037</td>\n",
       "      <td>0.984114</td>\n",
       "      <td>0.997590</td>\n",
       "      <td>0.997344</td>\n",
       "      <td>0.987424</td>\n",
       "      <td>0.975897</td>\n",
       "      <td>0.991398</td>\n",
       "      <td>0.986964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           basalt   carbonate    chloride  iron_oxide     oxalate  \\\n",
       "count  804.000000  804.000000  804.000000  804.000000  804.000000   \n",
       "mean     0.078483    0.118999    0.097082    0.169565    0.035201   \n",
       "std      0.181363    0.268019    0.199739    0.262904    0.174971   \n",
       "min      0.002834    0.001488    0.001984    0.003239    0.000069   \n",
       "25%      0.009633    0.006857    0.007400    0.017270    0.000134   \n",
       "50%      0.019441    0.014295    0.017098    0.041274    0.000231   \n",
       "75%      0.047220    0.040816    0.055102    0.159096    0.000868   \n",
       "max      0.977683    0.982626    0.982037    0.984114    0.997590   \n",
       "\n",
       "       oxychlorine  phyllosilicate    silicate     sulfate     sulfide  \n",
       "count   804.000000      804.000000  804.000000  804.000000  804.000000  \n",
       "mean      0.188936        0.296443    0.103878    0.162384    0.037270  \n",
       "std       0.328374        0.350624    0.206015    0.293087    0.146485  \n",
       "min       0.002775        0.002868    0.002762    0.001790    0.000368  \n",
       "25%       0.010645        0.031280    0.010037    0.008597    0.000691  \n",
       "50%       0.025661        0.099027    0.021206    0.018716    0.001284  \n",
       "75%       0.094257        0.582143    0.064119    0.098102    0.004943  \n",
       "max       0.997344        0.987424    0.975897    0.991398    0.986964  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_template_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_template_df.to_csv('catboost_preds_on_test_imp_fts-2.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int64)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# clf.classes_"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "518a807dccee1bb2cf00c0cea9388abbe4210a3c385c01718c979ef7759eaf87"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}