baseline script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "f7971b7f-d99e-4b55-854d-7d3823404dbb",
    "_uuid": "ccc855613dc9c2986afc718508e97d21213b7213",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import string\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import scipy\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "a58e8629-e64f-4073-818b-b88dbe41583d",
    "_uuid": "d20f7162e1adb56bc3b304f61b7891e7cc3d0521",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gp = pd.read_csv('aggregated_features.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "bc719fcf-f418-4db5-a4fa-ebfa27716521",
    "_uuid": "02aad0b977053435e089bd189ea45e4a88731e87"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>region</th>\n",
       "      <th>city</th>\n",
       "      <th>parent_category_name</th>\n",
       "      <th>category_name</th>\n",
       "      <th>param_1</th>\n",
       "      <th>param_2</th>\n",
       "      <th>param_3</th>\n",
       "      <th>title</th>\n",
       "      <th>...</th>\n",
       "      <th>activation_date</th>\n",
       "      <th>user_type</th>\n",
       "      <th>image</th>\n",
       "      <th>image_top_1</th>\n",
       "      <th>deal_probability</th>\n",
       "      <th>avg_days_up_user</th>\n",
       "      <th>avg_times_up_user</th>\n",
       "      <th>med_days_up_user</th>\n",
       "      <th>med_times_up_user</th>\n",
       "      <th>n_user_items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b912c3c6a6ad</td>\n",
       "      <td>e00f8ff2eaf9</td>\n",
       "      <td>Свердловская область</td>\n",
       "      <td>Екатеринбург</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Постельные принадлежности</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Кокоби(кокон для сна)</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-28</td>\n",
       "      <td>Private</td>\n",
       "      <td>d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...</td>\n",
       "      <td>1008.0</td>\n",
       "      <td>0.12789</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2dac0150717d</td>\n",
       "      <td>39aeb48f0017</td>\n",
       "      <td>Самарская область</td>\n",
       "      <td>Самара</td>\n",
       "      <td>Для дома и дачи</td>\n",
       "      <td>Мебель и интерьер</td>\n",
       "      <td>Другое</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Стойка для Одежды</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-26</td>\n",
       "      <td>Private</td>\n",
       "      <td>79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...</td>\n",
       "      <td>692.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ba83aefab5dc</td>\n",
       "      <td>91e2f88dd6e3</td>\n",
       "      <td>Ростовская область</td>\n",
       "      <td>Ростов-на-Дону</td>\n",
       "      <td>Бытовая электроника</td>\n",
       "      <td>Аудио и видео</td>\n",
       "      <td>Видео, DVD и Blu-ray плееры</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Philips bluray</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-20</td>\n",
       "      <td>Private</td>\n",
       "      <td>b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...</td>\n",
       "      <td>3032.0</td>\n",
       "      <td>0.43177</td>\n",
       "      <td>4.428571</td>\n",
       "      <td>1.142857</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02996f1dd2ea</td>\n",
       "      <td>bf5cccea572d</td>\n",
       "      <td>Татарстан</td>\n",
       "      <td>Набережные Челны</td>\n",
       "      <td>Личные вещи</td>\n",
       "      <td>Товары для детей и игрушки</td>\n",
       "      <td>Автомобильные кресла</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Автокресло</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-25</td>\n",
       "      <td>Company</td>\n",
       "      <td>e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...</td>\n",
       "      <td>796.0</td>\n",
       "      <td>0.80323</td>\n",
       "      <td>16.714286</td>\n",
       "      <td>2.642857</td>\n",
       "      <td>18.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7c90be56d2ab</td>\n",
       "      <td>ef50846afc0b</td>\n",
       "      <td>Волгоградская область</td>\n",
       "      <td>Волгоград</td>\n",
       "      <td>Транспорт</td>\n",
       "      <td>Автомобили</td>\n",
       "      <td>С пробегом</td>\n",
       "      <td>ВАЗ (LADA)</td>\n",
       "      <td>2110</td>\n",
       "      <td>ВАЗ 2110, 2003</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-03-16</td>\n",
       "      <td>Private</td>\n",
       "      <td>54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...</td>\n",
       "      <td>2264.0</td>\n",
       "      <td>0.20797</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_id       user_id                 region              city  \\\n",
       "0  b912c3c6a6ad  e00f8ff2eaf9   Свердловская область      Екатеринбург   \n",
       "1  2dac0150717d  39aeb48f0017      Самарская область            Самара   \n",
       "2  ba83aefab5dc  91e2f88dd6e3     Ростовская область    Ростов-на-Дону   \n",
       "3  02996f1dd2ea  bf5cccea572d              Татарстан  Набережные Челны   \n",
       "4  7c90be56d2ab  ef50846afc0b  Волгоградская область         Волгоград   \n",
       "\n",
       "  parent_category_name               category_name  \\\n",
       "0          Личные вещи  Товары для детей и игрушки   \n",
       "1      Для дома и дачи           Мебель и интерьер   \n",
       "2  Бытовая электроника               Аудио и видео   \n",
       "3          Личные вещи  Товары для детей и игрушки   \n",
       "4            Транспорт                  Автомобили   \n",
       "\n",
       "                       param_1     param_2 param_3                  title  \\\n",
       "0    Постельные принадлежности         NaN     NaN  Кокоби(кокон для сна)   \n",
       "1                       Другое         NaN     NaN      Стойка для Одежды   \n",
       "2  Видео, DVD и Blu-ray плееры         NaN     NaN         Philips bluray   \n",
       "3         Автомобильные кресла         NaN     NaN             Автокресло   \n",
       "4                   С пробегом  ВАЗ (LADA)    2110         ВАЗ 2110, 2003   \n",
       "\n",
       "       ...      activation_date  user_type  \\\n",
       "0      ...           2017-03-28    Private   \n",
       "1      ...           2017-03-26    Private   \n",
       "2      ...           2017-03-20    Private   \n",
       "3      ...           2017-03-25    Company   \n",
       "4      ...           2017-03-16    Private   \n",
       "\n",
       "                                               image image_top_1  \\\n",
       "0  d10c7e016e03247a3bf2d13348fe959fe6f436c1caf64c...      1008.0   \n",
       "1  79c9392cc51a9c81c6eb91eceb8e552171db39d7142700...       692.0   \n",
       "2  b7f250ee3f39e1fedd77c141f273703f4a9be59db4b48a...      3032.0   \n",
       "3  e6ef97e0725637ea84e3d203e82dadb43ed3cc0a1c8413...       796.0   \n",
       "4  54a687a3a0fc1d68aed99bdaaf551c5c70b761b16fd0a2...      2264.0   \n",
       "\n",
       "  deal_probability avg_days_up_user  avg_times_up_user  med_days_up_user  \\\n",
       "0          0.12789         8.000000           2.000000               8.0   \n",
       "1          0.00000              NaN                NaN               NaN   \n",
       "2          0.43177         4.428571           1.142857               3.0   \n",
       "3          0.80323        16.714286           2.642857              18.0   \n",
       "4          0.20797              NaN                NaN               NaN   \n",
       "\n",
       "   med_times_up_user  n_user_items  \n",
       "0                2.0           2.0  \n",
       "1                NaN           NaN  \n",
       "2                1.0           9.0  \n",
       "3                3.0          32.0  \n",
       "4                NaN           NaN  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('data/train.csv', parse_dates=['activation_date'])\n",
    "test = pd.read_csv('data/test.csv', parse_dates=['activation_date'])\n",
    "\n",
    "train = train.merge(gp, on='user_id', how='left')\n",
    "test = test.merge(gp, on='user_id', how='left')\n",
    "\n",
    "agg_cols = list(gp.columns)[1:]\n",
    "\n",
    "del gp; gc.collect()\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "29edf671-7daf-4037-a286-6ad09992cb1b",
    "_uuid": "6fe0d03951f67ff3f28c7c7240089f3631c83c73",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train.sort_values(['activation_date']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "1985e85e-d8d7-4a58-b0c0-7317abd125fe",
    "_uuid": "9fb7f90bc8d3ed7a2e0d5513050bc634d945db67",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, valid_index = train_test_split(np.arange(len(train)), test_size=0.1, random_state=519)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "10353cf1-5b93-4fd1-82ea-f252d054e9d0",
    "_uuid": "88c2d490a28a8a82a0258f1343dd5fed455a2f95"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    207048\n",
       "6    205318\n",
       "1    201131\n",
       "2    196631\n",
       "3    191293\n",
       "4    176527\n",
       "5    175133\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[train_index, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_cell_guid": "c89bd334-407a-4b13-b78b-82752d38ae3a",
    "_uuid": "89044abdca31af70a5373f007ee10920c5477613"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    23007\n",
       "6    22614\n",
       "1    22289\n",
       "2    21884\n",
       "3    21422\n",
       "4    19599\n",
       "5    19528\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[valid_index, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_cell_guid": "64125deb-a81f-459e-b915-975768ac8f38",
    "_uuid": "285fa47da4734b9ecfe7a7a2b8bbc93e9f8c3d3b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    81888\n",
       "1    81114\n",
       "0    80191\n",
       "3    77177\n",
       "4    70366\n",
       "6    58909\n",
       "5    58793\n",
       "Name: activation_date, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.loc[:, 'activation_date'].dt.weekday.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e624308f-c957-48e8-87f6-f9e78fc5b69f",
    "_uuid": "abbecd367f012736903c150e6fb989de3e135cca"
   },
   "source": [
    "One more thing about the approach that I haven't mentioned yet is that we will have quite some NaN values because not every ID in `train` and `test` occurs in `train_active` and `test_active`. Let's check how big that problem is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "d6a65f65-04a0-4d0d-81ec-4ed0551308bc",
    "_uuid": "65a2f2fe6ca76cc9f742affe57d4bd491f29c3bc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.407185198586692"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[agg_cols].isnull().any(axis=1).sum() / len(train) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_cell_guid": "5e26f756-3f9e-4eb7-bcbe-5eaedc7d9f35",
    "_uuid": "e088f55ab84dbd80c5a466e551d9eeb44ca9e623"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24.354198545348694"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[agg_cols].isnull().any(axis=1).sum() / len(test) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(item_id                      0\n",
       " user_id                      0\n",
       " region                       0\n",
       " city                         0\n",
       " parent_category_name         0\n",
       " category_name                0\n",
       " param_1                  61576\n",
       " param_2                 654542\n",
       " param_3                 862565\n",
       " title                        0\n",
       " description             116276\n",
       " price                    85362\n",
       " item_seq_number              0\n",
       " activation_date              0\n",
       " user_type                    0\n",
       " image                   112588\n",
       " image_top_1             112588\n",
       " deal_probability             0\n",
       " avg_days_up_user        336875\n",
       " avg_times_up_user       336875\n",
       " med_days_up_user        336875\n",
       " med_times_up_user       336875\n",
       " n_user_items            336875\n",
       " dtype: int64, item_id                      0\n",
       " user_id                      0\n",
       " region                       0\n",
       " city                         0\n",
       " parent_category_name         0\n",
       " category_name                0\n",
       " param_1                  22910\n",
       " param_2                 233229\n",
       " param_3                 306331\n",
       " title                        0\n",
       " description                  0\n",
       " price                    30585\n",
       " item_seq_number              0\n",
       " activation_date              0\n",
       " user_type                    0\n",
       " image                    42609\n",
       " image_top_1              42609\n",
       " avg_days_up_user        123826\n",
       " avg_times_up_user       123826\n",
       " med_days_up_user        123826\n",
       " med_times_up_user       123826\n",
       " n_user_items            123826\n",
       " dtype: int64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum(), test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "06a9a6c2-a602-48c5-bcb5-d7ed40cc463d",
    "_uuid": "813b1e3a93f96e3bc7d9f2375e452f8b109112fa"
   },
   "source": [
    "We have missing features for 22.41% of train and 24.35% of test data. That's not perfect but certainly acceptable. Onto some more basic feature engineering with ideas from [a great kernel](https://www.kaggle.com/tunguz/bow-meta-text-and-dense-features-lb-0-2241?scriptVersionId=3603709)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_cell_guid": "36649982-801e-4e2b-a954-a0c8aac37702",
    "_uuid": "ea823fe2ee45c3492ca70d75ce3d9a7041d57024",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_cell_guid": "41e1111d-674e-4c86-b232-c0086832ee6d",
    "_uuid": "9c60409e1dc6c618f229e15c2b368660385c2ec1",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])\n",
    "\n",
    "\n",
    "for df in [train, test]:\n",
    "    df['description'].fillna('unknowndescription', inplace=True)\n",
    "    df['title'].fillna('unknowntitle', inplace=True)\n",
    "\n",
    "    df['weekday'] = pd.to_datetime(df['activation_date']).dt.day\n",
    "    \n",
    "    for col in ['description', 'title']:\n",
    "        df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))\n",
    "        df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "    df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100\n",
    "    df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100\n",
    "    \n",
    "    df['city'] = df['region'] + '_' + df['city']\n",
    "    df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "    \n",
    "    for col in agg_cols:\n",
    "        df[col].fillna(-1, inplace=True)\n",
    "    \n",
    "    for col in ['price', 'image_top_1']:\n",
    "         df[col].fillna(-1, inplace=True)\n",
    "            \n",
    "    for col in ['param_1', 'param_2', 'param_3']:\n",
    "         df[col].fillna('khwinkaggle', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\khyeh\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_cell_guid": "8d61ada6-8f3a-4b58-baad-199589122985",
    "_uuid": "cc558a03cbddc3954b01783636785cdee6cf954a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1503424, 16447), (1503424, 15000))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=25)\n",
    "\n",
    "title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))\n",
    "\n",
    "train_title_counts = title_counts[:len(train)]\n",
    "test_title_counts = title_counts[len(train):]\n",
    "\n",
    "\n",
    "count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'), \n",
    "                                        lowercase=True, ngram_range=(1, 2),\n",
    "                                        max_features=15000)\n",
    "\n",
    "desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))\n",
    "\n",
    "train_desc_counts = desc_counts[:len(train)]\n",
    "test_desc_counts = desc_counts[len(train):]\n",
    "\n",
    "train_title_counts.shape, train_desc_counts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_cell_guid": "56338e56-c266-4342-a8f3-10621db49dee",
    "_uuid": "cc6f15f981f05ad9b1e89ab7acb54ea2350278c6",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "predictors = [\n",
    "    'num_desc_punct', \n",
    "    'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',\n",
    "    'avg_times_up_user', 'avg_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n",
    "]\n",
    "\n",
    "predictors = predictors + categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_cell_guid": "7695c1a1-06cf-40e6-8792-77b62f48262a",
    "_uuid": "0bcde9bb9b9b26347c7db7da47123c6fec40860e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming image_top_1...\n",
      "Transforming param_1...\n",
      "Transforming param_2...\n",
      "Transforming param_3...\n",
      "Transforming city...\n",
      "Transforming region...\n",
      "Transforming category_name...\n",
      "Transforming parent_category_name...\n",
      "Transforming user_type...\n"
     ]
    }
   ],
   "source": [
    "for feature in categorical:\n",
    "    print(f'Transforming {feature}...')\n",
    "    encoder = LabelEncoder()\n",
    "    encoder.fit(train[feature].append(test[feature]).astype(str))\n",
    "    \n",
    "    train[feature] = encoder.transform(train[feature].astype(str))\n",
    "    test[feature] = encoder.transform(test[feature].astype(str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "edf2ac02-764e-42d3-bd00-8c1f79878d52",
    "_uuid": "312ce17bcca12df526c85cb9c85e7ac67c9246c7"
   },
   "source": [
    "After some hyperparameter definitions and creating train / valid / test matrices, we can finally train the model. Let's see if the aggregated features helped.\n",
    "\n",
    "*Note: For further feature engineering, I would recommend restricting the max_depth further (5 worked well for me) and increasing the learning rate (to ~ 0.1) so you don't have to wait forever for the training to finish.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_cell_guid": "7e94bcc4-ab56-4a38-bdcf-218791e95c2c",
    "_uuid": "ac57e30cf4452005a2cb20db424d7bcf1f11d1df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features: 31467\n"
     ]
    }
   ],
   "source": [
    "rounds = 24000\n",
    "early_stop_rounds = 200\n",
    "params = {\n",
    "    'objective' : 'regression',\n",
    "    'metric' : 'rmse',\n",
    "    'num_leaves' : 48,\n",
    "    'max_depth': 15,\n",
    "    'learning_rate' : 0.02,\n",
    "    'feature_fraction' : 0.6,\n",
    "    'verbosity' : -1\n",
    "}\n",
    "\n",
    "feature_names = np.hstack([\n",
    "    count_vectorizer_desc.get_feature_names(),\n",
    "    count_vectorizer_title.get_feature_names(),\n",
    "    predictors\n",
    "])\n",
    "print('Number of features:', len(feature_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kf = KFold(5, shuffle=True, random_state=411)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test = scipy.sparse.hstack([\n",
    "    test_desc_counts,\n",
    "    test_title_counts,\n",
    "    test.loc[:, predictors]\n",
    "], format='csr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = scipy.sparse.hstack([\n",
    "        train_desc_counts,\n",
    "        train_title_counts,\n",
    "        train.loc[:, predictors]\n",
    "    ], format='csr')\n",
    "y_train = train.loc[:, target].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from GridSearcher import data_loader, model_loader, fit_params, get_oof_predictions\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ml = model_loader('lgb')\n",
    "SEED=719"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\khyeh\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.224391\tvalid's rmse: 0.227166\n",
      "[200]\ttrain's rmse: 0.221526\tvalid's rmse: 0.225369\n",
      "[300]\ttrain's rmse: 0.219622\tvalid's rmse: 0.224341\n",
      "[400]\ttrain's rmse: 0.218164\tvalid's rmse: 0.223607\n",
      "[500]\ttrain's rmse: 0.217122\tvalid's rmse: 0.223182\n",
      "[600]\ttrain's rmse: 0.216129\tvalid's rmse: 0.222764\n",
      "[700]\ttrain's rmse: 0.215395\tvalid's rmse: 0.222504\n",
      "[800]\ttrain's rmse: 0.214642\tvalid's rmse: 0.222241\n",
      "[900]\ttrain's rmse: 0.214063\tvalid's rmse: 0.222058\n",
      "[1000]\ttrain's rmse: 0.213542\tvalid's rmse: 0.221903\n",
      "[1100]\ttrain's rmse: 0.213032\tvalid's rmse: 0.221764\n",
      "[1200]\ttrain's rmse: 0.212517\tvalid's rmse: 0.221632\n",
      "[1300]\ttrain's rmse: 0.212001\tvalid's rmse: 0.221525\n",
      "[1400]\ttrain's rmse: 0.211595\tvalid's rmse: 0.221439\n",
      "[1500]\ttrain's rmse: 0.211067\tvalid's rmse: 0.221308\n",
      "[1600]\ttrain's rmse: 0.210671\tvalid's rmse: 0.221219\n",
      "[1700]\ttrain's rmse: 0.210166\tvalid's rmse: 0.221079\n",
      "[1800]\ttrain's rmse: 0.209833\tvalid's rmse: 0.221029\n",
      "[1900]\ttrain's rmse: 0.209403\tvalid's rmse: 0.220948\n",
      "[2000]\ttrain's rmse: 0.20904\tvalid's rmse: 0.220896\n",
      "{'random_state': 719} train loss: 0.208969, valid loss:0.220810, loss_diff:0.011841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\khyeh\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.224413\tvalid's rmse: 0.226451\n",
      "[200]\ttrain's rmse: 0.221576\tvalid's rmse: 0.224644\n",
      "[300]\ttrain's rmse: 0.219836\tvalid's rmse: 0.223765\n",
      "[400]\ttrain's rmse: 0.218348\tvalid's rmse: 0.223004\n",
      "[500]\ttrain's rmse: 0.217374\tvalid's rmse: 0.222555\n",
      "[600]\ttrain's rmse: 0.216409\tvalid's rmse: 0.222183\n",
      "[700]\ttrain's rmse: 0.215622\tvalid's rmse: 0.221895\n",
      "[800]\ttrain's rmse: 0.214996\tvalid's rmse: 0.221696\n",
      "[900]\ttrain's rmse: 0.214231\tvalid's rmse: 0.221418\n",
      "[1000]\ttrain's rmse: 0.213698\tvalid's rmse: 0.221266\n",
      "[1100]\ttrain's rmse: 0.213144\tvalid's rmse: 0.221109\n",
      "[1200]\ttrain's rmse: 0.212701\tvalid's rmse: 0.221016\n",
      "[1300]\ttrain's rmse: 0.212097\tvalid's rmse: 0.220831\n",
      "[1400]\ttrain's rmse: 0.211681\tvalid's rmse: 0.220745\n",
      "[1500]\ttrain's rmse: 0.211177\tvalid's rmse: 0.220635\n",
      "[1600]\ttrain's rmse: 0.210712\tvalid's rmse: 0.220548\n",
      "[1700]\ttrain's rmse: 0.210369\tvalid's rmse: 0.220499\n",
      "[1800]\ttrain's rmse: 0.209997\tvalid's rmse: 0.220443\n",
      "[1900]\ttrain's rmse: 0.20956\tvalid's rmse: 0.220349\n",
      "[2000]\ttrain's rmse: 0.209237\tvalid's rmse: 0.220283\n",
      "{'random_state': 719} train loss: 0.209168, valid loss:0.220192, loss_diff:0.011024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\khyeh\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.224687\tvalid's rmse: 0.226328\n",
      "[200]\ttrain's rmse: 0.221914\tvalid's rmse: 0.224582\n",
      "[300]\ttrain's rmse: 0.220136\tvalid's rmse: 0.2236\n",
      "[400]\ttrain's rmse: 0.218703\tvalid's rmse: 0.222927\n",
      "[500]\ttrain's rmse: 0.217561\tvalid's rmse: 0.222439\n",
      "[600]\ttrain's rmse: 0.216586\tvalid's rmse: 0.222044\n",
      "[700]\ttrain's rmse: 0.215818\tvalid's rmse: 0.221773\n",
      "[800]\ttrain's rmse: 0.215078\tvalid's rmse: 0.221566\n",
      "[900]\ttrain's rmse: 0.214482\tvalid's rmse: 0.221379\n",
      "[1000]\ttrain's rmse: 0.213868\tvalid's rmse: 0.221191\n",
      "[1100]\ttrain's rmse: 0.213176\tvalid's rmse: 0.220966\n",
      "[1200]\ttrain's rmse: 0.212674\tvalid's rmse: 0.220842\n",
      "[1300]\ttrain's rmse: 0.212186\tvalid's rmse: 0.22073\n",
      "[1400]\ttrain's rmse: 0.211769\tvalid's rmse: 0.220648\n",
      "[1500]\ttrain's rmse: 0.211209\tvalid's rmse: 0.220522\n",
      "[1600]\ttrain's rmse: 0.210816\tvalid's rmse: 0.22045\n",
      "[1700]\ttrain's rmse: 0.210481\tvalid's rmse: 0.220392\n",
      "[1800]\ttrain's rmse: 0.210112\tvalid's rmse: 0.22035\n",
      "[1900]\ttrain's rmse: 0.20973\tvalid's rmse: 0.220279\n",
      "[2000]\ttrain's rmse: 0.209342\tvalid's rmse: 0.220219\n",
      "{'random_state': 719} train loss: 0.209273, valid loss:0.220135, loss_diff:0.010862\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\khyeh\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.224314\tvalid's rmse: 0.22669\n",
      "[200]\ttrain's rmse: 0.221403\tvalid's rmse: 0.224903\n",
      "[300]\ttrain's rmse: 0.219697\tvalid's rmse: 0.224033\n",
      "[400]\ttrain's rmse: 0.218381\tvalid's rmse: 0.223409\n",
      "[500]\ttrain's rmse: 0.21725\tvalid's rmse: 0.222901\n",
      "[600]\ttrain's rmse: 0.216342\tvalid's rmse: 0.222531\n",
      "[700]\ttrain's rmse: 0.215613\tvalid's rmse: 0.222276\n",
      "[800]\ttrain's rmse: 0.214852\tvalid's rmse: 0.222026\n",
      "[900]\ttrain's rmse: 0.214238\tvalid's rmse: 0.221831\n",
      "[1000]\ttrain's rmse: 0.213685\tvalid's rmse: 0.221669\n",
      "[1100]\ttrain's rmse: 0.213138\tvalid's rmse: 0.22152\n",
      "[1200]\ttrain's rmse: 0.21271\tvalid's rmse: 0.221419\n",
      "[1300]\ttrain's rmse: 0.212193\tvalid's rmse: 0.221306\n",
      "[1400]\ttrain's rmse: 0.211751\tvalid's rmse: 0.221217\n",
      "[1500]\ttrain's rmse: 0.211205\tvalid's rmse: 0.22106\n",
      "[1600]\ttrain's rmse: 0.210831\tvalid's rmse: 0.220977\n",
      "[1700]\ttrain's rmse: 0.210424\tvalid's rmse: 0.220908\n",
      "[1800]\ttrain's rmse: 0.210016\tvalid's rmse: 0.220825\n",
      "[1900]\ttrain's rmse: 0.209663\tvalid's rmse: 0.220757\n",
      "[2000]\ttrain's rmse: 0.209328\tvalid's rmse: 0.220699\n",
      "{'random_state': 719} train loss: 0.209262, valid loss:0.220617, loss_diff:0.011355\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\khyeh\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1007: UserWarning: categorical_feature in Dataset is overrided. New categorical_feature is ['category_name', 'city', 'image_top_1', 'param_1', 'param_2', 'param_3', 'parent_category_name', 'region', 'user_type']\n",
      "  warnings.warn('categorical_feature in Dataset is overrided. New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 50 rounds.\n",
      "[100]\ttrain's rmse: 0.224699\tvalid's rmse: 0.22563\n",
      "[200]\ttrain's rmse: 0.221786\tvalid's rmse: 0.22386\n",
      "[300]\ttrain's rmse: 0.219973\tvalid's rmse: 0.222908\n",
      "[400]\ttrain's rmse: 0.218578\tvalid's rmse: 0.222245\n",
      "[500]\ttrain's rmse: 0.217459\tvalid's rmse: 0.221759\n",
      "[600]\ttrain's rmse: 0.216474\tvalid's rmse: 0.221379\n",
      "[700]\ttrain's rmse: 0.215708\tvalid's rmse: 0.221116\n",
      "[800]\ttrain's rmse: 0.215004\tvalid's rmse: 0.220874\n",
      "[900]\ttrain's rmse: 0.214244\tvalid's rmse: 0.220615\n",
      "[1000]\ttrain's rmse: 0.213658\tvalid's rmse: 0.220458\n",
      "[1100]\ttrain's rmse: 0.213139\tvalid's rmse: 0.220323\n",
      "[1200]\ttrain's rmse: 0.212697\tvalid's rmse: 0.220237\n",
      "[1300]\ttrain's rmse: 0.212221\tvalid's rmse: 0.220128\n",
      "[1400]\ttrain's rmse: 0.211746\tvalid's rmse: 0.220025\n",
      "[1500]\ttrain's rmse: 0.211333\tvalid's rmse: 0.219929\n",
      "[1600]\ttrain's rmse: 0.210835\tvalid's rmse: 0.219824\n",
      "[1700]\ttrain's rmse: 0.21042\tvalid's rmse: 0.219741\n",
      "[1800]\ttrain's rmse: 0.210059\tvalid's rmse: 0.219696\n",
      "[1900]\ttrain's rmse: 0.209712\tvalid's rmse: 0.219655\n",
      "[2000]\ttrain's rmse: 0.209333\tvalid's rmse: 0.219595\n",
      "{'random_state': 719} train loss: 0.209263, valid loss:0.219507, loss_diff:0.010244\n",
      "=================>{'random_state': 719} loss:0.220252\n",
      "Best params: {'random_state': 719} \tbest loss: 0.220252129404\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>param</th>\n",
       "      <th>val_loss_mean</th>\n",
       "      <th>val_loss_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'random_state': 719}</td>\n",
       "      <td>0.220252</td>\n",
       "      <td>0.000451</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   param  val_loss_mean  val_loss_std\n",
       "0  {'random_state': 719}       0.220252      0.000451"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_params = {\n",
    "    'boosting_type':'gbdt', \n",
    "    'num_leaves':31, \n",
    "    'max_depth':5, \n",
    "    'learning_rate':0.1, \n",
    "    'n_estimators':2000, \n",
    "    'min_split_gain':0.0, \n",
    "    'min_child_weight':0.001, \n",
    "    'min_child_samples':20, \n",
    "    'subsample':1., \n",
    "    'subsample_freq':1, \n",
    "    'colsample_bytree':.6, \n",
    "    'reg_alpha':0.0, \n",
    "    'reg_lambda':0.0, \n",
    "    'random_state':SEED, \n",
    "    'n_jobs': 3\n",
    "}\n",
    "fit_param = {\n",
    "    'feature_name': list(feature_names), \n",
    "    'categorical_feature': categorical,\n",
    "}\n",
    "fit_param.update({\n",
    "    'early_stopping_rounds': 50,\n",
    "    'verbose': 100,\n",
    "    'eval_metric': 'rmse'\n",
    "})\n",
    "\n",
    "try_params = {\n",
    "    'random_state': [719]\n",
    "}\n",
    "\n",
    "fit_params(x_train, y_train, ml, default_params, try_params, fit_params=fit_param, seed=SEED, use_eval_set=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9e910115-312f-4437-8ba3-9fb6f7426af8",
    "_uuid": "1a50b3e34d76019fdbfd45c3566eec09556f24cb",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\lightgbm\\basic.py:1005: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's rmse: 0.228141\tvalid's rmse: 0.229112\n",
      "[200]\ttrain's rmse: 0.22355\tvalid's rmse: 0.225547\n",
      "[300]\ttrain's rmse: 0.220796\tvalid's rmse: 0.223783\n",
      "[400]\ttrain's rmse: 0.219092\tvalid's rmse: 0.222865\n",
      "[500]\ttrain's rmse: 0.217913\tvalid's rmse: 0.22228\n",
      "[600]\ttrain's rmse: 0.216963\tvalid's rmse: 0.221866\n",
      "[700]\ttrain's rmse: 0.216158\tvalid's rmse: 0.221532\n",
      "[800]\ttrain's rmse: 0.215481\tvalid's rmse: 0.221277\n",
      "[900]\ttrain's rmse: 0.214836\tvalid's rmse: 0.221023\n",
      "[1000]\ttrain's rmse: 0.214268\tvalid's rmse: 0.220826\n",
      "[1100]\ttrain's rmse: 0.213764\tvalid's rmse: 0.220649\n",
      "[1200]\ttrain's rmse: 0.213273\tvalid's rmse: 0.22049\n",
      "[1300]\ttrain's rmse: 0.212822\tvalid's rmse: 0.22036\n",
      "[1400]\ttrain's rmse: 0.212349\tvalid's rmse: 0.220228\n",
      "[1500]\ttrain's rmse: 0.211926\tvalid's rmse: 0.220111\n",
      "[1600]\ttrain's rmse: 0.211526\tvalid's rmse: 0.220012\n",
      "[1700]\ttrain's rmse: 0.211131\tvalid's rmse: 0.219911\n",
      "[1800]\ttrain's rmse: 0.210756\tvalid's rmse: 0.219827\n",
      "[1900]\ttrain's rmse: 0.210408\tvalid's rmse: 0.21975\n",
      "[2000]\ttrain's rmse: 0.210054\tvalid's rmse: 0.219679\n",
      "[2100]\ttrain's rmse: 0.209743\tvalid's rmse: 0.219619\n",
      "[2200]\ttrain's rmse: 0.209419\tvalid's rmse: 0.21956\n",
      "[2300]\ttrain's rmse: 0.209102\tvalid's rmse: 0.219496\n",
      "[2400]\ttrain's rmse: 0.208817\tvalid's rmse: 0.219449\n",
      "[2500]\ttrain's rmse: 0.208537\tvalid's rmse: 0.219393\n",
      "[2600]\ttrain's rmse: 0.208257\tvalid's rmse: 0.219345\n",
      "[2700]\ttrain's rmse: 0.207985\tvalid's rmse: 0.219294\n",
      "[2800]\ttrain's rmse: 0.207702\tvalid's rmse: 0.21925\n",
      "[2900]\ttrain's rmse: 0.207432\tvalid's rmse: 0.219209\n",
      "[3000]\ttrain's rmse: 0.20718\tvalid's rmse: 0.219173\n",
      "[3100]\ttrain's rmse: 0.206928\tvalid's rmse: 0.219137\n",
      "[3200]\ttrain's rmse: 0.206663\tvalid's rmse: 0.219097\n",
      "[3300]\ttrain's rmse: 0.206416\tvalid's rmse: 0.219066\n",
      "[3400]\ttrain's rmse: 0.206166\tvalid's rmse: 0.219036\n",
      "[3500]\ttrain's rmse: 0.205914\tvalid's rmse: 0.219003\n",
      "[3600]\ttrain's rmse: 0.205664\tvalid's rmse: 0.218972\n",
      "[3700]\ttrain's rmse: 0.205425\tvalid's rmse: 0.218938\n",
      "[3800]\ttrain's rmse: 0.205203\tvalid's rmse: 0.218914\n",
      "[3900]\ttrain's rmse: 0.204968\tvalid's rmse: 0.218888\n",
      "[4000]\ttrain's rmse: 0.204753\tvalid's rmse: 0.218867\n",
      "[4100]\ttrain's rmse: 0.204526\tvalid's rmse: 0.21884\n",
      "[4200]\ttrain's rmse: 0.204307\tvalid's rmse: 0.218813\n",
      "[4300]\ttrain's rmse: 0.204094\tvalid's rmse: 0.218794\n",
      "[4400]\ttrain's rmse: 0.203876\tvalid's rmse: 0.218771\n",
      "[4500]\ttrain's rmse: 0.203654\tvalid's rmse: 0.218749\n",
      "[4600]\ttrain's rmse: 0.203441\tvalid's rmse: 0.21873\n",
      "[4700]\ttrain's rmse: 0.203234\tvalid's rmse: 0.21871\n",
      "[4800]\ttrain's rmse: 0.203027\tvalid's rmse: 0.218695\n",
      "[4900]\ttrain's rmse: 0.202822\tvalid's rmse: 0.21868\n",
      "[5000]\ttrain's rmse: 0.202615\tvalid's rmse: 0.218661\n",
      "[5100]\ttrain's rmse: 0.202425\tvalid's rmse: 0.218648\n",
      "[5200]\ttrain's rmse: 0.202235\tvalid's rmse: 0.218632\n",
      "[5300]\ttrain's rmse: 0.202036\tvalid's rmse: 0.218615\n",
      "[5400]\ttrain's rmse: 0.20183\tvalid's rmse: 0.218598\n",
      "[5500]\ttrain's rmse: 0.201632\tvalid's rmse: 0.218578\n",
      "[5600]\ttrain's rmse: 0.201439\tvalid's rmse: 0.218566\n",
      "[5700]\ttrain's rmse: 0.201248\tvalid's rmse: 0.218554\n",
      "[5800]\ttrain's rmse: 0.20107\tvalid's rmse: 0.218542\n",
      "[5900]\ttrain's rmse: 0.200893\tvalid's rmse: 0.218533\n",
      "[6000]\ttrain's rmse: 0.200684\tvalid's rmse: 0.218519\n",
      "[6100]\ttrain's rmse: 0.200492\tvalid's rmse: 0.218504\n",
      "[6200]\ttrain's rmse: 0.200302\tvalid's rmse: 0.218494\n",
      "[6300]\ttrain's rmse: 0.200109\tvalid's rmse: 0.21848\n",
      "[6400]\ttrain's rmse: 0.199929\tvalid's rmse: 0.218472\n",
      "[6500]\ttrain's rmse: 0.199745\tvalid's rmse: 0.218461\n",
      "[6600]\ttrain's rmse: 0.199572\tvalid's rmse: 0.218451\n",
      "[6700]\ttrain's rmse: 0.199386\tvalid's rmse: 0.218442\n",
      "[6800]\ttrain's rmse: 0.199212\tvalid's rmse: 0.21843\n",
      "[6900]\ttrain's rmse: 0.19903\tvalid's rmse: 0.218422\n",
      "[7000]\ttrain's rmse: 0.198855\tvalid's rmse: 0.218413\n",
      "[7100]\ttrain's rmse: 0.198674\tvalid's rmse: 0.218403\n",
      "[7200]\ttrain's rmse: 0.198493\tvalid's rmse: 0.218392\n",
      "[7300]\ttrain's rmse: 0.198329\tvalid's rmse: 0.218383\n",
      "[7400]\ttrain's rmse: 0.19816\tvalid's rmse: 0.218376\n",
      "[7500]\ttrain's rmse: 0.197979\tvalid's rmse: 0.218366\n",
      "[7600]\ttrain's rmse: 0.197815\tvalid's rmse: 0.218361\n",
      "[7700]\ttrain's rmse: 0.197615\tvalid's rmse: 0.218348\n",
      "[7800]\ttrain's rmse: 0.197457\tvalid's rmse: 0.218343\n",
      "[7900]\ttrain's rmse: 0.197281\tvalid's rmse: 0.218336\n",
      "[8000]\ttrain's rmse: 0.19711\tvalid's rmse: 0.218329\n",
      "[8100]\ttrain's rmse: 0.196942\tvalid's rmse: 0.218319\n",
      "[8200]\ttrain's rmse: 0.19676\tvalid's rmse: 0.218314\n",
      "[8300]\ttrain's rmse: 0.196586\tvalid's rmse: 0.218306\n",
      "[8400]\ttrain's rmse: 0.196409\tvalid's rmse: 0.218297\n",
      "[8500]\ttrain's rmse: 0.196261\tvalid's rmse: 0.218294\n",
      "[8600]\ttrain's rmse: 0.196098\tvalid's rmse: 0.21829\n",
      "[8700]\ttrain's rmse: 0.195933\tvalid's rmse: 0.218283\n",
      "[8800]\ttrain's rmse: 0.195775\tvalid's rmse: 0.218274\n",
      "[8900]\ttrain's rmse: 0.195626\tvalid's rmse: 0.21827\n",
      "[9000]\ttrain's rmse: 0.195472\tvalid's rmse: 0.218261\n",
      "[9100]\ttrain's rmse: 0.195306\tvalid's rmse: 0.218257\n",
      "[9200]\ttrain's rmse: 0.195158\tvalid's rmse: 0.21825\n",
      "[9300]\ttrain's rmse: 0.195002\tvalid's rmse: 0.218247\n",
      "[9400]\ttrain's rmse: 0.194844\tvalid's rmse: 0.218245\n",
      "[9500]\ttrain's rmse: 0.194681\tvalid's rmse: 0.218244\n",
      "[9600]\ttrain's rmse: 0.194534\tvalid's rmse: 0.218242\n",
      "[9700]\ttrain's rmse: 0.194375\tvalid's rmse: 0.218236\n",
      "[9800]\ttrain's rmse: 0.194224\tvalid's rmse: 0.218231\n",
      "[9900]\ttrain's rmse: 0.194075\tvalid's rmse: 0.218224\n",
      "[10000]\ttrain's rmse: 0.193921\tvalid's rmse: 0.218221\n",
      "[10100]\ttrain's rmse: 0.193752\tvalid's rmse: 0.218214\n",
      "[10200]\ttrain's rmse: 0.193593\tvalid's rmse: 0.218208\n",
      "[10300]\ttrain's rmse: 0.193442\tvalid's rmse: 0.218206\n",
      "[10400]\ttrain's rmse: 0.193299\tvalid's rmse: 0.218204\n",
      "[10500]\ttrain's rmse: 0.193163\tvalid's rmse: 0.2182\n",
      "[10600]\ttrain's rmse: 0.193009\tvalid's rmse: 0.21819\n",
      "[10700]\ttrain's rmse: 0.192876\tvalid's rmse: 0.218189\n",
      "[10800]\ttrain's rmse: 0.192734\tvalid's rmse: 0.218187\n",
      "[10900]\ttrain's rmse: 0.192586\tvalid's rmse: 0.218182\n",
      "[11000]\ttrain's rmse: 0.192435\tvalid's rmse: 0.218178\n",
      "[11100]\ttrain's rmse: 0.192285\tvalid's rmse: 0.218171\n",
      "[11200]\ttrain's rmse: 0.192138\tvalid's rmse: 0.218168\n",
      "[11300]\ttrain's rmse: 0.191987\tvalid's rmse: 0.218162\n",
      "[11400]\ttrain's rmse: 0.191829\tvalid's rmse: 0.218159\n",
      "[11500]\ttrain's rmse: 0.191686\tvalid's rmse: 0.218157\n",
      "[11600]\ttrain's rmse: 0.191552\tvalid's rmse: 0.218154\n",
      "[11700]\ttrain's rmse: 0.191407\tvalid's rmse: 0.218157\n",
      "[11800]\ttrain's rmse: 0.191265\tvalid's rmse: 0.218154\n",
      "[11900]\ttrain's rmse: 0.191135\tvalid's rmse: 0.218153\n",
      "[12000]\ttrain's rmse: 0.190989\tvalid's rmse: 0.218153\n",
      "Early stopping, best iteration is:\n",
      "[11856]\ttrain's rmse: 0.191193\tvalid's rmse: 0.218151\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's rmse: 0.227866\tvalid's rmse: 0.229895\n",
      "[200]\ttrain's rmse: 0.223275\tvalid's rmse: 0.226444\n",
      "[300]\ttrain's rmse: 0.220538\tvalid's rmse: 0.224718\n",
      "[400]\ttrain's rmse: 0.218821\tvalid's rmse: 0.223792\n",
      "[500]\ttrain's rmse: 0.21768\tvalid's rmse: 0.223223\n",
      "[600]\ttrain's rmse: 0.216707\tvalid's rmse: 0.222814\n",
      "[700]\ttrain's rmse: 0.215924\tvalid's rmse: 0.222506\n",
      "[800]\ttrain's rmse: 0.215223\tvalid's rmse: 0.222233\n",
      "[900]\ttrain's rmse: 0.214609\tvalid's rmse: 0.222021\n",
      "[1000]\ttrain's rmse: 0.214044\tvalid's rmse: 0.221824\n",
      "[1100]\ttrain's rmse: 0.21351\tvalid's rmse: 0.221648\n",
      "[1200]\ttrain's rmse: 0.213027\tvalid's rmse: 0.221505\n",
      "[1300]\ttrain's rmse: 0.212522\tvalid's rmse: 0.221356\n",
      "[1400]\ttrain's rmse: 0.21209\tvalid's rmse: 0.22124\n",
      "[1500]\ttrain's rmse: 0.211669\tvalid's rmse: 0.221124\n",
      "[1600]\ttrain's rmse: 0.211274\tvalid's rmse: 0.22103\n",
      "[1700]\ttrain's rmse: 0.210874\tvalid's rmse: 0.220931\n",
      "[1800]\ttrain's rmse: 0.210513\tvalid's rmse: 0.220848\n",
      "[1900]\ttrain's rmse: 0.210185\tvalid's rmse: 0.220785\n",
      "[2000]\ttrain's rmse: 0.209845\tvalid's rmse: 0.220725\n",
      "[2100]\ttrain's rmse: 0.209505\tvalid's rmse: 0.220658\n",
      "[2200]\ttrain's rmse: 0.209194\tvalid's rmse: 0.220591\n",
      "[2300]\ttrain's rmse: 0.208896\tvalid's rmse: 0.22054\n",
      "[2400]\ttrain's rmse: 0.208584\tvalid's rmse: 0.220486\n",
      "[2500]\ttrain's rmse: 0.208281\tvalid's rmse: 0.220439\n",
      "[2600]\ttrain's rmse: 0.208006\tvalid's rmse: 0.220406\n",
      "[2700]\ttrain's rmse: 0.20773\tvalid's rmse: 0.220365\n",
      "[2800]\ttrain's rmse: 0.207434\tvalid's rmse: 0.220318\n",
      "[2900]\ttrain's rmse: 0.207178\tvalid's rmse: 0.220284\n",
      "[3000]\ttrain's rmse: 0.206919\tvalid's rmse: 0.220243\n",
      "[3100]\ttrain's rmse: 0.206656\tvalid's rmse: 0.220202\n",
      "[3200]\ttrain's rmse: 0.206415\tvalid's rmse: 0.220179\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3300]\ttrain's rmse: 0.20616\tvalid's rmse: 0.220148\n",
      "[3400]\ttrain's rmse: 0.205919\tvalid's rmse: 0.220123\n",
      "[3500]\ttrain's rmse: 0.205682\tvalid's rmse: 0.220096\n",
      "[3600]\ttrain's rmse: 0.205444\tvalid's rmse: 0.220066\n",
      "[3700]\ttrain's rmse: 0.20522\tvalid's rmse: 0.220043\n",
      "[3800]\ttrain's rmse: 0.204996\tvalid's rmse: 0.220018\n",
      "[3900]\ttrain's rmse: 0.204773\tvalid's rmse: 0.219992\n",
      "[4000]\ttrain's rmse: 0.204541\tvalid's rmse: 0.219971\n",
      "[4100]\ttrain's rmse: 0.204327\tvalid's rmse: 0.219952\n",
      "[4200]\ttrain's rmse: 0.204075\tvalid's rmse: 0.219921\n",
      "[4300]\ttrain's rmse: 0.203869\tvalid's rmse: 0.2199\n",
      "[4400]\ttrain's rmse: 0.203649\tvalid's rmse: 0.219878\n",
      "[4500]\ttrain's rmse: 0.203441\tvalid's rmse: 0.219855\n",
      "[4600]\ttrain's rmse: 0.203231\tvalid's rmse: 0.219834\n",
      "[4700]\ttrain's rmse: 0.202997\tvalid's rmse: 0.219806\n",
      "[4800]\ttrain's rmse: 0.202796\tvalid's rmse: 0.219793\n",
      "[4900]\ttrain's rmse: 0.202589\tvalid's rmse: 0.219775\n",
      "[5000]\ttrain's rmse: 0.202387\tvalid's rmse: 0.219764\n",
      "[5100]\ttrain's rmse: 0.202183\tvalid's rmse: 0.219751\n",
      "[5200]\ttrain's rmse: 0.20199\tvalid's rmse: 0.219736\n",
      "[5300]\ttrain's rmse: 0.201799\tvalid's rmse: 0.21972\n",
      "[5400]\ttrain's rmse: 0.201599\tvalid's rmse: 0.219698\n",
      "[5500]\ttrain's rmse: 0.201403\tvalid's rmse: 0.21968\n",
      "[5600]\ttrain's rmse: 0.201207\tvalid's rmse: 0.219674\n",
      "[5700]\ttrain's rmse: 0.20101\tvalid's rmse: 0.219659\n",
      "[5800]\ttrain's rmse: 0.200825\tvalid's rmse: 0.219649\n",
      "[5900]\ttrain's rmse: 0.200635\tvalid's rmse: 0.219638\n",
      "[6000]\ttrain's rmse: 0.200438\tvalid's rmse: 0.219622\n",
      "[6100]\ttrain's rmse: 0.200253\tvalid's rmse: 0.219608\n",
      "[6200]\ttrain's rmse: 0.200068\tvalid's rmse: 0.219599\n",
      "[6300]\ttrain's rmse: 0.199898\tvalid's rmse: 0.219587\n",
      "[6400]\ttrain's rmse: 0.199716\tvalid's rmse: 0.219573\n",
      "[6500]\ttrain's rmse: 0.199542\tvalid's rmse: 0.219561\n",
      "[6600]\ttrain's rmse: 0.199347\tvalid's rmse: 0.219548\n",
      "[6700]\ttrain's rmse: 0.199173\tvalid's rmse: 0.219543\n",
      "[6800]\ttrain's rmse: 0.198999\tvalid's rmse: 0.219536\n",
      "[6900]\ttrain's rmse: 0.198839\tvalid's rmse: 0.21953\n",
      "[7000]\ttrain's rmse: 0.198669\tvalid's rmse: 0.219522\n",
      "[7100]\ttrain's rmse: 0.19849\tvalid's rmse: 0.219513\n",
      "[7200]\ttrain's rmse: 0.198317\tvalid's rmse: 0.219504\n",
      "[7300]\ttrain's rmse: 0.198146\tvalid's rmse: 0.219495\n",
      "[7400]\ttrain's rmse: 0.197977\tvalid's rmse: 0.219489\n",
      "[7500]\ttrain's rmse: 0.197802\tvalid's rmse: 0.219476\n",
      "[7600]\ttrain's rmse: 0.197632\tvalid's rmse: 0.219468\n",
      "[7700]\ttrain's rmse: 0.197476\tvalid's rmse: 0.219467\n",
      "[7800]\ttrain's rmse: 0.197306\tvalid's rmse: 0.219462\n",
      "[7900]\ttrain's rmse: 0.197142\tvalid's rmse: 0.219452\n",
      "[8000]\ttrain's rmse: 0.19697\tvalid's rmse: 0.219446\n",
      "[8100]\ttrain's rmse: 0.196805\tvalid's rmse: 0.219442\n",
      "[8200]\ttrain's rmse: 0.196632\tvalid's rmse: 0.219435\n",
      "[8300]\ttrain's rmse: 0.196475\tvalid's rmse: 0.219431\n",
      "[8400]\ttrain's rmse: 0.196316\tvalid's rmse: 0.219423\n",
      "[8500]\ttrain's rmse: 0.196155\tvalid's rmse: 0.219415\n",
      "[8600]\ttrain's rmse: 0.195992\tvalid's rmse: 0.219412\n",
      "[8700]\ttrain's rmse: 0.195833\tvalid's rmse: 0.219408\n",
      "[8800]\ttrain's rmse: 0.195687\tvalid's rmse: 0.219404\n",
      "[8900]\ttrain's rmse: 0.19552\tvalid's rmse: 0.219395\n",
      "[9000]\ttrain's rmse: 0.195361\tvalid's rmse: 0.219389\n",
      "[9100]\ttrain's rmse: 0.195203\tvalid's rmse: 0.219384\n",
      "[9200]\ttrain's rmse: 0.195055\tvalid's rmse: 0.21938\n",
      "[9300]\ttrain's rmse: 0.194901\tvalid's rmse: 0.219375\n",
      "[9400]\ttrain's rmse: 0.194743\tvalid's rmse: 0.21937\n",
      "[9500]\ttrain's rmse: 0.194572\tvalid's rmse: 0.219364\n",
      "[9600]\ttrain's rmse: 0.19442\tvalid's rmse: 0.219364\n",
      "[9700]\ttrain's rmse: 0.194263\tvalid's rmse: 0.21936\n",
      "[9800]\ttrain's rmse: 0.194091\tvalid's rmse: 0.219352\n",
      "[9900]\ttrain's rmse: 0.193938\tvalid's rmse: 0.219348\n",
      "[10000]\ttrain's rmse: 0.193793\tvalid's rmse: 0.219346\n",
      "[10100]\ttrain's rmse: 0.193636\tvalid's rmse: 0.219343\n",
      "[10200]\ttrain's rmse: 0.19348\tvalid's rmse: 0.219338\n",
      "[10300]\ttrain's rmse: 0.193325\tvalid's rmse: 0.219334\n",
      "[10400]\ttrain's rmse: 0.193185\tvalid's rmse: 0.219331\n",
      "[10500]\ttrain's rmse: 0.193046\tvalid's rmse: 0.219327\n",
      "[10600]\ttrain's rmse: 0.192908\tvalid's rmse: 0.219326\n",
      "[10700]\ttrain's rmse: 0.19276\tvalid's rmse: 0.219324\n",
      "[10800]\ttrain's rmse: 0.19261\tvalid's rmse: 0.219323\n",
      "[10900]\ttrain's rmse: 0.192464\tvalid's rmse: 0.219321\n",
      "[11000]\ttrain's rmse: 0.192313\tvalid's rmse: 0.219317\n",
      "[11100]\ttrain's rmse: 0.192157\tvalid's rmse: 0.219314\n",
      "[11200]\ttrain's rmse: 0.19202\tvalid's rmse: 0.219313\n",
      "[11300]\ttrain's rmse: 0.191885\tvalid's rmse: 0.219311\n",
      "[11400]\ttrain's rmse: 0.191742\tvalid's rmse: 0.21931\n",
      "[11500]\ttrain's rmse: 0.191596\tvalid's rmse: 0.219309\n",
      "[11600]\ttrain's rmse: 0.19145\tvalid's rmse: 0.219307\n",
      "[11700]\ttrain's rmse: 0.191303\tvalid's rmse: 0.219305\n",
      "[11800]\ttrain's rmse: 0.191165\tvalid's rmse: 0.219303\n",
      "[11900]\ttrain's rmse: 0.191017\tvalid's rmse: 0.219302\n",
      "[12000]\ttrain's rmse: 0.190876\tvalid's rmse: 0.219301\n",
      "[12100]\ttrain's rmse: 0.190738\tvalid's rmse: 0.2193\n",
      "[12200]\ttrain's rmse: 0.190597\tvalid's rmse: 0.219298\n",
      "[12300]\ttrain's rmse: 0.190462\tvalid's rmse: 0.219297\n",
      "[12400]\ttrain's rmse: 0.190324\tvalid's rmse: 0.219298\n",
      "[12500]\ttrain's rmse: 0.1902\tvalid's rmse: 0.219295\n",
      "[12600]\ttrain's rmse: 0.190065\tvalid's rmse: 0.219297\n",
      "[12700]\ttrain's rmse: 0.189928\tvalid's rmse: 0.219298\n",
      "[12800]\ttrain's rmse: 0.189791\tvalid's rmse: 0.219295\n",
      "[12900]\ttrain's rmse: 0.189644\tvalid's rmse: 0.219297\n",
      "Early stopping, best iteration is:\n",
      "[12791]\ttrain's rmse: 0.189806\tvalid's rmse: 0.219294\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[100]\ttrain's rmse: 0.228305\tvalid's rmse: 0.228421\n",
      "[200]\ttrain's rmse: 0.2237\tvalid's rmse: 0.224961\n",
      "[300]\ttrain's rmse: 0.22094\tvalid's rmse: 0.223238\n",
      "[400]\ttrain's rmse: 0.219235\tvalid's rmse: 0.222339\n",
      "[500]\ttrain's rmse: 0.218056\tvalid's rmse: 0.221763\n",
      "[600]\ttrain's rmse: 0.21712\tvalid's rmse: 0.221377\n",
      "[700]\ttrain's rmse: 0.216275\tvalid's rmse: 0.221014\n",
      "[800]\ttrain's rmse: 0.215599\tvalid's rmse: 0.220749\n",
      "[900]\ttrain's rmse: 0.214997\tvalid's rmse: 0.22054\n",
      "[1000]\ttrain's rmse: 0.214417\tvalid's rmse: 0.220339\n",
      "[1100]\ttrain's rmse: 0.213912\tvalid's rmse: 0.220186\n",
      "[1200]\ttrain's rmse: 0.2134\tvalid's rmse: 0.220025\n",
      "[1300]\ttrain's rmse: 0.212919\tvalid's rmse: 0.219886\n",
      "[1400]\ttrain's rmse: 0.212464\tvalid's rmse: 0.219766\n",
      "[1500]\ttrain's rmse: 0.212046\tvalid's rmse: 0.219655\n",
      "[1600]\ttrain's rmse: 0.211645\tvalid's rmse: 0.219558\n",
      "[1700]\ttrain's rmse: 0.211265\tvalid's rmse: 0.219464\n",
      "[1800]\ttrain's rmse: 0.210887\tvalid's rmse: 0.219381\n",
      "[1900]\ttrain's rmse: 0.210508\tvalid's rmse: 0.219304\n",
      "[2000]\ttrain's rmse: 0.210169\tvalid's rmse: 0.219236\n",
      "[2100]\ttrain's rmse: 0.209831\tvalid's rmse: 0.219169\n",
      "[2200]\ttrain's rmse: 0.209525\tvalid's rmse: 0.219115\n",
      "[2300]\ttrain's rmse: 0.209231\tvalid's rmse: 0.219061\n",
      "[2400]\ttrain's rmse: 0.208936\tvalid's rmse: 0.219007\n",
      "[2500]\ttrain's rmse: 0.208632\tvalid's rmse: 0.218956\n",
      "[2600]\ttrain's rmse: 0.208322\tvalid's rmse: 0.218897\n",
      "[2700]\ttrain's rmse: 0.208044\tvalid's rmse: 0.218849\n",
      "[2800]\ttrain's rmse: 0.20776\tvalid's rmse: 0.218807\n",
      "[2900]\ttrain's rmse: 0.207492\tvalid's rmse: 0.21877\n",
      "[3000]\ttrain's rmse: 0.207227\tvalid's rmse: 0.218733\n",
      "[3100]\ttrain's rmse: 0.206965\tvalid's rmse: 0.218694\n",
      "[3200]\ttrain's rmse: 0.206703\tvalid's rmse: 0.218651\n",
      "[3300]\ttrain's rmse: 0.206432\tvalid's rmse: 0.218617\n",
      "[3400]\ttrain's rmse: 0.206189\tvalid's rmse: 0.218585\n",
      "[3500]\ttrain's rmse: 0.205955\tvalid's rmse: 0.218562\n",
      "[3600]\ttrain's rmse: 0.205716\tvalid's rmse: 0.218532\n",
      "[3700]\ttrain's rmse: 0.205465\tvalid's rmse: 0.218503\n",
      "[3800]\ttrain's rmse: 0.205223\tvalid's rmse: 0.218477\n",
      "[3900]\ttrain's rmse: 0.20498\tvalid's rmse: 0.218443\n",
      "[4000]\ttrain's rmse: 0.204779\tvalid's rmse: 0.218419\n",
      "[4100]\ttrain's rmse: 0.204562\tvalid's rmse: 0.218399\n",
      "[4200]\ttrain's rmse: 0.204347\tvalid's rmse: 0.218378\n",
      "[4300]\ttrain's rmse: 0.20414\tvalid's rmse: 0.218356\n",
      "[4400]\ttrain's rmse: 0.203933\tvalid's rmse: 0.218337\n",
      "[4500]\ttrain's rmse: 0.203738\tvalid's rmse: 0.218317\n",
      "[4600]\ttrain's rmse: 0.203531\tvalid's rmse: 0.218299\n",
      "[4700]\ttrain's rmse: 0.203324\tvalid's rmse: 0.218278\n",
      "[4800]\ttrain's rmse: 0.203119\tvalid's rmse: 0.218264\n",
      "[4900]\ttrain's rmse: 0.202912\tvalid's rmse: 0.218242\n",
      "[5000]\ttrain's rmse: 0.202706\tvalid's rmse: 0.218228\n",
      "[5100]\ttrain's rmse: 0.2025\tvalid's rmse: 0.218218\n",
      "[5200]\ttrain's rmse: 0.202311\tvalid's rmse: 0.218203\n",
      "[5300]\ttrain's rmse: 0.202122\tvalid's rmse: 0.218194\n",
      "[5400]\ttrain's rmse: 0.201918\tvalid's rmse: 0.218171\n",
      "[5500]\ttrain's rmse: 0.201731\tvalid's rmse: 0.218158\n",
      "[5600]\ttrain's rmse: 0.201537\tvalid's rmse: 0.218146\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5700]\ttrain's rmse: 0.201328\tvalid's rmse: 0.21813\n",
      "[5800]\ttrain's rmse: 0.201137\tvalid's rmse: 0.218118\n",
      "[5900]\ttrain's rmse: 0.200966\tvalid's rmse: 0.21811\n",
      "[6000]\ttrain's rmse: 0.200778\tvalid's rmse: 0.218094\n",
      "[6100]\ttrain's rmse: 0.200592\tvalid's rmse: 0.218085\n",
      "[6200]\ttrain's rmse: 0.200415\tvalid's rmse: 0.218081\n",
      "[6300]\ttrain's rmse: 0.200235\tvalid's rmse: 0.218068\n",
      "[6400]\ttrain's rmse: 0.20006\tvalid's rmse: 0.21806\n",
      "[6500]\ttrain's rmse: 0.199886\tvalid's rmse: 0.218051\n"
     ]
    }
   ],
   "source": [
    "ret = np.zeros((train.shape[0],))\n",
    "ret_test = np.zeros((x_test.shape[0],))\n",
    "ret_models = []\n",
    "    \n",
    "for train_ix, val_ix in kf.split(train):\n",
    "    x_train = scipy.sparse.hstack([\n",
    "        train_desc_counts[train_ix],\n",
    "        train_title_counts[train_ix],\n",
    "        train.loc[train_ix, predictors]\n",
    "    ], format='csr')\n",
    "    y_train = train.loc[train_ix, target]\n",
    "    \n",
    "    x_valid = scipy.sparse.hstack([\n",
    "        train_desc_counts[val_ix],\n",
    "        train_title_counts[val_ix],\n",
    "        train.loc[val_ix, predictors]\n",
    "    ], format='csr')\n",
    "    y_valid = train.loc[val_ix, target]\n",
    "    \n",
    "    dtrain = lgb.Dataset(x_train, label=y_train,\n",
    "                         feature_name=list(feature_names), \n",
    "                         categorical_feature=categorical)\n",
    "    dvalid = lgb.Dataset(x_valid, label=y_valid,\n",
    "                         feature_name=list(feature_names), \n",
    "                         categorical_feature=categorical)\n",
    "    \n",
    "    model = lgb.train(params, dtrain, \n",
    "                      valid_sets=[dtrain, dvalid], \n",
    "                      valid_names=['train', 'valid'],\n",
    "                      num_boost_round=rounds, \n",
    "                      early_stopping_rounds=early_stop_rounds, \n",
    "                      verbose_eval=100)\n",
    "    \n",
    "    ret[val_ix] = model.predict(x_valid)\n",
    "    ret_test += model.predict(x_test)\n",
    "    ret_models.append(model)\n",
    "    \n",
    "    del x_train, y_train, x_valid, y_valid, dtrain, dvalid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_test = ret_test / 5."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cfc958c3-f045-49d4-8017-be7a60d7e18b",
    "_uuid": "40209bda4fe8e803d8e56e0c98ed4ec0375eac94",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for model in ret_models:\n",
    "    fig, ax = plt.subplots(figsize=(10, 14))\n",
    "    lgb.plot_importance(model, max_num_features=50, ax=ax)\n",
    "    plt.title(\"Light GBM Feature Importance\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "c1870612-7d3a-4871-a1de-27b379c9e970",
    "_uuid": "723d20c1ee33b7ccbeb3e0a428048300f0a86944"
   },
   "source": [
    "That looks good. But the model is kind of a black box. It is a good idea to plot the feature importances for our model now."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "959449c6-6712-4447-9223-9ac8dd094e22",
    "_uuid": "1b9e07f74d5599578910c4d7486f60dc1f6e5688"
   },
   "source": [
    "`avg_days_up`, `avg_times_up_user` and `n_user_items` are our most important engineered features! Looks like we were successful. Now we just have to predict the test matrix and submit!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f41e4221-1654-4ce8-86ed-ae39b6099591",
    "_uuid": "5e934b64b9340e9666bc339c605a4b0438e3fef6",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=['simple_feature2_lgb_pred']).to_csv('simple_feature2_lgb_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=['simple_feature2_lgb_pred']).to_csv('simple_feature2_lgb_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9d49a8cc-4b52-47f4-868a-f7b3006cd9f3",
    "_uuid": "3c93c6232801704e05d2a94c38b7df77d7ed14f4",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subm = pd.read_csv('sample_submission.csv')\n",
    "subm['deal_probability'] = np.clip(ret_test, 0, 1)\n",
    "subm.to_csv('simple_feature2_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "for md in ret_models:\n",
    "    with open('models/simple2_feature_lgb.model', 'wb') as handle:\n",
    "        pickle.dump(md, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ret_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_cell_guid": "e41f2802-c21c-471b-a7b3-f190aec1a7f2",
    "_uuid": "478c61d627e87de36aa762a69b1c10ff6c718788"
   },
   "source": [
    "I'll end this kernel with some ideas to improve it:\n",
    "- Use K-Fold cross validation.\n",
    "- Try other methods than mean for reducing the aggregated features to one per user (e. g. modus or median).\n",
    "- Try other gradient boosting libraries like CatBoost or XGBoost.\n",
    "- Add a temporal dimension to engineered features (e. g. # of items a user put up for sale *per day*).\n",
    "- Add more advanced text features like pretrained word embeddings.\n",
    "- Add image features. At the moment we completely ignore images! (as discussed [here](https://www.kaggle.com/c/avito-demand-prediction/discussion/56678), two promising approaches could be [NIMA: Neural Image Assessment](https://arxiv.org/abs/1709.05424) and [Multimedia Features for Click Prediction](https://storage.googleapis.com/kaggle-forum-message-attachments/328059/9411/dimitri-clickadvert.pdf)).\n",
    "- Normalize text before creating the Tf-Idf matrix (e. g. using [stemming](http://www.nltk.org/howto/stem.html)).\n",
    "- ~~Learn russian and do in-depth text analysis.~~\n",
    "\n",
    "Thanks for reading and have fun in this competition!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}