xgboost_regressor_flo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named xgboost",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mImportError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-6bb017303816>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[1;32mimport\u001b[0m \u001b[0mxgboost\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mxgb\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcross_validation\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgrid_search\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named xgboost"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from sklearn.cross_validation import train_test_split \n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.metrics import mean_absolute_error,mean_squared_error\n",
    "import time\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('version4.csv',encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def price_outlier(data, thresold):\n",
    "    y=data['price']\n",
    "    m = np.median(y)\n",
    "    abs_dev = np.abs(y - m)\n",
    "    y_mad=np.median(abs_dev)\n",
    "    control_var=abs_dev/y_mad\n",
    "    data[\"control_var\"]=control_var\n",
    "    data = data[data['control_var']<thresold]\n",
    "    data = data.drop(['control_var'], axis=1)\n",
    "    return data\n",
    "    \n",
    "data=price_outlier(data, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y =data['price']\n",
    "X = data.drop(['price'], axis=1)\n",
    "#X=np.array(X)\n",
    "#y=np.array(y)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"parameters CValidated and that are as by default:\\n    'reg_alpha': 0,\\n    'reg_lambda': 1\\n\\n    Parameters CValidated and are slightly different:\\n    'seed': 1,  default 0\\n    'gamma': 0.2, default 0\""
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {\n",
    "    'max_depth': [10]\n",
    "            }\n",
    "\n",
    "ind_params = {\n",
    "    \n",
    "    #'max_depth': [10]\n",
    "    'min_child_weight': 3,\n",
    "    'learning_rate': 0.01,\n",
    "    'n_estimators': 900,\n",
    "    'objective': 'reg:linear',  \n",
    "    'subsample': 0.7, \n",
    "    'colsample_bytree': 0.7,\n",
    "    'silent': 0,\n",
    "    'seed': 1,\n",
    "    'gamma': 0.2,\n",
    "    'reg_alpha': 0,\n",
    "    'reg_lambda': 1\n",
    "             }\n",
    "\n",
    "'''parameters CValidated and that are as by default:\n",
    "    'reg_alpha': 0,\n",
    "    'reg_lambda': 1\n",
    "\n",
    "    Parameters CValidated and are slightly different:\n",
    "    'seed': 1,  default 0\n",
    "    'gamma': 0.2, default 0'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)\n",
    "\n",
    "#To see the paramters set for this estimator, including the default ones:\n",
    "#optimized_GBM.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 1 candidates, totalling 2 fits\n",
      "Fitting finished:\n",
      "elapsed time 185.913767\n",
      "[mean: 0.78161, std: 0.00708, params: {'max_depth': 10}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   2 | elapsed:  1.1min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:  1.1min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Without dropping some property_type and relative few flats:\n",
    "\n",
    "Barcelona\n",
    "[mean: 0.77698, std: 0.01030, params: {'max_depth': 10}]\n",
    "Listings number: 14855\n",
    "\n",
    "Madrid\n",
    "[mean: 0.74218, std: 0.00927, params: {'max_depth': 10}]\n",
    "Listings number:7446\n",
    "\n",
    "London\n",
    "[mean: 0.80706, std: 0.00098, params: {'max_depth': 10}]\n",
    "Listings number:42646 and 3 more fields (that we are not importing, they should be considered to improve the model)\n",
    "\n",
    "Berlin\n",
    "[mean: 0.68971, std: 0.01060, params: {'max_depth': 10}]\n",
    "Listings number:15373\n",
    "\n",
    "Amsterdam\n",
    "[mean: 0.67187, std: 0.00509, params: {'max_depth': 10}]\n",
    "Listings number:11362\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'max_depth': [6, 8, 10], 'min_child_weight': [1, 3, 5]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "Fitting finished:\n",
      "elapsed time 228.682761\n",
      "[mean: 0.77252, std: 0.00917, params: {'max_depth': 6, 'min_child_weight': 1}, mean: 0.77652, std: 0.00676, params: {'max_depth': 6, 'min_child_weight': 3}, mean: 0.77457, std: 0.00827, params: {'max_depth': 6, 'min_child_weight': 5}, mean: 0.76954, std: 0.00308, params: {'max_depth': 8, 'min_child_weight': 1}, mean: 0.77106, std: 0.00495, params: {'max_depth': 8, 'min_child_weight': 3}, mean: 0.77161, std: 0.00759, params: {'max_depth': 8, 'min_child_weight': 5}, mean: 0.75809, std: 0.00712, params: {'max_depth': 10, 'min_child_weight': 1}, mean: 0.75953, std: 0.00466, params: {'max_depth': 10, 'min_child_weight': 3}, mean: 0.76139, std: 0.00572, params: {'max_depth': 10, 'min_child_weight': 5}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed:  2.8min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'max_depth': [6, 7, 8], 'min_child_weight': [2, 3, 4]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "Fitting finished:\n",
      "elapsed time 164.969713\n",
      "[mean: 0.77700, std: 0.00662, params: {'max_depth': 6, 'min_child_weight': 2}, mean: 0.77652, std: 0.00676, params: {'max_depth': 6, 'min_child_weight': 3}, mean: 0.77481, std: 0.00795, params: {'max_depth': 6, 'min_child_weight': 4}, mean: 0.77362, std: 0.00558, params: {'max_depth': 7, 'min_child_weight': 2}, mean: 0.77320, std: 0.00755, params: {'max_depth': 7, 'min_child_weight': 3}, mean: 0.77250, std: 0.00465, params: {'max_depth': 7, 'min_child_weight': 4}, mean: 0.77165, std: 0.00795, params: {'max_depth': 8, 'min_child_weight': 2}, mean: 0.77106, std: 0.00495, params: {'max_depth': 8, 'min_child_weight': 3}, mean: 0.77096, std: 0.00477, params: {'max_depth': 8, 'min_child_weight': 4}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed:  2.5min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'gamma': [0, 0.1, 0.2, 0.3]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n",
      "Fitting finished:\n",
      "elapsed time 74.078125\n",
      "[mean: 0.77700, std: 0.00662, params: {'gamma': 0}, mean: 0.77698, std: 0.00766, params: {'gamma': 0.1}, mean: 0.77599, std: 0.00779, params: {'gamma': 0.2}, mean: 0.77646, std: 0.00806, params: {'gamma': 0.3}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:   59.9s finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'subsample': [0.6, 0.7, 0.8], 'colsample_bytree': [0.6, 0.7, 0.8]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "    'gamma': 0,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 9 candidates, totalling 18 fits\n",
      "Fitting finished:\n",
      "elapsed time 125.131995\n",
      "[mean: 0.77340, std: 0.00635, params: {'subsample': 0.6, 'colsample_bytree': 0.6}, mean: 0.77518, std: 0.00786, params: {'subsample': 0.7, 'colsample_bytree': 0.6}, mean: 0.77492, std: 0.00478, params: {'subsample': 0.8, 'colsample_bytree': 0.6}, mean: 0.77430, std: 0.00926, params: {'subsample': 0.6, 'colsample_bytree': 0.7}, mean: 0.77387, std: 0.00588, params: {'subsample': 0.7, 'colsample_bytree': 0.7}, mean: 0.77490, std: 0.00555, params: {'subsample': 0.8, 'colsample_bytree': 0.7}, mean: 0.77579, std: 0.01066, params: {'subsample': 0.6, 'colsample_bytree': 0.8}, mean: 0.77509, std: 0.00523, params: {'subsample': 0.7, 'colsample_bytree': 0.8}, mean: 0.77773, std: 0.00534, params: {'subsample': 0.8, 'colsample_bytree': 0.8}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  18 out of  18 | elapsed:  1.9min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'reg_alpha': [0.0001, 0.01, 0.1, 1, 100]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "    'gamma': 0,\n",
    "    'subsample': 0.8,\n",
    "    'colsample_bytree': 0.7,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 5 candidates, totalling 10 fits\n",
      "Fitting finished:\n",
      "elapsed time 122.544637\n",
      "[mean: 0.77486, std: 0.00541, params: {'reg_alpha': 0.0001}, mean: 0.77550, std: 0.00489, params: {'reg_alpha': 0.01}, mean: 0.77704, std: 0.00469, params: {'reg_alpha': 0.1}, mean: 0.77479, std: 0.00686, params: {'reg_alpha': 1}, mean: 0.77581, std: 0.00472, params: {'reg_alpha': 100}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:  1.1min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'reg_lambda': [0.01, 0.1, 0, 10, 100]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "    'gamma': 0,\n",
    "    'subsample': 0.8,\n",
    "    'colsample_bytree': 0.7,\n",
    "    'reg_alpha': 0.1,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 5 candidates, totalling 10 fits\n",
      "Fitting finished:\n",
      "elapsed time 116.600561\n",
      "[mean: 0.77714, std: 0.00309, params: {'reg_lambda': 0.01}, mean: 0.77501, std: 0.00361, params: {'reg_lambda': 0.1}, mean: 0.77670, std: 0.00324, params: {'reg_lambda': 0}, mean: 0.77836, std: 0.00918, params: {'reg_lambda': 10}, mean: 0.77737, std: 0.00841, params: {'reg_lambda': 100}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:  1.1min finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'learning_rate': [0.2, 0.1, 0.05, 0.01]}\n",
    "\n",
    "ind_params = {\n",
    "    'n_estimators': 900,\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "    'gamma': 0,\n",
    "    'subsample': 0.8,\n",
    "    'colsample_bytree': 0.7,\n",
    "    'reg_alpha': 0.1,\n",
    "    'reg_lambda': 1,\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n",
      "Fitting finished:\n",
      "elapsed time 69.306702\n",
      "[mean: 0.76397, std: 0.00594, params: {'learning_rate': 0.2}, mean: 0.77704, std: 0.00469, params: {'learning_rate': 0.1}, mean: 0.78563, std: 0.00651, params: {'learning_rate': 0.05}, mean: 0.78074, std: 0.00620, params: {'learning_rate': 0.01}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   8 out of   8 | elapsed:   57.0s finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#XGboost and CV parameters\n",
    "\n",
    "cv_params = {'n_estimators': [1000]}\n",
    "\n",
    "ind_params = {\n",
    "    'max_depth': 6, \n",
    "    'min_child_weight': 2,\n",
    "    'gamma': 0,\n",
    "    'subsample': 0.8,\n",
    "    'colsample_bytree': 0.7,\n",
    "    'reg_alpha': 0.1,\n",
    "    'reg_lambda': 1,\n",
    "    'learning_rate': 0.05\n",
    "             }\n",
    "\n",
    "optimized_GBM = GridSearchCV(xgb.XGBRegressor(**ind_params),\n",
    "                             cv_params,\n",
    "                             scoring = 'r2',\n",
    "                             cv = 2,\n",
    "                             n_jobs = -1,\n",
    "                             error_score='raise',\n",
    "                             verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting...\n",
      "Fitting 2 folds for each of 1 candidates, totalling 2 fits\n",
      "Fitting finished:\n",
      "elapsed time 38.714836\n",
      "[mean: 0.77435, std: 0.00330, params: {'n_estimators': 1000}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   2 | elapsed:   25.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:   25.0s finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model\n",
    "t0=time.clock()\n",
    "print \"Fitting...\"\n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train)\n",
    "    #GBM = xgb.XGBRegressor(**ind_params)\n",
    "print \"Fitting finished:\"\n",
    "t1=time.clock()\n",
    "print 'elapsed time %f'% (t1-t0)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Predicting on the whole testing set\n",
    "\n",
    "features=X_test.columns.values\n",
    "y_pred=optimized_GBM.predict(X_test.loc[:,features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20.7794539121 31.8739246458 26.2237078415\n"
     ]
    }
   ],
   "source": [
    "#Getting the corresponding errors\n",
    "error_1=mean_absolute_error(y_test, y_pred)\n",
    "error_2=mean_squared_error(y_test, y_pred)\n",
    "if np.count_nonzero(y_test)== y_test.size:\n",
    "    rel_error=np.mean(np.abs((y_test - y_pred) / y_test)) * 100\n",
    "else:\n",
    "    print 'something is free'\n",
    "print error_1, np.sqrt(error_2), rel_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    284\n",
      "1     18\n",
      "2     25\n",
      "3     85\n",
      "4     40\n",
      "5     45\n",
      "6     30\n",
      "7     50\n",
      "8     80\n",
      "9     33\n",
      "Name: price, dtype: float64\n",
      "0    240\n",
      "1     65\n",
      "2     42\n",
      "3    200\n",
      "4     58\n",
      "5     35\n",
      "6    110\n",
      "7     65\n",
      "8    115\n",
      "9     30\n",
      "Name: price, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Let's check the errors by shuffling the prices\n",
    "y_train_shuffled=y_train.sample(frac=1).reset_index(drop=True).copy()\n",
    "print y_train_shuffled.head(10)\n",
    "print y_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 1 candidates, totalling 2 fits\n",
      "[mean: -0.12062, std: 0.00312, params: {'n_estimators': 1000}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   3 out of   2 | elapsed:   26.3s remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   2 | elapsed:   26.3s finished\n"
     ]
    }
   ],
   "source": [
    "#Fitting the model with shuffled prices \n",
    "if __name__ == '__main__':\n",
    "    optimized_GBM.fit(X_train, y_train_shuffled)\n",
    "print optimized_GBM.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58.15634114 76.930502256 99.5787206502\n"
     ]
    }
   ],
   "source": [
    "#Predicting on the whole testing set and getting the corresponding errors with shuffled prices\n",
    "\n",
    "y_pred=optimized_GBM.predict(X_test.loc[:,features])\n",
    "error_1=mean_absolute_error(y_test, y_pred)\n",
    "error_2=mean_squared_error(y_test, y_pred)\n",
    "if np.count_nonzero(y_test)== y_test.size:\n",
    "    rel_error=np.mean(np.abs((y_test - y_pred) / y_test)) * 100\n",
    "print error_1, np.sqrt(error_2), rel_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "with host_verification and security_deposit as 0 and 1:\n",
    "[mean: 0.78999, std: 0.00528, params: {'max_depth': 10}]\n",
    "\n",
    "with host_verification as categorical encoding:\n",
    "[mean: 0.79142, std: 0.00491, params: {'max_depth': 10}]\n",
    "\n",
    "with security_deposit as categorical encoding:\n",
    "!!!!!!! [mean: 0.79850, std: 0.00411, params: {'max_depth': 10}]   !!!!!!!\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PREDICT\n",
    "review_scores_rating should be sent as empty. 5000 flats have an empty value in the dataframe\n",
    "number_of_reviews and review_per_month should be sent as 0. 3800 flats have a 0 value in the dataframe\n",
    "first_review and last_review: empty\n",
    "calendar_updated to be set to \"today\" encoded in number\n",
    "street to be encoded in number too.\n",
    "\n",
    "Could be added: host_acceptance_rate with a 0 value for the user (only 273 flats with 0% acceptance rate)\n",
    "\n",
    "Only to check the uplift: host_response_rate (no flats at 0%)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results\n",
    "\n",
    "with step3_output (excluding neighboorhoods, property_type, amenities):\n",
    "mean: 0.65593, std: 0.00551, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.65584, std: 0.00512, params: {'max_depth': 10, 'min_child_weight': 8}]\n",
    "\n",
    "with step3_output, including all the neighboorhoods:\n",
    "mean: 0.65597, std: 0.00877, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.65546, std: 0.00911, params: {'max_depth': 10, 'min_child_weight': 8}]\n",
    "\n",
    "with step3_output, including all the property_type:\n",
    "mean: 0.64399, std: 0.00516, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.64315, std: 0.00530, params: {'max_depth': 10, 'min_child_weight': 8}]\n",
    "\n",
    "with step3_output, including all the amenities:\n",
    "mean: 0.66044, std: 0.00681, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.66028, std: 0.00682, params: {'max_depth': 10, 'min_child_weight': 8}]\n",
    "\n",
    "Conclusion:\n",
    "We don't exclude neighboorhoods nor amenities, but we do exclude the property type:\n",
    "\n",
    "mean: 0.65475, std: 0.01172, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.65485, std: 0.01135, params: {'max_depth': 10, 'min_child_weight': 8}]\n",
    "\n",
    "\n",
    "Conclusion 2: The mean is lower, what does it means? I guess the improvement by excluding / including features is more related to so specific flats than to main features.\n",
    "\n",
    "Consider adding ALL the amenities we had at the beginning in the file.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding the price and the cleaning fee:\n",
    "mean: 0.69731, std: 0.01100, params: {'max_depth': 10, 'min_child_weight': 7}, \n",
    "mean: 0.69731, std: 0.01060, params: {'max_depth': 10, 'min_child_weight': 8}]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Considering the proice outliers removed per apartement type:\n",
    "\n",
    "[mean: 0.73642, std: 0.00349, params: {'max_depth': 10, 'min_child_weight': 7}, mean: 0.73601, std: 0.00320, params: {'max_depth': 10, 'min_child_weight': 8}]\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}