RNN Self-Trained WordVec with raw Image.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing import text, sequence\n",
    "import numpy as np\n",
    "from keras.layers import Input, SpatialDropout1D,Dropout, GlobalAveragePooling1D, GlobalMaxPooling1D, \\\n",
    "                            CuDNNGRU, GRU, Bidirectional, LSTM, Dense, Embedding, concatenate, Embedding, \\\n",
    "                            Flatten, Activation, BatchNormalization, regularizers, Conv1D, Conv2D, MaxPooling2D\n",
    "from keras.constraints import max_norm\n",
    "\n",
    "from keras.initializers import Orthogonal\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LambdaCallback, Callback, LearningRateScheduler\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import pickle\n",
    "import gc; gc.enable()\n",
    "import matplotlib \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords                \n",
    "from nltk.stem.snowball import RussianStemmer\n",
    "from scipy.stats import boxcox\n",
    "import re\n",
    "#from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check GPU Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 13424623503592292352\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 3174131302\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 12561443380628681287\n",
      "physical_device_desc: \"device: 0, name: GeForce GTX 1050 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1\"\n",
      "]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/job:localhost/replica:0/task:0/device:GPU:0']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "K.tensorflow_backend._get_available_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess Training and Testing Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* > train.csv, test.csv, train_jpg.zip, test_jpg.zip: Download from kaggle competition website\n",
    "* > aggregated_features, self trained wordvec: https://drive.google.com/drive/folders/1yO_W-m0fF_PludrnScdgyTGsPFoDsA6_?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you could download it on the google drive we shared. More columns than in public kernel\n",
    "agg_features_path = 'aggregated_features.csv'\n",
    "train_data_path = 'train.csv'\n",
    "test_data_path = 'test.csv'\n",
    "embedding_file = 'avito_description_300.w2v' #'cc.ru.300.vec'\n",
    "title_embedding_file = 'avito_title_100_ver2.w2v'\n",
    "seed = 411\n",
    "rnn_train_epochs = 10\n",
    "batch_size=2 # 32 or 64 is good (too huge for my PC), 128 is worse in the past experiments\n",
    "cpu_count=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_features = pd.read_csv(agg_features_path)\n",
    "agg_cols = list(agg_features.columns)[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', parse_dates=['activation_date']) \\\n",
    "          .sort_values('activation_date') \\\n",
    "          .reset_index(drop=True)\n",
    "    \n",
    "test = pd.read_csv('test.csv', parse_dates=['activation_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.merge(agg_features, on='user_id', how='left')\n",
    "test = test.merge(agg_features, on='user_id', how='left')\n",
    "del agg_features; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge to faciliate data cleaning and transformation\n",
    "train_len = train.shape[0]\n",
    "train_y = train.deal_probability.values\n",
    "\n",
    "train.drop('deal_probability', axis=1, inplace=True)\n",
    "all_features = pd.concat([train, test]).reset_index(drop=True)\n",
    "\n",
    "del train, test; gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = RussianStemmer(ignore_stopwords=False)\n",
    "'''\n",
    "def clean_text(txt):\n",
    "    words = str(txt).lower().strip().split(\" \\t\\r.,!?^+-*/@~:;/\\\\\\\"\\'&{}[]()#$%\") #str(txt).split(\" \") #\n",
    "    words = [stemmer.stem(wrd) for wrd in words \\\n",
    "                if wrd not in stopwords.words('russian') and len(wrd) > 1]\n",
    "    txt = u\" \".join(words)\n",
    "    return txt\n",
    "'''\n",
    "def clean_text(txt):\n",
    "    return u\" \".join([stemmer.stem(re.sub(r'\\b\\d+\\b', '', wrd)) for wrd in str(txt).lower().strip().split(string.punctuation)\n",
    "                         if wrd not in stopwords.words('russian')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features['description'].fillna('unknowndescription', inplace=True)\n",
    "all_features['description'] = [clean_text(txt) for txt in all_features['description'].values] # clean text\n",
    "\n",
    "all_features['title'].fillna('unknowntitle', inplace=True)\n",
    "all_features['title'] = [clean_text(txt) for txt in all_features['title'].values]\n",
    "\n",
    "all_features['weekday'] = pd.to_datetime(all_features['activation_date']).dt.day\n",
    "\n",
    "for col in ['description', 'title']:\n",
    "    all_features['num_words_' + col] = all_features[col].apply(lambda comment: len(comment.split()))\n",
    "    all_features['num_unique_words_' + col] = all_features[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "all_features['words_vs_unique_title'] = all_features['num_unique_words_title'] / all_features['num_words_title'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['num_unique_words_description'] / all_features['num_words_description'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['words_vs_unique_description'].fillna(0.)\n",
    "\n",
    "all_features['city'] = all_features['region'] + '_' + all_features['city'] # city is repeated in different region\n",
    "all_features['num_desc_punct'] = all_features['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "\n",
    "for col in agg_cols:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['price', 'image_top_1']:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['param_1', 'param_2', 'param_3']:\n",
    "    all_features[col].fillna('unknwonparam', inplace=True)\n",
    "    \n",
    "for col in ['image']:\n",
    "    all_features[col].fillna('no-image', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "item_id                           0\n",
       "user_id                           0\n",
       "region                            0\n",
       "city                              0\n",
       "parent_category_name              0\n",
       "category_name                     0\n",
       "param_1                           0\n",
       "param_2                           0\n",
       "param_3                           0\n",
       "title                             0\n",
       "description                       0\n",
       "price                             0\n",
       "item_seq_number                   0\n",
       "activation_date                   0\n",
       "user_type                         0\n",
       "image                             0\n",
       "image_top_1                       0\n",
       "avg_days_up_user                  0\n",
       "avg_times_up_user                 0\n",
       "med_days_up_user                  0\n",
       "med_times_up_user                 0\n",
       "n_user_items                      0\n",
       "weekday                           0\n",
       "num_words_description             0\n",
       "num_unique_words_description      0\n",
       "num_words_title                   0\n",
       "num_unique_words_title            0\n",
       "words_vs_unique_title           166\n",
       "words_vs_unique_description       0\n",
       "num_desc_punct                    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1bd07f08860>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEJCAYAAABR4cpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHS1JREFUeJzt3X+Q1PWd5/HnKxAN6ioY4xQH7GEubHZRb4nMKbdWUqNkEdxUMFd6B+XJxHA10cKreGfVibm6Iqexytyd8UrLuEvCnJAlEtcfC5XFEIrY66ZKDaKuSNRlRFZHOFgdRCcavXHf98f3M5uvk57p6f7MTDfx9ajq6u739/P5ft9tqa/5/uhvKyIwMzPL8ZFmN2BmZsc+h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWbbJzW5gopx22mkxe/bshuf/8pe/5MQTTxy7hsaI+6pPq/YFrdub+6pfq/bWSF+7du16LSI+UXNgRHwoHvPnz48cDz/8cNb88eK+6tOqfUW0bm/uq36t2lsjfQFPxCj+H+vDXGZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtlqhomkWZIelvScpD2Svpbqp0raLmlvep6W6pJ0u6QeSc9IOqe0rs40fq+kzlJ9vqTdac7tktToNszMbOKNZs9kALguIv4AWACskjQXWA3siIg5wI70HmAJMCc9uoC7oAgGYA1wHnAusGYwHNKYrtK8xale1zbMzKw5aoZJRByMiCfT67eA54AZwFJgfRq2HrgkvV4KbEjfd3kMmCppOnARsD0i+iLiCLAdWJyWnRwRj6YvyGwYsq56tmFmZk1Q1+1UJM0GPgM8DrRFxEEoAkfS6WnYDOCV0rTeVBup3lulTgPbOFjP56nH7leP8uXVfzVeqx/W/lv+ZMK3aWZWr1GHiaSTgPuBayPizXRao+rQKrVooD5iO6OZI6mL4jAYbW1tVCqVGqsdXtsUuO7sgYbnN6pWz/39/Vmfa7y4r/q1am/uq36t2tt49jWqMJH0UYog2RgRD6TyIUnT0x7DdOBwqvcCs0rTZwIHUr1jSL2S6jOrjG9kGx8QEWuBtQDt7e3R0dExdMio3bFxM7funvj7Yu6/vGPE5ZVKhZzPNV7cV/1atTf3Vb9W7W08+xrN1VwC1gHPRcS3S4u2AINXZHUCm0v1FemKqwXA0XSoahuwSNK0dOJ9EbAtLXtL0oK0rRVD1lXPNszMrAlG86f2+cAVwG5JT6fa14FbgHslrQReBi5Ly7YCFwM9wNvAlQAR0SfpJmBnGndjRPSl11cDdwNTgIfSg3q3YWZmzVEzTCLiZ1Q/RwGwsMr4AFYNs65uoLtK/QngrCr11+vdhpmZTTx/A97MzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyjeY34LslHZb0bKn2Q0lPp8f+wZ/zlTRb0julZX9amjNf0m5JPZJuT7/3jqRTJW2XtDc9T0t1pXE9kp6RdE5pXZ1p/F5JnZiZWVONZs/kbmBxuRAR/y4i5kXEPOB+4IHS4hcHl0XEVaX6XUAXMCc9Bte5GtgREXOAHek9wJLS2K40H0mnAmuA84BzgTWDAWRmZs1RM0wi4hGgr9qytHfxb4F7RlqHpOnAyRHxaPr99g3AJWnxUmB9er1+SH1DFB4Dpqb1XARsj4i+iDgCbGdI2JmZ2cTKPWfyWeBQROwt1c6Q9JSkv5b02VSbAfSWxvSmGkBbRBwESM+nl+a8UmXOcHUzM2uSyZnzl/PBvZKDwO9GxOuS5gN/KelMQFXmRo11Dzdn1OuS1EVxiIy2tjYqlUqNTQ6vbQpcd/ZAw/MbVavn/v7+rM81XtxX/Vq1N/dVv1btbTz7ajhMJE0G/g0wf7AWEe8C76bXuyS9CPwexd7DzNL0mcCB9PqQpOkRcTAdxjqc6r3ArCpzeoGOIfVKtR4jYi2wFqC9vT06OjqqDRuVOzZu5tbdudlbv/2Xd4y4vFKpkPO5xov7ql+r9ua+6teqvY1nXzmHuT4PPB8R/3T4StInJE1Krz9JcfJ8Xzp89ZakBek8ywpgc5q2BRi8IqtzSH1FuqprAXA0rWcbsEjStHTifVGqmZlZk9T8U1vSPRR7AqdJ6gXWRMQ6YBm/eeL9c8CNkgaA94GrImLw5P3VFFeGTQEeSg+AW4B7Ja0EXgYuS/WtwMVAD/A2cCVARPRJugnYmcbdWNqGmZk1Qc0wiYjlw9S/XKV2P8WlwtXGPwGcVaX+OrCwSj2AVcOsqxvoHqlvMzObOP4GvJmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWWrGSaSuiUdlvRsqfYNSa9Kejo9Li4tu0FSj6QXJF1Uqi9OtR5Jq0v1MyQ9LmmvpB9KOi7Vj0/ve9Ly2bW2YWZmzTGaPZO7gcVV6rdFxLz02AogaS7Fb8OfmeZ8R9IkSZOAO4ElwFxgeRoL8K20rjnAEWBlqq8EjkTEp4Db0rhht1HfxzYzs7FUM0wi4hGgb5TrWwpsioh3I+IloAc4Nz16ImJfRLwHbAKWShJwIXBfmr8euKS0rvXp9X3AwjR+uG2YmVmTTM6Ye42kFcATwHURcQSYATxWGtObagCvDKmfB3wceCMiBqqMnzE4JyIGJB1N40faxgdI6gK6ANra2qhUKvV/yqRtClx39kDtgWOsVs/9/f1Zn2u8uK/6tWpv7qt+rdrbePbVaJjcBdwERHq+FfgKoCpjg+p7QDHCeEZYNtKcDxYj1gJrAdrb26Ojo6PasFG5Y+Nmbt2dk72N2X95x4jLK5UKOZ9rvLiv+rVqb+6rfq3a23j21dDVXBFxKCLej4h/BL7Lrw8z9QKzSkNnAgdGqL8GTJU0eUj9A+tKy0+hONw23LrMzKxJGgoTSdNLb78EDF7ptQVYlq7EOgOYA/wc2AnMSVduHUdxAn1LRATwMHBpmt8JbC6tqzO9vhT4aRo/3DbMzKxJah63kXQP0AGcJqkXWAN0SJpHcXhpP/BVgIjYI+le4BfAALAqIt5P67kG2AZMArojYk/axPXAJknfBJ4C1qX6OuD7knoo9kiW1dqGmZk1R80wiYjlVcrrqtQGx98M3FylvhXYWqW+jypXY0XEr4DL6tmGmZk1h78Bb2Zm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpatZphI6pZ0WNKzpdr/lPS8pGckPShpaqrPlvSOpKfT409Lc+ZL2i2pR9LtkpTqp0raLmlvep6W6krjetJ2zimtqzON3yupEzMza6rR7JncDSweUtsOnBUR/xL4O+CG0rIXI2JeelxVqt8FdFH8Zvuc0jpXAzsiYg6wI70HWFIa25XmI+lUip8OPo/iFxrXDAaQmZk1R80wiYhHKH6DvVz7SUQMpLePATNHWoek6cDJEfFoRASwAbgkLV4KrE+v1w+pb4jCY8DUtJ6LgO0R0RcRRyiCbWjYmZnZBBqLcyZfAR4qvT9D0lOS/lrSZ1NtBtBbGtObagBtEXEQID2fXprzSpU5w9XNzKxJJudMlvRfgQFgYyodBH43Il6XNB/4S0lnAqoyPWqtfpg5o16XpC6KQ2S0tbVRqVRqbHJ4bVPgurMHag8cY7V67u/vz/pc48V91a9Ve3Nf9WvV3sazr4bDJJ34/gKwMB26IiLeBd5Nr3dJehH4PYq9h/KhsJnAgfT6kKTpEXEwHcY6nOq9wKwqc3qBjiH1SrUeI2ItsBagvb09Ojo6qg0blTs2bubW3VnZ25D9l3eMuLxSqZDzucaL+6pfq/bmvurXqr2NZ18NHeaStBi4HvhiRLxdqn9C0qT0+pMUJ8/3pcNXb0lakK7iWgFsTtO2AINXZHUOqa9IV3UtAI6m9WwDFkmalk68L0o1MzNrkpp/aku6h2JP4DRJvRRXUt0AHA9sT1f4Ppau3PoccKOkAeB94KqIGDx5fzXFlWFTKM6xDJ5nuQW4V9JK4GXgslTfClwM9ABvA1cCRESfpJuAnWncjaVtmJlZE9QMk4hYXqW8bpix9wP3D7PsCeCsKvXXgYVV6gGsGmZd3UD38F2bmdlE8jfgzcwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLNuowkRSt6TDkp4t1U6VtF3S3vQ8LdUl6XZJPZKekXROaU5nGr9XUmepPl/S7jTn9vQ78Q1tw8zMJt5o90zuBhYPqa0GdkTEHGBHeg+wBJiTHl3AXVAEA8Xvx58HnAusGQyHNKarNG9xI9swM7PmGFWYRMQjQN+Q8lJgfXq9HrikVN8QhceAqZKmAxcB2yOiLyKOANuBxWnZyRHxaPrd9w1D1lXPNszMrAlyzpm0RcRBgPR8eqrPAF4pjetNtZHqvVXqjWzDzMyaYPI4rFNVatFAvZFtfHCQ1EVxGIy2tjYqlUqN1Q6vbQpcd/ZAw/MbVavn/v7+rM81XtxX/Vq1N/dVv1btbTz7ygmTQ5KmR8TBdIjpcKr3ArNK42YCB1K9Y0i9kuozq4xvZBsfEBFrgbUA7e3t0dHRMXTIqN2xcTO37h6P7B3Z/ss7RlxeqVTI+VzjxX3Vr1V7c1/1a9XexrOvnMNcW4DBK7I6gc2l+op0xdUC4Gg6RLUNWCRpWjrxvgjYlpa9JWlBuoprxZB11bMNMzNrglH9qS3pHoq9itMk9VJclXULcK+klcDLwGVp+FbgYqAHeBu4EiAi+iTdBOxM426MiMGT+ldTXDE2BXgoPah3G2Zm1hyjCpOIWD7MooVVxgawapj1dAPdVepPAGdVqb9e7zbMzGzi+RvwZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmlq3hMJH0aUlPlx5vSrpW0jckvVqqX1yac4OkHkkvSLqoVF+caj2SVpfqZ0h6XNJeST+UdFyqH5/e96Tlsxv9HGZmlq/hMImIFyJiXkTMA+ZT/Bb7g2nxbYPLImIrgKS5wDLgTGAx8B1JkyRNAu4ElgBzgeVpLMC30rrmAEeAlam+EjgSEZ8CbkvjzMysScbqMNdC4MWI+PsRxiwFNkXEuxHxEtADnJsePRGxLyLeAzYBSyUJuBC4L81fD1xSWtf69Po+YGEab2ZmTTBWYbIMuKf0/hpJz0jqljQt1WYAr5TG9KbacPWPA29ExMCQ+gfWlZYfTePNzKwJFBF5KyjOYxwAzoyIQ5LagNeAAG4CpkfEVyTdCTwaEX+e5q0DtlIE2kUR8R9S/QqKvZUb0/hPpfosYGtEnC1pT5rTm5a9CJwbEa8P6a0L6AJoa2ubv2nTpoY/5+G+oxx6p+HpDTt7xikjLu/v7+ekk06aoG5Gz33Vr1V7c1/1a9XeGunrggsu2BUR7bXGTW64q19bAjwZEYcABp8BJH0X+FF62wvMKs2bSRFCDFN/DZgqaXLa+yiPH1xXr6TJwClA39DGImItsBagvb09Ojo6Gv6Qd2zczK27x+IfV332X94x4vJKpULO5xov7qt+rdqb+6pfq/Y2nn2NxWGu5ZQOcUmaXlr2JeDZ9HoLsCxdiXUGMAf4ObATmJOu3DqO4pDZlih2mR4GLk3zO4HNpXV1pteXAj+N3F0sMzNrWNaf2pJOAP4Y+Gqp/D8kzaM4zLV/cFlE7JF0L/ALYABYFRHvp/VcA2wDJgHdEbEnret6YJOkbwJPAetSfR3wfUk9FHsky3I+h5mZ5ckKk4h4myEnviPiihHG3wzcXKW+leL8ydD6PorzJ0PrvwIua6BlMzMbB/4GvJmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWXLDhNJ+yXtlvS0pCdS7VRJ2yXtTc/TUl2SbpfUI+kZSeeU1tOZxu+V1Fmqz0/r70lzNdI2zMxs4o3VnskFETEvItrT+9XAjoiYA+xI7wGWAHPSowu4C4pgANYA51H8TO+aUjjclcYOzltcYxtmZjbBxusw11JgfXq9HrikVN8QhceAqZKmAxcB2yOiLyKOANuBxWnZyRHxaEQEsGHIuqptw8zMJpiK/0dnrEB6CTgCBPBnEbFW0hsRMbU05khETJP0I+CWiPhZqu8Argc6gI9FxDdT/b8B7wCVNP7zqf5Z4PqI+MJw2xjSWxfFXg1tbW3zN23a1PDnPNx3lEPvNDy9YWfPOGXE5f39/Zx00kkT1M3oua/6tWpv7qt+rdpbI31dcMEFu0pHnYY1ueGufu38iDgg6XRgu6TnRxirKrVooD4qEbEWWAvQ3t4eHR0do536G+7YuJlbd4/FP6767L+8Y8TllUqFnM81XtxX/Vq1N/dVv1btbTz7yj7MFREH0vNh4EGKcx6H0iEq0vPhNLwXmFWaPhM4UKM+s0qdEbZhZmYTLCtMJJ0o6XcGXwOLgGeBLcDgFVmdwOb0eguwIl3VtQA4GhEHgW3AIknT0on3RcC2tOwtSQvSVVwrhqyr2jbMzGyC5R63aQMeTFfrTgZ+EBE/lrQTuFfSSuBl4LI0fitwMdADvA1cCRARfZJuAnamcTdGRF96fTVwNzAFeCg9AG4ZZhtmZjbBssIkIvYBf1il/jqwsEo9gFXDrKsb6K5SfwI4a7TbMDOziedvwJuZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllazhMJM2S9LCk5yTtkfS1VP+GpFclPZ0eF5fm3CCpR9ILki4q1RenWo+k1aX6GZIel7RX0g8lHZfqx6f3PWn57EY/h5mZ5cvZMxkArouIPwAWAKskzU3LbouIeemxFSAtWwacCSwGviNpkqRJwJ3AEmAusLy0nm+ldc0BjgArU30lcCQiPgXclsaZmVmTNBwmEXEwIp5Mr98CngNmjDBlKbApIt6NiJcofgf+3PToiYh9EfEesAlYquKH5S8E7kvz1wOXlNa1Pr2+D1iYxpuZWROMyTmTdJjpM8DjqXSNpGckdUualmozgFdK03pTbbj6x4E3ImJgSP0D60rLj6bxZmbWBJNzVyDpJOB+4NqIeFPSXcBNQKTnW4GvANX2HILqgRYjjKfGsnJvXUAXQFtbG5VKZcTPMpK2KXDd2QO1B46xWj339/dnfa7x4r7q16q9ua/6tWpv49lXVphI+ihFkGyMiAcAIuJQafl3gR+lt73ArNL0mcCB9Lpa/TVgqqTJae+jPH5wXb2SJgOnAH1D+4uItcBagPb29ujo6Gj4s96xcTO37s7O3rrtv7xjxOWVSoWczzVe3Ff9WrU391W/Vu1tPPvKuZpLwDrguYj4dqk+vTTsS8Cz6fUWYFm6EusMYA7wc2AnMCdduXUcxUn6LRERwMPApWl+J7C5tK7O9PpS4KdpvJmZNUHOn9rnA1cAuyU9nWpfp7gaax7FYaf9wFcBImKPpHuBX1BcCbYqIt4HkHQNsA2YBHRHxJ60vuuBTZK+CTxFEV6k5+9L6qHYI1mW8TnMzCxTw2ESET+j+rmLrSPMuRm4uUp9a7V5EbGP4mqvofVfAZfV06+ZmY0ffwPezMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsh3TYSJpsaQXJPVIWt3sfszMPqyO2TCRNAm4E1gCzKX47fm5ze3KzOzD6ZgNE4rfhu+JiH0R8R6wCVja5J7MzD6UjuUwmQG8Unrfm2pmZjbBJje7gQyqUosPDJC6gK70tl/SCxnbOw14LWN+Q/StmkOa0tcouK/6tWpv7qt+rdpbI33989EMOpbDpBeYVXo/EzhQHhARa4G1Y7ExSU9ERPtYrGssua/6tGpf0Lq9ua/6tWpv49nXsXyYaycwR9IZko4DlgFbmtyTmdmH0jG7ZxIRA5KuAbYBk4DuiNjT5LbMzD6UjtkwAYiIrcDWCdrcmBwuGwfuqz6t2he0bm/uq36t2tu49aWIqD3KzMxsBMfyORMzM2sRDpMaWvWWLZK6JR2W9GyzeymTNEvSw5Kek7RH0tea3ROApI9J+rmkv019/fdm91QmaZKkpyT9qNm9lEnaL2m3pKclPdHsfgZJmirpPknPp3/X/nUL9PTp9M9p8PGmpGub3ReApP+U/r1/VtI9kj425tvwYa7hpVu2/B3wxxSXIu8ElkfEL5raGCDpc0A/sCEizmp2P4MkTQemR8STkn4H2AVc0ux/ZpIEnBgR/ZI+CvwM+FpEPNbMvgZJ+s9AO3ByRHyh2f0MkrQfaI+IlvrOhKT1wN9ExPfS1ZwnRMQbze5rUPp/x6vAeRHx903uZQbFv+9zI+IdSfcCWyPi7rHcjvdMRtayt2yJiEeAvmb3MVREHIyIJ9Prt4DnaIE7E0ShP739aHq0xF9SkmYCfwJ8r9m9HAsknQx8DlgHEBHvtVKQJAuBF5sdJCWTgSmSJgMnMOQ7eWPBYTIy37Ilg6TZwGeAx5vbSSEdSnoaOAxsj4iW6Av438B/Af6x2Y1UEcBPJO1Kd5RoBZ8E/gH4P+nQ4PckndjspoZYBtzT7CYAIuJV4H8BLwMHgaMR8ZOx3o7DZGQ1b9li1Uk6CbgfuDYi3mx2PwAR8X5EzKO4W8K5kpp+eFDSF4DDEbGr2b0M4/yIOIfi7tyr0uHVZpsMnAPcFRGfAX4JtNL5zOOALwJ/0exeACRNoziicgbwz4ATJf37sd6Ow2RkNW/ZYr8pnZO4H9gYEQ80u5+h0iGRCrC4ya0AnA98MZ2b2ARcKOnPm9vSr0XEgfR8GHiQ4tBvs/UCvaU9y/sowqVVLAGejIhDzW4k+TzwUkT8Q0T8P+AB4I/GeiMOk5H5li11Sie61wHPRcS3m93PIEmfkDQ1vZ5C8R/Y883tCiLihoiYGRGzKf79+mlEjPlfjY2QdGK6iIJ0GGkR0PSrByPi/wKvSPp0Ki0Emn5RTMlyWuQQV/IysEDSCem/z4UU5zLH1DH9Dfjx1sq3bJF0D9ABnCapF1gTEeua2xVQ/KV9BbA7nZ8A+Hq6W0EzTQfWp6tsPgLcGxEtdRluC2oDHiz+/8Nk4AcR8ePmtvRP/iOwMf2Rtw+4ssn9ACDpBIqrP7/a7F4GRcTjku4DngQGgKcYh2/C+9JgMzPL5sNcZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmx7B6bvoq6XOSnpQ0IOnSIcs6Je1Nj856+3CYmJkd2+5m9F/AfRn4MvCDclHSqcAa4DyKL6auSd+cHzWHiZnZMazaTV8l/QtJP073VPsbSb+fxu6PiGf4zfvAXURxv7q+iDgCbKfOO0T4S4tmZr991gJXRcReSecB3wEuHGF89k1tHSZmZr9F0k1W/wj4i3T3AoDja02rUqvrG+0OEzOz3y4fAd5Id8gerV6K2zMNmklxM9S6NmpmZr8l0k8+vCTpMihuvirpD2tM2wYskjQtnXhflGqj5jAxMzuGpZu+Pgp8WlKvpJXA5cBKSX8L7CH9Qqykf5VuDHsZ8GeS9gBERB9wE8Wd0ncCN6ba6PvwjR7NzCyX90zMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLL9f4nbkCuZOeo3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_features.price.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1bcea420dd8>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD8CAYAAACyyUlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG5hJREFUeJzt3X+MXfWZ3/H3p3agbn5hQhhZtrc2zWy1DrQOjMCrNNE0dI3xVmtSQWsLxbPE0iTUqIlEpZjNHyAIEtnKQYImJI4YYSIvPwphbaWmjuVwN4oUfpjgtXEI8eB4w8SWLbBDmLAhHfr0j/vM6nhy78yd+x3PncGfl3R0z33O98e5j+/4mfPjzlVEYGZmVuKfdXoHzMxs9nMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFZvb6R2YLhdccEEsWbKk7f6//e1vee973zt1O/Qu5Ty1xnlqjfPUmjOZp+eff/61iPjwRO3OmmKyZMkS9u7d23b/Wq1Gb2/v1O3Qu5Tz1BrnqTXOU2vOZJ4k/UMr7Xyay8zMirmYmJlZMRcTMzMr5mJiZmbFJiwmkhZLekrSS5IOSvpCxs+XtFvSoXycn3FJukfSoKT9ki6tjNWX7Q9J6qvEL5N0IPvcI0ntzmFmZtOvlSOTEeDmiPgTYAWwUdIyYBOwJyK6gT35HOBqoDuXfuA+qBcG4FbgCuBy4NbR4pBt+iv9VmV8UnOYmVlnTFhMIuJYRPwk198EXgIWAmuArdlsK3BNrq8BHoy6p4HzJC0ArgJ2R8TJiDgF7AZW5bYPRMSPo/61jw+OGWsyc5iZWQdM6pqJpCXAx4BngK6IOAb1ggNcmM0WAq9Wug1lbLz4UIM4bcxhZmYd0PKHFiW9D3gc+GJE/CYvazRs2iAWbcTH3Z1W+kjqp34ajK6uLmq12gTDNjc8PFzU/2zhPLXGeWqN89SamZCnloqJpPdQLyTbIuK7GT4uaUFEHMtTTCcyPgQsrnRfBBzNeO+YeC3jixq0b2eO00TEFmALQE9PT5R8QvTebdvZ/KPftt2/XUfu+vNpn7OEP7HcGuepNc5Ta2ZCnlq5m0vA/cBLEfG1yqYdwOgdWX3A9kp8fd5xtQJ4I09R7QJWSpqfF95XArty25uSVuRc68eMNZk5zMysA1o5Mvk48BnggKR9Gfsr4C7gUUkbgF8C1+W2ncBqYBB4C7gBICJOSroDeC7b3R4RJ3P9RuABYB7wZC5Mdg4zM+uMCYtJRPyIxtcoAK5s0D6AjU3GGgAGGsT3Ahc3iL8+2TnMzGz6+RPwZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWrJXvgB+QdELSi5XYI5L25XJk9Ot8JS2R9I+Vbd+s9LlM0gFJg5Luye97R9L5knZLOpSP8zOubDcoab+kSytj9WX7Q5L6MDOzjmrlyOQBYFU1EBH/JSKWR8Ry4HHgu5XNr4xui4jPV+L3Af1Ady6jY24C9kREN7AnnwNcXWnbn/2RdD5wK3AFcDlw62gBMjOzzpiwmETED4GTjbbl0cV/Bh4abwxJC4APRMSP8/vbHwSuyc1rgK25vnVM/MGoexo4L8e5CtgdEScj4hSwmzHFzszMplfpNZNPAMcj4lAltlTSC5L+TtInMrYQGKq0GcoYQFdEHAPIxwsrfV5t0KdZ3MzMOmRuYf91nH5Ucgz4o4h4XdJlwN9K+iigBn1jgrGb9Wl5LEn91E+R0dXVRa1Wm2DK5rrmwc2XjLTdv10l+9wJw8PDs26fO8F5ao3z1JqZkKe2i4mkucB/Ai4bjUXE28Dbuf68pFeAP6Z+9LCo0n0RcDTXj0taEBHH8jTWiYwPAYsb9BkCesfEa432MSK2AFsAenp6ore3t1Gzlty7bTubD5TW3sk7cn3vtM9ZolarUZLns4Xz1BrnqTUzIU8lp7n+A/CziPin01eSPixpTq5fRP3i+eE8ffWmpBV5nWU9sD277QBG78jqGxNfn3d1rQDeyHF2ASslzc8L7yszZmZmHTLhr9qSHqJ+JHCBpCHg1oi4H1jLH154/yRwu6QR4B3g8xExevH+Rup3hs0DnswF4C7gUUkbgF8C12V8J7AaGATeAm4AiIiTku4Anst2t1fmMDOzDpiwmETEuibxv2wQe5z6rcKN2u8FLm4Qfx24skE8gI1NxhoABsbbbzMzmz7+BLyZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRWbsJhIGpB0QtKLldhtkn4laV8uqyvbbpE0KOllSVdV4qsyNihpUyW+VNIzkg5JekTSORk/N58P5vYlE81hZmad0cqRyQPAqgbxuyNieS47ASQto/7d8B/NPt+QNEfSHODrwNXAMmBdtgX4ao7VDZwCNmR8A3AqIj4C3J3tms4xuZdtZmZTacJiEhE/BE62ON4a4OGIeDsifgEMApfnMhgRhyPi98DDwBpJAj4FPJb9twLXVMbamuuPAVdm+2ZzmJlZh5RcM7lJ0v48DTY/YwuBVytthjLWLP4h4NcRMTImftpYuf2NbN9sLDMz65C5bfa7D7gDiHzcDHwWUIO2QeOiFeO0Z5xt4/U5jaR+oB+gq6uLWq3WqFlLuubBzZeMTNxwipXscycMDw/Pun3uBOepNc5Ta2ZCntoqJhFxfHRd0reB7+XTIWBxpeki4GiuN4q/BpwnaW4efVTbj441JGku8EHqp9vGm2Psfm4BtgD09PREb2/vpF5n1b3btrP5QLu1t31Hru+d9jlL1Go1SvJ8tnCeWuM8tWYm5Kmt01ySFlSefhoYvdNrB7A278RaCnQDzwLPAd1559Y51C+g74iIAJ4Crs3+fcD2ylh9uX4t8INs32wOMzPrkAl/1Zb0ENALXCBpCLgV6JW0nPrppSPA5wAi4qCkR4GfAiPAxoh4J8e5CdgFzAEGIuJgTvEl4GFJXwFeAO7P+P3AdyQNUj8iWTvRHGZm1hkTFpOIWNcgfH+D2Gj7O4E7G8R3AjsbxA/T4G6siPgdcN1k5jAzs87wJ+DNzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKzYhMVE0oCkE5JerMT+h6SfSdov6QlJ52V8iaR/lLQvl29W+lwm6YCkQUn3SFLGz5e0W9KhfJyfcWW7wZzn0spYfdn+kKQ+zMyso1o5MnkAWDUmthu4OCL+DfBz4JbKtlciYnkun6/E7wP6ge5cRsfcBOyJiG5gTz4HuLrStj/7I+l86t9DfwX1r/u9dbQAmZlZZ0xYTCLih8DJMbHvR8RIPn0aWDTeGJIWAB+IiB9HRAAPAtfk5jXA1lzfOib+YNQ9DZyX41wF7I6IkxFxinphG1vszMxsGk3FNZPPAk9Wni+V9IKkv5P0iYwtBIYqbYYyBtAVEccA8vHCSp9XG/RpFjczsw6ZW9JZ0peBEWBbho4BfxQRr0u6DPhbSR8F1KB7TDR8kz4tjyWpn/opMrq6uqjVahNM2VzXPLj5kpGJG06xkn3uhOHh4Vm3z53gPLXGeWrNTMhT28UkL3z/R+DKPHVFRLwNvJ3rz0t6Bfhj6kcP1VNhi4CjuX5c0oKIOJansU5kfAhY3KDPENA7Jl5rtI8RsQXYAtDT0xO9vb2NmrXk3m3b2XygqPa25cj1vdM+Z4larUZJns8WzlNrnKfWzIQ8tXWaS9Iq4EvAX0TEW5X4hyXNyfWLqF88P5ynr96UtCLv4loPbM9uO4DRO7L6xsTX511dK4A3cpxdwEpJ8/PC+8qMmZlZh0z4q7akh6gfCVwgaYj6nVS3AOcCu/MO36fzzq1PArdLGgHeAT4fEaMX72+kfmfYPOrXWEavs9wFPCppA/BL4LqM7wRWA4PAW8ANABFxUtIdwHPZ7vbKHGZm1gETFpOIWNcgfH+Tto8DjzfZthe4uEH8deDKBvEANjYZawAYaL7XZmY2nfwJeDMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK9ZSMZE0IOmEpBcrsfMl7ZZ0KB/nZ1yS7pE0KGm/pEsrffqy/SFJfZX4ZZIOZJ978nvi25rDzMymX6tHJg8Aq8bENgF7IqIb2JPPAa4GunPpB+6DemGg/v3xVwCXA7eOFods01/pt6qdOczMrDNaKiYR8UPg5JjwGmBrrm8FrqnEH4y6p4HzJC0ArgJ2R8TJiDgF7AZW5bYPRMSP83vfHxwz1mTmMDOzDii5ZtIVEccA8vHCjC8EXq20G8rYePGhBvF25jAzsw6YewbGVINYtBFvZ47TG0n91E+D0dXVRa1Wm2DY5rrmwc2XjLTdv10l+9wJw8PDs26fO8F5ao3z1JqZkKeSYnJc0oKIOJanmE5kfAhYXGm3CDia8d4x8VrGFzVo384cp4mILcAWgJ6enujt7R3bpGX3btvO5gNnovaO78j1vdM+Z4larUZJns8WzlNrnKfWzIQ8lZzm2gGM3pHVB2yvxNfnHVcrgDfyFNUuYKWk+XnhfSWwK7e9KWlF3sW1fsxYk5nDzMw6oKVftSU9RP2o4gJJQ9TvyroLeFTSBuCXwHXZfCewGhgE3gJuAIiIk5LuAJ7LdrdHxOhF/Rup3zE2D3gyFyY7h5mZdUZLxSQi1jXZdGWDtgFsbDLOADDQIL4XuLhB/PXJzmFmZtPPn4A3M7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NibRcTSf9a0r7K8htJX5R0m6RfVeKrK31ukTQo6WVJV1XiqzI2KGlTJb5U0jOSDkl6RNI5GT83nw/m9iXtvg4zMyvXdjGJiJcjYnlELAcuo/5d7E/k5rtHt0XETgBJy4C1wEeBVcA3JM2RNAf4OnA1sAxYl20BvppjdQOngA0Z3wCcioiPAHdnOzMz65CpOs11JfBKRPzDOG3WAA9HxNsR8QtgELg8l8GIOBwRvwceBtZIEvAp4LHsvxW4pjLW1lx/DLgy25uZWQdMVTFZCzxUeX6TpP2SBiTNz9hC4NVKm6GMNYt/CPh1RIyMiZ82Vm5/I9ubmVkHzC0dIK9j/AVwS4buA+4AIh83A58FGh05BI0LWozTngm2VfetH+gH6OrqolarNXsZE+qaBzdfMjJxwylWss+dMDw8POv2uROcp9Y4T62ZCXkqLibUr3X8JCKOA4w+Akj6NvC9fDoELK70WwQczfVG8deA8yTNzaOPavvRsYYkzQU+CJwcu2MRsQXYAtDT0xO9vb1tv8h7t21n84GpSNfkHLm+d9rnLFGr1SjJ89nCeWqN89SamZCnqTjNtY7KKS5JCyrbPg28mOs7gLV5J9ZSoBt4FngO6M47t86hfspsR0QE8BRwbfbvA7ZXxurL9WuBH2R7MzPrgKJftSX9C+DPgM9Vwn8taTn1005HRrdFxEFJjwI/BUaAjRHxTo5zE7ALmAMMRMTBHOtLwMOSvgK8ANyf8fuB70gapH5EsrbkdZiZWZmiYhIRbzHmwndEfGac9ncCdzaI7wR2Nogfpn6319j474Dr2thlMzM7A/wJeDMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK1ZcTCQdkXRA0j5JezN2vqTdkg7l4/yMS9I9kgYl7Zd0aWWcvmx/SFJfJX5Zjj+YfTXeHGZmNv2m6sjk30fE8ojoyeebgD0R0Q3syecAVwPdufQD90G9MAC3AldQ/5reWyvF4b5sO9pv1QRzmJnZNDtTp7nWAFtzfStwTSX+YNQ9DZwnaQFwFbA7Ik5GxClgN7Aqt30gIn4cEQE8OGasRnOYmdk0m4piEsD3JT0vqT9jXRFxDCAfL8z4QuDVSt+hjI0XH2oQH28OMzObZnOnYIyPR8RRSRcCuyX9bJy2ahCLNuItyeLWD9DV1UWtVmu16x/omgc3XzLSdv92lexzJwwPD8+6fe4E56k1zlNrZkKeiotJRBzNxxOSnqB+zeO4pAURcSxPVZ3I5kPA4kr3RcDRjPeOidcyvqhBe8aZo7pvW4AtAD09PdHb2zu2Scvu3badzQemovZOzpHre6d9zhK1Wo2SPJ8tnKfWOE+tmQl5KjrNJem9kt4/ug6sBF4EdgCjd2T1AdtzfQewPu/qWgG8kaeodgErJc3PC+8rgV257U1JK/IurvVjxmo0h5mZTbPSX7W7gCfybt25wN9ExP+R9BzwqKQNwC+B67L9TmA1MAi8BdwAEBEnJd0BPJftbo+Ik7l+I/AAMA94MheAu5rMYWZm06yomETEYeDfNoi/DlzZIB7AxiZjDQADDeJ7gYtbncPMzKafPwFvZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbF2i4mkhZLekrSS5IOSvpCxm+T9CtJ+3JZXelzi6RBSS9LuqoSX5WxQUmbKvGlkp6RdEjSI5LOyfi5+Xwwty9p93WYmVm5kiOTEeDmiPgTYAWwUdKy3HZ3RCzPZSdAblsLfBRYBXxD0hxJc4CvA1cDy4B1lXG+mmN1A6eADRnfAJyKiI8Ad2c7MzPrkLaLSUQci4if5PqbwEvAwnG6rAEejoi3I+IXwCBweS6DEXE4In4PPAyskSTgU8Bj2X8rcE1lrK25/hhwZbY3M7MOmJJrJnma6WPAMxm6SdJ+SQOS5mdsIfBqpdtQxprFPwT8OiJGxsRPGyu3v5HtzcysA+aWDiDpfcDjwBcj4jeS7gPuACIfNwOfBRodOQSNC1qM054JtlX3rR/oB+jq6qJWq437WsbTNQ9uvmRk4oZTrGSfO2F4eHjW7XMnOE+tcZ5aMxPyVFRMJL2HeiHZFhHfBYiI45Xt3wa+l0+HgMWV7ouAo7neKP4acJ6kuXn0UW0/OtaQpLnAB4GTY/cvIrYAWwB6enqit7e37dd677btbD5QXHsn7cj1vdM+Z4larUZJns8WzlNrnKfWzIQ8ldzNJeB+4KWI+FolvqDS7NPAi7m+A1ibd2ItBbqBZ4HngO68c+sc6hfpd0REAE8B12b/PmB7Zay+XL8W+EG2NzOzDij5VfvjwGeAA5L2ZeyvqN+NtZz6aacjwOcAIuKgpEeBn1K/E2xjRLwDIOkmYBcwBxiIiIM53peAhyV9BXiBevEiH78jaZD6EcnagtdhZmaF2i4mEfEjGl+72DlOnzuBOxvEdzbqFxGHqd/tNTb+O+C6yeyvmZmdOf4EvJmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFZvVxUTSKkkvSxqUtKnT+2NmdraatcVE0hzg68DVwDLq3z2/rLN7ZWZ2dpq1xYT6d8MPRsThiPg98DCwpsP7ZGZ2Vprb6R0osBB4tfJ8CLiiQ/tyxizZ9L87vQuTcvMlI/zlLNvnUUfu+vNO74LZrDWbi4kaxOK0BlI/0J9PhyW9XDDfBcBrBf3PCv9tFudJX53W6WZtnqaZ89SaM5mnf9lKo9lcTIaAxZXni4Cj1QYRsQXYMhWTSdobET1TMda7mfPUGuepNc5Ta2ZCnmbzNZPngG5JSyWdA6wFdnR4n8zMzkqz9sgkIkYk3QTsAuYAAxFxsMO7ZWZ2Vpq1xQQgInYCO6dpuik5XXYWcJ5a4zy1xnlqTcfzpIiYuJWZmdk4ZvM1EzMzmyFcTCZwtv7JFklHJB2QtE/S3oydL2m3pEP5OD/jknRP5mi/pEsr4/Rl+0OS+irxy3L8wezb6FbvGUfSgKQTkl6sxM54XprNMVM1ydNtkn6V76l9klZXtt2Sr/llSVdV4g1//vLGm2cyH4/kTThIOjefD+b2JdPzitsjabGkpyS9JOmgpC9kfPa9pyLCS5OF+oX9V4CLgHOAvweWdXq/pum1HwEuGBP7a2BTrm8Cvprrq4EnqX/2ZwXwTMbPBw7n4/xcn5/bngX+NPs8CVzd6dfcYl4+CVwKvDideWk2x0xdmuTpNuC/N2i7LH+2zgWW5s/cnPF+/oBHgbW5/k3gxlz/r8A3c30t8EinczFBnhYAl+b6+4GfZz5m3Xuq48mcyUv+A+yqPL8FuKXT+zVNr/0If1hMXgYW5PoC4OVc/xawbmw7YB3wrUr8WxlbAPysEj+t3UxfgCVj/pM843lpNsdMXhrk6TYaF5PTfq6o36H5p81+/vI/xdeAuRn/p3ajfXN9brZTp3MxiZxtB/5sNr6nfJprfI3+ZMvCDu3LdAvg+5KeV/0vCQB0RcQxgHy8MOPN8jRefKhBfLaajrw0m2O2uSlPzwxUTqtMNk8fAn4dESNj4qeNldvfyPYzXp6S+xjwDLPwPeViMr4J/2TLu9jHI+JS6n+VeaOkT47TtlmeJht/t3FeTncf8K+A5cAxYHPGpzJPszKHkt4HPA58MSJ+M17TBrEZ8Z5yMRnfhH+y5d0qIo7m4wngCep/pfm4pAUA+XgimzfL03jxRQ3is9V05KXZHLNGRByPiHci4v8B36b+noLJ5+k14DxJc8fETxsrt38QODn1r2bqSHoP9UKyLSK+m+FZ955yMRnfWfknWyS9V9L7R9eBlcCL1F/76F0ifdTP75Lx9XmnyQrgjTxs3gWslDQ/T2mspH5u+xjwpqQVeWfJ+spYs9F05KXZHLPG6H9c6dPU31NQf21r806spUA39YvGDX/+on6S/yng2uw/NuejeboW+EG2n5Hy3/l+4KWI+Fpl0+x7T3X6gtNMX6jfPfFz6neVfLnT+zNNr/ki6nfO/D1wcPR1Uz/3vAc4lI/nZ1zUv6jsFeAA0FMZ67PAYC43VOI91P8zeQX4n8ySi6TAQ9RP0fxf6r/1bZiOvDSbY6YuTfL0nczDfur/kS2otP9yvuaXqdzZ1+znL9+jz2b+/hdwbsb/eT4fzO0XdToXE+Tp31E/7bQf2JfL6tn4nvIn4M3MrJhPc5mZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIr9f6myGMVAdhI1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_features.item_seq_number.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Since price and item seq number are highly skewed, we'll transform it into more normal like by using boxcox (more robust to outliers compared to np.log1p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGS5JREFUeJzt3X+s3XWd5/Hne1oZkVlsxeGGbZstExtHpFHhBjpjMrlrnVLEWP6QDYQZCmnSjUHFsclYJps0q0OCyTiMJErSSMey64pdRkOj1U6DnmwmUSwgK0IlvYMdeqcdUFuQK1H2Ou/943yqx8v3nPu5F+79nt4+H8nJ+Z739/P9fr7nw7m8+v1xzjcyE0mSavxO2xsgSTp9GBqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqotbXsDXm1vfOMbc/Xq1W1vxiv285//nHPOOaftzRhKjk0zx6U/x6ZZ77g8/PDDP8nM359pmUUXGqtXr+ahhx5qezNesU6nw9jYWNubMZQcm2aOS3+OTbPecYmIf6lZxsNTkqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqL7hvhkl5u9favtdLvkduvaqVfzZ+qPY2I+IuIeDwifhARX4yI10bEhRHxYEQcjogvRcRZpe3vltfjZf7qnvXcWupPRsQVPfWNpTYeEdt76o19SJLaMWNoRMQK4MPAaGZeDCwBrgU+CdyRmWuAk8CWssgW4GRmvgm4o7QjIi4qy70V2Ah8NiKWRMQS4DPAlcBFwHWlLQP6kCS1oPacxlLg7IhYCrwOOA68C7ivzN8NXF2mN5XXlPnrIyJK/d7M/GVm/ggYBy4rj/HMfCozXwLuBTaVZfr1IUlqwYyhkZn/CvwN8DTdsHgeeBh4LjOnSrMJYEWZXgEcLctOlfbn9danLdOvft6APiRJLZjxRHhELKe7l3Ah8Bzwv+keSpouTy3SZ16/elNwDWrftI1bga0AIyMjdDqdpmanlcnJyUXxPuaDY9Ns0LhsWzvVWJ9vw/Lfyc9Ms7mMS83VU+8GfpSZPwaIiC8Dfwwsi4ilZU9gJXCstJ8AVgET5XDW64ETPfVTepdpqv9kQB+/JTN3AjsBRkdHczH8br6//9+fY9Ns0Ljc2NbVU9ePtdLvdH5mms1lXGrOaTwNrIuI15XzDOuBJ4BvAe8vbTYD95fpveU1Zf43MzNL/dpyddWFwBrgu8BBYE25UuosuifL95Zl+vUhSWpBzTmNB+mejH4EeKwssxP4GPDRiBine/7h7rLI3cB5pf5RYHtZz+PAHrqB8w3g5sz8VdmL+CCwHzgE7CltGdCHJKkFVV/uy8wdwI5p5afoXvk0ve0vgGv6rOc24LaG+j5gX0O9sQ9JUjv8GRFJUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1WYMjYh4c0Q82vP4WUR8JCLeEBEHIuJweV5e2kdE3BkR4xHx/Yi4pGddm0v7wxGxuad+aUQ8Vpa5s9xWln59SJLaUXO71ycz8+2Z+XbgUuBF4Ct0b+P6QGauAR4orwGupHv/7zXAVuAu6AYA3bv/XU73bnw7ekLgrtL21HIbS71fH5KkFsz28NR64J8z81+ATcDuUt8NXF2mNwH3ZNd3gGURcQFwBXAgM09k5kngALCxzDs3M7+dmQncM21dTX1Iklow29C4FvhimR7JzOMA5fn8Ul8BHO1ZZqLUBtUnGuqD+pAktWBpbcOIOAt4H3DrTE0bajmHerWI2Er38BYjIyN0Op3ZLD6UJicnF8X7mA+OTbNB47Jt7dTCbkwxLP+d/Mw0m8u4VIcG3XMVj2TmM+X1MxFxQWYeL4eYni31CWBVz3IrgWOlPjat3in1lQ3tB/XxWzJzJ7ATYHR0NMfGxpqanVY6nQ6L4X3MB8em2aBxuXH71xZ2Y4oj14+10u90fmaazWVcZnN46jp+c2gKYC9w6gqozcD9PfUbylVU64Dny6Gl/cCGiFheToBvAPaXeS9ExLpy1dQN09bV1IckqQVVexoR8TrgT4H/2lO+HdgTEVuAp4FrSn0f8B5gnO6VVjcBZOaJiPgEcLC0+3hmnijTHwA+D5wNfL08BvUhSWpBVWhk5ovAedNqP6V7NdX0tgnc3Gc9u4BdDfWHgIsb6o19SJLa4TfCJUnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1apCIyKWRcR9EfHDiDgUEX8UEW+IiAMRcbg8Ly9tIyLujIjxiPh+RFzSs57Npf3hiNjcU780Ih4ry9xZ7hVOvz4kSe2o3dP4NPCNzPxD4G3AIWA78EBmrgEeKK8BrgTWlMdW4C7oBgCwA7gcuAzY0RMCd5W2p5bbWOr9+pAktWDG0IiIc4E/Ae4GyMyXMvM5YBOwuzTbDVxdpjcB92TXd4BlEXEBcAVwIDNPZOZJ4ACwscw7NzO/Xe4vfs+0dTX1IUlqwdKKNn8A/Bj4+4h4G/AwcAswkpnHATLzeEScX9qvAI72LD9RaoPqEw11BvTxWyJiK909FUZGRuh0OhVva7hNTk4uivcxHxybZoPGZdvaqYXdmGJY/jv5mWk2l3GpCY2lwCXAhzLzwYj4NIMPE0VDLedQr5aZO4GdAKOjozk2NjabxYdSp9NhMbyP+eDYNBs0Ljdu/9rCbkxx5PqxVvqdzs9Ms7mMS805jQlgIjMfLK/voxsiz5RDS5TnZ3var+pZfiVwbIb6yoY6A/qQJLVgxtDIzH8DjkbEm0tpPfAEsBc4dQXUZuD+Mr0XuKFcRbUOeL4cYtoPbIiI5eUE+AZgf5n3QkSsK1dN3TBtXU19SJJaUHN4CuBDwBci4izgKeAmuoGzJyK2AE8D15S2+4D3AOPAi6UtmXkiIj4BHCztPp6ZJ8r0B4DPA2cDXy8PgNv79CFJakFVaGTmo8Bow6z1DW0TuLnPenYBuxrqDwEXN9R/2tSHJKkdfiNcklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWq0IiIIxHxWEQ8GhEPldobIuJARBwuz8tLPSLizogYj4jvR8QlPevZXNofjojNPfVLy/rHy7IxqA9JUjtms6fxnzPz7Zl56g5+24EHMnMN8EB5DXAlsKY8tgJ3QTcAgB3A5cBlwI6eELirtD213MYZ+pAkteCVHJ7aBOwu07uBq3vq92TXd4BlEXEBcAVwIDNPZOZJ4ACwscw7NzO/XW4Ve8+0dTX1IUlqQW1oJPCPEfFwRGwttZHMPA5Qns8v9RXA0Z5lJ0ptUH2ioT6oD0lSC5ZWtntnZh6LiPOBAxHxwwFto6GWc6hXK0G2FWBkZIROpzObxYfS5OTkongf88GxaTZoXLatnVrYjSmG5b+Tn5lmcxmXqtDIzGPl+dmI+ArdcxLPRMQFmXm8HGJ6tjSfAFb1LL4SOFbqY9PqnVJf2dCeAX1M376dwE6A0dHRHBsba2p2Wul0OiyG9zEfHJtmg8blxu1fW9iNKY5cP9ZKv9P5mWk2l3GZ8fBURJwTEf/h1DSwAfgBsBc4dQXUZuD+Mr0XuKFcRbUOeL4cWtoPbIiI5eUE+AZgf5n3QkSsK1dN3TBtXU19SJJaULOnMQJ8pVwFuxT4X5n5jYg4COyJiC3A08A1pf0+4D3AOPAicBNAZp6IiE8AB0u7j2fmiTL9AeDzwNnA18sD4PY+fUiSWjBjaGTmU8DbGuo/BdY31BO4uc+6dgG7GuoPARfX9iFJaoffCJckVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdVqfxpdkmZtdUu/rgtw5ParWut7MXNPQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVK06NCJiSUR8LyK+Wl5fGBEPRsThiPhSRJxV6r9bXo+X+at71nFrqT8ZEVf01DeW2nhEbO+pN/YhSWrHbPY0bgEO9bz+JHBHZq4BTgJbSn0LcDIz3wTcUdoRERcB1wJvBTYCny1BtAT4DHAlcBFwXWk7qA9JUguqQiMiVgJXAZ8rrwN4F3BfabIbuLpMbyqvKfPXl/abgHsz85eZ+SO69xC/rDzGM/OpzHwJuBfYNEMfkqQW1O5p/B3wl8C/l9fnAc9l5lR5PQGsKNMrgKMAZf7zpf2v69OW6Vcf1IckqQUzfiM8It4LPJuZD0fE2KlyQ9OcYV6/elNwDWrftI1bga0AIyMjdDqdpmanlcnJyUXxPuaDY9Ns0LhsWzvVWF/MesfCz0yzuYxLzc+IvBN4X0S8B3gtcC7dPY9lEbG07AmsBI6V9hPAKmAiIpYCrwdO9NRP6V2mqf6TAX38lszcCewEGB0dzbGxsYq3Ndw6nQ6L4X3MB8em2aBxubHFn/Noy5Hrx3497Wem2VzGZcbDU5l5a2auzMzVdE9kfzMzrwe+Bby/NNsM3F+m95bXlPnfzMws9WvL1VUXAmuA7wIHgTXlSqmzSh97yzL9+pAkteCVfE/jY8BHI2Kc7vmHu0v9buC8Uv8osB0gMx8H9gBPAN8Abs7MX5W9iA8C++lenbWntB3UhySpBbP6ldvM7ACdMv0U3Sufprf5BXBNn+VvA25rqO8D9jXUG/uQJLXDb4RLkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKnarO6nIWnuVs/zLVe3rZ06I2/rqoU1455GRLw2Ir4bEf83Ih6PiP9e6hdGxIMRcTgivlRu1Uq5neuXImK8zF/ds65bS/3JiLiip76x1MYjYntPvbEPSVI7ag5P/RJ4V2a+DXg7sDEi1gGfBO7IzDXASWBLab8FOJmZbwLuKO2IiIvo3v/7rcBG4LMRsSQilgCfAa4ELgKuK20Z0IckqQUzhkZ2TZaXrymPBN4F3Ffqu4Gry/Sm8poyf31ERKnfm5m/zMwfAeN0b+V6GTCemU9l5kvAvcCmsky/PiRJLag6EV72CB4FngUOAP8MPJeZU6XJBLCiTK8AjgKU+c8D5/XWpy3Tr37egD4kSS2oOhGemb8C3h4Ry4CvAG9palaeo8+8fvWm4BrU/mUiYiuwFWBkZIROp9PU7LQyOTm5KN7HfDhdx2bb2qmZG70CI2fPfx+nk97PyOn6mZlvcxmXWV09lZnPRUQHWAcsi4ilZU9gJXCsNJsAVgETEbEUeD1woqd+Su8yTfWfDOhj+nbtBHYCjI6O5tjY2Gze1lDqdDoshvcxH07XsZnvK5u2rZ3iU495QeQpR64f+/X06fqZmW9zGZeaq6d+v+xhEBFnA+8GDgHfAt5fmm0G7i/Te8tryvxvZmaW+rXl6qoLgTXAd4GDwJpypdRZdE+W7y3L9OtDktSCmn+WXADsLlc5/Q6wJzO/GhFPAPdGxF8D3wPuLu3vBv5HRIzT3cO4FiAzH4+IPcATwBRwcznsRUR8ENgPLAF2ZebjZV0f69OHJKkFM4ZGZn4feEdD/Sm6Vz5Nr/8CuKbPum4Dbmuo7wP21fYhSWqHPyMiSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKq+ZOYkhal3nuyL+T904/cftWC9NMW9zQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWae4SviohvRcShiHg8Im4p9TdExIGIOFyel5d6RMSdETEeEd+PiEt61rW5tD8cEZt76pdGxGNlmTsjIgb1IUlqR82exhSwLTPfAqwDbo6Ii4DtwAOZuQZ4oLwGuBJYUx5bgbugGwDADuByurdw3dETAneVtqeW21jq/fqQJLVgxtDIzOOZ+UiZfgE4BKwANgG7S7PdwNVlehNwT3Z9B1gWERcAVwAHMvNEZp4EDgAby7xzM/PbmZnAPdPW1dSHJKkFs/pGeESsBt4BPAiMZOZx6AZLRJxfmq0AjvYsNlFqg+oTDXUG9DF9u7bS3VNhZGSETqczm7c1lCYnJxfF+5gPp+vYbFs7Na/rHzl7/vs4XS3k2JxOn825/C1Vh0ZE/B7wD8BHMvNn5bRDY9OGWs6hXi0zdwI7AUZHR3NsbGw2iw+lTqfDYngf8+F0HZv5/hmLbWun+NRj/jJQk4UcmyPXjy1IP6+GufwtVV09FRGvoRsYX8jML5fyM+XQEuX52VKfAFb1LL4SODZDfWVDfVAfkqQW1Fw9FcDdwKHM/NueWXuBU1dAbQbu76nfUK6iWgc8Xw4x7Qc2RMTycgJ8A7C/zHshItaVvm6Ytq6mPiRJLajZX3sn8OfAYxHxaKn9FXA7sCcitgBPA9eUefuA9wDjwIvATQCZeSIiPgEcLO0+npknyvQHgM8DZwNfLw8G9CFJasGMoZGZ/0TzeQeA9Q3tE7i5z7p2Absa6g8BFzfUf9rUhySpHX4jXJJUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVK1GW/CFBG7gPcCz2bmxaX2BuBLwGrgCPBfMvNkuV3rp+neue9F4MbMfKQssxn4b2W1f52Zu0v9Un5z1759wC2Zmf36eMXveIDV2782n6vv68jtV7XSryTNVs2exueBjdNq24EHMnMN8EB5DXAlsKY8tgJ3wa9DZgdwOXAZsKPcJ5zSZmvPchtn6EOS1JIZQyMz/w9wYlp5E7C7TO8Gru6p35Nd3wGWRcQFwBXAgcw8UfYWDgAby7xzM/Pb5Tax90xbV1MfkqSWzPWcxkhmHgcoz+eX+grgaE+7iVIbVJ9oqA/qQ5LUkhnPacxSNNRyDvXZdRqxle4hLkZGRuh0OrNdBQDb1k7NablXqml7Jycn5/w+FrvTdWzm+/M1cnZ7n+Fht5Bjczp9NufytzTX0HgmIi7IzOPlENOzpT4BrOpptxI4Vupj0+qdUl/Z0H5QHy+TmTuBnQCjo6M5NjbWr+lAN7Z1Ivz6sZfVOp0Oc30fi93pOjbz/fnatnaKTz32av87cHFYyLFp+nseVnP5W5rr4am9wOYyvRm4v6d+Q3StA54vh5b2AxsiYnk5Ab4B2F/mvRAR68qVVzdMW1dTH5KkltRccvtFunsJb4yICbpXQd0O7ImILcDTwDWl+T66l9uO073k9iaAzDwREZ8ADpZ2H8/MUyfXP8BvLrn9enkwoA9JUktmDI3MvK7PrPUNbRO4uc96dgG7GuoPARc31H/a1IckqT1+I1ySVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1f3xfZ5zVLd03RVoMDA1JehW19Y+SI7dftSD9eHhKklTN0JAkVRv60IiIjRHxZESMR8T2trdHks5kQ31OIyKWAJ8B/hSYAA5GxN7MfKLdLXt1NR0D3bZ2ihsX4NjoQh0HlbQ4DHVoAJcB45n5FEBE3AtsAhZVaJyJXsnJwoUKVEkvN+yhsQI42vN6Ari8pW1ZlLz8VNJsDHtoREMtX9YoYiuwtbycjIgn53WrFsCH4Y3AT9rejmHk2DRzXPo7E8YmPjmnxXrH5T/VLDDsoTEBrOp5vRI4Nr1RZu4Edi7URi2EiHgoM0fb3o5h5Ng0c1z6c2yazWVchv3qqYPAmoi4MCLOAq4F9ra8TZJ0xhrqPY3MnIqIDwL7gSXArsx8vOXNkqQz1lCHBkBm7gP2tb0dLVhUh9teZY5NM8elP8em2azHJTJfdl5ZkqRGw35OQ5I0RAyNIeRPp7xcRKyKiG9FxKGIeDwibml7m4ZJRCyJiO9FxFfb3pZhEhHLIuK+iPhh+ez8UdvbNAwi4i/K39EPIuKLEfHa2mUNjSHT89MpVwIXAddFxEXtbtVQmAK2ZeZbgHXAzY7Lb7kFONT2RgyhTwPfyMw/BN6GY0RErAA+DIxm5sV0LzK6tnZ5Q2P4/PqnUzLzJeDUT6ec0TLzeGY+UqZfoPvHv6LdrRoOEbESuAr4XNvbMkwi4lzgT4C7ATLzpcx8rt2tGhpLgbMjYinwOhq+/9aPoTF8mn46xf859oiI1cA7gAfb3ZKh8XfAXwL/3vaGDJk/AH4M/H05dPe5iDin7Y1qW2b+K/A3wNPAceD5zPzH2uUNjeFT9dMpZ6qI+D3gH4CPZObP2t6etkXEe4FnM/PhtrdlCC0FLgHuysx3AD8HzvhzhBGxnO7RiwuB/wicExF/Vru8oTF8qn465UwUEa+hGxhfyMwvt709Q+KdwPsi4gjdQ5nvioj/2e4mDY0JYCIzT+2R3kc3RM507wZ+lJk/zsz/B3wZ+OPahQ2N4eNPpzSIiKB7bPpQZv5t29szLDLz1sxcmZmr6X5WvpmZ1f9qXMwy89+AoxHx5lJaj7dVgO5hqXUR8bryd7WeWVwgMPTfCD/T+NMpfb0T+HPgsYh4tNT+qvxigNTPh4AvlH+APQXc1PL2tC4zH4yI+4BH6F6V+D1m8c1wvxEuSarm4SlJUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdX+P8snVDPgznqBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEmxJREFUeJzt3W+MnWWZx/HvZQtSYbEI7IS0zRZjY6w2qzDBbsiaEVwoQiwvJIGwUgybbgwajE2kmGyIf0jqC8RglKShXcsua+2ihgaqbAOduCYqUERrqSwjdmWEpYstyOAfMnrti3MXzw5n5twznc5zTvl+kpM5z3Xu57mvmZz2N8+f80xkJpIk1Xhd0w1IkvqHoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqdr8phuYbaeddlouXbp0Ruu+9NJLnHjiibPb0FHUb/1C//Xcb/1C//Vsv0dfTc+7d+9+LjNP77qxzDymHmeffXbO1K5du2a8bhP6rd/M/uu53/rN7L+e7ffoq+kZeDgr/o/18JQkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSp2jF3GxGpm6Xr760eu27FOFdPY/xU9m+4eFa2IzXJPQ1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrXq0IiIeRHxo4i4pyyfGRE/jIgnIuLrEXF8qb++LI+U15e2beOGUn88Ii5sq68qtZGIWN9W7ziHJKkZ09nTuA7Y17b8eeCWzFwGHAKuKfVrgEOZ+RbgljKOiFgOXA68HVgFfKUE0Tzgy8BFwHLgijJ2qjkkSQ2oCo2IWAxcDNxelgM4D7irDNkCXFqery7LlNfPL+NXA1sz8w+Z+QtgBDinPEYy88nMfBnYCqzuMockqQG1expfBD4J/Kksnwo8n5njZXkUWFSeLwKeAiivv1DGv1KfsM5k9anmkCQ1YH63ARFxCXAgM3dHxNDhcoeh2eW1yeqdgmuq8Z16XAusBRgYGGB4eLjTsK7GxsZmvG4T+q1f6I2e160Y7z6oGFgwvfFTmavvuxd+xtNhv0ffbPbcNTSAc4EPRMT7gROAk2nteSyMiPllT2Ax8HQZPwosAUYjYj7wRuBgW/2w9nU61Z+bYo7/JzM3AhsBBgcHc2hoqOLberXh4WFmum4T+q1f6I2er15/b/XYdSvGuXlPzT+T7vZfOTQr2+mmF37G02G/R99s9tz18FRm3pCZizNzKa0T2Q9k5pXALuCDZdga4O7yfHtZprz+QGZmqV9erq46E1gGPAg8BCwrV0odX+bYXtaZbA5JUgOO5HMa1wOfiIgRWucfNpX6JuDUUv8EsB4gM/cC24DHgO8A12bmH8texEeB+2hdnbWtjJ1qDklSA6a1352Zw8Bwef4krSufJo75PXDZJOvfBNzUob4D2NGh3nEOSVIz/ES4JKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqs3O/RGkaVo6jVt5SOod7mlIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKq+Zf7pDkyV3+tcN2Kca6eMNf+DRfPydw69rmnIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSarWNTQi4oSIeDAifhwReyPi06V+ZkT8MCKeiIivR8Txpf76sjxSXl/atq0bSv3xiLiwrb6q1EYiYn1bveMckqRm1Oxp/AE4LzP/GngnsCoiVgKfB27JzGXAIeCaMv4a4FBmvgW4pYwjIpYDlwNvB1YBX4mIeRExD/gycBGwHLiijGWKOSRJDegaGtkyVhaPK48EzgPuKvUtwKXl+eqyTHn9/IiIUt+amX/IzF8AI8A55TGSmU9m5svAVmB1WWeyOSRJDag6p1H2CB4FDgA7gZ8Dz2fmeBkyCiwqzxcBTwGU118ATm2vT1hnsvqpU8whSWpA1V1uM/OPwDsjYiHwLeBtnYaVrzHJa5PVOwXXVONfJSLWAmsBBgYGGB4e7jSsq7GxsRmv24R+6xf+3PO6FePdB/eAgQX0Ta+Hdeq5l98n/fY+7rd+YXZ7ntat0TPz+YgYBlYCCyNiftkTWAw8XYaNAkuA0YiYD7wRONhWP6x9nU7156aYY2JfG4GNAIODgzk0NDSdb+sVw8PDzHTdJvRbv/DnnifeurtXrVsxzs17+usvCHTqef+VQ800U6Hf3sf91i/Mbs81V0+dXvYwiIgFwPuAfcAu4INl2Brg7vJ8e1mmvP5AZmapX16urjoTWAY8CDwELCtXSh1P62T59rLOZHNIkhpQ8yvUGcCWcpXT64BtmXlPRDwGbI2IzwE/AjaV8ZuAf4mIEVp7GJcDZObeiNgGPAaMA9eWw15ExEeB+4B5wObM3Fu2df0kc0iSGtA1NDLzJ8C7OtSfpHXl08T674HLJtnWTcBNHeo7gB21c0iSmuEnwiVJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRV668/fqxZt3SO/1b3uhXjffP3wSW9mnsakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqtY1NCJiSUTsioh9EbE3Iq4r9TdFxM6IeKJ8PaXUIyJujYiRiPhJRJzVtq01ZfwTEbGmrX52ROwp69waETHVHJKkZtTsaYwD6zLzbcBK4NqIWA6sB+7PzGXA/WUZ4CJgWXmsBW6DVgAANwLvBs4BbmwLgdvK2MPrrSr1yeaQJDWga2hk5jOZ+Uh5/iKwD1gErAa2lGFbgEvL89XAHdnyA2BhRJwBXAjszMyDmXkI2AmsKq+dnJnfz8wE7piwrU5zSJIaEK3/pysHRywFvgu8A/hlZi5se+1QZp4SEfcAGzLze6V+P3A9MASckJmfK/V/An4HDJfx7yv1vwWuz8xLIuL5TnN06GstrT0VBgYGzt66dWv199TuwMEXePZ3M1r1iKxY9MYZrTc2NsZJJ510RHPv+dULR7T+dA0soJGf8Uz1W7/QueeZvsfmwmy8j+dSv/ULdT2/973v3Z2Zg922Nb920og4CfgG8PHM/E057dBxaIdazqBeLTM3AhsBBgcHc2hoaDqrv+JLd97NzXuqfySzZv+VQzNab3h4mJl+r4ddvf7eI1p/utatGG/kZzxT/dYvdO55pu+xuTAb7+O51G/9wuz2XHX1VEQcRysw7szMb5bys+XQEuXrgVIfBZa0rb4YeLpLfXGH+lRzSJIaUHP1VACbgH2Z+YW2l7YDh6+AWgPc3Va/qlxFtRJ4ITOfAe4DLoiIU8oJ8AuA+8prL0bEyjLXVRO21WkOSVIDava7zwU+BOyJiEdL7VPABmBbRFwD/BK4rLy2A3g/MAL8FvgwQGYejIjPAg+VcZ/JzIPl+UeArwILgG+XB1PMIUlqQNfQKCe0JzuBcX6H8QlcO8m2NgObO9QfpnVyfWL9153mkCQ1w0+ES5KqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkavObbkDS0bd0/b2NzLt/w8WNzKujxz0NSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTNS257wEwvh1y3YpyrG7qUUtJrk3sakqRqhoYkqZqhIUmqZmhIkqoZGpKkal1DIyI2R8SBiPhpW+1NEbEzIp4oX08p9YiIWyNiJCJ+EhFnta2zpox/IiLWtNXPjog9ZZ1bIyKmmkOS1JyaPY2vAqsm1NYD92fmMuD+sgxwEbCsPNYCt0ErAIAbgXcD5wA3toXAbWXs4fVWdZlDktSQrqGRmd8FDk4orwa2lOdbgEvb6ndkyw+AhRFxBnAhsDMzD2bmIWAnsKq8dnJmfj8zE7hjwrY6zSFJashMz2kMZOYzAOXrX5b6IuCptnGjpTZVfbRDfao5JEkNme1PhEeHWs6gPr1JI9bSOsTFwMAAw8PD090EAAMLWp+y7hf91i/0X8/91i/0Vs81/xbHxsZm/G+2Cf3WL8xuzzMNjWcj4ozMfKYcYjpQ6qPAkrZxi4GnS31oQn241Bd3GD/VHK+SmRuBjQCDg4M5NDQ02dApfenOu7l5T//cWWXdivG+6hf6r+d+6xd6q+f9Vw51HTM8PMxM/802od/6hdnteaaHp7YDh6+AWgPc3Va/qlxFtRJ4oRxaug+4ICJOKSfALwDuK6+9GBEry1VTV03YVqc5JEkN6frrSER8jdZewmkRMUrrKqgNwLaIuAb4JXBZGb4DeD8wAvwW+DBAZh6MiM8CD5Vxn8nMwyfXP0LrCq0FwLfLgynmkCQ1pGtoZOYVk7x0foexCVw7yXY2A5s71B8G3tGh/utOc0iSmuMnwiVJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVeuOuZpKOSUvX39t1zLoV41xdMW669m+4eNa3Kfc0JEnTYGhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqnkbEUnHpJpbmMxEt9ueHOu3L3FPQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVfM2IpI0i47W7Uu6mavbl7inIUmqZmhIkqoZGpKkaj0fGhGxKiIej4iRiFjfdD+S9FrW06EREfOALwMXAcuBKyJiebNdSdJrV0+HBnAOMJKZT2bmy8BWYHXDPUnSa1avh8Yi4Km25dFSkyQ1IDKz6R4mFRGXARdm5j+U5Q8B52TmxyaMWwusLYtvBR6f4ZSnAc/NcN0m9Fu/0H8991u/0H892+/RV9PzX2Xm6d021Osf7hsFlrQtLwaenjgoMzcCG490soh4ODMHj3Q7c6Xf+oX+67nf+oX+69l+j77Z7LnXD089BCyLiDMj4njgcmB7wz1J0mtWT+9pZOZ4RHwUuA+YB2zOzL0NtyVJr1k9HRoAmbkD2DFH0x3xIa451m/9Qv/13G/9Qv/1bL9H36z13NMnwiVJvaXXz2lIknqIoVH00+1KImJzRByIiJ823UuNiFgSEbsiYl9E7I2I65ruqZuIOCEiHoyIH5eeP910TzUiYl5E/Cgi7mm6lxoRsT8i9kTEoxHxcNP9dBMRCyPiroj4WXk//03TPU0mIt5afq6HH7+JiI8f8XY9PPXK7Ur+C/g7Wpf5PgRckZmPNdrYJCLiPcAYcEdmvqPpfrqJiDOAMzLzkYj4C2A3cGmv/nwBIiKAEzNzLCKOA74HXJeZP2i4tSlFxCeAQeDkzLyk6X66iYj9wGBm9sXnHiJiC/CfmXl7uaLzDZn5fNN9dVP+j/sV8O7M/O8j2ZZ7Gi19dbuSzPwucLDpPmpl5jOZ+Uh5/iKwjx7/ZH+2jJXF48qjp3/DiojFwMXA7U33ciyKiJOB9wCbADLz5X4IjOJ84OdHGhhgaBzm7UrmSEQsBd4F/LDZTrorh3oeBQ4AOzOz13v+IvBJ4E9NNzINCfxHROwud3boZW8G/hf453II8PaIOLHppipdDnxtNjZkaLREh1pP/1bZjyLiJOAbwMcz8zdN99NNZv4xM99J604E50REzx4KjIhLgAOZubvpXqbp3Mw8i9adrK8th1571XzgLOC2zHwX8BLQ0+c/AcphtA8A/z4b2zM0WqpuV6KZK+cFvgHcmZnfbLqf6SiHIIaBVQ23MpVzgQ+UcwRbgfMi4l+bbam7zHy6fD0AfIvWoeJeNQqMtu1x3kUrRHrdRcAjmfnsbGzM0GjxdiVHUTmpvAnYl5lfaLqfGhFxekQsLM8XAO8DftZsV5PLzBsyc3FmLqX1/n0gM/++4bamFBEnlgsjKId5LgB69orAzPwf4KmIeGspnQ/07MUcba5glg5NQR98Inwu9NvtSiLia8AQcFpEjAI3ZuamZrua0rnAh4A95RwBwKfKp/171RnAlnLVyeuAbZnZF5ex9pEB4Fut3ymYD/xbZn6n2Za6+hhwZ/nl8kngww33M6WIeAOtq0L/cda26SW3kqRaHp6SJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTt/wDOR+pwWRt7JQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for col in ['price', 'item_seq_number']:\n",
    "    select_filter = all_features[col] > 0\n",
    "    all_features.loc[select_filter, col], _ = boxcox(all_features.loc[select_filter, col])\n",
    "    all_features[col].hist()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['item_id',\n",
       " 'user_id',\n",
       " 'region',\n",
       " 'city',\n",
       " 'parent_category_name',\n",
       " 'category_name',\n",
       " 'param_1',\n",
       " 'param_2',\n",
       " 'param_3',\n",
       " 'title',\n",
       " 'description',\n",
       " 'price',\n",
       " 'item_seq_number',\n",
       " 'activation_date',\n",
       " 'user_type',\n",
       " 'image',\n",
       " 'image_top_1',\n",
       " 'avg_days_up_user',\n",
       " 'avg_times_up_user',\n",
       " 'med_days_up_user',\n",
       " 'med_times_up_user',\n",
       " 'n_user_items',\n",
       " 'weekday',\n",
       " 'num_words_description',\n",
       " 'num_unique_words_description',\n",
       " 'num_words_title',\n",
       " 'num_unique_words_title',\n",
       " 'words_vs_unique_title',\n",
       " 'words_vs_unique_description',\n",
       " 'num_desc_punct']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "numerical = [\n",
    "    'weekday', 'num_words_title', 'num_words_description',\n",
    "    'num_unique_words_title', 'num_unique_words_description',\n",
    "    'words_vs_unique_title', 'words_vs_unique_description',\n",
    "    'num_desc_punct', 'avg_times_up_user', 'avg_days_up_user', \n",
    "    'med_times_up_user', 'med_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n",
    "]\n",
    "\n",
    "features = numerical+categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming image_top_1...\n",
      "Transforming param_1...\n",
      "Transforming param_2...\n",
      "Transforming param_3...\n",
      "Transforming city...\n",
      "Transforming region...\n",
      "Transforming category_name...\n",
      "Transforming parent_category_name...\n",
      "Transforming user_type...\n"
     ]
    }
   ],
   "source": [
    "# label encode categorical data\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "for feature in categorical:\n",
    "    print('Transforming {}...'.format(feature))\n",
    "    encoder = LabelEncoder()\n",
    "    all_features.loc[:, feature] = encoder.fit_transform(all_features[feature].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features.loc[:, numerical] = all_features[numerical].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "weekday                         0\n",
       "num_words_title                 0\n",
       "num_words_description           0\n",
       "num_unique_words_title          0\n",
       "num_unique_words_description    0\n",
       "words_vs_unique_title           0\n",
       "words_vs_unique_description     0\n",
       "num_desc_punct                  0\n",
       "avg_times_up_user               0\n",
       "avg_days_up_user                0\n",
       "med_times_up_user               0\n",
       "med_days_up_user                0\n",
       "n_user_items                    0\n",
       "price                           0\n",
       "item_seq_number                 0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features[numerical].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardize numerical data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "all_features.loc[:, numerical] = scaler.fit_transform(all_features[numerical].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare  pre-trained embeddings and text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 500000\n",
    "maxlen = 150\n",
    "embed_size = 300\n",
    "\n",
    "title_max_features = 200000\n",
    "title_maxlen = 80\n",
    "title_embed_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import word2vec\n",
    "\n",
    "def get_embed_matrix(emb_file, texts, max_feat_num, max_len, emb_size):\n",
    "\n",
    "    print('getting embeddings')\n",
    "    embeddings_index = word2vec.Word2Vec.load(emb_file)\n",
    "    \n",
    "    print('fitting tokenizer')\n",
    "    tokenizer = text.Tokenizer(num_words=max_feat_num)\n",
    "    tokenizer.fit_on_texts(texts)\n",
    "    \n",
    "    print('convert to sequences')\n",
    "    texts = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "    print('padding')\n",
    "    texts = sequence.pad_sequences(texts, maxlen=max_len)\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    nb_words = min(max_feat_num, len(word_index))\n",
    "    embedding_matrix = np.zeros((nb_words, emb_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_feat_num: continue\n",
    "        try:\n",
    "            embedding_vector = embeddings_index[word]\n",
    "        except KeyError:\n",
    "            embedding_vector = None\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "      \n",
    "    \n",
    "    return {\n",
    "        'text': texts,\n",
    "        'emb_matrix': embedding_matrix,\n",
    "        'nb_words': nb_words\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting embeddings\n",
      "fitting tokenizer\n",
      "convert to sequences\n",
      "padding\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py:24: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting embeddings\n",
      "fitting tokenizer\n",
      "convert to sequences\n",
      "padding\n"
     ]
    }
   ],
   "source": [
    "desc_embed_info = get_embed_matrix(embedding_file, \n",
    "                                   all_features.description.values, \n",
    "                                   max_features, \n",
    "                                   maxlen, \n",
    "                                   embed_size)\n",
    "\n",
    "title_embed_info = get_embed_matrix(title_embedding_file, \n",
    "                                    all_features.title.values, \n",
    "                                    title_max_features, \n",
    "                                    title_maxlen, \n",
    "                                    title_embed_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_dump = {\n",
    "    'desc_embed_info': desc_embed_info,\n",
    "    'title_embed_info': title_embed_info\n",
    "}\n",
    "\n",
    "import pickle\n",
    "pickle.dump(embed_dump, open('rnn_embed.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setup max info for embedding in categorical variables\n"
     ]
    }
   ],
   "source": [
    "print('setup max info for embedding in categorical variables')\n",
    "max_info = dict((col, all_features[col].max()+1) for col in categorical)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(y_true - y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        print(input_shape)\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_rmse(true, prediction):\n",
    "    return np.sqrt(metrics.mean_squared_error(true, np.clip(prediction, 0., 1.)))\n",
    "    \n",
    "class NBatchEvalLogger(Callback):\n",
    "    def __init__(self, display, val_X, val_y, save_path=None, save_start=1000):\n",
    "        self.step = 0\n",
    "        self.display = display\n",
    "        self.val_X = val_X\n",
    "        self.val_y = val_y\n",
    "        self.best_loss = None\n",
    "        self.save_path = save_path\n",
    "        self.save_start = save_start\n",
    "        self.record_count = 0\n",
    "        \n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.step += 1\n",
    "        if self.step % self.display == 0 and self.step >= self.save_start:\n",
    "            #loss, metric = self.model.evaluate(self.val_X, self.val_y, batch_size=128, verbose=1)\n",
    "            prediction = self.model.predict(self.val_X, batch_size=128, verbose=0)\n",
    "            loss = clip_rmse(self.val_y, prediction)\n",
    "            \n",
    "            if self.best_loss is None:\n",
    "                self.best_loss = loss\n",
    "            else:\n",
    "                if loss < self.best_loss:\n",
    "                    self.best_loss = loss\n",
    "                    if self.save_path is not None:\n",
    "                        self.model.save(self.save_path, overwrite=True)\n",
    "                        self.record_count += 1\n",
    "                    \n",
    "            print('\\rstep: {} val loss={:.5f}, best loss={:.5f}'.format(self.step, loss, self.best_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from copy import deepcopy as cp\n",
    "import os\n",
    "from zipfile import ZipFile\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from dask import bag, threaded\n",
    "from dask.diagnostics import ProgressBar\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.applications.resnet50 import preprocess_input\n",
    "import concurrent.futures\n",
    "\n",
    "class DataGenerator(keras.utils.Sequence):\n",
    "    #'Generates data for Keras'\n",
    "    def __init__(self, list_IDs, X, y, img_arch, img_path, batch_size=32, shuffle=True, is_train=True):\n",
    "        #'Initialization'\n",
    "        self.batch_size = batch_size\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.list_IDs = list_IDs\n",
    "        self.shuffle = shuffle\n",
    "        self.img_path = img_path\n",
    "        self.is_train = is_train\n",
    "        self.on_epoch_end()\n",
    "        self.zipped = ZipFile(img_arch)\n",
    "        print('file names:\\n', self.zipped.namelist()[1:10], '\\n...')\n",
    "        self.img_path = img_path\n",
    "        \n",
    "    def __len__(self):\n",
    "        #'Denotes the number of batches per epoch'\n",
    "        return int(np.ceil(len(self.list_IDs) / self.batch_size))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        #'Generate one batch of data'\n",
    "        # Generate indexes of the batch\n",
    "        start = index*self.batch_size\n",
    "        end = min((index+1)*self.batch_size, len(self.indexes))\n",
    "        indexes = self.indexes[start: end]\n",
    "\n",
    "        # Generate data\n",
    "        return self.__data_generation(indexes)\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        #'Updates indexes after each epoch'\n",
    "        self.indexes = cp(list(self.list_IDs))\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "\n",
    "    def load_img_from_zipped(self, img_id):\n",
    "        \n",
    "        invalid_img_ids = ['4f029e2a00e892aa2cac27d98b52ef8b13d91471f613c8d3c38e3f29d4da0b0c', \n",
    "                           '8513a91e55670c709069b5f85e12a59095b802877715903abef16b7a6f306e58', \n",
    "                           '60d310a42e87cdf799afcd89dc1b11ae3fdc3d0233747ec7ef78d82c87002e83', \n",
    "                           'b98b291bd04c3d92165ca515e00468fd9756af9a8f1df42505deed1dcfb5d7ae']\n",
    "        \n",
    "        try:\n",
    "            if img_id in invalid_img_ids or img_id == 'no-image':\n",
    "                imz = None\n",
    "            else:\n",
    "                exfile = self.zipped.read(self.img_path+img_id+'.jpg')\n",
    "                arr = np.frombuffer(exfile, np.uint8)\n",
    "                imz = cv2.imdecode(arr, flags=cv2.IMREAD_UNCHANGED)\n",
    "                imz = cv2.resize(imz, (224,224), interpolation=cv2.INTER_AREA)\n",
    "        except:\n",
    "            print(img_id, ' is invalid')\n",
    "            imz = None\n",
    "            \n",
    "        if imz is None:\n",
    "            imz = np.zeros((1, 224, 224, 3))\n",
    "        else:\n",
    "            imz = img_to_array(imz)\n",
    "            imz = np.expand_dims(imz, axis=0)\n",
    "            \n",
    "        imz = preprocess_input(imz) # adjust to mean of rgb to some value\n",
    "            \n",
    "        return imz\n",
    "\n",
    "    def parallel_load_imgs(self, img_ids):\n",
    "        global cpu_count\n",
    "        '''\n",
    "        with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count) as executor:\n",
    "            # Process the list of files, but split the work across the process pool to use all CPUs!\n",
    "            imgs = np.concatenate([img for img in executor.map(self.load_img_from_zipped, img_ids, chunksize=len(img_ids)//3)])    \n",
    "        '''\n",
    "        imgs = np.concatenate([self.load_img_from_zipped(imgid) for imgid in img_ids])    \n",
    "        \n",
    "        return imgs\n",
    "    \n",
    "    def __data_generation(self, list_IDs_temp):\n",
    "        #'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n",
    "        \n",
    "        # Generate data\n",
    "        X = dict((col, self.X.loc[list_IDs_temp, col].values) for col in features)\n",
    "        X['desc'] = desc_embed_info['text'][list_IDs_temp,:]\n",
    "        X['title'] = title_embed_info['text'][list_IDs_temp,:]\n",
    "        X['imgs'] = self.parallel_load_imgs(self.X.loc[list_IDs_temp, 'image'].values)\n",
    "        \n",
    "        if self.is_train:\n",
    "            y = cp(self.y[list_IDs_temp])\n",
    "            return X, y\n",
    "        else:\n",
    "            return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nzipped = ZipFile('train_jpg.zip')\\nprint(zipped.namelist()[1:10])\\n\\nimg_id = '2809fd6afd6d3cae4dd4ad93a7f905a0db32292f4df4b3f19fa5492e08cbfd90'\\ntarget_size=(224,224)\\ntry:\\n    exfile = zipped.read('data/competition_files/train_jpg/'+img_id+'.jpg')\\n    arr = np.frombuffer(exfile, np.uint8)\\n    imz = cv2.imdecode(arr, flags=cv2.IMREAD_UNCHANGED)\\n    imz = cv2.resize(imz, target_size, interpolation=cv2.INTER_AREA)\\nexcept:\\n    print(img_id, ' is invalid')\\n    imz = None\\nimz\\n\""
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 'train_jpg.zip', 'data/competition_files/train_jpg/', \n",
    "# debug use\n",
    "'''\n",
    "zipped = ZipFile('train_jpg.zip')\n",
    "print(zipped.namelist()[1:10])\n",
    "\n",
    "img_id = '2809fd6afd6d3cae4dd4ad93a7f905a0db32292f4df4b3f19fa5492e08cbfd90'\n",
    "target_size=(224,224)\n",
    "try:\n",
    "    exfile = zipped.read('data/competition_files/train_jpg/'+img_id+'.jpg')\n",
    "    arr = np.frombuffer(exfile, np.uint8)\n",
    "    imz = cv2.imdecode(arr, flags=cv2.IMREAD_UNCHANGED)\n",
    "    imz = cv2.resize(imz, target_size, interpolation=cv2.INTER_AREA)\n",
    "except:\n",
    "    print(img_id, ' is invalid')\n",
    "    imz = None\n",
    "imz\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(X, categorical_features, numerical_features):\n",
    "    \n",
    "    # non-cat features\n",
    "    non_cat_inputs = []\n",
    "    for col in numerical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        non_cat_inputs.append(f)\n",
    "        \n",
    "    # cat features\n",
    "    cat_inputs = []\n",
    "    cat_embeds = []\n",
    "    for col in categorical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        embed_dim = max_info[col].max()\n",
    "        if max_info[col] > 10:\n",
    "            reduced_dim = 10\n",
    "        else:\n",
    "            reduced_dim = 1\n",
    "        embed_f = Embedding(embed_dim, reduced_dim)(f)\n",
    "        flatten_f = Flatten()(embed_f)\n",
    "        cat_inputs.append(f)\n",
    "        cat_embeds.append(flatten_f)\n",
    "      \n",
    "    # text features: architecture of text to try here!!!\n",
    "    \n",
    "    # description\n",
    "    text_inp = Input(shape = (maxlen, ), name='desc')\n",
    "    text_emb = Embedding(desc_embed_info['nb_words'], embed_size, weights = [desc_embed_info['emb_matrix']],\n",
    "                    input_length = maxlen, trainable = False)(text_inp)\n",
    "    text_emb = SpatialDropout1D(0.3)(text_emb)\n",
    "    text_gru = Bidirectional(CuDNNGRU(128, return_sequences = True))(text_emb)\n",
    "    text_gru = Conv1D(64, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(text_gru)\n",
    "    text_gru_avg = GlobalAveragePooling1D()(text_gru)\n",
    "    text_gru_max = GlobalMaxPooling1D()(text_gru)\n",
    "    text_gru = concatenate([text_gru_avg, text_gru_max]) \n",
    "    text_gru = Dropout(0.1)(text_gru)\n",
    "    \n",
    "    # title\n",
    "    title_inp = Input(shape = (title_maxlen, ), name='title')\n",
    "    title_emb = Embedding(title_embed_info['nb_words'], title_embed_size, weights = [title_embed_info['emb_matrix']],\n",
    "                    input_length = title_maxlen, trainable = False)(title_inp)\n",
    "    title_emb = SpatialDropout1D(0.1)(title_emb)\n",
    "    title_gru = Bidirectional(CuDNNGRU(32, return_sequences = True))(title_emb)\n",
    "    title_gru = Conv1D(16, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(title_gru)\n",
    "    title_gru_avg = GlobalAveragePooling1D()(title_gru)\n",
    "    title_gru_max = GlobalMaxPooling1D()(title_gru)\n",
    "    title_gru = concatenate([title_gru_avg, title_gru_max]) \n",
    "    title_gru = Dropout(0.1)(title_gru)\n",
    "    \n",
    "    # add image architecture\n",
    "    # reference: https://keras.io/getting-started/functional-api-guide/#more-examples, Visual question answering model\n",
    "    img_inp = Input(shape = (224, 224, 3 ), name='imgs')\n",
    "    img_ch = Conv2D(64, (3, 3), activation='relu', padding='same', W_constraint=max_norm(3))(img_inp)\n",
    "    img_ch = Conv2D(64, (3, 3), activation='relu')(img_ch)\n",
    "    img_ch = MaxPooling2D((2, 2))(img_ch)\n",
    "    '''\n",
    "    img_ch = Conv2D(128, (3, 3), activation='relu', padding='same', W_constraint=max_norm(3))(img_ch)\n",
    "    img_ch = Conv2D(128, (3, 3), activation='relu')(img_ch)\n",
    "    img_ch = MaxPooling2D((2, 2))(img_ch)\n",
    "    img_ch = Conv2D(256, (3, 3), activation='relu', padding='same', W_constraint=max_norm(3))(img_ch)\n",
    "    img_ch = Conv2D(256, (3, 3), activation='relu')(img_ch)\n",
    "    img_ch = Conv2D(256, (3, 3), activation='relu')(img_ch)\n",
    "    img_ch = MaxPooling2D((2, 2))(img_ch)\n",
    "    '''\n",
    "    img_ch = Flatten()(img_ch)\n",
    "\n",
    "    # merge each branch: non-cat, cat, text, img\n",
    "    concat_main = non_cat_inputs+cat_embeds+[text_gru, title_gru, img_ch]\n",
    "    main = concatenate(concat_main)\n",
    "    main = BatchNormalization()(main)\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(512, activation='relu')(main))\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(64, activation='relu')(main))\n",
    "    out = Dense(1, activation = \"sigmoid\")(main)\n",
    "\n",
    "    concat_input = non_cat_inputs+cat_inputs+[text_inp, title_inp, img_inp]\n",
    "    model = Model(concat_input, out)\n",
    "    model.regularizers = [regularizers.l2(0.0001)]\n",
    "    model.compile(optimizer = Adam(lr=0.001), loss = root_mean_squared_error,\n",
    "                  metrics =[root_mean_squared_error])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import warnings; warnings.filterwarnings('ignore') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indices = np.arange(0, train_len)\n",
    "test_indices = np.arange(train_len, all_features.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from ipywidgets import IntProgress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_fold = 0 # <= 0 for invalid, train from fold 1, > 0: used to train from fold=start_fold\n",
    "resume_file_prefix = '0618_rnn' # whatever we like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "WARNING:tensorflow:From E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\util\\deprecation.py:497: calling conv1d (from tensorflow.python.ops.nn_ops) with data_format=NHWC is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "`NHWC` for data_format is deprecated, use `NWC` instead\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "desc (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "title (InputLayer)              (None, 80)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 150, 300)     150000000   desc[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_11 (Embedding)        (None, 80, 100)      20000000    title[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 150, 300)     0           embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_2 (SpatialDro (None, 80, 100)      0           embedding_11[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 150, 256)     330240      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 80, 64)       25728       spatial_dropout1d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "imgs (InputLayer)               (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 148, 64)      49216       bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 78, 16)       3088        bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 224, 224, 64) 1792        imgs[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_1 (Glo (None, 64)           0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 64)           0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_2 (Glo (None, 16)           0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 16)           0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 222, 222, 64) 36928       conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 1, 10)        30640       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1, 10)        3720        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1, 10)        2780        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 1, 10)        12770       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 1, 10)        18240       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 1, 10)        280         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 1, 10)        470         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 1, 1)         9           parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_9 (Embedding)         (None, 1, 1)         3           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 128)          0           global_average_pooling1d_1[0][0] \n",
      "                                                                 global_max_pooling1d_1[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 32)           0           global_average_pooling1d_2[0][0] \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 111, 111, 64) 0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 10)           0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 10)           0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 10)           0           embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 10)           0           embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 10)           0           embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 10)           0           embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 10)           0           embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 1)            0           embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 1)            0           embedding_9[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 128)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 32)           0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 788544)       0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 788791)       0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "                                                                 flatten_5[0][0]                  \n",
      "                                                                 flatten_6[0][0]                  \n",
      "                                                                 flatten_7[0][0]                  \n",
      "                                                                 flatten_8[0][0]                  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                 flatten_9[0][0]                  \n",
      "                                                                 dropout_1[0][0]                  \n",
      "                                                                 dropout_2[0][0]                  \n",
      "                                                                 flatten_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 788791)       3155164     concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 788791)       0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          403861504   dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 512)          2048        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 512)          0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           32832       dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 64)           256         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1)            65          batch_normalization_3[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 577,567,773\n",
      "Trainable params: 405,989,039\n",
      "Non-trainable params: 171,578,734\n",
      "__________________________________________________________________________________________________\n",
      "file names:\n",
      " ['data/competition_files/train_jpg/0b8eed559572527d972b4d959e8f4c107fdd9bc19cca04903854ac315f74615e.jpg', 'data/competition_files/train_jpg/856e74b8c46edcf0c0e23444eab019bfda63687bb70a3481955cc6ab86e39df2.jpg', 'data/competition_files/train_jpg/122d198cf11ab32d2346bff455d6702f1ea519df957cea2625aa50842fe14ad1.jpg', 'data/competition_files/train_jpg/2809fd6afd6d3cae4dd4ad93a7f905a0db32292f4df4b3f19fa5492e08cbfd90.jpg', 'data/competition_files/train_jpg/5ef4a19afe4ad593464931734ff43c1112cf94c6bdb4593f3b754fee46739515.jpg', 'data/competition_files/train_jpg/c37787b5cc6c3052130c6f390aa5b57462b558a204d5c4124bc89447c9e1b4b0.jpg', 'data/competition_files/train_jpg/0f8ae17e177ed82363ed3dba7d277ed6227ac0c935cb52c17f02d638a92aef6e.jpg', 'data/competition_files/train_jpg/ba126be25858022d3cddf07d27288f9d35c495458ec49aa9820708379b7ffc1e.jpg', 'data/competition_files/train_jpg/4cc05cb70bcdde73e34718020f2ef4c69063af4098602bfed8a00e7f53a523ed.jpg'] \n",
      "...\n",
      "file names:\n",
      " ['data/competition_files/train_jpg/0b8eed559572527d972b4d959e8f4c107fdd9bc19cca04903854ac315f74615e.jpg', 'data/competition_files/train_jpg/856e74b8c46edcf0c0e23444eab019bfda63687bb70a3481955cc6ab86e39df2.jpg', 'data/competition_files/train_jpg/122d198cf11ab32d2346bff455d6702f1ea519df957cea2625aa50842fe14ad1.jpg', 'data/competition_files/train_jpg/2809fd6afd6d3cae4dd4ad93a7f905a0db32292f4df4b3f19fa5492e08cbfd90.jpg', 'data/competition_files/train_jpg/5ef4a19afe4ad593464931734ff43c1112cf94c6bdb4593f3b754fee46739515.jpg', 'data/competition_files/train_jpg/c37787b5cc6c3052130c6f390aa5b57462b558a204d5c4124bc89447c9e1b4b0.jpg', 'data/competition_files/train_jpg/0f8ae17e177ed82363ed3dba7d277ed6227ac0c935cb52c17f02d638a92aef6e.jpg', 'data/competition_files/train_jpg/ba126be25858022d3cddf07d27288f9d35c495458ec49aa9820708379b7ffc1e.jpg', 'data/competition_files/train_jpg/4cc05cb70bcdde73e34718020f2ef4c69063af4098602bfed8a00e7f53a523ed.jpg'] \n",
      "...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4435c48737be41b78f1973376d4a5935",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[788791,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: training/Adam/Variable_31/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@training/Adam/Variable_31\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](training/Adam/Variable_31, training/Adam/zeros_31)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'training/Adam/Variable_31/Assign', defined at:\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 486, in start\n    self.io_loop.start()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 127, in start\n    self.asyncio_loop.run_forever()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\base_events.py\", line 421, in run_forever\n    self._run_once()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\base_events.py\", line 1425, in _run_once\n    handle._run()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\events.py\", line 127, in _run\n    self._callback(*self._args)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\ioloop.py\", line 759, in _run_callback\n    ret = callback()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 536, in <lambda>\n    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2903, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2963, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-35-104f91fa3312>\", line 39, in <module>\n    callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\interfaces.py\", line 91, in wrapper\n    return func(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\", line 2080, in fit_generator\n    self._make_train_function()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\", line 992, in _make_train_function\n    loss=self.total_loss)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\interfaces.py\", line 91, in wrapper\n    return func(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\optimizers.py\", line 457, in get_updates\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\optimizers.py\", line 457, in <listcomp>\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\", line 695, in zeros\n    return variable(v, dtype=dtype, name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\", line 396, in variable\n    v = tf.Variable(value, dtype=tf.as_dtype(dtype), name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\variables.py\", line 235, in __init__\n    constraint=constraint)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\variables.py\", line 381, in _init_from_args\n    validate_shape=validate_shape).op\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\state_ops.py\", line 281, in assign\n    validate_shape=validate_shape)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\gen_state_ops.py\", line 64, in assign\n    use_locking=use_locking, name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3290, in create_op\n    op_def=op_def)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1654, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[788791,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: training/Adam/Variable_31/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@training/Adam/Variable_31\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](training/Adam/Variable_31, training/Adam/zeros_31)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1326\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1327\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1328\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1311\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1312\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1313\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1419\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1420\u001b[1;33m             status, run_metadata)\n\u001b[0m\u001b[0;32m   1421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[0;32m    515\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 516\u001b[1;33m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[0;32m    517\u001b[0m     \u001b[1;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[788791,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: training/Adam/Variable_31/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@training/Adam/Variable_31\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](training/Adam/Variable_31, training/Adam/zeros_31)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-104f91fa3312>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     37\u001b[0m                                   \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrnn_train_epochs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m                                   \u001b[0mverbose\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m                                   callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n\u001b[0m\u001b[0;32m     40\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 91\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   2188\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstateful_metric_functions\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2189\u001b[0m                     \u001b[0mm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_states\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2190\u001b[1;33m                 \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2191\u001b[0m                 \u001b[0msteps_done\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2192\u001b[0m                 \u001b[0mbatch_index\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_epoch_begin\u001b[1;34m(self, epoch, logs)\u001b[0m\n\u001b[0;32m     61\u001b[0m         \u001b[0mlogs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m             \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_t_batch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_ts_batch_begin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdeque\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mqueue_length\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_epoch_begin\u001b[1;34m(self, epoch, logs)\u001b[0m\n\u001b[0;32m    616\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'lr'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    617\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Optimizer must have a \"lr\" attribute.'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 618\u001b[1;33m         \u001b[0mlr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    619\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# new API\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    620\u001b[0m             \u001b[0mlr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36mget_value\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m   2312\u001b[0m         \u001b[0mA\u001b[0m \u001b[0mNumpy\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2313\u001b[0m     \"\"\"\n\u001b[1;32m-> 2314\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2315\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2316\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[1;34m()\u001b[0m\n\u001b[0;32m    198\u001b[0m                     \u001b[0mv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_keras_initialized\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0muninitialized_vars\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m                     \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvariables_initializer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muninitialized_vars\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m     \u001b[1;31m# hack for list_devices() function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;31m# list_devices() function is not available under tensorflow r1.3.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    903\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 905\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    906\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1139\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1140\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1141\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1142\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1319\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1320\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1321\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1322\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1323\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mE:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1338\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1339\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1340\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1341\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1342\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[788791,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: training/Adam/Variable_31/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@training/Adam/Variable_31\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](training/Adam/Variable_31, training/Adam/zeros_31)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'training/Adam/Variable_31/Assign', defined at:\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 486, in start\n    self.io_loop.start()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 127, in start\n    self.asyncio_loop.run_forever()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\base_events.py\", line 421, in run_forever\n    self._run_once()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\base_events.py\", line 1425, in _run_once\n    handle._run()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\asyncio\\events.py\", line 127, in _run\n    self._callback(*self._args)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\ioloop.py\", line 759, in _run_callback\n    ret = callback()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 536, in <lambda>\n    self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n    self._handle_recv()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n    callback(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tornado\\stack_context.py\", line 276, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2903, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2963, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-35-104f91fa3312>\", line 39, in <module>\n    callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\interfaces.py\", line 91, in wrapper\n    return func(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\", line 2080, in fit_generator\n    self._make_train_function()\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\", line 992, in _make_train_function\n    loss=self.total_loss)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\legacy\\interfaces.py\", line 91, in wrapper\n    return func(*args, **kwargs)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\optimizers.py\", line 457, in get_updates\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\optimizers.py\", line 457, in <listcomp>\n    ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\", line 695, in zeros\n    return variable(v, dtype=dtype, name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\", line 396, in variable\n    v = tf.Variable(value, dtype=tf.as_dtype(dtype), name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\variables.py\", line 235, in __init__\n    constraint=constraint)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\variables.py\", line 381, in _init_from_args\n    validate_shape=validate_shape).op\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\state_ops.py\", line 281, in assign\n    validate_shape=validate_shape)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\gen_state_ops.py\", line 64, in assign\n    use_locking=use_locking, name=name)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3290, in create_op\n    op_def=op_def)\n  File \"E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1654, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[788791,512] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: training/Adam/Variable_31/Assign = Assign[T=DT_FLOAT, _class=[\"loc:@training/Adam/Variable_31\"], use_locking=true, validate_shape=true, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](training/Adam/Variable_31, training/Adam/zeros_31)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n"
     ]
    }
   ],
   "source": [
    "if start_fold > 0:\n",
    "    import pickle\n",
    "    ret = pickle.load(open(resume_file_prefix+'_oof_val_pred', 'rb'))\n",
    "    ret_test = pickle.load(open(resume_file_prefix+'_oof_test_pred', 'rb'))\n",
    "    print(ret)\n",
    "    print(ret_test)\n",
    "else:\n",
    "    ret = np.zeros((train_len,))\n",
    "    ret_test = np.zeros((all_features.shape[0]-train_len,))\n",
    "\n",
    "fold = 0    \n",
    "for tr_ix, val_ix in KFold(5, shuffle=True, random_state=seed).split(train_indices):\n",
    "    fold += 1\n",
    "    \n",
    "    if start_fold > 0 and fold < start_fold:\n",
    "        continue\n",
    "    else:\n",
    "        pass\n",
    "    \n",
    "    model = build_model(all_features, categorical, numerical)\n",
    "    file_path = \"rnn_weights/model_self_train_with_image_fold_{}.hdf5\".format(fold)\n",
    "     \n",
    "    # customized batch loader\n",
    "    training_generator = DataGenerator(tr_ix, all_features, train_y, \n",
    "                                       'train_jpg.zip', 'data/competition_files/train_jpg/', \n",
    "                                       batch_size=batch_size, shuffle=True)\n",
    "    validation_generator = DataGenerator(val_ix, all_features, train_y, \n",
    "                                         'train_jpg.zip', 'data/competition_files/train_jpg/', \n",
    "                                         batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    lr_schd = LearningRateScheduler(lambda epoch: 0.001*(0.2**(epoch//6)), verbose=1)\n",
    "    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", mode = \"min\", save_best_only = True, verbose = 1)\n",
    "    history = model.fit_generator(generator=training_generator,\n",
    "                                  validation_data=validation_generator,\n",
    "                                  use_multiprocessing=False,\n",
    "                                  workers=1, \n",
    "                                  epochs=rnn_train_epochs,\n",
    "                                  verbose = 0, \n",
    "                                  callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n",
    "    \n",
    "    \n",
    "    # Predict val + test oofs\n",
    "    model.load_weights(file_path) # load weight with best validation score\n",
    "    \n",
    "    del validation_generator\n",
    "    validation_generator = DataGenerator(val_ix, all_features, None, \n",
    "                                         'train_jpg.zip', 'data/competition_files/train_jpg/', \n",
    "                                         batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    test_generator = DataGenerator(test_indices, all_features, None, \n",
    "                                   'test_jpg.zip', 'data/competition_files/test_jpg/',       \n",
    "                                   batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    \n",
    "    ret[val_ix] = model.predict_generator(validation_generator, use_multiprocessing=False, workers=1).reshape((len(val_ix),))\n",
    "    ret_test += model.predict_generator(test_generator, use_multiprocessing=False, workers=1).reshape((ret_test.shape[0],))\n",
    "    \n",
    "    del model, history, training_generator, validation_generator, test_generator; gc.collect()\n",
    "    \n",
    "ret_test /= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment these to dump files if OOM (out-of-mem) happens\n",
    "import pickle\n",
    "pickle.dump(ret, open(resume_file_prefix+'_oof_val_pred', 'wb'))\n",
    "pickle.dump(ret_test, open(resume_file_prefix+'_oof_test_pred', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# public:  cv = .2220, lb = .2247 \n",
    "# bigru-conv1d: cv =.2185 , lb = .2235\n",
    "# bigru-attention: cv =.2186 , lb = .2235\n",
    "# 2gru: lb: .2239\n",
    "# self-trained wordvec: cv .217232, lb: .2229"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate OOFs and Submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=['selftrained_bigru_conv1d_img_rnn_pred']).to_csv('selftrained_bigru_conv1d_img_rnn_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=['selftrained_bigru_conv1d_img_rnn_pred']).to_csv('selftrained_bigru_conv1d_img_rnn_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subm = pd.read_csv('sample_submission.csv')\n",
    "subm['deal_probability'] = np.clip(ret_test, 0, 1)\n",
    "subm.to_csv('selftrained_bigru_conv1d_img_rnn_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.5 tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}