keras_examples_imdb_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/fchollet/keras/blob/master/examples/imdb_lstm.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "25000 train sequences\n",
      "25000 test sequences\n"
     ]
    }
   ],
   "source": [
    "'''Trains an LSTM model on the IMDB sentiment classification task.\n",
    "The dataset is actually too small for LSTM to be of any advantage\n",
    "compared to simpler, much faster methods such as TF-IDF + LogReg.\n",
    "# Notes\n",
    "- RNNs are tricky. Choice of batch size is important,\n",
    "choice of loss and optimizer is critical, etc.\n",
    "Some configurations won't converge.\n",
    "- LSTM loss decrease patterns during training can be quite different\n",
    "from what you see with CNNs/MLPs/etc.\n",
    "'''\n",
    "from __future__ import print_function\n",
    "\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding\n",
    "from keras.layers import LSTM\n",
    "from keras.datasets import imdb\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import log_loss, accuracy_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "max_features = 20000\n",
    "maxlen = 80  # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "print('Loading data...')\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)\n",
    "print(len(x_train), 'train sequences')\n",
    "print(len(x_test), 'test sequences')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pad sequences (samples x time)\n",
      "x_train shape: (25000, 80)\n",
      "x_test shape: (25000, 80)\n"
     ]
    }
   ],
   "source": [
    "print('Pad sequences (samples x time)')\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = imdb.get_word_index()\n",
    "index_word = dict((v, k) for k, v in word_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"point try invisible not of homage he very to of snuff very america have can ideal sheesh parker's his for watching so quirky this as taylor not of becomes horror very to of little find actor altman's find is him mark profound that end in michigan when defense end in deck dvds that city me is married russian wrong not from off much film that when from after to charles of films much though for lose i i of here\""
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join('0' if i==0 else index_word[i] for i in x_train[np.random.choice(len(x_train))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:526: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg = LogisticRegression()\n",
    "logreg.fit(x_train, np.array(y_train).reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_probs = logreg.predict_proba(x_test)\n",
    "valid_preds = logreg.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.69248894829043217"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(y_test, valid_probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.51983999999999997"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, valid_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### rnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, 128))\n",
    "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, None, 128)         2560000   \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 128)               131584    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 2,691,713\n",
      "Trainable params: 2,691,713\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------\n",
      "(20000, 128)\n",
      "-------------\n",
      "(128, 512)\n",
      "(128, 512)\n",
      "(512,)\n",
      "-------------\n",
      "(128, 1)\n",
      "(1,)\n"
     ]
    }
   ],
   "source": [
    "utilities.print_weights_shape(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"304pt\" viewBox=\"0.00 0.00 436.00 304.00\" width=\"436pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 300)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-300 432,-300 432,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 140425721930752 -->\n",
       "<g class=\"node\" id=\"node1\"><title>140425721930752</title>\n",
       "<polygon fill=\"none\" points=\"-2.84217e-14,-249.5 -2.84217e-14,-295.5 428,-295.5 428,-249.5 -2.84217e-14,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"123.5\" y=\"-268.8\">embedding_2_input: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"247,-249.5 247,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"281\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"247,-272.5 315,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"281\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"315,-249.5 315,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"371.5\" y=\"-280.3\">(None, None)</text>\n",
       "<polyline fill=\"none\" points=\"315,-272.5 428,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"371.5\" y=\"-257.3\">(None, None)</text>\n",
       "</g>\n",
       "<!-- 140425721933384 -->\n",
       "<g class=\"node\" id=\"node2\"><title>140425721933384</title>\n",
       "<polygon fill=\"none\" points=\"4,-166.5 4,-212.5 424,-212.5 424,-166.5 4,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105.5\" y=\"-185.8\">embedding_2: Embedding</text>\n",
       "<polyline fill=\"none\" points=\"207,-166.5 207,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"207,-189.5 275,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"275,-166.5 275,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"349.5\" y=\"-197.3\">(None, None)</text>\n",
       "<polyline fill=\"none\" points=\"275,-189.5 424,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"349.5\" y=\"-174.3\">(None, None, 128)</text>\n",
       "</g>\n",
       "<!-- 140425721930752&#45;&gt;140425721933384 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>140425721930752-&gt;140425721933384</title>\n",
       "<path d=\"M214,-249.366C214,-241.152 214,-231.658 214,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"217.5,-222.607 214,-212.607 210.5,-222.607 217.5,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140425721931144 -->\n",
       "<g class=\"node\" id=\"node3\"><title>140425721931144</title>\n",
       "<polygon fill=\"none\" points=\"47.5,-83.5 47.5,-129.5 380.5,-129.5 380.5,-83.5 47.5,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105.5\" y=\"-102.8\">lstm_2: LSTM</text>\n",
       "<polyline fill=\"none\" points=\"163.5,-83.5 163.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"197.5\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"163.5,-106.5 231.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"197.5\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"231.5,-83.5 231.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"306\" y=\"-114.3\">(None, None, 128)</text>\n",
       "<polyline fill=\"none\" points=\"231.5,-106.5 380.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"306\" y=\"-91.3\">(None, 128)</text>\n",
       "</g>\n",
       "<!-- 140425721933384&#45;&gt;140425721931144 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>140425721933384-&gt;140425721931144</title>\n",
       "<path d=\"M214,-166.366C214,-158.152 214,-148.658 214,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"217.5,-139.607 214,-129.607 210.5,-139.607 217.5,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 140425721931928 -->\n",
       "<g class=\"node\" id=\"node4\"><title>140425721931928</title>\n",
       "<polygon fill=\"none\" points=\"65.5,-0.5 65.5,-46.5 362.5,-46.5 362.5,-0.5 65.5,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"129.5\" y=\"-19.8\">dense_2: Dense</text>\n",
       "<polyline fill=\"none\" points=\"193.5,-0.5 193.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"227.5\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"193.5,-23.5 261.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"227.5\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"261.5,-0.5 261.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-31.3\">(None, 128)</text>\n",
       "<polyline fill=\"none\" points=\"261.5,-23.5 362.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"312\" y=\"-8.3\">(None, 1)</text>\n",
       "</g>\n",
       "<!-- 140425721931144&#45;&gt;140425721931928 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>140425721931144-&gt;140425721931928</title>\n",
       "<path d=\"M214,-83.3664C214,-75.1516 214,-65.6579 214,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"217.5,-56.6068 214,-46.6068 210.5,-56.6069 217.5,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "utilities.plot_keras_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/15\n",
      "25000/25000 [==============================] - 88s 4ms/step - loss: 0.4671 - acc: 0.7774 - val_loss: 0.3856 - val_acc: 0.8315\n",
      "Epoch 2/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.3068 - acc: 0.8744 - val_loss: 0.3701 - val_acc: 0.8361\n",
      "Epoch 3/15\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.2186 - acc: 0.9139 - val_loss: 0.4138 - val_acc: 0.8291\n",
      "Epoch 4/15\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.1502 - acc: 0.9448 - val_loss: 0.4869 - val_acc: 0.8297\n",
      "Epoch 5/15\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.1087 - acc: 0.9608 - val_loss: 0.6374 - val_acc: 0.8227\n",
      "Epoch 6/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0799 - acc: 0.9717 - val_loss: 0.7185 - val_acc: 0.8217\n",
      "Epoch 7/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0570 - acc: 0.9793 - val_loss: 0.7389 - val_acc: 0.8174\n",
      "Epoch 8/15\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0387 - acc: 0.9874 - val_loss: 0.7983 - val_acc: 0.8180\n",
      "Epoch 9/15\n",
      "25000/25000 [==============================] - 87s 3ms/step - loss: 0.0315 - acc: 0.9899 - val_loss: 0.8938 - val_acc: 0.8090\n",
      "Epoch 10/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0241 - acc: 0.9923 - val_loss: 0.9554 - val_acc: 0.8148\n",
      "Epoch 11/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0232 - acc: 0.9922 - val_loss: 1.0091 - val_acc: 0.8160\n",
      "Epoch 12/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0219 - acc: 0.9932 - val_loss: 0.9008 - val_acc: 0.8142\n",
      "Epoch 13/15\n",
      "25000/25000 [==============================] - 86s 3ms/step - loss: 0.0124 - acc: 0.9963 - val_loss: 1.1988 - val_acc: 0.8014\n",
      "Epoch 14/15\n",
      "25000/25000 [==============================] - 84s 3ms/step - loss: 0.0088 - acc: 0.9973 - val_loss: 1.0630 - val_acc: 0.8127\n",
      "Epoch 15/15\n",
      "25000/25000 [==============================] - 85s 3ms/step - loss: 0.0092 - acc: 0.9972 - val_loss: 1.1278 - val_acc: 0.8093\n",
      "25000/25000 [==============================] - 14s 560us/step\n",
      "Test score: 1.12782008769\n",
      "Test accuracy: 0.80932\n"
     ]
    }
   ],
   "source": [
    "print('Train...')\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=15,\n",
    "          validation_data=(x_test, y_test))\n",
    "score, acc = model.evaluate(x_test, y_test,\n",
    "                            batch_size=batch_size)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}