Seq2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.3.0\n",
      "Default GPU Device: /gpu:0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from distutils.version import LooseVersion\n",
    "import warnings\n",
    "import tensorflow as tf\n",
    "import gensim\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))\n",
    "\n",
    "# Check for a GPU\n",
    "if not tf.test.gpu_device_name():\n",
    "    warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n",
    "else:\n",
    "    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load word2vec model\n",
    "w2v = gensim.models.KeyedVectors.load('data/w2v-773752559-1000000-300-5-5-OpenSubtitles2016.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_inputs(output_dim=300):\n",
    "    \"\"\"\n",
    "    Create TF Placeholders for input, targets, learning_rate and input_sequence_length.\n",
    "    :return: Tuple (input_, targets, learning_rate, keep_prob, input_sequence_length)\n",
    "    \"\"\"\n",
    "\n",
    "    input_ = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    targets = tf.placeholder(tf.float32, [None, output_dim])\n",
    "    learning_rate = tf.placeholder(tf.float32)\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    input_sequence_length = tf.placeholder(tf.int32, [None], name=\"input_sequence_length\")\n",
    "    \n",
    "    return (input_, targets, learning_rate, keep_prob, input_sequence_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_lstm(lstm_size, num_layers, batch_size, keep_prob, inputs, num_classes, input_sequence_length):\n",
    "    ''' Build LSTM cell.\n",
    "    \n",
    "        Arguments\n",
    "        ---------\n",
    "        keep_prob: Scalar tensor (tf.placeholder) for the dropout keep probability\n",
    "        lstm_size: Size of the hidden layers in the LSTM cells\n",
    "        num_layers: Number of LSTM layers\n",
    "        batch_size: Batch size\n",
    "\n",
    "    '''\n",
    "    \n",
    "    # one_hot encode input\n",
    "    x_one_hot = tf.one_hot(inputs, num_classes) # num_classes = len(vocab)\n",
    "    \n",
    "    def build_cell(rnn_size):\n",
    "        cell = tf.contrib.rnn.LSTMCell(rnn_size, initializer=tf.contrib.layers.xavier_initializer())\n",
    "        return cell\n",
    "    \n",
    "    # Construct a stacked tf.contrib.rnn.LSTMCell...\n",
    "    stacked_cell = tf.contrib.rnn.MultiRNNCell([build_cell(lstm_size) for _ in range(num_layers)])\n",
    "    # ...wrapped in a tf.contrib.rnn.DropoutWrapper\n",
    "    cell = tf.contrib.rnn.DropoutWrapper(stacked_cell, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Pass cell and embedded input to tf.nn.dynamic_rnn()\n",
    "    rnn_output, rnn_state = tf.nn.dynamic_rnn(cell, x_one_hot, sequence_length=input_sequence_length, dtype=tf.float32)\n",
    "    \n",
    "    # Initial state\n",
    "    initial_state = tf.identity(stacked_cell.zero_state(batch_size, tf.float32), name=\"initial_state\")\n",
    "    \n",
    "    return rnn_output, rnn_state, initial_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only care about the final rnn cell output. So we need to grab it with outputs[:, -1]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_output(cell, keep_prob, hidden_dim=1024, output_dim=300):\n",
    "    input_ = cell[:, -1]\n",
    "    dense = tf.contrib.layers.fully_connected(inputs=input_, num_outputs=hidden_dim, activation_fn=tf.nn.tanh)\n",
    "    # dense = tf.nn.dropout(dense, keep_prob)\n",
    "    dense = tf.layers.batch_normalization(dense)\n",
    "    return tf.contrib.layers.fully_connected(dense, num_outputs=output_dim, activation_fn=None)\n",
    "#     return tf.contrib.layers.fully_connected(inputs=cell[:, -1], num_outputs=output_dim, activation_fn=tf.nn.relu) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_loss(pred, Y):\n",
    "    pred=tf.nn.l2_normalize(pred,0)\n",
    "    Y=tf.nn.l2_normalize(Y,0)\n",
    "    return tf.reduce_mean (1 - tf.reduce_sum(tf.multiply(pred,Y), axis=(1,), keep_dims=True))\n",
    "#     return tf.losses.cosine_distance(pred, Y, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# build vocabulary\n",
    "vocab = sorted(set(\" \".join(w2v.wv.index2word)))\n",
    "vocab_to_int = {c: i for i, c in enumerate(vocab,1 )}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size=256\n",
    "lstm_size=1024\n",
    "num_layers=3\n",
    "keep_probability=0.8\n",
    "num_classes=len(vocab)\n",
    "output_dim=300\n",
    "# learning_rate=0.001\n",
    "learning_rate=0.0005\n",
    "save_dir = './model/seq2vec'\n",
    "\n",
    "# Create the graph object\n",
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default():\n",
    "    (input_, targets, lr, keep_prob, input_sequence_length) = get_inputs()\n",
    "    with tf.variable_scope('LSTM'):\n",
    "        rnn_output, rnn_state, initial_state = build_lstm(lstm_size, num_layers, batch_size, keep_prob, input_, num_classes, input_sequence_length)\n",
    "    with tf.variable_scope('OUTPUT'):\n",
    "        output = build_output(rnn_output, keep_prob, output_dim)\n",
    "    with tf.variable_scope('LOSS'):\n",
    "        loss = get_loss(output, targets)\n",
    "    \n",
    "    with tf.variable_scope('OPTIMIZER'):\n",
    "        # Optimizer\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "        gradients = optimizer.compute_gradients(loss)\n",
    "        # clip gradients\n",
    "        capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]\n",
    "        train_op = optimizer.apply_gradients(capped_gradients)\n",
    "    \n",
    "with tf.Session(graph=graph) as sess:\n",
    "    #writer = tf.summary.FileWriter(\"log\", sess.graph)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # tensorboard summary scalar for loss\n",
    "    tf.summary.scalar(\"loss\", loss)\n",
    "    merged_summary = tf.summary.merge_all()\n",
    "    \n",
    "    # tensorboard writer graph\n",
    "    writer=tf.summary.FileWriter(\"writer/1\")\n",
    "    writer.add_graph(sess.graph)\n",
    "    \n",
    "    # Save session\n",
    "    saver = tf.train.Saver()\n",
    "    saver.save(sess, save_dir, global_step=0)\n",
    "\n",
    "#     # Optimizer for training, using gradient clipping to control exploding gradients\n",
    "#     tvars = tf.trainable_variables()\n",
    "#     grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars), grad_clip)\n",
    "#     train_op = tf.train.AdamOptimizer(learning_rate)\n",
    "#     optimizer = train_op.apply_gradients(zip(grads, tvars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def word2seq(word):\n",
    "#     return np.array([vocab_to_int.get(c,0) for c in word])\n",
    "\n",
    "# def get_train_subset(model=w2v, seed_words=500, topn=7):\n",
    "#     top_words=model.wv.index2word[120:120+seed_words]\n",
    "#     top_words=np.append(np.array(top_words),np.array([np.array(model.most_similar_cosmul(w, topn=topn))[:,0] for w in top_words]))\n",
    "#     top_words=top_words.flatten()\n",
    "#     top_words=set(top_words)\n",
    "#     return top_words\n",
    "\n",
    "# input_list=list(get_train_subset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_padded_int_batch(input_batch, vocab_to_int=vocab_to_int):\n",
    "    max_len = max([len(word) for word in input_batch])\n",
    "    int_batch =  [[0] * (max_len - len(l)) + [vocab_to_int[w] for w in l] for l in input_batch]\n",
    "    return int_batch\n",
    "\n",
    "\n",
    "def get_batch(input_list=w2v.wv.index2word, batch_size=batch_size, vocab=vocab, vocab_to_int=vocab_to_int, model=w2v):\n",
    "    \"\"\"\n",
    "    Batch generator.\n",
    "    Input: train_set - list of words\n",
    "    Returns touple:\n",
    "    (pad_input_batch, pad_input_lengths, output_batch)\n",
    "    \"\"\"\n",
    "    for batch_i in range(0, len(input_list)//batch_size):\n",
    "        start_i = batch_i * batch_size\n",
    "\n",
    "        # Slice the right amount for the batch\n",
    "        input_batch = input_list[start_i:start_i + batch_size]\n",
    "\n",
    "        # Pad\n",
    "        pad_input_batch = np.array(get_padded_int_batch(input_batch, vocab_to_int))\n",
    "\n",
    "        # Need the lengths for the _lengths parameters\n",
    "        pad_input_lengths = []\n",
    "        for line in pad_input_batch:\n",
    "            pad_input_lengths.append(len(line))\n",
    "    \n",
    "        # output batch\n",
    "        output_batch=np.array([w2v.wv.word_vec(w) for w in input_batch])\n",
    "\n",
    "        yield (pad_input_batch, pad_input_lengths, output_batch)\n",
    "\n",
    "\n",
    "# for (batch_i, (pad_input_batch, pad_input_lengths, output)) in enumerate(get_batch(w2v.wv.index2word[:1000], batch_size=50)):\n",
    "#     print (batch_i)\n",
    "#     pass\n",
    "\n",
    "train_size= 6000#00\n",
    "\n",
    "train_input = w2v.wv.index2word[:train_size]\n",
    "valid_input = w2v.wv.index2word[train_size:train_size+batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/seq2vec-0\n",
      "Epoch:   1 | Loss: 0.6965\t validation loss: 0.6934\n",
      "Epoch:   2 | Loss: 0.6851\t validation loss: 0.6849\n",
      "Epoch:   3 | Loss: 0.6849\t validation loss: 0.6854\n",
      "Epoch:   4 | Loss: 0.6831\t validation loss: 0.6845\n",
      "Epoch:   5 | Loss: 0.6801\t validation loss: 0.6828\n",
      "Epoch:   6 | Loss: 0.68\t validation loss: 0.6828\n",
      "Epoch:   7 | Loss: 0.6774\t validation loss: 0.6802\n",
      "Epoch:   8 | Loss: 0.677\t validation loss: 0.6805\n",
      "Epoch:   9 | Loss: 0.6776\t validation loss: 0.6803\n",
      "Epoch:  10 | Loss: 0.6757\t validation loss: 0.6773\n",
      "Epoch:  11 | Loss: 0.6765\t validation loss: 0.6741\n",
      "Epoch:  12 | Loss: 0.669\t validation loss: 0.672\n",
      "Epoch:  13 | Loss: 0.6656\t validation loss: 0.667\n",
      "Epoch:  14 | Loss: 0.6679\t validation loss: 0.6645\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-af9a3843295a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     19\u001b[0m                 \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                 \u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mkeep_probability\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m                 \u001b[0minput_sequence_length\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpad_input_lengths\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m             })\n\u001b[1;32m     23\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mepoch_i\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdisplay_step\u001b[0m \u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/envs/tensorflow/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1304\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#%%time\n",
    "#tf.reset_default_graph()\n",
    "\n",
    "num_epochs=100\n",
    "display_step=1\n",
    "\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "#     sess.run(tf.global_variables_initializer())\n",
    "#     loader = tf.train.import_meta_graph(save_dir + '.meta')\n",
    "    saver.restore(sess, saver.last_checkpoints[-1])\n",
    "\n",
    "    for epoch_i in range(1, num_epochs):\n",
    "        for batch_i, (pad_input_batch, pad_input_lengths, out_vec) in enumerate(get_batch(train_input)):\n",
    "\n",
    "            _, _, l = sess.run([initial_state, train_op, loss], {\n",
    "                input_: pad_input_batch,\n",
    "                targets: out_vec,\n",
    "                lr: learning_rate, \n",
    "                keep_prob: keep_probability,\n",
    "                input_sequence_length: pad_input_lengths,\n",
    "            })\n",
    "        if (epoch_i % display_step ==0):\n",
    "            (pad_input_batch, pad_input_lengths, out_vec) = next(get_batch(valid_input))\n",
    "            _, valid_loss, ms = sess.run([initial_state, loss, merged_summary],{\n",
    "                input_: pad_input_batch,\n",
    "                targets: out_vec,\n",
    "                keep_prob: 1.0,\n",
    "                input_sequence_length: pad_input_lengths,\n",
    "            })\n",
    "            print(\"Epoch: {:3} | Loss: {:2.4}\\t validation loss: {:2.4}\".format(epoch_i, l, valid_loss))\n",
    "            writer.add_summary(ms, epoch_i)\n",
    "        # save model\n",
    "        saver.save(sess, save_dir, global_step=epoch_i)\n",
    "\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.last_checkpoints[-1]\n",
    "writer.add_graph(sess.graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/seq2vec\n",
      "Epoch:   0 | Loss: 0.6172\t validation loss: 0.6232\n",
      "Epoch:   1 | Loss: 0.6153\t validation loss: 0.6178\n",
      "Epoch:   2 | Loss: 0.6096\t validation loss: 0.6153\n",
      "Epoch:   3 | Loss: 0.6051\t validation loss: 0.6125\n",
      "Epoch:   4 | Loss: 0.601\t validation loss: 0.6093\n",
      "Epoch:   5 | Loss: 0.5958\t validation loss: 0.606\n",
      "Epoch:   6 | Loss: 0.5926\t validation loss: 0.6031\n",
      "Epoch:   7 | Loss: 0.5874\t validation loss: 0.6006\n",
      "Epoch:   8 | Loss: 0.5817\t validation loss: 0.597\n",
      "Epoch:   9 | Loss: 0.5738\t validation loss: 0.5933\n",
      "Epoch:  10 | Loss: 0.5714\t validation loss: 0.5892\n",
      "Epoch:  11 | Loss: 0.5612\t validation loss: 0.586\n",
      "Epoch:  12 | Loss: 0.559\t validation loss: 0.5843\n",
      "Epoch:  13 | Loss: 0.5493\t validation loss: 0.5778\n",
      "Epoch:  14 | Loss: 0.5435\t validation loss: 0.5752\n",
      "Epoch:  15 | Loss: 0.5339\t validation loss: 0.5707\n",
      "Epoch:  16 | Loss: 0.524\t validation loss: 0.5711\n",
      "Epoch:  17 | Loss: 0.5156\t validation loss: 0.5658\n",
      "Epoch:  18 | Loss: 0.507\t validation loss: 0.5629\n",
      "Epoch:  19 | Loss: 0.5108\t validation loss: 0.5638\n",
      "Epoch:  20 | Loss: 0.4972\t validation loss: 0.56\n",
      "Epoch:  21 | Loss: 0.4832\t validation loss: 0.5543\n",
      "Epoch:  22 | Loss: 0.4674\t validation loss: 0.5585\n",
      "Epoch:  23 | Loss: 0.4574\t validation loss: 0.5592\n",
      "Epoch:  24 | Loss: 0.4563\t validation loss: 0.5591\n",
      "Epoch:  25 | Loss: 0.4527\t validation loss: 0.552\n",
      "Epoch:  26 | Loss: 0.4368\t validation loss: 0.5527\n",
      "Epoch:  27 | Loss: 0.4308\t validation loss: 0.5491\n",
      "Epoch:  28 | Loss: 0.4188\t validation loss: 0.5463\n",
      "Epoch:  29 | Loss: 0.406\t validation loss: 0.5434\n",
      "Epoch:  30 | Loss: 0.3961\t validation loss: 0.5438\n",
      "Epoch:  31 | Loss: 0.391\t validation loss: 0.5375\n",
      "Epoch:  32 | Loss: 0.3773\t validation loss: 0.5402\n",
      "Epoch:  33 | Loss: 0.3695\t validation loss: 0.5385\n",
      "Epoch:  34 | Loss: 0.3558\t validation loss: 0.5365\n",
      "Epoch:  35 | Loss: 0.3432\t validation loss: 0.5347\n",
      "Epoch:  36 | Loss: 0.3233\t validation loss: 0.5318\n",
      "Epoch:  37 | Loss: 0.3139\t validation loss: 0.5294\n",
      "Epoch:  38 | Loss: 0.3064\t validation loss: 0.533\n",
      "Epoch:  39 | Loss: 0.2939\t validation loss: 0.5348\n",
      "Epoch:  40 | Loss: 0.2827\t validation loss: 0.5313\n",
      "Epoch:  41 | Loss: 0.2657\t validation loss: 0.5326\n",
      "Epoch:  42 | Loss: 0.2581\t validation loss: 0.5325\n",
      "Epoch:  43 | Loss: 0.2475\t validation loss: 0.5321\n",
      "Epoch:  44 | Loss: 0.2383\t validation loss: 0.5329\n",
      "Epoch:  45 | Loss: 0.231\t validation loss: 0.5277\n",
      "Epoch:  46 | Loss: 0.2234\t validation loss: 0.5319\n",
      "Epoch:  47 | Loss: 0.2149\t validation loss: 0.5306\n",
      "Epoch:  48 | Loss: 0.1992\t validation loss: 0.5272\n",
      "Epoch:  49 | Loss: 0.1952\t validation loss: 0.5296\n",
      "Epoch:  50 | Loss: 0.1871\t validation loss: 0.5308\n",
      "Epoch:  51 | Loss: 0.1882\t validation loss: 0.5285\n",
      "Epoch:  52 | Loss: 0.1892\t validation loss: 0.5287\n",
      "Epoch:  53 | Loss: 0.1787\t validation loss: 0.5281\n",
      "Epoch:  54 | Loss: 0.1693\t validation loss: 0.5251\n",
      "Epoch:  55 | Loss: 0.1639\t validation loss: 0.5236\n",
      "Epoch:  56 | Loss: 0.1602\t validation loss: 0.5228\n",
      "Epoch:  57 | Loss: 0.1522\t validation loss: 0.5245\n",
      "Epoch:  58 | Loss: 0.1419\t validation loss: 0.5214\n",
      "Epoch:  59 | Loss: 0.1376\t validation loss: 0.5229\n",
      "Epoch:  60 | Loss: 0.1338\t validation loss: 0.5239\n",
      "Epoch:  61 | Loss: 0.1295\t validation loss: 0.5244\n",
      "Epoch:  62 | Loss: 0.1199\t validation loss: 0.5236\n",
      "Epoch:  63 | Loss: 0.1132\t validation loss: 0.5245\n",
      "Epoch:  64 | Loss: 0.1119\t validation loss: 0.5246\n",
      "Epoch:  65 | Loss: 0.111\t validation loss: 0.5254\n",
      "Epoch:  66 | Loss: 0.1251\t validation loss: 0.5249\n",
      "Epoch:  67 | Loss: 0.1146\t validation loss: 0.5234\n",
      "Epoch:  68 | Loss: 0.118\t validation loss: 0.5252\n",
      "Epoch:  69 | Loss: 0.1018\t validation loss: 0.5266\n",
      "Epoch:  70 | Loss: 0.1047\t validation loss: 0.5265\n",
      "Epoch:  71 | Loss: 0.08491\t validation loss: 0.5272\n",
      "Epoch:  72 | Loss: 0.08207\t validation loss: 0.5301\n",
      "Epoch:  73 | Loss: 0.07518\t validation loss: 0.5319\n",
      "Epoch:  74 | Loss: 0.07674\t validation loss: 0.5316\n",
      "Epoch:  75 | Loss: 0.06545\t validation loss: 0.5291\n",
      "Epoch:  76 | Loss: 0.06091\t validation loss: 0.5286\n",
      "Epoch:  77 | Loss: 0.05759\t validation loss: 0.5265\n",
      "Epoch:  78 | Loss: 0.05539\t validation loss: 0.5258\n",
      "Epoch:  79 | Loss: 0.05333\t validation loss: 0.526\n",
      "Epoch:  80 | Loss: 0.05147\t validation loss: 0.5259\n",
      "Epoch:  81 | Loss: 0.05078\t validation loss: 0.5268\n",
      "Epoch:  82 | Loss: 0.05021\t validation loss: 0.5269\n",
      "Epoch:  83 | Loss: 0.04482\t validation loss: 0.5269\n",
      "Epoch:  84 | Loss: 0.03767\t validation loss: 0.5261\n",
      "Epoch:  85 | Loss: 0.03912\t validation loss: 0.5255\n",
      "Epoch:  86 | Loss: 0.03442\t validation loss: 0.5269\n",
      "Epoch:  87 | Loss: 0.03062\t validation loss: 0.5265\n",
      "Epoch:  88 | Loss: 0.02986\t validation loss: 0.5262\n",
      "Epoch:  89 | Loss: 0.02875\t validation loss: 0.5262\n",
      "Epoch:  90 | Loss: 0.02635\t validation loss: 0.5279\n",
      "Epoch:  91 | Loss: 0.02216\t validation loss: 0.5277\n",
      "Epoch:  92 | Loss: 0.01943\t validation loss: 0.5281\n",
      "Epoch:  93 | Loss: 0.01679\t validation loss: 0.5282\n",
      "Epoch:  94 | Loss: 0.01547\t validation loss: 0.5303\n",
      "Epoch:  95 | Loss: 0.01206\t validation loss: 0.5328\n",
      "Epoch:  96 | Loss: 0.01155\t validation loss: 0.5361\n",
      "Epoch:  97 | Loss: 0.009801\t validation loss: 0.5403\n",
      "Epoch:  98 | Loss: 0.007907\t validation loss: 0.5395\n",
      "Epoch:  99 | Loss: 0.005958\t validation loss: 0.5401\n",
      "CPU times: user 5min 37s, sys: 1min 38s, total: 7min 15s\n",
      "Wall time: 7min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tf.reset_default_graph()\n",
    "learning_rate=0.00001\n",
    "\n",
    "num_epochs=100\n",
    "display_step=1\n",
    "\n",
    "# tensorboard writer graph\n",
    "writer=tf.summary.FileWriter(\"writer/3\")\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "#     sess.run(tf.global_variables_initializer())\n",
    "#    loader = tf.train.import_meta_graph(save_dir + '.meta')\n",
    "    saver.restore(sess, saver.last_checkpoints[-1])\n",
    "\n",
    "    for epoch_i in range(num_epochs):\n",
    "        for batch_i, (pad_input_batch, pad_input_lengths, out_vec) in enumerate(get_batch(train_input)):\n",
    "\n",
    "            _, _, l = sess.run([initial_state, train_op, loss], {\n",
    "                input_: pad_input_batch,\n",
    "                targets: out_vec,\n",
    "                lr: learning_rate, \n",
    "                keep_prob: keep_probability,\n",
    "                input_sequence_length: pad_input_lengths,\n",
    "            })\n",
    "        if (epoch_i % display_step ==0):\n",
    "            (pad_input_batch, pad_input_lengths, out_vec) = next(get_batch(valid_input))\n",
    "            _, valid_loss, ms = sess.run([initial_state, loss, merged_summary],{\n",
    "                input_: pad_input_batch,\n",
    "                targets: out_vec,\n",
    "                keep_prob: 1.0,\n",
    "                input_sequence_length: pad_input_lengths,\n",
    "            })\n",
    "            print(\"Epoch: {:3} | Loss: {:2.4}\\t validation loss: {:2.4}\".format(epoch_i, l, valid_loss))\n",
    "            writer.add_summary(ms, epoch_i)\n",
    "        # save model\n",
    "        saver.save(sess, save_dir)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/seq2vec\n",
      "CPU times: user 716 ms, sys: 192 ms, total: 908 ms\n",
      "Wall time: 641 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tf.reset_default_graph()\n",
    "# Load model and use a global session\n",
    "sess = tf.Session(graph=graph)\n",
    "#loader = tf.train.import_meta_graph(save_dir + '.meta')\n",
    "saver.restore(sess, saver.last_checkpoints[-1])\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%%time\n",
    "tf.reset_default_graph()\n",
    "# Load model and use a global session\n",
    "sess = tf.Session()\n",
    "loader = tf.train.import_meta_graph(save_dir + '.meta')\n",
    "loader.restore(sess, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_word2vec(word, sess=sess):\n",
    "    pad_input_batch=get_padded_int_batch([word])\n",
    "    #print(len(pad_input_batch[0]))\n",
    "    _, outputs = sess.run([initial_state, output],{\n",
    "        input_: pad_input_batch,\n",
    "        keep_prob: 1.0,\n",
    "        input_sequence_length: [len(pad_input_batch[0])],\n",
    "    })\n",
    "    return outputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kapitan\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('konfabulacje', 0.8141539692878723),\n",
       " ('samochodzik-zabaweczka', 0.8116906881332397),\n",
       " ('wykrad', 0.8109175562858582),\n",
       " ('bio-organiczny', 0.8095035552978516),\n",
       " ('skrotu', 0.8080761432647705),\n",
       " ('geniejalny', 0.8075656294822693),\n",
       " ('dtf', 0.8071662783622742),\n",
       " ('panteizm', 0.8054572343826294),\n",
       " ('zwyrolem', 0.8048051595687866),\n",
       " ('mier', 0.8045909404754639)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#word=train_input[200]\n",
    "\n",
    "word=\"kapitan\"\n",
    "\n",
    "wordvec=get_word2vec(word)\n",
    "\n",
    "print(word)\n",
    "w2v.wv.similar_by_vector(wordvec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['takiej',\n",
       " 'historii',\n",
       " 'cel',\n",
       " 'około',\n",
       " 'siostra',\n",
       " 'tom',\n",
       " 'wolno',\n",
       " 'złego',\n",
       " 'myśl',\n",
       " 'moment',\n",
       " 'wieku',\n",
       " 'boli',\n",
       " 'f',\n",
       " 'spróbować',\n",
       " 'mimo',\n",
       " 'niby',\n",
       " 'prawdziwy',\n",
       " 'kapitan',\n",
       " 'przyjacielem',\n",
       " 'czyż']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_word2vec(\"ula\")\n",
    "w2v.wv.index2word[990:1010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}