one_million/all-word/Model-aw-lex-local_attention-fast-v2-7.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fast gated local attention \n",
    "\n",
    "#### batch size 128, learning rate = 0.001, windows size 5, num_gpus = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time\n",
    "\n",
    "VERY_BIG_NUMBER = 1e30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# Model Description\n",
    "model_name = 'model-aw-lex-local-att-fast-v2-7'\n",
    "model_dir = '../output/all-word/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "with open('../../../dataset/train_val_data_fine/all_word_lex','rb') as f:\n",
    "    train_data, val_data = pickle.load(f)    \n",
    "    \n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = 45\n",
    "num_pos = 12\n",
    "batch_size = 128\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 256\n",
    "num_filter = 256\n",
    "window_size = 5\n",
    "kernel_size = 5\n",
    "keep_prob = 0.3\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 500\n",
    "decay_rate = 0.99\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6\n",
    "width = int(window_size/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, axis=0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/clip_ops.py:110: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "# MODEL\n",
    "device_num = 0\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "sense_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='sense_mask')\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "pretrain = tf.placeholder('bool', [], name=\"pretrain\")\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "summaries = []\n",
    "\n",
    "def global_attention(input_x, input_mask, W_att):\n",
    "    h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "    h_tanh = tf.tanh(h_masked)\n",
    "    u = tf.matmul(h_tanh, W_att)\n",
    "    a = tf.nn.softmax(u)\n",
    "    c = tf.reduce_sum(tf.multiply(h_masked, a), axis=0)  \n",
    "    return c\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        if gpu_idx>=3:\n",
    "            device_num = 1\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % device_num):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), axis=1)\n",
    "            \n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, Wx, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "            \n",
    "            with tf.variable_scope(\"local_attention\"):\n",
    "                W_att_local = tf.get_variable(\"W_att_local\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                flat_h = tf.reshape(h, [batch_size*max_sent_size, tf.shape(h)[2]])\n",
    "                h_tanh = tf.tanh(flat_h)\n",
    "                u_flat = tf.matmul(h_tanh, W_att_local)\n",
    "                u_local = tf.reshape(u_flat, [batch_size, max_sent_size])\n",
    "                final_u = (tf.cast(x_mask[gpu_idx], 'float') -1)*VERY_BIG_NUMBER + u_local\n",
    "                c_local = tf.map_fn(lambda i:tf.reduce_sum(tf.multiply(h[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)],\n",
    "                    tf.expand_dims(tf.nn.softmax(final_u[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)], 1), 2)), axis=1),\n",
    "                    tf.range(max_sent_size), dtype=tf.float32)  \n",
    "                \n",
    "                c_local = tf.transpose(c_local, perm=[1,0,2]) \n",
    "\n",
    "            with tf.variable_scope(\"gate\"):\n",
    "                W_gate_c = tf.get_variable(\"W_gate_c\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                W_gate_h = tf.get_variable(\"W_gate_h\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                flat_c_local = tf.reshape(c_local, [batch_size*max_sent_size, 2*hidden_size])\n",
    "                flat_h = tf.reshape(h, [batch_size*max_sent_size, 2*hidden_size])\n",
    "                attention_gate = tf.sigmoid(tf.matmul(flat_c_local, W_gate_c) + tf.matmul(flat_h, W_gate_h))\n",
    "                flat_h_final = tf.multiply(flat_c_local, attention_gate) + tf.multiply(flat_h, 1-attention_gate)\n",
    "                \n",
    "            with tf.variable_scope(\"hidden_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, 2*hidden_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[2*hidden_size], initializer=tf.zeros_initializer())\n",
    "                drop_flat_h_final = tf.nn.dropout(flat_h_final, input_keep_prob)\n",
    "                flat_hl = tf.matmul(drop_flat_h_final, W) + b\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_sense = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits = tf.reshape(flat_logits_sense, [batch_size, max_sent_size, num_senses])\n",
    "                predictions.append(tf.argmax(logits, 2))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*30))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, tf.shape(h1)[2]])\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                predictions_pos.append(tf.argmax(logits_pos, 2))\n",
    "\n",
    "\n",
    "            float_sense_mask = tf.cast(sense_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.contrib.seq2seq.sequence_loss(logits, y[gpu_idx], float_sense_mask, name=\"loss\")\n",
    "            loss_pos = tf.contrib.seq2seq.sequence_loss(logits_pos, y_pos[gpu_idx], float_x_mask, name=\"loss_\")\n",
    "\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = tf.cond(pretrain, lambda:loss_pos, lambda:loss + loss_pos + l2_loss)\n",
    "\n",
    "            summaries.append(tf.summary.scalar(\"loss_{}\".format(gpu_idx), loss))\n",
    "            summaries.append(tf.summary.scalar(\"loss_pos_{}\".format(gpu_idx), loss_pos))\n",
    "            summaries.append(tf.summary.scalar(\"total_loss_{}\".format(gpu_idx), total_loss))\n",
    "\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    tower_grads = average_gradients(tower_grads)\n",
    "    losses = tf.add_n(losses)/len(losses)\n",
    "    apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "    summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "    summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "    train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2,3\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_period = 100\n",
    "log_period = 100\n",
    "\n",
    "def model(xx, yy, yy_pos, mask, smask, train_cond=True, pretrain_cond=False):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    temp_loss = 0\n",
    "    preds_sense = []\n",
    "    true_sense = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_pos_re = yy_pos[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        smask_re = smask[s:e].reshape([num_gpus, batch_size, -1])\n",
    " \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:yy_pos_re, x_mask:mask_re, sense_mask:smask_re, pretrain:pretrain_cond, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)\n",
    "            \n",
    "            temp_loss += _loss\n",
    "            if((j+1)%log_period==0):\n",
    "                print(\"Steps: {}\".format(step), \"Loss:{0:.4f}\".format(temp_loss/log_period), \", Current Loss: {0:.4f}\".format(_loss))\n",
    "                temp_loss = 0\n",
    "            if((j+1)%save_period==0):\n",
    "                saver.save(sess, save_path=save_dir)                         \n",
    "                \n",
    "        else:\n",
    "            _loss, pred, pred_pos = sess.run([total_loss, predictions, predictions_pos], feed_dict)\n",
    "            for i in range(num_gpus):\n",
    "                preds_sense.append(pred[i][smask_re[i]])\n",
    "                true_sense.append(yy_re[i][smask_re[i]])\n",
    "                preds_pos.append(pred_pos[i][mask_re[i]])\n",
    "                true_pos.append(yy_pos_re[i][mask_re[i]])\n",
    "\n",
    "        _losses +=_loss\n",
    "\n",
    "    if(train_cond==False): \n",
    "        sense_preds = []\n",
    "        sense_true = []\n",
    "        pos_preds = []\n",
    "        pos_true = []\n",
    "        \n",
    "        for preds in preds_sense:\n",
    "            for ps in preds:      \n",
    "                sense_preds.append(ps)  \n",
    "        for trues in true_sense:\n",
    "            for ts in trues:\n",
    "                sense_true.append(ts)\n",
    "        \n",
    "        for preds in preds_pos:\n",
    "            for ps in preds:      \n",
    "                pos_preds.append(ps)      \n",
    "        for trues in true_pos:\n",
    "            for ts in trues:\n",
    "                pos_true.append(ts)\n",
    "                \n",
    "        return _losses/num_batches, sense_preds, sense_true, pos_preds, pos_true\n",
    "\n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, yy_pos, pred_pos):\n",
    "    f1 = f1_score(yy, pred, average='macro')\n",
    "    accu = accuracy_score(yy, pred)\n",
    "    f1_pos = f1_score(yy_pos, pred_pos, average='macro')\n",
    "    accu_pos = accuracy_score(yy_pos, pred_pos)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_id_train = train_data['x']\n",
    "mask_train = train_data['x_mask']\n",
    "sense_mask_train = train_data['sense_mask']\n",
    "y_train = train_data['y']\n",
    "y_pos_train = train_data['pos']\n",
    "\n",
    "x_id_val = val_data['x']\n",
    "mask_val = val_data['x_mask']\n",
    "sense_mask_val = val_data['sense_mask']\n",
    "y_val = val_data['y']\n",
    "y_pos_val = val_data['pos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testing():\n",
    "    start_time = time.time()\n",
    "    val_loss, val_pred, val_true, val_pred_pos, val_true_pos = model(x_id_val, y_val, y_pos_val, mask_val, sense_mask_val, train_cond=False)        \n",
    "    f1_, accu_, f1_pos_, accu_pos_ = eval_score(val_true, val_pred, val_true_pos, val_pred_pos)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Val: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "    return f1_, accu_, f1_pos_, accu_pos_\n",
    "\n",
    "def training(current_epoch, pre_train_cond):\n",
    "        random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "        x_id_train_tmp = x_id_train[random]\n",
    "        y_train_tmp = y_train[random]\n",
    "        mask_train_tmp = mask_train[random]    \n",
    "        sense_mask_train_tmp = sense_mask_train[random]\n",
    "        y_pos_train_tmp = y_pos_train[random]\n",
    "\n",
    "        start_time = time.time()\n",
    "        train_loss, step = model(x_id_train_tmp, y_train_tmp, y_pos_train_tmp, mask_train_tmp, sense_mask_train_tmp, pretrain_cond=pre_train_cond)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Epoch: {}\".format(current_epoch+1),\", Step: {}\".format(step), \", loss: {0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        return [step, train_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 100 Loss:1.5798 , Current Loss: 0.7919\n",
      "Steps: 200 Loss:0.6341 , Current Loss: 0.5442\n",
      "Steps: 300 Loss:0.4903 , Current Loss: 0.4556\n",
      "Steps: 400 Loss:0.4163 , Current Loss: 0.3903\n",
      "Steps: 500 Loss:0.3722 , Current Loss: 0.3571\n",
      "Steps: 600 Loss:0.3446 , Current Loss: 0.3428\n",
      "Steps: 700 Loss:0.3219 , Current Loss: 0.3079\n",
      "Steps: 800 Loss:0.3057 , Current Loss: 0.3015\n",
      "Epoch: 1 , Step: 885 , loss: 0.5327 , Time: 5787.2\n",
      "Model Saved\n",
      "Steps: 985 Loss:0.2823 , Current Loss: 0.2743\n",
      "Steps: 1085 Loss:0.2727 , Current Loss: 0.2586\n",
      "Steps: 1185 Loss:0.2635 , Current Loss: 0.2620\n",
      "Steps: 1285 Loss:0.2565 , Current Loss: 0.2545\n",
      "Steps: 1385 Loss:0.2501 , Current Loss: 0.2447\n",
      "Steps: 1485 Loss:0.2445 , Current Loss: 0.2409\n",
      "Steps: 1585 Loss:0.2378 , Current Loss: 0.2281\n",
      "Steps: 1685 Loss:0.2341 , Current Loss: 0.2311\n",
      "Epoch: 2 , Step: 1770 , loss: 0.2527 , Time: 5730.0\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:1.26 Accuracy:1.68  POS: F1 Score:86.78 Accuracy:92.30 Loss:6.4994 , Time: 1122.8\n",
      "Steps: 1870 Loss:2.9046 , Current Loss: 2.5561\n",
      "Steps: 1970 Loss:2.5323 , Current Loss: 2.3815\n",
      "Steps: 2070 Loss:2.4303 , Current Loss: 2.3735\n",
      "Steps: 2170 Loss:2.3705 , Current Loss: 2.3562\n",
      "Steps: 2270 Loss:2.3178 , Current Loss: 2.2702\n",
      "Steps: 2370 Loss:2.2950 , Current Loss: 2.2255\n",
      "Steps: 2470 Loss:2.2683 , Current Loss: 2.3279\n",
      "Steps: 2570 Loss:2.2499 , Current Loss: 2.2570\n",
      "Epoch: 1 , Step: 2655 , loss: 2.4023 , Time: 5706.0\n",
      "Model Saved\n",
      "Steps: 2755 Loss:2.2023 , Current Loss: 2.1899\n",
      "Steps: 2855 Loss:2.1995 , Current Loss: 2.1790\n",
      "Steps: 2955 Loss:2.1744 , Current Loss: 2.2110\n",
      "Steps: 3055 Loss:2.1595 , Current Loss: 2.0980\n",
      "Steps: 3155 Loss:2.1529 , Current Loss: 2.2096\n",
      "Steps: 3255 Loss:2.1402 , Current Loss: 2.1259\n",
      "Steps: 3355 Loss:2.1268 , Current Loss: 2.1010\n",
      "Steps: 3455 Loss:2.1091 , Current Loss: 2.1158\n",
      "Epoch: 2 , Step: 3540 , loss: 2.1538 , Time: 5686.4\n",
      "Model Saved\n",
      "Val: F1 Score:30.38 Accuracy:43.62  POS: F1 Score:82.47 Accuracy:90.72 Loss:2.1049 , Time: 1118.1\n",
      "Steps: 3640 Loss:2.0901 , Current Loss: 2.0902\n",
      "Steps: 3740 Loss:2.0797 , Current Loss: 2.1053\n",
      "Steps: 3840 Loss:2.0774 , Current Loss: 2.1093\n",
      "Steps: 3940 Loss:2.0640 , Current Loss: 2.1120\n",
      "Steps: 4040 Loss:2.0596 , Current Loss: 1.9818\n",
      "Steps: 4140 Loss:2.0565 , Current Loss: 1.9972\n",
      "Steps: 4240 Loss:2.0460 , Current Loss: 2.0258\n",
      "Steps: 4340 Loss:2.0404 , Current Loss: 2.0506\n",
      "Epoch: 3 , Step: 4425 , loss: 2.0618 , Time: 5891.6\n",
      "Model Saved\n",
      "Steps: 4525 Loss:2.0181 , Current Loss: 1.9439\n",
      "Steps: 4625 Loss:2.0219 , Current Loss: 2.0081\n",
      "Steps: 4725 Loss:2.0168 , Current Loss: 2.0942\n",
      "Steps: 4825 Loss:2.0100 , Current Loss: 2.0338\n",
      "Steps: 4925 Loss:2.0106 , Current Loss: 1.9703\n",
      "Steps: 5025 Loss:2.0004 , Current Loss: 1.9900\n",
      "Steps: 5125 Loss:1.9933 , Current Loss: 2.0115\n",
      "Steps: 5225 Loss:1.9879 , Current Loss: 1.9458\n",
      "Epoch: 4 , Step: 5310 , loss: 2.0048 , Time: 5774.9\n",
      "Model Saved\n",
      "Val: F1 Score:34.81 Accuracy:46.54  POS: F1 Score:82.72 Accuracy:91.05 Loss:1.9931 , Time: 1160.2\n",
      "Steps: 5410 Loss:1.9785 , Current Loss: 2.0306\n",
      "Steps: 5510 Loss:1.9705 , Current Loss: 1.9568\n",
      "Steps: 5610 Loss:1.9665 , Current Loss: 1.9874\n",
      "Steps: 5710 Loss:1.9651 , Current Loss: 1.9504\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "# Pretraining POS Tags\n",
    "training(0, True)\n",
    "training(1, True)\n",
    "testing()\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 5810 Loss:1.9610 , Current Loss: 1.9610\n",
      "Steps: 5910 Loss:1.9617 , Current Loss: 1.9839\n",
      "Steps: 6010 Loss:1.9578 , Current Loss: 1.9475\n",
      "Steps: 6110 Loss:1.9536 , Current Loss: 2.0081\n",
      "Steps: 6210 Loss:1.9459 , Current Loss: 1.9396\n",
      "Steps: 6310 Loss:1.9354 , Current Loss: 1.8898\n",
      "Steps: 6410 Loss:1.9345 , Current Loss: 1.7894\n",
      "Steps: 6510 Loss:1.9359 , Current Loss: 1.9386\n",
      "Epoch: 1 , Step: 6595 , loss: 1.9457 , Time: 7062.2\n",
      "Model Saved\n",
      "Steps: 6695 Loss:1.9209 , Current Loss: 1.8152\n",
      "Steps: 6795 Loss:1.9226 , Current Loss: 1.8139\n",
      "Steps: 6895 Loss:1.9201 , Current Loss: 1.9704\n",
      "Steps: 6995 Loss:1.9130 , Current Loss: 1.9559\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 7095 Loss:1.9092 , Current Loss: 1.8735\n",
      "Steps: 7195 Loss:1.9030 , Current Loss: 1.9451\n",
      "Steps: 7295 Loss:1.9070 , Current Loss: 1.9532\n",
      "Steps: 7395 Loss:1.9039 , Current Loss: 1.8896\n",
      "Steps: 7495 Loss:1.9058 , Current Loss: 1.8813\n",
      "Steps: 7595 Loss:1.8959 , Current Loss: 1.9200\n",
      "Steps: 7695 Loss:1.8903 , Current Loss: 1.9260\n",
      "Steps: 7795 Loss:1.8936 , Current Loss: 1.8576\n",
      "Epoch: 1 , Step: 7880 , loss: 1.9006 , Time: 6227.5\n",
      "Model Saved\n",
      "Steps: 7980 Loss:1.8857 , Current Loss: 1.9079\n",
      "Steps: 8080 Loss:1.8730 , Current Loss: 1.8205\n",
      "Steps: 8180 Loss:1.8780 , Current Loss: 1.8552\n",
      "Steps: 8280 Loss:1.8791 , Current Loss: 1.9359\n",
      "Steps: 8380 Loss:1.8776 , Current Loss: 1.9627\n",
      "Steps: 8480 Loss:1.8749 , Current Loss: 1.9455\n",
      "Steps: 8580 Loss:1.8666 , Current Loss: 1.8703\n",
      "Steps: 8680 Loss:1.8710 , Current Loss: 1.8687\n",
      "Epoch: 2 , Step: 8765 , loss: 1.8755 , Time: 6100.4\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:38.39 Accuracy:49.77  POS: F1 Score:82.97 Accuracy:91.39 Loss:1.8857 , Time: 1190.9\n",
      "Steps: 8865 Loss:1.8592 , Current Loss: 1.9508\n",
      "Steps: 8965 Loss:1.8656 , Current Loss: 1.8866\n",
      "Steps: 9065 Loss:1.8580 , Current Loss: 1.7608\n",
      "Steps: 9165 Loss:1.8581 , Current Loss: 1.9059\n",
      "Steps: 9265 Loss:1.8493 , Current Loss: 1.8979\n",
      "Steps: 9365 Loss:1.8447 , Current Loss: 1.8226\n",
      "Steps: 9465 Loss:1.8527 , Current Loss: 1.7729\n",
      "Steps: 9565 Loss:1.8414 , Current Loss: 1.9348\n",
      "Epoch: 3 , Step: 9650 , loss: 1.8533 , Time: 6070.7\n",
      "Model Saved\n",
      "Steps: 9750 Loss:1.8517 , Current Loss: 1.9460\n",
      "Steps: 9850 Loss:1.8338 , Current Loss: 1.8472\n",
      "Steps: 9950 Loss:1.8377 , Current Loss: 1.8456\n",
      "Steps: 10050 Loss:1.8389 , Current Loss: 1.7439\n",
      "Steps: 10150 Loss:1.8331 , Current Loss: 1.8670\n",
      "Steps: 10250 Loss:1.8254 , Current Loss: 1.7736\n",
      "Steps: 10350 Loss:1.8307 , Current Loss: 1.8882\n",
      "Steps: 10450 Loss:1.8294 , Current Loss: 1.8811\n",
      "Epoch: 4 , Step: 10535 , loss: 1.8339 , Time: 6074.2\n",
      "Model Saved\n",
      "Val: F1 Score:39.43 Accuracy:50.60  POS: F1 Score:83.32 Accuracy:91.53 Loss:1.8455 , Time: 1201.8\n",
      "Steps: 10635 Loss:1.8223 , Current Loss: 1.8117\n",
      "Steps: 10735 Loss:1.8158 , Current Loss: 1.7981\n",
      "Steps: 10835 Loss:1.8070 , Current Loss: 1.8575\n",
      "Steps: 10935 Loss:1.8201 , Current Loss: 1.8166\n",
      "Steps: 11035 Loss:1.8288 , Current Loss: 1.8574\n",
      "Steps: 11135 Loss:1.8184 , Current Loss: 1.9200\n",
      "Steps: 11235 Loss:1.8188 , Current Loss: 1.7779\n",
      "Steps: 11335 Loss:1.8092 , Current Loss: 1.7363\n",
      "Epoch: 5 , Step: 11420 , loss: 1.8177 , Time: 6050.6\n",
      "Model Saved\n",
      "Steps: 11520 Loss:1.8049 , Current Loss: 1.8705\n",
      "Steps: 11620 Loss:1.8065 , Current Loss: 1.7991\n",
      "Steps: 11720 Loss:1.8061 , Current Loss: 1.8279\n",
      "Steps: 11820 Loss:1.8080 , Current Loss: 1.6910\n",
      "Steps: 11920 Loss:1.7994 , Current Loss: 1.7811\n",
      "Steps: 12020 Loss:1.8007 , Current Loss: 1.7230\n",
      "Steps: 12120 Loss:1.8034 , Current Loss: 1.8541\n",
      "Steps: 12220 Loss:1.8009 , Current Loss: 1.7498\n",
      "Epoch: 6 , Step: 12305 , loss: 1.8024 , Time: 6079.7\n",
      "Model Saved\n",
      "Val: F1 Score:41.62 Accuracy:51.38  POS: F1 Score:83.53 Accuracy:91.65 Loss:1.8135 , Time: 1200.8\n",
      "Steps: 12405 Loss:1.7999 , Current Loss: 1.8756\n",
      "Steps: 12505 Loss:1.7929 , Current Loss: 1.8408\n",
      "Steps: 12605 Loss:1.7845 , Current Loss: 1.7957\n",
      "Steps: 12705 Loss:1.7880 , Current Loss: 1.7907\n",
      "Steps: 12805 Loss:1.7782 , Current Loss: 1.8660\n",
      "Steps: 12905 Loss:1.7777 , Current Loss: 1.7932\n",
      "Steps: 13005 Loss:1.7937 , Current Loss: 1.8258\n",
      "Steps: 13105 Loss:1.7841 , Current Loss: 1.7262\n",
      "Epoch: 7 , Step: 13190 , loss: 1.7866 , Time: 6034.0\n",
      "Model Saved\n",
      "Steps: 13290 Loss:1.7706 , Current Loss: 1.7608\n",
      "Steps: 13390 Loss:1.7749 , Current Loss: 1.7121\n",
      "Steps: 13490 Loss:1.7794 , Current Loss: 1.7191\n",
      "Steps: 13590 Loss:1.7744 , Current Loss: 1.8228\n",
      "Steps: 13690 Loss:1.7746 , Current Loss: 1.8266\n",
      "Steps: 13790 Loss:1.7739 , Current Loss: 1.8688\n",
      "Steps: 13890 Loss:1.7725 , Current Loss: 1.8138\n",
      "Steps: 13990 Loss:1.7694 , Current Loss: 1.7080\n",
      "Epoch: 8 , Step: 14075 , loss: 1.7737 , Time: 6073.5\n",
      "Model Saved\n",
      "Val: F1 Score:42.33 Accuracy:52.27  POS: F1 Score:83.64 Accuracy:91.77 Loss:1.7803 , Time: 1195.4\n",
      "Steps: 14175 Loss:1.7699 , Current Loss: 1.7361\n",
      "Steps: 14275 Loss:1.7621 , Current Loss: 1.9498\n",
      "Steps: 14375 Loss:1.7652 , Current Loss: 1.6372\n",
      "Steps: 14475 Loss:1.7733 , Current Loss: 1.6647\n",
      "Steps: 14575 Loss:1.7618 , Current Loss: 1.7154\n",
      "Steps: 14675 Loss:1.7624 , Current Loss: 1.7980\n",
      "Steps: 14775 Loss:1.7638 , Current Loss: 1.7775\n",
      "Steps: 14875 Loss:1.7620 , Current Loss: 1.7652\n",
      "Epoch: 9 , Step: 14960 , loss: 1.7637 , Time: 6045.2\n",
      "Model Saved\n",
      "Steps: 15060 Loss:1.7512 , Current Loss: 1.7584\n",
      "Steps: 15160 Loss:1.7449 , Current Loss: 1.6968\n",
      "Steps: 15260 Loss:1.7522 , Current Loss: 1.8537\n",
      "Steps: 15360 Loss:1.7582 , Current Loss: 1.7480\n",
      "Steps: 15460 Loss:1.7521 , Current Loss: 1.8520\n",
      "Steps: 15560 Loss:1.7537 , Current Loss: 1.7485\n",
      "Steps: 15660 Loss:1.7423 , Current Loss: 1.7622\n",
      "Steps: 15760 Loss:1.7422 , Current Loss: 1.7138\n",
      "Epoch: 10 , Step: 15845 , loss: 1.7497 , Time: 6084.7\n",
      "Model Saved\n",
      "Val: F1 Score:43.92 Accuracy:52.70  POS: F1 Score:83.89 Accuracy:91.85 Loss:1.7603 , Time: 1204.9\n",
      "Steps: 15945 Loss:1.7528 , Current Loss: 1.7500\n",
      "Steps: 16045 Loss:1.7424 , Current Loss: 1.7023\n",
      "Steps: 16145 Loss:1.7475 , Current Loss: 1.7689\n",
      "Steps: 16245 Loss:1.7348 , Current Loss: 1.7812\n",
      "Steps: 16345 Loss:1.7412 , Current Loss: 1.7304\n",
      "Steps: 16445 Loss:1.7412 , Current Loss: 1.6575\n",
      "Steps: 16545 Loss:1.7402 , Current Loss: 1.7268\n",
      "Steps: 16645 Loss:1.7441 , Current Loss: 1.6853\n",
      "Epoch: 11 , Step: 16730 , loss: 1.7427 , Time: 6057.9\n",
      "Model Saved\n",
      "Steps: 16830 Loss:1.7373 , Current Loss: 1.7110\n",
      "Steps: 16930 Loss:1.7292 , Current Loss: 1.7207\n",
      "Steps: 17030 Loss:1.7241 , Current Loss: 1.7406\n",
      "Steps: 17130 Loss:1.7400 , Current Loss: 1.6555\n",
      "Steps: 17230 Loss:1.7336 , Current Loss: 1.7918\n",
      "Steps: 17330 Loss:1.7312 , Current Loss: 1.7318\n",
      "Steps: 17430 Loss:1.7400 , Current Loss: 1.7271\n",
      "Steps: 17530 Loss:1.7241 , Current Loss: 1.6741\n",
      "Epoch: 12 , Step: 17615 , loss: 1.7322 , Time: 6072.1\n",
      "Model Saved\n",
      "Val: F1 Score:44.17 Accuracy:53.07  POS: F1 Score:84.01 Accuracy:91.94 Loss:1.7458 , Time: 1196.4\n",
      "Steps: 17715 Loss:1.7282 , Current Loss: 1.6457\n",
      "Steps: 17815 Loss:1.7273 , Current Loss: 1.7675\n",
      "Steps: 17915 Loss:1.7228 , Current Loss: 1.7641\n",
      "Steps: 18015 Loss:1.7265 , Current Loss: 1.6964\n",
      "Steps: 18115 Loss:1.7240 , Current Loss: 1.6845\n",
      "Steps: 18215 Loss:1.7208 , Current Loss: 1.7907\n",
      "Steps: 18315 Loss:1.7269 , Current Loss: 1.7108\n",
      "Steps: 18415 Loss:1.7314 , Current Loss: 1.6982\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 20485 Loss:1.6997 , Current Loss: 1.6319\n",
      "Steps: 20585 Loss:1.7035 , Current Loss: 1.6649\n",
      "Steps: 20685 Loss:1.6931 , Current Loss: 1.6423\n",
      "Steps: 20785 Loss:1.6937 , Current Loss: 1.6412\n",
      "Steps: 20885 Loss:1.7026 , Current Loss: 1.7167\n",
      "Steps: 20985 Loss:1.7014 , Current Loss: 1.7413\n",
      "Steps: 21085 Loss:1.6890 , Current Loss: 1.5999\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "train_loss, train_pred, train_true, train_pred_pos, train_true_pos = model(x_id_train, y_train, y_pos_train, mask_train, sense_mask_train, train_cond=False)        \n",
    "f1_, accu_, f1_pos_, accu_pos_ = etrain_score(train_true, train_pred, train_true_pos, train_pred_pos)\n",
    "time_taken = time.time() - start_time\n",
    "print(\"train: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "envs",
   "language": "python",
   "name": "cs771"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}