one_million/all-word/Model-aw-lex-1.4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### learning rate = 0.001, used convolution for extracting features using word embeddings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# Model Description\n",
    "model_name = 'model-aw-lex-1-4'\n",
    "model_dir = '../output/all-word/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "with open('/data/aviraj/dataset/train_val_data_fine/all_word_lex','rb') as f:\n",
    "    train_data, val_data = pickle.load(f)    \n",
    "    \n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = 45\n",
    "num_pos = 12\n",
    "batch_size = 64\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 256\n",
    "num_filter = 256\n",
    "kernel_size = 5\n",
    "keep_prob = 0.3\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 500\n",
    "decay_rate = 0.9\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, 0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MODEL\n",
    "device_num = 0\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "sense_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='sense_mask')\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "pretrain = tf.placeholder('bool', [], name=\"pretrain\")\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "summaries = []\n",
    "\n",
    "def global_attention(input_x, input_mask, W_att):\n",
    "    h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "    h_tanh = tf.tanh(h_masked)\n",
    "    u = tf.matmul(h_tanh, W_att)\n",
    "    a = tf.nn.softmax(u)\n",
    "    c = tf.reduce_sum(tf.multiply(h_tanh, a), 0)  \n",
    "    return c\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        if gpu_idx>int(num_gpus/2)-1:\n",
    "            device_num = 1\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % device_num):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "            tile_x_mask = tf.tile(tf.expand_dims(float_x_mask, 2), [1, 1, word_emb_size])\n",
    "            Wx_masked = tf.multiply(Wx, tile_x_mask)\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), 1)\n",
    "            \n",
    "            with tf.variable_scope(\"convolution\"):\n",
    "                conv1 = tf.layers.conv1d(inputs=Wx_masked, filters=num_filter, kernel_size=[kernel_size], padding='same', activation=tf.nn.relu)\n",
    "                conv2 = tf.layers.conv1d(inputs=conv1, filters=num_filter, kernel_size=[kernel_size], padding='same')\n",
    "                \n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, conv2, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "\n",
    "            with tf.variable_scope(\"global_attention\"):\n",
    "                W_att = tf.get_variable(\"W_att\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                c = tf.expand_dims(global_attention(h[0], x_mask[gpu_idx][0], W_att), 0)\n",
    "                for i in range(1, batch_size):\n",
    "                    c = tf.concat([c, tf.expand_dims(global_attention(h[i], x_mask[gpu_idx][i], W_att), 0)], 0)\n",
    "                cc = tf.expand_dims(c, 1)\n",
    "                c_final = tf.tile(cc, [1, max_sent_size, 1])\n",
    "\n",
    "            h_final = tf.concat([c_final, h], 2)\n",
    "            flat_h_final = tf.reshape(h_final, [-1, tf.shape(h_final)[2]])\n",
    "\n",
    "            with tf.variable_scope(\"hidden_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[4*hidden_size, 2*hidden_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[2*hidden_size], initializer=tf.zeros_initializer())\n",
    "                drop_flat_h_final = tf.nn.dropout(flat_h_final, input_keep_prob)\n",
    "                flat_hl = tf.matmul(drop_flat_h_final, W) + b\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_sense = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits = tf.reshape(flat_logits_sense, [batch_size, max_sent_size, num_senses])\n",
    "                predictions.append(tf.arg_max(logits, 2))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*30))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, tf.shape(h1)[2]])\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                predictions_pos.append(tf.arg_max(logits_pos, 2))\n",
    "\n",
    "\n",
    "            float_sense_mask = tf.cast(sense_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.contrib.seq2seq.sequence_loss(logits, y[gpu_idx], float_sense_mask, name=\"loss\")\n",
    "            loss_pos = tf.contrib.seq2seq.sequence_loss(logits_pos, y_pos[gpu_idx], float_x_mask, name=\"loss_\")\n",
    "\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = tf.cond(pretrain, lambda:loss_pos, lambda:loss + loss_pos + l2_loss)\n",
    "\n",
    "            summaries.append(tf.summary.scalar(\"loss_{}\".format(gpu_idx), loss))\n",
    "            summaries.append(tf.summary.scalar(\"loss_pos_{}\".format(gpu_idx), loss_pos))\n",
    "            summaries.append(tf.summary.scalar(\"total_loss_{}\".format(gpu_idx), total_loss))\n",
    "\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "tower_grads = average_gradients(tower_grads)\n",
    "losses = tf.add_n(losses)/len(losses)\n",
    "apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "for var in tf.trainable_variables():\n",
    "    summaries.append(tf.summary.histogram(var.op.name, var))\n",
    "\n",
    "variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_period = 100\n",
    "log_period = 100\n",
    "\n",
    "def model(xx, yy, yy_pos, mask, smask, train_cond=True, pretrain_cond=False):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    temp_loss = 0\n",
    "    preds_sense = []\n",
    "    true_sense = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_pos_re = yy_pos[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        smask_re = smask[s:e].reshape([num_gpus, batch_size, -1])\n",
    " \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:yy_pos_re, x_mask:mask_re, sense_mask:smask_re, pretrain:pretrain_cond, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)\n",
    "            \n",
    "            temp_loss += _loss\n",
    "            if((j+1)%log_period==0):\n",
    "                print(\"Steps: {}\".format(step), \"Loss:{0:.4f}\".format(temp_loss/log_period), \", Current Loss: {0:.4f}\".format(_loss))\n",
    "                temp_loss = 0\n",
    "            if((j+1)%save_period==0):\n",
    "                saver.save(sess, save_path=save_dir)                         \n",
    "                \n",
    "        else:\n",
    "            _loss, pred, pred_pos = sess.run([total_loss, predictions, predictions_pos], feed_dict)\n",
    "            for i in range(num_gpus):\n",
    "                preds_sense.append(pred[i][smask_re[i]])\n",
    "                true_sense.append(yy_re[i][smask_re[i]])\n",
    "                preds_pos.append(pred_pos[i][mask_re[i]])\n",
    "                true_pos.append(yy_pos_re[i][mask_re[i]])\n",
    "\n",
    "        _losses +=_loss\n",
    "\n",
    "    if(train_cond==False): \n",
    "        sense_preds = []\n",
    "        sense_true = []\n",
    "        pos_preds = []\n",
    "        pos_true = []\n",
    "        \n",
    "        for preds in preds_sense:\n",
    "            for ps in preds:      \n",
    "                sense_preds.append(ps)  \n",
    "        for trues in true_sense:\n",
    "            for ts in trues:\n",
    "                sense_true.append(ts)\n",
    "        \n",
    "        for preds in preds_pos:\n",
    "            for ps in preds:      \n",
    "                pos_preds.append(ps)      \n",
    "        for trues in true_pos:\n",
    "            for ts in trues:\n",
    "                pos_true.append(ts)\n",
    "                \n",
    "        return _losses/num_batches, sense_preds, sense_true, pos_preds, pos_true\n",
    "\n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, yy_pos, pred_pos):\n",
    "    f1 = f1_score(yy, pred, average='macro')\n",
    "    accu = accuracy_score(yy, pred)\n",
    "    f1_pos = f1_score(yy_pos, pred_pos, average='macro')\n",
    "    accu_pos = accuracy_score(yy_pos, pred_pos)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_id_train = train_data['x']\n",
    "mask_train = train_data['x_mask']\n",
    "sense_mask_train = train_data['sense_mask']\n",
    "y_train = train_data['y']\n",
    "y_pos_train = train_data['pos']\n",
    "\n",
    "x_id_val = val_data['x']\n",
    "mask_val = val_data['x_mask']\n",
    "sense_mask_val = val_data['sense_mask']\n",
    "y_val = val_data['y']\n",
    "y_pos_val = val_data['pos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testing():\n",
    "    start_time = time.time()\n",
    "    val_loss, val_pred, val_true, val_pred_pos, val_true_pos = model(x_id_val, y_val, y_pos_val, mask_val, sense_mask_val, train_cond=False)        \n",
    "    f1_, accu_, f1_pos_, accu_pos_ = eval_score(val_true, val_pred, val_true_pos, val_pred_pos)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Val: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "    return f1_, accu_, f1_pos_, accu_pos_\n",
    "\n",
    "def training(current_epoch, pre_train_cond):\n",
    "        random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "        x_id_train_tmp = x_id_train[random]\n",
    "        y_train_tmp = y_train[random]\n",
    "        mask_train_tmp = mask_train[random]    \n",
    "        sense_mask_train_tmp = sense_mask_train[random]\n",
    "        y_pos_train_tmp = y_pos_train[random]\n",
    "\n",
    "        start_time = time.time()\n",
    "        train_loss, step = model(x_id_train_tmp, y_train_tmp, y_pos_train_tmp, mask_train_tmp, sense_mask_train_tmp, pretrain_cond=pre_train_cond)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Epoch: {}\".format(current_epoch+1),\", Step: {}\".format(step), \", loss: {0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        return [step, train_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 100 Loss:9.1989 , Current Loss: 2.1383\n",
      "Steps: 200 Loss:2.0983 , Current Loss: 2.0462\n",
      "Steps: 300 Loss:1.9160 , Current Loss: 1.7626\n",
      "Steps: 400 Loss:1.3839 , Current Loss: 1.1323\n",
      "Steps: 500 Loss:0.9359 , Current Loss: 0.7643\n",
      "Steps: 600 Loss:0.6473 , Current Loss: 0.5642\n",
      "Steps: 700 Loss:0.5082 , Current Loss: 0.4730\n",
      "Steps: 800 Loss:0.4297 , Current Loss: 0.4146\n",
      "Steps: 900 Loss:0.3753 , Current Loss: 0.3453\n",
      "Steps: 1000 Loss:0.3316 , Current Loss: 0.3059\n",
      "Steps: 1100 Loss:0.2968 , Current Loss: 0.2864\n",
      "Steps: 1200 Loss:0.2672 , Current Loss: 0.2495\n",
      "Steps: 1300 Loss:0.2478 , Current Loss: 0.2552\n",
      "Steps: 1400 Loss:0.2293 , Current Loss: 0.2242\n",
      "Steps: 1500 Loss:0.2067 , Current Loss: 0.1914\n",
      "Steps: 1600 Loss:0.1906 , Current Loss: 0.1830\n",
      "Steps: 1700 Loss:0.1763 , Current Loss: 0.1711\n",
      "Epoch: 64 , Step: 1771 , loss: 1.1043 , Time: 11749.2\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:1.62 Accuracy:2.00  POS: F1 Score:88.83 Accuracy:95.04 Loss:8.6611 , Time: 2263.9\n",
      "Steps: 1871 Loss:0.1562 , Current Loss: 0.1427\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 10\n",
    "val_period = 2\n",
    "\n",
    "# Pretraining POS Tags\n",
    "loss_collection.append(training(i, True))\n",
    "val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 2052 Loss:3.3164 , Current Loss: 2.9249\n",
      "Steps: 2152 Loss:2.6982 , Current Loss: 2.6539\n",
      "Steps: 2252 Loss:2.5134 , Current Loss: 2.3444\n",
      "Steps: 2352 Loss:2.3963 , Current Loss: 2.4742\n",
      "Steps: 2452 Loss:2.3296 , Current Loss: 2.2533\n",
      "Steps: 2552 Loss:2.2754 , Current Loss: 2.3117\n",
      "Steps: 2652 Loss:2.2334 , Current Loss: 2.2473\n",
      "Steps: 2752 Loss:2.1891 , Current Loss: 2.0999\n",
      "Steps: 2852 Loss:2.1574 , Current Loss: 2.1592\n",
      "Steps: 2952 Loss:2.1154 , Current Loss: 2.0235\n",
      "Steps: 3052 Loss:2.0940 , Current Loss: 2.0204\n",
      "Steps: 3152 Loss:2.0549 , Current Loss: 2.0922\n",
      "Steps: 3252 Loss:2.0362 , Current Loss: 2.1125\n",
      "Steps: 3352 Loss:2.0035 , Current Loss: 1.9530\n",
      "Steps: 3452 Loss:1.9732 , Current Loss: 1.8338\n",
      "Steps: 3552 Loss:1.9511 , Current Loss: 1.8576\n",
      "Steps: 3652 Loss:1.9250 , Current Loss: 2.1319\n",
      "Epoch: 1 , Step: 3723 , loss: 2.2360 , Time: 16815.5\n",
      "Model Saved\n",
      "Steps: 3823 Loss:1.8577 , Current Loss: 1.6996\n",
      "Steps: 3923 Loss:1.8623 , Current Loss: 1.8257\n",
      "Steps: 4023 Loss:1.8193 , Current Loss: 1.7376\n",
      "Steps: 4123 Loss:1.8162 , Current Loss: 1.8388\n",
      "Steps: 4223 Loss:1.7939 , Current Loss: 1.8030\n",
      "Steps: 4323 Loss:1.7654 , Current Loss: 1.8652\n",
      "Steps: 4423 Loss:1.7522 , Current Loss: 1.7473\n",
      "Steps: 4523 Loss:1.7183 , Current Loss: 1.4910\n",
      "Steps: 4623 Loss:1.7088 , Current Loss: 1.8784\n",
      "Steps: 4723 Loss:1.6879 , Current Loss: 1.6822\n",
      "Steps: 4823 Loss:1.6847 , Current Loss: 1.6440\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 4923 Loss:1.6377 , Current Loss: 1.5147\n",
      "Steps: 5023 Loss:1.6370 , Current Loss: 1.5468\n",
      "Steps: 5123 Loss:1.6022 , Current Loss: 1.5690\n",
      "Steps: 5223 Loss:1.5983 , Current Loss: 1.5429\n",
      "Steps: 5323 Loss:1.5897 , Current Loss: 1.6596\n",
      "Steps: 5423 Loss:1.5729 , Current Loss: 1.4462\n",
      "Steps: 5523 Loss:1.5633 , Current Loss: 1.5260\n",
      "Steps: 5623 Loss:1.5391 , Current Loss: 1.6116\n",
      "Steps: 5723 Loss:1.5308 , Current Loss: 1.4481\n",
      "Steps: 5823 Loss:1.5139 , Current Loss: 1.5606\n",
      "Steps: 5923 Loss:1.5042 , Current Loss: 1.4563\n",
      "Steps: 6023 Loss:1.4825 , Current Loss: 1.3770\n",
      "Steps: 6123 Loss:1.4966 , Current Loss: 1.4541\n",
      "Steps: 6223 Loss:1.4893 , Current Loss: 1.3478\n",
      "Steps: 6323 Loss:1.4766 , Current Loss: 1.4357\n",
      "Steps: 6423 Loss:1.4601 , Current Loss: 1.5925\n",
      "Steps: 6523 Loss:1.4403 , Current Loss: 1.3285\n",
      "Epoch: 1 , Step: 6594 , loss: 1.5332 , Time: 11766.0\n",
      "Model Saved\n",
      "Steps: 6694 Loss:1.4052 , Current Loss: 1.3481\n",
      "Steps: 6794 Loss:1.4008 , Current Loss: 1.4364\n",
      "Steps: 6894 Loss:1.3823 , Current Loss: 1.4883\n",
      "Steps: 6994 Loss:1.3862 , Current Loss: 1.5302\n",
      "Steps: 7094 Loss:1.3835 , Current Loss: 1.4114\n",
      "Steps: 7194 Loss:1.3649 , Current Loss: 1.2698\n",
      "Steps: 7294 Loss:1.3650 , Current Loss: 1.3610\n",
      "Steps: 7394 Loss:1.3507 , Current Loss: 1.2269\n",
      "Steps: 7494 Loss:1.3524 , Current Loss: 1.3383\n",
      "Steps: 7594 Loss:1.3444 , Current Loss: 1.2951\n",
      "Steps: 7694 Loss:1.3350 , Current Loss: 1.4110\n",
      "Steps: 7794 Loss:1.3386 , Current Loss: 1.3949\n",
      "Steps: 7894 Loss:1.3183 , Current Loss: 1.3976\n",
      "Steps: 7994 Loss:1.3056 , Current Loss: 1.0915\n",
      "Steps: 8094 Loss:1.3046 , Current Loss: 1.2166\n",
      "Steps: 8194 Loss:1.2933 , Current Loss: 1.1658\n",
      "Steps: 8294 Loss:1.2906 , Current Loss: 1.2906\n",
      "Epoch: 2 , Step: 8365 , loss: 1.3464 , Time: 11717.8\n",
      "Model Saved\n",
      "Steps: 8465 Loss:1.2608 , Current Loss: 1.2724\n",
      "Steps: 8565 Loss:1.2477 , Current Loss: 1.2298\n",
      "Steps: 8665 Loss:1.2540 , Current Loss: 1.2229\n",
      "Steps: 8765 Loss:1.2537 , Current Loss: 1.0456\n",
      "Steps: 8865 Loss:1.2518 , Current Loss: 1.2165\n",
      "Steps: 8965 Loss:1.2471 , Current Loss: 1.3002\n",
      "Steps: 9065 Loss:1.2372 , Current Loss: 1.3861\n",
      "Steps: 9165 Loss:1.2497 , Current Loss: 1.3938\n",
      "Steps: 9265 Loss:1.2288 , Current Loss: 1.2985\n",
      "Steps: 9365 Loss:1.2340 , Current Loss: 1.3649\n",
      "Steps: 9465 Loss:1.2173 , Current Loss: 1.2675\n",
      "Steps: 9565 Loss:1.2248 , Current Loss: 1.2929\n",
      "Steps: 9665 Loss:1.2190 , Current Loss: 1.1746\n",
      "Steps: 9765 Loss:1.2209 , Current Loss: 1.2578\n",
      "Steps: 9865 Loss:1.2109 , Current Loss: 1.2249\n",
      "Steps: 9965 Loss:1.2171 , Current Loss: 1.1401\n",
      "Steps: 10065 Loss:1.2105 , Current Loss: 1.2644\n",
      "Epoch: 3 , Step: 10136 , loss: 1.2333 , Time: 11714.5\n",
      "Model Saved\n",
      "Steps: 10236 Loss:1.1663 , Current Loss: 1.1692\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:55.67 Accuracy:67.24  POS: F1 Score:91.11 Accuracy:95.95 Loss:1.2112 , Time: 2268.4\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(55.667867534724493, 67.24365329462168, 91.114662306819355, 95.950279386802848)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 10341 Loss:1.1559 , Current Loss: 1.1919\n",
      "Steps: 10441 Loss:1.1635 , Current Loss: 1.1936\n",
      "Steps: 10541 Loss:1.1677 , Current Loss: 1.0781\n",
      "Steps: 10641 Loss:1.1611 , Current Loss: 1.1736\n",
      "Steps: 10741 Loss:1.1605 , Current Loss: 1.2458\n",
      "Steps: 10841 Loss:1.1578 , Current Loss: 1.1275\n",
      "Steps: 10941 Loss:1.1515 , Current Loss: 0.9577\n",
      "Steps: 11041 Loss:1.1716 , Current Loss: 1.3048\n",
      "Steps: 11141 Loss:1.1651 , Current Loss: 1.3566\n",
      "Steps: 11241 Loss:1.1490 , Current Loss: 1.1201\n",
      "Steps: 11341 Loss:1.1500 , Current Loss: 1.1885\n",
      "Steps: 11441 Loss:1.1336 , Current Loss: 1.1917\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 11541 Loss:1.1244 , Current Loss: 1.0213\n",
      "Steps: 11641 Loss:1.1196 , Current Loss: 1.0545\n",
      "Steps: 11741 Loss:1.1205 , Current Loss: 1.0195\n",
      "Steps: 11841 Loss:1.1306 , Current Loss: 1.1614\n",
      "Steps: 11941 Loss:1.1325 , Current Loss: 1.1343\n",
      "Steps: 12041 Loss:1.1127 , Current Loss: 1.1756\n",
      "Steps: 12141 Loss:1.1217 , Current Loss: 1.1518\n",
      "Steps: 12241 Loss:1.1279 , Current Loss: 1.1131\n",
      "Steps: 12341 Loss:1.0970 , Current Loss: 1.0538\n",
      "Steps: 12441 Loss:1.1231 , Current Loss: 1.1238\n",
      "Steps: 12541 Loss:1.1076 , Current Loss: 1.0790\n",
      "Steps: 12641 Loss:1.1010 , Current Loss: 1.0229\n",
      "Steps: 12741 Loss:1.1092 , Current Loss: 1.1688\n",
      "Steps: 12841 Loss:1.0961 , Current Loss: 1.2620\n",
      "Steps: 12941 Loss:1.0944 , Current Loss: 1.0215\n",
      "Steps: 13041 Loss:1.0984 , Current Loss: 1.1725\n",
      "Steps: 13141 Loss:1.0841 , Current Loss: 1.0826\n",
      "Epoch: 1 , Step: 13212 , loss: 1.1112 , Time: 11709.9\n",
      "Model Saved\n",
      "Steps: 13312 Loss:1.0673 , Current Loss: 0.9973\n",
      "Steps: 13412 Loss:1.0582 , Current Loss: 1.0824\n",
      "Steps: 13512 Loss:1.0671 , Current Loss: 1.0505\n",
      "Steps: 13612 Loss:1.0675 , Current Loss: 1.2026\n",
      "Steps: 13712 Loss:1.0610 , Current Loss: 1.0264\n",
      "Steps: 13812 Loss:1.0732 , Current Loss: 1.0295\n",
      "Steps: 13912 Loss:1.0732 , Current Loss: 1.1329\n",
      "Steps: 14012 Loss:1.0650 , Current Loss: 1.0918\n",
      "Steps: 14112 Loss:1.0648 , Current Loss: 1.1886\n",
      "Steps: 14212 Loss:1.0561 , Current Loss: 1.0024\n",
      "Steps: 14312 Loss:1.0686 , Current Loss: 1.0577\n",
      "Steps: 14412 Loss:1.0569 , Current Loss: 1.0271\n",
      "Steps: 14512 Loss:1.0663 , Current Loss: 1.1015\n",
      "Steps: 14612 Loss:1.0725 , Current Loss: 1.1059\n",
      "Steps: 14712 Loss:1.0466 , Current Loss: 1.1324\n",
      "Steps: 14812 Loss:1.0593 , Current Loss: 1.0861\n",
      "Steps: 14912 Loss:1.0567 , Current Loss: 1.0737\n",
      "Epoch: 2 , Step: 14983 , loss: 1.0632 , Time: 11652.9\n",
      "Model Saved\n",
      "Val: F1 Score:62.38 Accuracy:71.32  POS: F1 Score:92.35 Accuracy:96.44 Loss:1.0645 , Time: 2260.8\n",
      "Steps: 15083 Loss:1.0193 , Current Loss: 1.1170\n",
      "Steps: 15183 Loss:1.0253 , Current Loss: 1.0609\n",
      "Steps: 15283 Loss:1.0320 , Current Loss: 0.9050\n",
      "Steps: 15383 Loss:1.0331 , Current Loss: 1.0234\n",
      "Steps: 15483 Loss:1.0397 , Current Loss: 0.9642\n",
      "Steps: 15583 Loss:1.0288 , Current Loss: 1.0888\n",
      "Steps: 15683 Loss:1.0288 , Current Loss: 1.0456\n",
      "Steps: 15783 Loss:1.0135 , Current Loss: 1.0987\n",
      "Steps: 15883 Loss:1.0332 , Current Loss: 1.0730\n",
      "Steps: 15983 Loss:1.0249 , Current Loss: 1.0781\n",
      "Steps: 16083 Loss:1.0247 , Current Loss: 0.9855\n",
      "Steps: 16183 Loss:1.0284 , Current Loss: 1.0041\n",
      "Steps: 16283 Loss:1.0127 , Current Loss: 1.1343\n",
      "Steps: 16383 Loss:1.0160 , Current Loss: 1.1660\n",
      "Steps: 16483 Loss:1.0129 , Current Loss: 0.8755\n",
      "Steps: 16583 Loss:1.0198 , Current Loss: 1.1038\n",
      "Steps: 16683 Loss:1.0231 , Current Loss: 0.8671\n",
      "Epoch: 3 , Step: 16754 , loss: 1.0241 , Time: 11938.8\n",
      "Model Saved\n",
      "Steps: 16854 Loss:0.9798 , Current Loss: 0.9586\n",
      "Steps: 16954 Loss:0.9975 , Current Loss: 1.0342\n",
      "Steps: 17054 Loss:0.9914 , Current Loss: 0.9918\n",
      "Steps: 17154 Loss:0.9969 , Current Loss: 0.9713\n",
      "Steps: 17254 Loss:0.9846 , Current Loss: 1.0262\n",
      "Steps: 17354 Loss:0.9965 , Current Loss: 1.0803\n",
      "Steps: 17454 Loss:0.9833 , Current Loss: 1.0012\n",
      "Steps: 17554 Loss:1.0068 , Current Loss: 0.9115\n",
      "Steps: 17654 Loss:0.9976 , Current Loss: 0.9897\n",
      "Steps: 17754 Loss:0.9967 , Current Loss: 1.0398\n",
      "Steps: 17854 Loss:0.9835 , Current Loss: 0.9109\n",
      "Steps: 17954 Loss:0.9934 , Current Loss: 1.0083\n",
      "Steps: 18054 Loss:1.0037 , Current Loss: 1.0476\n",
      "Steps: 18154 Loss:0.9966 , Current Loss: 0.9086\n",
      "Steps: 18254 Loss:0.9901 , Current Loss: 0.9791\n",
      "Steps: 18354 Loss:0.9808 , Current Loss: 1.0058\n",
      "Steps: 18454 Loss:0.9831 , Current Loss: 0.8462\n",
      "Epoch: 4 , Step: 18525 , loss: 0.9915 , Time: 11984.6\n",
      "Model Saved\n",
      "Val: F1 Score:64.62 Accuracy:72.72  POS: F1 Score:92.77 Accuracy:96.65 Loss:1.0340 , Time: 2259.6\n",
      "Steps: 18625 Loss:0.9658 , Current Loss: 0.9919\n",
      "Steps: 18725 Loss:0.9618 , Current Loss: 1.0812\n",
      "Steps: 18825 Loss:0.9659 , Current Loss: 1.0484\n",
      "Steps: 18925 Loss:0.9703 , Current Loss: 0.9512\n",
      "Steps: 19025 Loss:0.9569 , Current Loss: 0.9130\n",
      "Steps: 19125 Loss:0.9687 , Current Loss: 0.9554\n",
      "Steps: 19225 Loss:0.9575 , Current Loss: 1.0713\n",
      "Steps: 19325 Loss:0.9805 , Current Loss: 1.1124\n",
      "Steps: 19425 Loss:0.9637 , Current Loss: 0.9639\n",
      "Steps: 19525 Loss:0.9718 , Current Loss: 0.8783\n",
      "Steps: 19625 Loss:0.9557 , Current Loss: 0.9521\n",
      "Steps: 19725 Loss:0.9625 , Current Loss: 0.9306\n",
      "Steps: 19825 Loss:0.9538 , Current Loss: 0.9414\n",
      "Steps: 19925 Loss:0.9499 , Current Loss: 0.9720\n",
      "Steps: 20025 Loss:0.9679 , Current Loss: 1.0832\n",
      "Steps: 20125 Loss:0.9599 , Current Loss: 1.0066\n",
      "Steps: 20225 Loss:0.9615 , Current Loss: 0.9462\n",
      "Epoch: 5 , Step: 20296 , loss: 0.9632 , Time: 11982.9\n",
      "Model Saved\n",
      "Steps: 20396 Loss:0.9306 , Current Loss: 0.9276\n",
      "Steps: 20496 Loss:0.9357 , Current Loss: 0.9334\n",
      "Steps: 20596 Loss:0.9357 , Current Loss: 0.9590\n",
      "Steps: 20696 Loss:0.9562 , Current Loss: 1.0155\n",
      "Steps: 20796 Loss:0.9370 , Current Loss: 0.8754\n",
      "Steps: 20896 Loss:0.9337 , Current Loss: 0.8607\n",
      "Steps: 20996 Loss:0.9425 , Current Loss: 0.9030\n",
      "Steps: 21096 Loss:0.9483 , Current Loss: 0.9322\n",
      "Steps: 21196 Loss:0.9295 , Current Loss: 1.0307\n",
      "Steps: 21296 Loss:0.9498 , Current Loss: 1.0252\n",
      "Steps: 21396 Loss:0.9416 , Current Loss: 0.9676\n",
      "Steps: 21496 Loss:0.9388 , Current Loss: 0.8670\n",
      "Steps: 21596 Loss:0.9407 , Current Loss: 1.0255\n",
      "Steps: 21696 Loss:0.9227 , Current Loss: 1.0455\n",
      "Steps: 21796 Loss:0.9428 , Current Loss: 0.9691\n",
      "Steps: 21896 Loss:0.9392 , Current Loss: 1.0668\n",
      "Steps: 21996 Loss:0.9386 , Current Loss: 1.0163\n",
      "Epoch: 6 , Step: 22067 , loss: 0.9392 , Time: 11924.5\n",
      "Model Saved\n",
      "Val: F1 Score:65.94 Accuracy:73.60  POS: F1 Score:93.12 Accuracy:96.80 Loss:0.9823 , Time: 2258.6\n",
      "Steps: 22167 Loss:0.9066 , Current Loss: 0.8368\n",
      "Steps: 22267 Loss:0.9186 , Current Loss: 0.9563\n",
      "Steps: 22367 Loss:0.9112 , Current Loss: 0.7872\n",
      "Steps: 22467 Loss:0.9360 , Current Loss: 0.8528\n",
      "Steps: 22567 Loss:0.9152 , Current Loss: 0.8052\n",
      "Steps: 22667 Loss:0.9224 , Current Loss: 1.0545\n",
      "Steps: 22767 Loss:0.9178 , Current Loss: 0.8799\n",
      "Steps: 22867 Loss:0.9187 , Current Loss: 1.0387\n",
      "Steps: 22967 Loss:0.9131 , Current Loss: 0.8865\n",
      "Steps: 23067 Loss:0.9230 , Current Loss: 0.7818\n",
      "Steps: 23167 Loss:0.9228 , Current Loss: 0.8659\n",
      "Steps: 23267 Loss:0.9340 , Current Loss: 0.9645\n",
      "Steps: 23367 Loss:0.9138 , Current Loss: 0.8436\n",
      "Steps: 23467 Loss:0.9088 , Current Loss: 0.9540\n",
      "Steps: 23567 Loss:0.9161 , Current Loss: 0.9944\n",
      "Steps: 23667 Loss:0.9168 , Current Loss: 1.0443\n",
      "Steps: 23767 Loss:0.9120 , Current Loss: 1.0047\n",
      "Epoch: 7 , Step: 23838 , loss: 0.9178 , Time: 11944.3\n",
      "Model Saved\n",
      "Steps: 23938 Loss:0.8973 , Current Loss: 0.9023\n",
      "Steps: 24038 Loss:0.8914 , Current Loss: 0.8713\n",
      "Steps: 24138 Loss:0.9065 , Current Loss: 0.9467\n",
      "Steps: 24238 Loss:0.8942 , Current Loss: 0.8632\n",
      "Steps: 24338 Loss:0.9102 , Current Loss: 0.8156\n",
      "Steps: 24438 Loss:0.8880 , Current Loss: 0.9161\n",
      "Steps: 24538 Loss:0.8979 , Current Loss: 0.8932\n",
      "Steps: 24638 Loss:0.9096 , Current Loss: 1.0207\n",
      "Steps: 24738 Loss:0.8954 , Current Loss: 0.7887\n",
      "Steps: 24838 Loss:0.8984 , Current Loss: 0.8352\n",
      "Steps: 24938 Loss:0.8838 , Current Loss: 0.7050\n",
      "Steps: 25038 Loss:0.9017 , Current Loss: 0.9377\n",
      "Steps: 25138 Loss:0.9052 , Current Loss: 1.0566\n",
      "Steps: 25238 Loss:0.9050 , Current Loss: 0.7313\n",
      "Steps: 25338 Loss:0.8989 , Current Loss: 0.7866\n",
      "Steps: 25438 Loss:0.9085 , Current Loss: 0.8984\n",
      "Steps: 25538 Loss:0.8889 , Current Loss: 0.9125\n",
      "Epoch: 8 , Step: 25609 , loss: 0.8984 , Time: 12212.4\n",
      "Model Saved\n",
      "Val: F1 Score:67.58 Accuracy:74.49  POS: F1 Score:93.50 Accuracy:96.94 Loss:0.9509 , Time: 2253.6\n",
      "Steps: 25709 Loss:0.8633 , Current Loss: 0.9256\n",
      "Steps: 25809 Loss:0.8607 , Current Loss: 0.8346\n",
      "Steps: 25909 Loss:0.8850 , Current Loss: 1.0033\n",
      "Steps: 26009 Loss:0.8861 , Current Loss: 0.9196\n",
      "Steps: 26109 Loss:0.8822 , Current Loss: 0.7631\n",
      "Steps: 26209 Loss:0.8783 , Current Loss: 0.8512\n",
      "Steps: 26309 Loss:0.8838 , Current Loss: 0.9644\n",
      "Steps: 26409 Loss:0.8842 , Current Loss: 0.8119\n",
      "Steps: 26509 Loss:0.8754 , Current Loss: 0.8477\n",
      "Steps: 26609 Loss:0.8892 , Current Loss: 0.8283\n",
      "Steps: 26709 Loss:0.8754 , Current Loss: 0.8691\n",
      "Steps: 26809 Loss:0.8819 , Current Loss: 0.8108\n",
      "Steps: 26909 Loss:0.8859 , Current Loss: 0.8357\n",
      "Steps: 27009 Loss:0.8895 , Current Loss: 0.9522\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 27409 Loss:0.8657 , Current Loss: 0.9147\n",
      "Steps: 27509 Loss:0.8580 , Current Loss: 0.7973\n",
      "Steps: 27609 Loss:0.8644 , Current Loss: 0.9745\n",
      "Steps: 27709 Loss:0.8687 , Current Loss: 0.9120\n",
      "Steps: 27809 Loss:0.8565 , Current Loss: 0.8624\n",
      "Steps: 27909 Loss:0.8643 , Current Loss: 0.9333\n",
      "Steps: 28009 Loss:0.8565 , Current Loss: 0.7526\n",
      "Steps: 28109 Loss:0.8686 , Current Loss: 0.8453\n",
      "Steps: 28209 Loss:0.8642 , Current Loss: 0.9026\n",
      "Steps: 28309 Loss:0.8658 , Current Loss: 0.8748\n",
      "Steps: 28409 Loss:0.8768 , Current Loss: 0.8207\n",
      "Steps: 28509 Loss:0.8549 , Current Loss: 0.8667\n",
      "Steps: 28609 Loss:0.8789 , Current Loss: 0.9216\n",
      "Steps: 28709 Loss:0.8722 , Current Loss: 0.8870\n",
      "Steps: 28809 Loss:0.8816 , Current Loss: 0.7889\n",
      "Steps: 28909 Loss:0.8655 , Current Loss: 0.8497\n",
      "Steps: 29009 Loss:0.8688 , Current Loss: 0.9067\n",
      "Epoch: 1 , Step: 29080 , loss: 0.8658 , Time: 14157.9\n",
      "Model Saved\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 29180 Loss:0.8276 , Current Loss: 0.8597\n",
      "Steps: 29280 Loss:0.8390 , Current Loss: 0.8335\n",
      "Steps: 29380 Loss:0.8426 , Current Loss: 0.8314\n",
      "Steps: 29480 Loss:0.8485 , Current Loss: 0.9055\n",
      "Steps: 29580 Loss:0.8572 , Current Loss: 0.8291\n",
      "Steps: 29680 Loss:0.8434 , Current Loss: 0.8401\n",
      "Steps: 29780 Loss:0.8576 , Current Loss: 0.9258\n",
      "Steps: 29880 Loss:0.8597 , Current Loss: 0.8240\n",
      "Steps: 29980 Loss:0.8574 , Current Loss: 0.8616\n",
      "Steps: 30080 Loss:0.8562 , Current Loss: 0.9875\n",
      "Steps: 30180 Loss:0.8577 , Current Loss: 0.9439\n",
      "Steps: 30280 Loss:0.8566 , Current Loss: 0.9347\n",
      "Steps: 30380 Loss:0.8541 , Current Loss: 0.9795\n",
      "Steps: 30480 Loss:0.8482 , Current Loss: 0.9477\n",
      "Steps: 30580 Loss:0.8625 , Current Loss: 0.8053\n",
      "Steps: 30680 Loss:0.8471 , Current Loss: 0.9104\n",
      "Steps: 30780 Loss:0.8610 , Current Loss: 0.7927\n",
      "Epoch: 1 , Step: 30851 , loss: 0.8516 , Time: 16681.4\n",
      "Model Saved\n",
      "Steps: 30951 Loss:0.8389 , Current Loss: 0.9358\n",
      "Steps: 31051 Loss:0.8285 , Current Loss: 0.8432\n",
      "Steps: 31151 Loss:0.8342 , Current Loss: 0.8119\n",
      "Steps: 31251 Loss:0.8387 , Current Loss: 0.7815\n",
      "Steps: 31351 Loss:0.8290 , Current Loss: 0.8719\n",
      "Steps: 31451 Loss:0.8418 , Current Loss: 0.8725\n",
      "Steps: 31551 Loss:0.8455 , Current Loss: 0.9882\n",
      "Steps: 31651 Loss:0.8346 , Current Loss: 0.9158\n",
      "Steps: 31751 Loss:0.8389 , Current Loss: 0.9137\n",
      "Steps: 31851 Loss:0.8378 , Current Loss: 1.0007\n",
      "Steps: 31951 Loss:0.8454 , Current Loss: 0.8010\n",
      "Steps: 32051 Loss:0.8480 , Current Loss: 0.8128\n",
      "Steps: 32151 Loss:0.8375 , Current Loss: 0.8440\n",
      "Steps: 32251 Loss:0.8398 , Current Loss: 0.8707\n",
      "Steps: 32351 Loss:0.8498 , Current Loss: 0.7784\n",
      "Steps: 32451 Loss:0.8443 , Current Loss: 0.9116\n",
      "Steps: 32551 Loss:0.8297 , Current Loss: 0.7175\n",
      "Epoch: 2 , Step: 32622 , loss: 0.8394 , Time: 12077.7\n",
      "Model Saved\n",
      "Val: F1 Score:69.48 Accuracy:75.81  POS: F1 Score:93.82 Accuracy:97.16 Loss:0.8961 , Time: 2261.5\n",
      "Steps: 32722 Loss:0.8173 , Current Loss: 0.8415\n",
      "Steps: 32822 Loss:0.8279 , Current Loss: 0.9122\n",
      "Steps: 32922 Loss:0.8306 , Current Loss: 0.7439\n",
      "Steps: 33022 Loss:0.8270 , Current Loss: 0.6963\n",
      "Steps: 33122 Loss:0.8268 , Current Loss: 0.8948\n",
      "Steps: 33222 Loss:0.8205 , Current Loss: 0.9141\n",
      "Steps: 33322 Loss:0.8335 , Current Loss: 0.8680\n",
      "Steps: 33422 Loss:0.8262 , Current Loss: 0.9292\n",
      "Steps: 33522 Loss:0.8176 , Current Loss: 0.9071\n",
      "Steps: 33622 Loss:0.8173 , Current Loss: 0.8130\n",
      "Steps: 33722 Loss:0.8221 , Current Loss: 0.9263\n",
      "Steps: 33822 Loss:0.8386 , Current Loss: 0.8053\n",
      "Steps: 33922 Loss:0.8207 , Current Loss: 0.8257\n",
      "Steps: 34022 Loss:0.8258 , Current Loss: 0.9010\n",
      "Steps: 34122 Loss:0.8246 , Current Loss: 0.8145\n",
      "Steps: 34222 Loss:0.8402 , Current Loss: 0.7823\n",
      "Steps: 34322 Loss:0.8270 , Current Loss: 0.8396\n",
      "Epoch: 3 , Step: 34393 , loss: 0.8265 , Time: 11665.4\n",
      "Model Saved\n",
      "Steps: 34493 Loss:0.8001 , Current Loss: 0.7734\n",
      "Steps: 34593 Loss:0.8070 , Current Loss: 0.8744\n",
      "Steps: 34693 Loss:0.8058 , Current Loss: 0.8391\n",
      "Steps: 34793 Loss:0.8089 , Current Loss: 0.8658\n",
      "Steps: 34893 Loss:0.8225 , Current Loss: 0.8787\n",
      "Steps: 34993 Loss:0.8187 , Current Loss: 0.6688\n",
      "Steps: 35093 Loss:0.8179 , Current Loss: 0.8923\n",
      "Steps: 35193 Loss:0.8117 , Current Loss: 0.9043\n",
      "Steps: 35293 Loss:0.8161 , Current Loss: 0.8281\n",
      "Steps: 35393 Loss:0.8186 , Current Loss: 0.8127\n",
      "Steps: 35493 Loss:0.8209 , Current Loss: 0.7587\n",
      "Steps: 35593 Loss:0.8259 , Current Loss: 0.7520\n",
      "Steps: 35693 Loss:0.8146 , Current Loss: 0.7946\n",
      "Steps: 35793 Loss:0.8267 , Current Loss: 0.8200\n",
      "Steps: 35893 Loss:0.8170 , Current Loss: 0.8674\n",
      "Steps: 35993 Loss:0.8187 , Current Loss: 0.7978\n",
      "Steps: 36093 Loss:0.8225 , Current Loss: 0.8209\n",
      "Epoch: 4 , Step: 36164 , loss: 0.8161 , Time: 11666.0\n",
      "Model Saved\n",
      "Val: F1 Score:69.87 Accuracy:76.15  POS: F1 Score:94.03 Accuracy:97.20 Loss:0.8928 , Time: 2260.0\n",
      "Steps: 36264 Loss:0.7996 , Current Loss: 0.7836\n",
      "Steps: 36364 Loss:0.8107 , Current Loss: 0.8295\n",
      "Steps: 36464 Loss:0.7911 , Current Loss: 0.8436\n",
      "Steps: 36564 Loss:0.8105 , Current Loss: 0.7572\n",
      "Steps: 36664 Loss:0.7999 , Current Loss: 0.8262\n",
      "Steps: 36764 Loss:0.8168 , Current Loss: 0.7534\n",
      "Steps: 36864 Loss:0.7925 , Current Loss: 0.8728\n",
      "Steps: 36964 Loss:0.8044 , Current Loss: 0.8842\n",
      "Steps: 37064 Loss:0.8052 , Current Loss: 0.8794\n",
      "Steps: 37164 Loss:0.8003 , Current Loss: 0.8103\n",
      "Steps: 37264 Loss:0.8184 , Current Loss: 0.9300\n",
      "Steps: 37364 Loss:0.8089 , Current Loss: 0.9162\n",
      "Steps: 37464 Loss:0.8171 , Current Loss: 0.8365\n",
      "Steps: 37564 Loss:0.8162 , Current Loss: 0.9292\n",
      "Steps: 37664 Loss:0.8109 , Current Loss: 0.8555\n",
      "Steps: 37764 Loss:0.8161 , Current Loss: 0.8062\n",
      "Steps: 37864 Loss:0.8049 , Current Loss: 0.8634\n",
      "Epoch: 5 , Step: 37935 , loss: 0.8071 , Time: 11803.9\n",
      "Model Saved\n",
      "Steps: 38035 Loss:0.7794 , Current Loss: 0.9035\n",
      "Steps: 38135 Loss:0.7903 , Current Loss: 0.7744\n",
      "Steps: 38235 Loss:0.7883 , Current Loss: 0.6843\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 38535 Loss:0.7951 , Current Loss: 0.7292\n",
      "Steps: 38635 Loss:0.7887 , Current Loss: 0.8071\n",
      "Steps: 38735 Loss:0.7925 , Current Loss: 0.7715\n",
      "Steps: 38835 Loss:0.7859 , Current Loss: 0.8336\n",
      "Steps: 38935 Loss:0.7952 , Current Loss: 0.7808\n",
      "Steps: 39035 Loss:0.7931 , Current Loss: 0.8661\n",
      "Steps: 39135 Loss:0.7966 , Current Loss: 0.8356\n",
      "Steps: 39235 Loss:0.7950 , Current Loss: 0.7296\n",
      "Steps: 39335 Loss:0.7950 , Current Loss: 0.8771\n",
      "Steps: 39435 Loss:0.7980 , Current Loss: 0.8438\n",
      "Steps: 39535 Loss:0.7938 , Current Loss: 0.7535\n",
      "Steps: 39635 Loss:0.7976 , Current Loss: 0.7921\n",
      "Steps: 39735 Loss:0.7978 , Current Loss: 0.8017\n",
      "Steps: 39835 Loss:0.7941 , Current Loss: 0.8090\n",
      "Steps: 39935 Loss:0.7975 , Current Loss: 0.7531\n",
      "Steps: 40035 Loss:0.7969 , Current Loss: 0.7633\n",
      "Steps: 40135 Loss:0.7976 , Current Loss: 0.7486\n",
      "Epoch: 1 , Step: 40206 , loss: 0.7948 , Time: 12631.6\n",
      "Model Saved\n",
      "Steps: 40306 Loss:0.7760 , Current Loss: 0.7280\n",
      "Steps: 40406 Loss:0.7711 , Current Loss: 0.7700\n",
      "Steps: 40506 Loss:0.7902 , Current Loss: 0.8324\n",
      "Steps: 40606 Loss:0.7827 , Current Loss: 0.8404\n",
      "Steps: 40706 Loss:0.7883 , Current Loss: 0.7134\n",
      "Steps: 40806 Loss:0.7817 , Current Loss: 0.8839\n",
      "Steps: 40906 Loss:0.7981 , Current Loss: 0.7610\n",
      "Steps: 41006 Loss:0.7731 , Current Loss: 0.7954\n",
      "Steps: 41106 Loss:0.7921 , Current Loss: 0.7748\n",
      "Steps: 41206 Loss:0.7878 , Current Loss: 0.8226\n",
      "Steps: 41306 Loss:0.7747 , Current Loss: 0.8407\n",
      "Steps: 41406 Loss:0.7850 , Current Loss: 0.8863\n",
      "Steps: 41506 Loss:0.7877 , Current Loss: 0.7940\n",
      "Steps: 41606 Loss:0.7884 , Current Loss: 0.8590\n",
      "Steps: 41706 Loss:0.8041 , Current Loss: 0.8364\n",
      "Steps: 41806 Loss:0.7818 , Current Loss: 0.7724\n",
      "Steps: 41906 Loss:0.7916 , Current Loss: 0.8357\n",
      "Epoch: 2 , Step: 41977 , loss: 0.7859 , Time: 23500.5\n",
      "Model Saved\n",
      "Val: F1 Score:70.08 Accuracy:76.44  POS: F1 Score:94.31 Accuracy:97.28 Loss:0.8781 , Time: 2691.4\n",
      "Steps: 42077 Loss:0.7663 , Current Loss: 0.7919\n",
      "Steps: 42177 Loss:0.7653 , Current Loss: 0.8146\n",
      "Steps: 42277 Loss:0.7808 , Current Loss: 0.8701\n",
      "Steps: 42377 Loss:0.7659 , Current Loss: 0.8130\n",
      "Steps: 42477 Loss:0.7748 , Current Loss: 0.8113\n",
      "Steps: 42577 Loss:0.7635 , Current Loss: 0.7487\n",
      "Steps: 42677 Loss:0.7702 , Current Loss: 0.8442\n",
      "Steps: 42777 Loss:0.7800 , Current Loss: 0.7103\n",
      "Steps: 42877 Loss:0.7818 , Current Loss: 0.8332\n",
      "Steps: 42977 Loss:0.7780 , Current Loss: 0.6998\n",
      "Steps: 43077 Loss:0.7789 , Current Loss: 0.8042\n",
      "Steps: 43177 Loss:0.7697 , Current Loss: 0.7036\n",
      "Steps: 43277 Loss:0.7923 , Current Loss: 0.8349\n",
      "Steps: 43377 Loss:0.7698 , Current Loss: 0.6862\n",
      "Steps: 43477 Loss:0.7868 , Current Loss: 0.7763\n",
      "Steps: 43577 Loss:0.7871 , Current Loss: 0.7402\n",
      "Steps: 43677 Loss:0.7861 , Current Loss: 0.8204\n",
      "Epoch: 3 , Step: 43748 , loss: 0.7759 , Time: 20690.1\n",
      "Model Saved\n",
      "Steps: 43848 Loss:0.7548 , Current Loss: 0.6757\n",
      "Steps: 43948 Loss:0.7559 , Current Loss: 0.8219\n",
      "Steps: 44048 Loss:0.7512 , Current Loss: 0.7016\n",
      "Steps: 44148 Loss:0.7730 , Current Loss: 0.7957\n",
      "Steps: 44248 Loss:0.7678 , Current Loss: 0.7524\n",
      "Steps: 44348 Loss:0.7667 , Current Loss: 0.7594\n",
      "Steps: 44448 Loss:0.7683 , Current Loss: 0.7125\n",
      "Steps: 44548 Loss:0.7681 , Current Loss: 0.8353\n",
      "Steps: 44648 Loss:0.7699 , Current Loss: 0.8095\n",
      "Steps: 44748 Loss:0.7672 , Current Loss: 0.7528\n",
      "Steps: 44848 Loss:0.7666 , Current Loss: 0.8954\n",
      "Steps: 44948 Loss:0.7751 , Current Loss: 0.8309\n",
      "Steps: 45048 Loss:0.7619 , Current Loss: 0.7145\n",
      "Steps: 45148 Loss:0.7810 , Current Loss: 0.7292\n",
      "Steps: 45248 Loss:0.7841 , Current Loss: 0.8041\n",
      "Steps: 45348 Loss:0.7747 , Current Loss: 0.8725\n",
      "Steps: 45448 Loss:0.7771 , Current Loss: 0.7387\n",
      "Epoch: 4 , Step: 45519 , loss: 0.7687 , Time: 25152.9\n",
      "Model Saved\n",
      "Val: F1 Score:70.78 Accuracy:76.76  POS: F1 Score:94.38 Accuracy:97.33 Loss:0.8678 , Time: 2840.5\n",
      "Steps: 45619 Loss:0.7508 , Current Loss: 0.8307\n",
      "Steps: 45719 Loss:0.7527 , Current Loss: 0.7303\n",
      "Steps: 45819 Loss:0.7592 , Current Loss: 0.7373\n",
      "Steps: 45919 Loss:0.7677 , Current Loss: 0.7043\n",
      "Steps: 46019 Loss:0.7588 , Current Loss: 0.7819\n",
      "Steps: 46119 Loss:0.7601 , Current Loss: 0.8760\n",
      "Steps: 46219 Loss:0.7567 , Current Loss: 0.6611\n",
      "Steps: 46319 Loss:0.7701 , Current Loss: 0.8594\n",
      "Steps: 46419 Loss:0.7571 , Current Loss: 0.7647\n",
      "Steps: 46519 Loss:0.7573 , Current Loss: 0.8578\n",
      "Steps: 46619 Loss:0.7559 , Current Loss: 0.7955\n",
      "Steps: 46719 Loss:0.7597 , Current Loss: 0.7321\n",
      "Steps: 46819 Loss:0.7711 , Current Loss: 0.8868\n",
      "Steps: 46919 Loss:0.7650 , Current Loss: 0.9891\n",
      "Steps: 47019 Loss:0.7635 , Current Loss: 0.7560\n",
      "Steps: 47119 Loss:0.7652 , Current Loss: 0.7632\n",
      "Steps: 47219 Loss:0.7745 , Current Loss: 0.8731\n",
      "Epoch: 5 , Step: 47290 , loss: 0.7617 , Time: 12568.5\n",
      "Model Saved\n",
      "Steps: 47390 Loss:0.7384 , Current Loss: 0.6738\n",
      "Steps: 47490 Loss:0.7369 , Current Loss: 0.7185\n",
      "Steps: 47590 Loss:0.7573 , Current Loss: 0.7331\n",
      "Steps: 47690 Loss:0.7353 , Current Loss: 0.8571\n",
      "Steps: 47790 Loss:0.7496 , Current Loss: 0.6775\n",
      "Steps: 47890 Loss:0.7529 , Current Loss: 0.7692\n",
      "Steps: 47990 Loss:0.7598 , Current Loss: 0.8561\n",
      "Steps: 48090 Loss:0.7613 , Current Loss: 0.7646\n",
      "Steps: 48190 Loss:0.7596 , Current Loss: 0.7345\n",
      "Steps: 48290 Loss:0.7561 , Current Loss: 0.8373\n",
      "Steps: 48390 Loss:0.7505 , Current Loss: 0.6993\n",
      "Steps: 48490 Loss:0.7642 , Current Loss: 0.7953\n",
      "Steps: 48590 Loss:0.7642 , Current Loss: 0.8727\n",
      "Steps: 48690 Loss:0.7553 , Current Loss: 0.8257\n",
      "Steps: 48790 Loss:0.7597 , Current Loss: 0.6279\n",
      "Steps: 48890 Loss:0.7533 , Current Loss: 0.7051\n",
      "Steps: 48990 Loss:0.7499 , Current Loss: 0.7970\n",
      "Epoch: 6 , Step: 49061 , loss: 0.7539 , Time: 11867.2\n",
      "Model Saved\n",
      "Val: F1 Score:71.00 Accuracy:76.93  POS: F1 Score:94.41 Accuracy:97.35 Loss:0.8558 , Time: 2267.2\n",
      "Steps: 49161 Loss:0.7437 , Current Loss: 0.7221\n",
      "Steps: 49261 Loss:0.7426 , Current Loss: 0.7161\n",
      "Steps: 49361 Loss:0.7390 , Current Loss: 0.5902\n",
      "Steps: 49461 Loss:0.7389 , Current Loss: 0.7407\n",
      "Steps: 49561 Loss:0.7442 , Current Loss: 0.7237\n",
      "Steps: 49661 Loss:0.7439 , Current Loss: 0.6234\n",
      "Steps: 49761 Loss:0.7439 , Current Loss: 0.7770\n",
      "Steps: 49861 Loss:0.7361 , Current Loss: 0.7343\n",
      "Steps: 49961 Loss:0.7558 , Current Loss: 0.7790\n",
      "Steps: 50061 Loss:0.7524 , Current Loss: 0.7928\n",
      "Steps: 50161 Loss:0.7482 , Current Loss: 0.8397\n",
      "Steps: 50261 Loss:0.7414 , Current Loss: 0.7084\n",
      "Steps: 50361 Loss:0.7588 , Current Loss: 0.7625\n",
      "Steps: 50461 Loss:0.7527 , Current Loss: 0.7931\n",
      "Steps: 50561 Loss:0.7502 , Current Loss: 0.7231\n",
      "Steps: 50661 Loss:0.7527 , Current Loss: 0.7390\n",
      "Steps: 50761 Loss:0.7589 , Current Loss: 0.7893\n",
      "Epoch: 7 , Step: 50832 , loss: 0.7480 , Time: 11663.5\n",
      "Model Saved\n",
      "Steps: 50932 Loss:0.7355 , Current Loss: 0.6930\n",
      "Steps: 51032 Loss:0.7304 , Current Loss: 0.7683\n",
      "Steps: 51132 Loss:0.7406 , Current Loss: 0.8554\n",
      "Steps: 51232 Loss:0.7588 , Current Loss: 0.7601\n",
      "Steps: 51332 Loss:0.7306 , Current Loss: 0.7024\n",
      "Steps: 51432 Loss:0.7413 , Current Loss: 0.6752\n",
      "Steps: 51532 Loss:0.7415 , Current Loss: 0.7714\n",
      "Steps: 51632 Loss:0.7365 , Current Loss: 0.7434\n",
      "Steps: 51732 Loss:0.7432 , Current Loss: 0.6774\n",
      "Steps: 51832 Loss:0.7395 , Current Loss: 0.8619\n",
      "Steps: 51932 Loss:0.7437 , Current Loss: 0.6459\n",
      "Steps: 52032 Loss:0.7384 , Current Loss: 0.6808\n",
      "Steps: 52132 Loss:0.7386 , Current Loss: 0.7578\n",
      "Steps: 52232 Loss:0.7497 , Current Loss: 0.7233\n",
      "Steps: 52332 Loss:0.7413 , Current Loss: 0.7359\n",
      "Steps: 52432 Loss:0.7465 , Current Loss: 0.8802\n",
      "Steps: 52532 Loss:0.7372 , Current Loss: 0.8083\n",
      "Epoch: 8 , Step: 52603 , loss: 0.7412 , Time: 11649.1\n",
      "Model Saved\n",
      "Val: F1 Score:71.06 Accuracy:77.20  POS: F1 Score:94.47 Accuracy:97.39 Loss:0.8529 , Time: 2259.5\n",
      "Steps: 52703 Loss:0.7270 , Current Loss: 0.7271\n",
      "Steps: 52803 Loss:0.7323 , Current Loss: 0.7625\n",
      "Steps: 52903 Loss:0.7333 , Current Loss: 0.7020\n",
      "Steps: 53003 Loss:0.7300 , Current Loss: 0.6456\n",
      "Steps: 53103 Loss:0.7268 , Current Loss: 0.6222\n",
      "Steps: 53203 Loss:0.7248 , Current Loss: 0.6911\n",
      "Steps: 53303 Loss:0.7296 , Current Loss: 0.6507\n",
      "Steps: 53403 Loss:0.7299 , Current Loss: 0.7313\n",
      "Steps: 53503 Loss:0.7430 , Current Loss: 0.6492\n",
      "Steps: 53603 Loss:0.7428 , Current Loss: 0.6496\n",
      "Steps: 53703 Loss:0.7377 , Current Loss: 0.8653\n",
      "Steps: 53803 Loss:0.7324 , Current Loss: 0.7173\n",
      "Steps: 53903 Loss:0.7392 , Current Loss: 0.6847\n",
      "Steps: 54003 Loss:0.7416 , Current Loss: 0.7360\n",
      "Steps: 54103 Loss:0.7488 , Current Loss: 0.6927\n",
      "Steps: 54203 Loss:0.7482 , Current Loss: 0.7708\n",
      "Steps: 54303 Loss:0.7289 , Current Loss: 0.7333\n",
      "Epoch: 9 , Step: 54374 , loss: 0.7356 , Time: 11870.8\n",
      "Model Saved\n",
      "Steps: 54474 Loss:0.7176 , Current Loss: 0.8152\n",
      "Steps: 54574 Loss:0.7330 , Current Loss: 0.6808\n",
      "Steps: 54674 Loss:0.7182 , Current Loss: 0.8317\n",
      "Steps: 54774 Loss:0.7238 , Current Loss: 0.7688\n",
      "Steps: 54874 Loss:0.7367 , Current Loss: 0.7859\n",
      "Steps: 54974 Loss:0.7350 , Current Loss: 0.7578\n",
      "Steps: 55074 Loss:0.7257 , Current Loss: 0.7791\n",
      "Steps: 55174 Loss:0.7289 , Current Loss: 0.7795\n",
      "Steps: 55274 Loss:0.7393 , Current Loss: 0.7647\n",
      "Steps: 55374 Loss:0.7270 , Current Loss: 0.6881\n",
      "Steps: 55474 Loss:0.7359 , Current Loss: 0.7492\n",
      "Steps: 55574 Loss:0.7307 , Current Loss: 0.6852\n",
      "Steps: 55674 Loss:0.7321 , Current Loss: 0.7210\n",
      "Steps: 55774 Loss:0.7290 , Current Loss: 0.7873\n",
      "Steps: 55874 Loss:0.7375 , Current Loss: 0.7574\n",
      "Steps: 55974 Loss:0.7302 , Current Loss: 0.7369\n",
      "Steps: 56074 Loss:0.7255 , Current Loss: 0.8171\n",
      "Epoch: 10 , Step: 56145 , loss: 0.7295 , Time: 11999.6\n",
      "Model Saved\n",
      "Val: F1 Score:71.29 Accuracy:77.22  POS: F1 Score:94.42 Accuracy:97.39 Loss:0.8456 , Time: 2277.2\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 56545 Loss:0.7085 , Current Loss: 0.6290\n",
      "Steps: 56645 Loss:0.7073 , Current Loss: 0.6460\n",
      "Steps: 56745 Loss:0.7227 , Current Loss: 0.6688\n",
      "Steps: 56845 Loss:0.7221 , Current Loss: 0.7113\n",
      "Steps: 56945 Loss:0.7254 , Current Loss: 0.6511\n",
      "Steps: 57045 Loss:0.7270 , Current Loss: 0.7832\n",
      "Steps: 57145 Loss:0.7149 , Current Loss: 0.6987\n",
      "Steps: 57245 Loss:0.7233 , Current Loss: 0.6848\n",
      "Steps: 57345 Loss:0.7170 , Current Loss: 0.7284\n",
      "Steps: 57445 Loss:0.7317 , Current Loss: 0.7737\n",
      "Steps: 57545 Loss:0.7304 , Current Loss: 0.7958\n",
      "Steps: 57645 Loss:0.7316 , Current Loss: 0.7285\n",
      "Steps: 57745 Loss:0.7226 , Current Loss: 0.6010\n",
      "Steps: 57845 Loss:0.7335 , Current Loss: 0.6583\n",
      "Steps: 57945 Loss:0.7232 , Current Loss: 0.8144\n",
      "Steps: 58045 Loss:0.7307 , Current Loss: 0.7244\n",
      "Steps: 58145 Loss:0.7222 , Current Loss: 0.7913\n",
      "Epoch: 1 , Step: 58216 , loss: 0.7233 , Time: 11850.3\n",
      "Model Saved\n",
      "Steps: 58316 Loss:0.7140 , Current Loss: 0.7330\n",
      "Steps: 58416 Loss:0.7123 , Current Loss: 0.7229\n",
      "Steps: 58516 Loss:0.7073 , Current Loss: 0.7370\n",
      "Steps: 58616 Loss:0.7088 , Current Loss: 0.6739\n",
      "Steps: 58716 Loss:0.7198 , Current Loss: 0.6764\n",
      "Steps: 58816 Loss:0.7151 , Current Loss: 0.6675\n",
      "Steps: 58916 Loss:0.7164 , Current Loss: 0.7431\n",
      "Steps: 59016 Loss:0.7235 , Current Loss: 0.6953\n",
      "Steps: 59116 Loss:0.7237 , Current Loss: 0.7729\n",
      "Steps: 59216 Loss:0.7282 , Current Loss: 0.6765\n",
      "Steps: 59316 Loss:0.7064 , Current Loss: 0.6471\n",
      "Steps: 59416 Loss:0.7249 , Current Loss: 0.7745\n",
      "Steps: 59516 Loss:0.7279 , Current Loss: 0.7280\n",
      "Steps: 59616 Loss:0.7268 , Current Loss: 0.6281\n",
      "Steps: 59716 Loss:0.7188 , Current Loss: 0.8666\n",
      "Steps: 59816 Loss:0.7173 , Current Loss: 0.6829\n",
      "Steps: 59916 Loss:0.7122 , Current Loss: 0.6341\n",
      "Epoch: 2 , Step: 59987 , loss: 0.7178 , Time: 11865.4\n",
      "Model Saved\n",
      "Val: F1 Score:71.65 Accuracy:77.43  POS: F1 Score:94.75 Accuracy:97.47 Loss:0.8448 , Time: 2281.0\n",
      "Steps: 60087 Loss:0.7088 , Current Loss: 0.6697\n",
      "Steps: 60187 Loss:0.7094 , Current Loss: 0.6717\n",
      "Steps: 60287 Loss:0.7168 , Current Loss: 0.6523\n",
      "Steps: 60387 Loss:0.7076 , Current Loss: 0.6057\n",
      "Steps: 60487 Loss:0.7076 , Current Loss: 0.7215\n",
      "Steps: 60587 Loss:0.7048 , Current Loss: 0.6996\n",
      "Steps: 60687 Loss:0.7150 , Current Loss: 0.7250\n",
      "Steps: 60787 Loss:0.7201 , Current Loss: 0.7771\n",
      "Steps: 60887 Loss:0.7134 , Current Loss: 0.7286\n",
      "Steps: 60987 Loss:0.7144 , Current Loss: 0.7202\n",
      "Steps: 61087 Loss:0.7223 , Current Loss: 0.7071\n",
      "Steps: 61187 Loss:0.7105 , Current Loss: 0.7703\n",
      "Steps: 61287 Loss:0.7159 , Current Loss: 0.6605\n",
      "Steps: 61387 Loss:0.7229 , Current Loss: 0.7184\n",
      "Steps: 61487 Loss:0.7237 , Current Loss: 0.7990\n",
      "Steps: 61587 Loss:0.7175 , Current Loss: 0.7409\n",
      "Steps: 61687 Loss:0.7257 , Current Loss: 0.6348\n",
      "Epoch: 3 , Step: 61758 , loss: 0.7149 , Time: 11763.7\n",
      "Model Saved\n",
      "Steps: 61858 Loss:0.6992 , Current Loss: 0.6003\n",
      "Steps: 61958 Loss:0.7064 , Current Loss: 0.7152\n",
      "Steps: 62058 Loss:0.7106 , Current Loss: 0.7584\n",
      "Steps: 62158 Loss:0.7055 , Current Loss: 0.7749\n",
      "Steps: 62258 Loss:0.7059 , Current Loss: 0.8045\n",
      "Steps: 62358 Loss:0.7009 , Current Loss: 0.7375\n",
      "Steps: 62458 Loss:0.7098 , Current Loss: 0.7426\n",
      "Steps: 62558 Loss:0.6997 , Current Loss: 0.6997\n",
      "Steps: 62658 Loss:0.7159 , Current Loss: 0.7834\n",
      "Steps: 62758 Loss:0.7127 , Current Loss: 0.6915\n",
      "Steps: 62858 Loss:0.7081 , Current Loss: 0.8189\n",
      "Steps: 62958 Loss:0.7088 , Current Loss: 0.6247\n",
      "Steps: 63058 Loss:0.7115 , Current Loss: 0.5765\n",
      "Steps: 63158 Loss:0.7078 , Current Loss: 0.6948\n",
      "Steps: 63258 Loss:0.7097 , Current Loss: 0.7634\n",
      "Steps: 63358 Loss:0.7092 , Current Loss: 0.6185\n",
      "Steps: 63458 Loss:0.7161 , Current Loss: 0.7122\n",
      "Epoch: 4 , Step: 63529 , loss: 0.7082 , Time: 11717.3\n",
      "Model Saved\n",
      "Val: F1 Score:72.06 Accuracy:77.70  POS: F1 Score:94.78 Accuracy:97.48 Loss:0.8337 , Time: 2266.2\n",
      "Steps: 63629 Loss:0.6985 , Current Loss: 0.7024\n",
      "Steps: 63729 Loss:0.6991 , Current Loss: 0.7405\n",
      "Steps: 63829 Loss:0.7124 , Current Loss: 0.7061\n",
      "Steps: 63929 Loss:0.7038 , Current Loss: 0.6888\n",
      "Steps: 64029 Loss:0.7059 , Current Loss: 0.7303\n",
      "Steps: 64129 Loss:0.7076 , Current Loss: 0.6357\n",
      "Steps: 64229 Loss:0.7070 , Current Loss: 0.6172\n",
      "Steps: 64329 Loss:0.7074 , Current Loss: 0.6469\n",
      "Steps: 64429 Loss:0.6930 , Current Loss: 0.7027\n",
      "Steps: 64529 Loss:0.6984 , Current Loss: 0.6250\n",
      "Steps: 64629 Loss:0.7106 , Current Loss: 0.6984\n",
      "Steps: 64729 Loss:0.7157 , Current Loss: 0.6090\n",
      "Steps: 64829 Loss:0.7105 , Current Loss: 0.5176\n",
      "Steps: 64929 Loss:0.7090 , Current Loss: 0.6985\n",
      "Steps: 65029 Loss:0.7090 , Current Loss: 0.7628\n",
      "Steps: 65129 Loss:0.7002 , Current Loss: 0.7831\n",
      "Steps: 65229 Loss:0.7046 , Current Loss: 0.7110\n",
      "Epoch: 5 , Step: 65300 , loss: 0.7054 , Time: 11719.4\n",
      "Model Saved\n",
      "Steps: 65400 Loss:0.6927 , Current Loss: 0.7277\n",
      "Steps: 65500 Loss:0.6830 , Current Loss: 0.6563\n",
      "Steps: 65600 Loss:0.7033 , Current Loss: 0.6827\n",
      "Steps: 65700 Loss:0.6902 , Current Loss: 0.7103\n",
      "Steps: 65800 Loss:0.6954 , Current Loss: 0.6474\n",
      "Steps: 65900 Loss:0.7000 , Current Loss: 0.6813\n",
      "Steps: 66000 Loss:0.6954 , Current Loss: 0.6635\n",
      "Steps: 66100 Loss:0.6969 , Current Loss: 0.7562\n",
      "Steps: 66200 Loss:0.7061 , Current Loss: 0.7368\n",
      "Steps: 66300 Loss:0.7094 , Current Loss: 0.6702\n",
      "Steps: 66400 Loss:0.6976 , Current Loss: 0.6229\n",
      "Steps: 66500 Loss:0.7066 , Current Loss: 0.7622\n",
      "Steps: 66600 Loss:0.7045 , Current Loss: 0.7817\n",
      "Steps: 66700 Loss:0.7036 , Current Loss: 0.7001\n",
      "Steps: 66800 Loss:0.7020 , Current Loss: 0.7192\n",
      "Steps: 66900 Loss:0.6943 , Current Loss: 0.7208\n",
      "Steps: 67000 Loss:0.6984 , Current Loss: 0.7556\n",
      "Epoch: 6 , Step: 67071 , loss: 0.6995 , Time: 11755.1\n",
      "Model Saved\n",
      "Val: F1 Score:71.98 Accuracy:77.69  POS: F1 Score:94.81 Accuracy:97.51 Loss:0.8361 , Time: 2276.1\n",
      "Steps: 67171 Loss:0.6923 , Current Loss: 0.6802\n",
      "Steps: 67271 Loss:0.6867 , Current Loss: 0.6918\n",
      "Steps: 67371 Loss:0.6786 , Current Loss: 0.6736\n",
      "Steps: 67471 Loss:0.6968 , Current Loss: 0.6739\n",
      "Steps: 67571 Loss:0.6969 , Current Loss: 0.6129\n",
      "Steps: 67671 Loss:0.6885 , Current Loss: 0.8122\n",
      "Steps: 67771 Loss:0.7024 , Current Loss: 0.6205\n",
      "Steps: 67871 Loss:0.6973 , Current Loss: 0.6659\n",
      "Steps: 67971 Loss:0.6913 , Current Loss: 0.8022\n",
      "Steps: 68071 Loss:0.6994 , Current Loss: 0.6944\n",
      "Steps: 68171 Loss:0.7073 , Current Loss: 0.7483\n",
      "Steps: 68271 Loss:0.6976 , Current Loss: 0.6610\n",
      "Steps: 68371 Loss:0.6943 , Current Loss: 0.7276\n",
      "Steps: 68471 Loss:0.7013 , Current Loss: 0.7402\n",
      "Steps: 68571 Loss:0.6984 , Current Loss: 0.6861\n",
      "Steps: 68671 Loss:0.7094 , Current Loss: 0.7681\n",
      "Steps: 68771 Loss:0.6992 , Current Loss: 0.8224\n",
      "Epoch: 7 , Step: 68842 , loss: 0.6968 , Time: 14002.0\n",
      "Model Saved\n",
      "Steps: 68942 Loss:0.6747 , Current Loss: 0.6638\n",
      "Steps: 69042 Loss:0.6841 , Current Loss: 0.6783\n",
      "Steps: 69142 Loss:0.6827 , Current Loss: 0.7015\n",
      "Steps: 69242 Loss:0.6879 , Current Loss: 0.6458\n",
      "Steps: 69342 Loss:0.6874 , Current Loss: 0.6590\n",
      "Steps: 69442 Loss:0.6915 , Current Loss: 0.5829\n",
      "Steps: 69542 Loss:0.6974 , Current Loss: 0.7448\n",
      "Steps: 69642 Loss:0.6964 , Current Loss: 0.6434\n",
      "Steps: 69742 Loss:0.6944 , Current Loss: 0.6843\n",
      "Steps: 69842 Loss:0.6863 , Current Loss: 0.6924\n",
      "Steps: 69942 Loss:0.6955 , Current Loss: 0.6861\n",
      "Steps: 70042 Loss:0.6928 , Current Loss: 0.7307\n",
      "Steps: 70142 Loss:0.6938 , Current Loss: 0.6889\n",
      "Steps: 70242 Loss:0.6908 , Current Loss: 0.7834\n",
      "Steps: 70342 Loss:0.7077 , Current Loss: 0.6833\n",
      "Steps: 70442 Loss:0.7034 , Current Loss: 0.7346\n",
      "Steps: 70542 Loss:0.6907 , Current Loss: 0.5835\n",
      "Epoch: 8 , Step: 70613 , loss: 0.6926 , Time: 20827.6\n",
      "Model Saved\n",
      "Val: F1 Score:72.03 Accuracy:77.77  POS: F1 Score:94.85 Accuracy:97.51 Loss:0.8227 , Time: 3152.2\n",
      "Steps: 70713 Loss:0.6737 , Current Loss: 0.7164\n",
      "Steps: 70813 Loss:0.6834 , Current Loss: 0.6922\n",
      "Steps: 70913 Loss:0.6866 , Current Loss: 0.7293\n",
      "Steps: 71013 Loss:0.6919 , Current Loss: 0.6861\n",
      "Steps: 71113 Loss:0.6763 , Current Loss: 0.7444\n",
      "Steps: 71213 Loss:0.6820 , Current Loss: 0.5853\n",
      "Steps: 71313 Loss:0.6863 , Current Loss: 0.7897\n",
      "Steps: 71413 Loss:0.6846 , Current Loss: 0.5857\n",
      "Steps: 71513 Loss:0.6906 , Current Loss: 0.7314\n",
      "Steps: 71613 Loss:0.6774 , Current Loss: 0.6346\n",
      "Steps: 71713 Loss:0.6805 , Current Loss: 0.5998\n",
      "Steps: 71813 Loss:0.6938 , Current Loss: 0.7864\n",
      "Steps: 71913 Loss:0.6947 , Current Loss: 0.6692\n",
      "Steps: 72013 Loss:0.6895 , Current Loss: 0.6868\n",
      "Steps: 72113 Loss:0.6884 , Current Loss: 0.7737\n",
      "Steps: 72213 Loss:0.6989 , Current Loss: 0.6331\n",
      "Steps: 72313 Loss:0.7032 , Current Loss: 0.7758\n",
      "Epoch: 9 , Step: 72384 , loss: 0.6873 , Time: 13535.2\n",
      "Model Saved\n",
      "Steps: 72484 Loss:0.6700 , Current Loss: 0.6157\n",
      "Steps: 72584 Loss:0.6802 , Current Loss: 0.7602\n",
      "Steps: 72684 Loss:0.6891 , Current Loss: 0.5675\n",
      "Steps: 72784 Loss:0.6744 , Current Loss: 0.6223\n",
      "Steps: 72884 Loss:0.6926 , Current Loss: 0.7445\n",
      "Steps: 72984 Loss:0.6794 , Current Loss: 0.5977\n",
      "Steps: 73084 Loss:0.6821 , Current Loss: 0.6480\n",
      "Steps: 73184 Loss:0.6748 , Current Loss: 0.6674\n",
      "Steps: 73284 Loss:0.6874 , Current Loss: 0.6323\n",
      "Steps: 73384 Loss:0.6836 , Current Loss: 0.6866\n",
      "Steps: 73484 Loss:0.6829 , Current Loss: 0.7178\n",
      "Steps: 73584 Loss:0.6922 , Current Loss: 0.7921\n",
      "Steps: 73684 Loss:0.6915 , Current Loss: 0.7594\n",
      "Steps: 73784 Loss:0.6824 , Current Loss: 0.7409\n",
      "Steps: 73884 Loss:0.6895 , Current Loss: 0.6244\n",
      "Steps: 73984 Loss:0.6904 , Current Loss: 0.7668\n",
      "Steps: 74084 Loss:0.6867 , Current Loss: 0.7151\n",
      "Epoch: 10 , Step: 74155 , loss: 0.6843 , Time: 13776.1\n",
      "Model Saved\n",
      "Val: F1 Score:72.25 Accuracy:77.85  POS: F1 Score:94.83 Accuracy:97.53 Loss:0.8295 , Time: 2261.8\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### deacy rate changed to 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 74255 Loss:0.6738 , Current Loss: 0.6780\n",
      "Steps: 74355 Loss:0.6674 , Current Loss: 0.6000\n",
      "Steps: 74455 Loss:0.6731 , Current Loss: 0.7061\n",
      "Steps: 74555 Loss:0.6607 , Current Loss: 0.7031\n",
      "Steps: 74655 Loss:0.6778 , Current Loss: 0.7995\n",
      "Steps: 74755 Loss:0.6785 , Current Loss: 0.8578\n",
      "Steps: 74855 Loss:0.6825 , Current Loss: 0.6331\n",
      "Steps: 74955 Loss:0.6709 , Current Loss: 0.7106\n",
      "Steps: 75055 Loss:0.6681 , Current Loss: 0.5998\n",
      "Steps: 75155 Loss:0.6732 , Current Loss: 0.6321\n",
      "Steps: 75255 Loss:0.6782 , Current Loss: 0.5446\n",
      "Steps: 75355 Loss:0.6778 , Current Loss: 0.7222\n",
      "Steps: 75455 Loss:0.6696 , Current Loss: 0.6757\n",
      "Steps: 75555 Loss:0.6553 , Current Loss: 0.6280\n",
      "Steps: 75655 Loss:0.6693 , Current Loss: 0.7712\n",
      "Steps: 75755 Loss:0.6861 , Current Loss: 0.7783\n",
      "Steps: 75855 Loss:0.6606 , Current Loss: 0.6652\n",
      "Epoch: 1 , Step: 75926 , loss: 0.6720 , Time: 14196.2\n",
      "Model Saved\n",
      "Steps: 76026 Loss:0.6716 , Current Loss: 0.7091\n",
      "Steps: 76126 Loss:0.6695 , Current Loss: 0.6860\n",
      "Steps: 76226 Loss:0.6704 , Current Loss: 0.6716\n",
      "Steps: 76326 Loss:0.6670 , Current Loss: 0.6763\n",
      "Steps: 76426 Loss:0.6792 , Current Loss: 0.5859\n",
      "Steps: 76526 Loss:0.6781 , Current Loss: 0.7030\n",
      "Steps: 76626 Loss:0.6694 , Current Loss: 0.5644\n",
      "Steps: 76726 Loss:0.6786 , Current Loss: 0.6643\n",
      "Steps: 76826 Loss:0.6742 , Current Loss: 0.6837\n",
      "Steps: 76926 Loss:0.6760 , Current Loss: 0.7378\n",
      "Steps: 77026 Loss:0.6693 , Current Loss: 0.7039\n",
      "Steps: 77126 Loss:0.6653 , Current Loss: 0.6925\n",
      "Steps: 77226 Loss:0.6540 , Current Loss: 0.7366\n",
      "Steps: 77326 Loss:0.6769 , Current Loss: 0.6089\n",
      "Steps: 77426 Loss:0.6812 , Current Loss: 0.6564\n",
      "Steps: 77526 Loss:0.6786 , Current Loss: 0.6305\n",
      "Steps: 77626 Loss:0.6646 , Current Loss: 0.6658\n",
      "Epoch: 2 , Step: 77697 , loss: 0.6721 , Time: 15516.2\n",
      "Model Saved\n",
      "Val: F1 Score:72.33 Accuracy:77.93  POS: F1 Score:94.84 Accuracy:97.54 Loss:0.8285 , Time: 2651.7\n",
      "Steps: 77797 Loss:0.6760 , Current Loss: 0.6373\n",
      "Steps: 77897 Loss:0.6798 , Current Loss: 0.5971\n",
      "Steps: 77997 Loss:0.6770 , Current Loss: 0.6174\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "val_period = 2\n",
    "loss_collection = []\n",
    "val_collection = []\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "train_loss, train_pred, train_true, train_pred_pos, train_true_pos = model(x_id_train, y_train, y_pos_train, mask_train, sense_mask_train, train_cond=False)        \n",
    "f1_, accu_, f1_pos_, accu_pos_ = etrain_score(train_true, train_pred, train_true_pos, train_pred_pos)\n",
    "time_taken = time.time() - start_time\n",
    "print(\"train: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "envs",
   "language": "python",
   "name": "cs771"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}