04_CBGRU_MEA_Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1337) \n",
    "import os \n",
    "from keras import backend as K\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Concatenate, Embedding, Input, Dropout, Dense, Activation, Bidirectional, GlobalMaxPooling1D\n",
    "from keras.layers import Convolution1D, LSTM, GRU, CuDNNGRU, CuDNNLSTM, Permute, Lambda, Flatten\n",
    "from keras import optimizers\n",
    "from keras.engine.topology import Layer\n",
    "import tensorflow as tf\n",
    "from keras import regularizers\n",
    "from keras.models import load_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data and Scorer path settings \n",
    "\n",
    "data_npy_path = './data/data_all.npy'\n",
    "scorer_perl_script = \"./corpus/SemEval2010_task8_scorer-v1.2/semeval2010_task8_scorer-v1.2.pl\"\n",
    "train_answer_key = \"./files/train_answer_keys.txt\"\n",
    "val_answer_key = \"./files/val_answer_keys.txt\"\n",
    "test_answer_key = \"./files/test_answer_keys.txt\"\n",
    "\n",
    "proposed_ans = \"./model/proposed_ans.txt\"\n",
    "scorer_output = \"./model/scorer_output.txt\"\n",
    "\n",
    "perl_exe_path = \"C:/Dwimperl/perl/bin/perl.exe\" # for windows \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x.shape (7208, 101)\n",
      "train_y.shape (7208,)\n",
      "val_x.shape (792, 101)\n",
      "val_y.shape (792,)\n",
      "test_x.shape (2717, 101)\n",
      "test_y.shape (2717,)\n",
      "embedding.shape (25656, 300)\n",
      "len(label_to_int) 19\n",
      "len(int_to_label) 19\n",
      "max_sentence_len 101\n",
      "n_out 19\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Data Loading\n",
    "\n",
    "train_set, val_set, test_set, embedding, label_to_int, int_to_label = np.load(data_npy_path)\n",
    "\n",
    "train_x, train_y = train_set\n",
    "val_x, val_y = val_set\n",
    "test_x, test_y = test_set\n",
    "\n",
    "train_x_copy = train_x\n",
    "train_y_copy = train_y\n",
    "\n",
    "max_sentence_len = train_x.shape[1]\n",
    "n_out = len(label_to_int)\n",
    "\n",
    "print(\"train_x.shape\", train_x.shape)\n",
    "print(\"train_y.shape\", train_y.shape)\n",
    "print(\"val_x.shape\", val_x.shape)\n",
    "print(\"val_y.shape\", val_y.shape)\n",
    "print(\"test_x.shape\", test_x.shape)\n",
    "print(\"test_y.shape\", test_y.shape)\n",
    "print(\"embedding.shape\", embedding.shape)\n",
    "print(\"len(label_to_int)\", len(label_to_int))\n",
    "print(\"len(int_to_label)\", len(int_to_label))\n",
    "print(\"max_sentence_len\", max_sentence_len)\n",
    "print(\"n_out\", n_out)\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_mask_entities(x):\n",
    "    ''' 1 for entity words, 0 otherwise '''\n",
    "    \n",
    "    ret = np.zeros_like(x)\n",
    "    #print(\"get_mask shape\", ret.shape)\n",
    "    # e1s 2409\n",
    "    # e1e 2408\n",
    "    # e2s 2411\n",
    "    # e2e 2410\n",
    "    for i in range(x.shape[0]): \n",
    "        e1 = [0, 0]\n",
    "        e2 = [0, 0]\n",
    "        for j in range(x.shape[1]):\n",
    "            if x[i][j] == 2409:\n",
    "                e1[0] = j\n",
    "            elif x[i][j] == 2408: \n",
    "                e1[1] = j\n",
    "            elif x[i][j] == 2411: \n",
    "                e2[0] = j\n",
    "            elif x[i][j] == 2410: \n",
    "                e2[1] = j\n",
    "                break\n",
    "        for j in range(e1[0]+1, e1[1]): \n",
    "            ret[i][j] = 1\n",
    "        for j in range(e2[0]+1, e2[1]): \n",
    "            ret[i][j] = 1\n",
    "        #print(ret[i][e1[0]:e1[1]+1], x[i][e1[0]:e1[1]+1])\n",
    "    \n",
    "    return ret\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_x_mask.shape (7208, 101)\n",
      "train_x_mask_copy.shape (7208, 101)\n",
      "val_x_mask.shape (792, 101)\n",
      "test_x_mask.shape (2717, 101)\n"
     ]
    }
   ],
   "source": [
    "train_x_mask = get_mask_entities(train_x)\n",
    "train_x_mask_copy = train_x_mask\n",
    "val_x_mask = get_mask_entities(val_x)\n",
    "test_x_mask = get_mask_entities(test_x)\n",
    "\n",
    "print(\"train_x_mask.shape\", train_x_mask.shape)\n",
    "print(\"train_x_mask_copy.shape\", train_x_mask_copy.shape)\n",
    "print(\"val_x_mask.shape\", val_x_mask.shape)\n",
    "print(\"test_x_mask.shape\", test_x_mask.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "units 64\n",
      "batch_size 128\n",
      "dropout_emb 0.64\n",
      "dropout_model 0.32\n",
      "dropout_pen 0.32\n",
      "l2_val 1e-05\n",
      "learning_rate 1.0\n",
      "activation_fn tanh\n",
      "nb_epoch 256\n",
      "adadelta <keras.optimizers.Adadelta object at 0x000001C3D499D470>\n",
      "save_model True\n",
      "es_epoch_stop 20\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Parameters \n",
    "\n",
    "units = 64\n",
    "batch_size = 128 #10\n",
    "dropout_emb = 0.64\n",
    "dropout_model = 0.32\n",
    "dropout_pen = 0.32\n",
    "l2_val = 0.00001\n",
    "learning_rate = 1.0\n",
    "activation_fn = 'tanh'\n",
    "nb_epoch = 256\n",
    "adadelta = optimizers.Adadelta(lr=learning_rate, decay=0.0)\n",
    "save_model = True\n",
    "es_epoch_stop = 20\n",
    "\n",
    "print(\"units\", units)\n",
    "print(\"batch_size\", batch_size)\n",
    "print(\"dropout_emb\", dropout_emb)\n",
    "print(\"dropout_model\", dropout_model)\n",
    "print(\"dropout_pen\", dropout_pen)\n",
    "print(\"l2_val\", l2_val)\n",
    "print(\"learning_rate\", learning_rate)\n",
    "print(\"activation_fn\", activation_fn)\n",
    "print(\"nb_epoch\", nb_epoch)\n",
    "print(\"adadelta\", adadelta)\n",
    "print(\"save_model\", save_model)\n",
    "print(\"es_epoch_stop\", es_epoch_stop)\n",
    "print()\n",
    "\n",
    "embeddings_reg = regularizers.l2(l2_val)\n",
    "kernel_reg = regularizers.l2(l2_val)\n",
    "recurrent_reg = regularizers.l2(l2_val)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# class AttnLayer(Layer):\n",
    "#     # Attention-Based Bidirectional Long Short-Term Memory Networks for Relation Classification\n",
    "#     # input: ?, 102, 200\n",
    "    \n",
    "#     def __init__(self, **kwargs):\n",
    "#         super(AttnLayer, self).__init__(**kwargs)\n",
    "    \n",
    "#     def build(self, input_shape):\n",
    "#         # Create a trainable weight variable for this layer.\n",
    "#         self.W = self.add_weight(name='W', shape=(input_shape[-1], 1), initializer='glorot_uniform', trainable=True, regularizer = kernel_reg)        \n",
    "#         super(AttnLayer, self).build(input_shape)  # Be sure to call this somewhere!\n",
    "\n",
    "#     def call(self, x):\n",
    "#         M = K.tanh(x) # ? 101 600\n",
    "#         alpha_bar = tf.matmul(M, self.W)  # ? 101 1\n",
    "#         alpha = K.permute_dimensions(alpha_bar, pattern=[0, 2, 1]) # ? 1 101\n",
    "\n",
    "#         alpha = K.softmax(alpha) # ? 1 101\n",
    "#         r_bar = tf.matmul(alpha, M) # ? 1 600\n",
    "#         r = K.squeeze(r_bar, axis=-2) # ? 600\n",
    "#         r = K.tanh(r)\n",
    "#         return r\n",
    "    \n",
    "#     def compute_output_shape(self, input_shape):\n",
    "#         return (input_shape[0], input_shape[-1])\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "# class MaskSumPoolingLayer(Layer):\n",
    "#     # input_shape [(None, 101, 600), (None, 101)]\n",
    "#     # input:  (?, 101, 600) (?, 101)\n",
    "#     # output: (?, 600)\n",
    "    \n",
    "#     def __init__(self, **kwargs):\n",
    "#         super(MaskSumPoolingLayer, self).__init__(**kwargs)\n",
    "    \n",
    "#     def build(self, input_shape):\n",
    "#         super(MaskSumPoolingLayer, self).build(input_shape)  # Be sure to call this somewhere!\n",
    "\n",
    "#     def call(self, x):\n",
    "#         x_1_float32 = K.cast(x[1], dtype='float32')\n",
    "#         #print(\"x_1_float32\", x_1_float32)\n",
    "#         x_0 = K.permute_dimensions(x[0], pattern=[2, 0, 1])\n",
    "#         #print(\"x_0\", x_0)\n",
    "#         x_0 = tf.multiply(x_0, x_1_float32)\n",
    "#         #print(\"x_0\", x_0)\n",
    "#         x_0 = K.permute_dimensions(x_0, pattern=[1, 2, 0])\n",
    "#         #print(\"x_0\", x_0)\n",
    "#         x_0 = K.sum(x_0, axis=-2)\n",
    "#         #print(\"x_0\", x_0)\n",
    "#         return x_0\n",
    "    \n",
    "#     def compute_output_shape(self, input_shape):\n",
    "#         # input_shape [(None, 101, 600), (None, 101)]\n",
    "#         return (input_shape[0][0], input_shape[0][-1])    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "class MaskMaxPoolingLayer(Layer):\n",
    "    # input_shape [(None, 101, 600), (None, 101)]\n",
    "    # input:  (?, 101, 600) (?, 101)\n",
    "    # output: (?, 600)\n",
    "    \n",
    "    def __init__(self, **kwargs):\n",
    "        super(MaskMaxPoolingLayer, self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        super(MaskMaxPoolingLayer, self).build(input_shape)  # Be sure to call this somewhere!\n",
    "\n",
    "    def call(self, x):\n",
    "        x_1_float32 = K.cast(x[1], dtype='float32')\n",
    "        #print(\"x_1_float32\", x_1_float32)\n",
    "        x_0 = K.permute_dimensions(x[0], pattern=[2, 0, 1])\n",
    "        #print(\"x_0\", x_0)\n",
    "        x_0 = tf.multiply(x_0, x_1_float32)\n",
    "        #print(\"x_0\", x_0)\n",
    "        x_0 = K.permute_dimensions(x_0, pattern=[1, 2, 0])\n",
    "        #print(\"x_0\", x_0)\n",
    "        x_0 = K.max(x_0, axis=-2)\n",
    "        #print(\"x_0\", x_0)\n",
    "        return x_0\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        # input_shape [(None, 101, 600), (None, 101)]\n",
    "        return (input_shape[0][0], input_shape[0][-1])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "________________________________________________________________________________________________________________________\n",
      "Layer (type)                           Output Shape               Param #       Connected to                            \n",
      "========================================================================================================================\n",
      "words_input (InputLayer)               (None, 101)                0                                                     \n",
      "________________________________________________________________________________________________________________________\n",
      "words_Embedding (Embedding)            (None, 101, 300)           7696800       words_input[0][0]                       \n",
      "________________________________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)                    (None, 101, 300)           0             words_Embedding[0][0]                   \n",
      "________________________________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)                      (None, 101, 256)           230656        dropout_1[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)                    (None, 101, 256)           0             conv1d_1[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional)        (None, 101, 128)           123648        dropout_2[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_1 (Activation)              (None, 101, 128)           0             bidirectional_1[0][0]                   \n",
      "________________________________________________________________________________________________________________________\n",
      "dense_1 (Dense)                        (None, 101, 1)             129           activation_1[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "permute_1 (Permute)                    (None, 1, 101)             0             dense_1[0][0]                           \n",
      "________________________________________________________________________________________________________________________\n",
      "attn_softmax (Activation)              (None, 1, 101)             0             permute_1[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "words_input_mask (InputLayer)          (None, 101)                0                                                     \n",
      "________________________________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)                      (None, 1, 128)             0             attn_softmax[0][0]                      \n",
      "                                                                                activation_1[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalMaxPooli (None, 128)                0             activation_1[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "mask_max_pooling_layer_1 (MaskMaxPooli (None, 128)                0             activation_1[0][0]                      \n",
      "                                                                                words_input_mask[0][0]                  \n",
      "________________________________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)                    (None, 128)                0             lambda_1[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)            (None, 384)                0             global_max_pooling1d_1[0][0]            \n",
      "                                                                                mask_max_pooling_layer_1[0][0]          \n",
      "                                                                                flatten_1[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)                    (None, 384)                0             concatenate_1[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "dense_2 (Dense)                        (None, 300)                115500        dropout_3[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "dense_3 (Dense)                        (None, 19)                 5719          dense_2[0][0]                           \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_2 (Activation)              (None, 19)                 0             dense_3[0][0]                           \n",
      "========================================================================================================================\n",
      "Total params: 8,172,452\n",
      "Trainable params: 8,172,452\n",
      "Non-trainable params: 0\n",
      "________________________________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model\n",
    "\n",
    "words_input = Input(shape=(max_sentence_len,), dtype='int32', name='words_input')\n",
    "words_input_mask = Input(shape=(max_sentence_len,), dtype='int32', name='words_input_mask')\n",
    "\n",
    "words = Embedding(embedding.shape[0], embedding.shape[1], weights=[embedding], trainable=True, embeddings_regularizer=embeddings_reg, name=\"words_Embedding\")(words_input)\n",
    "words = Dropout(dropout_emb)(words)\n",
    "\n",
    "output = Convolution1D(filters=256, kernel_size=3, activation=activation_fn, padding='same', strides=1, kernel_regularizer=kernel_reg)(words)\n",
    "output = Dropout(dropout_model)(output)\n",
    "\n",
    "output = Bidirectional( CuDNNGRU(units, return_sequences=True, recurrent_regularizer=recurrent_reg), merge_mode='concat') (output)\n",
    "output_h = Activation('tanh') (output)\n",
    "\n",
    "output1 = GlobalMaxPooling1D()(output_h) \n",
    "output2 = MaskMaxPoolingLayer()([output_h, words_input_mask]) \n",
    "\n",
    "output = Dense(1, kernel_regularizer=kernel_reg)(output_h)\n",
    "output = Permute((2, 1))(output)\n",
    "output = Activation('softmax', name=\"attn_softmax\")(output)\n",
    "output = Lambda(lambda x: tf.matmul(x[0], x[1])) ([output, output_h])\n",
    "output3 = Flatten() (output)\n",
    "\n",
    "output = Concatenate()([output1, output2, output3])\n",
    "output = Dropout(dropout_pen)(output)\n",
    "\n",
    "output = Dense(300, kernel_regularizer=kernel_reg, activation='relu')(output)\n",
    "output = Dense(n_out, kernel_regularizer=kernel_reg)(output)\n",
    "output = Activation('softmax')(output)\n",
    "\n",
    "model = Model(inputs=[words_input, words_input_mask], outputs=[output])\n",
    "model.compile(loss='sparse_categorical_crossentropy', optimizer=adadelta, metrics=['accuracy'])\n",
    "model.summary(line_length=120)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# Evaluation functions \n",
    "\n",
    "def get_precision(test_y_pred, test_y, label): \n",
    "    label_count = 0\n",
    "    label_count_correct = 0\n",
    "    \n",
    "    for i in range(len(test_y_pred)):\n",
    "        if test_y_pred[i] == label: \n",
    "            label_count += 1 \n",
    "            if test_y_pred[i] == test_y[i]: \n",
    "                label_count_correct += 1\n",
    "    \n",
    "    if label_count_correct == 0: \n",
    "        return 0.0\n",
    "    else: \n",
    "        ret = float(label_count_correct) / float(label_count)\n",
    "        return ret\n",
    "        \n",
    "\n",
    "def get_macro_f1(test_y_pred, test_y, n_out):\n",
    "    f1_sum = 0\n",
    "    f1_count = 0\n",
    "    for label in range(1, n_out):        \n",
    "        prec = get_precision(test_y_pred, test_y, label)\n",
    "        recall = get_precision(test_y, test_y_pred, label)\n",
    "        f1 = 0 if float(prec+recall)==float(0) else float(2*prec*recall/float(prec+recall))\n",
    "        f1_sum += f1\n",
    "        f1_count += 1\n",
    "    macro_f1 = float(f1_sum) / float(f1_count)    \n",
    "    return macro_f1\n",
    "    \n",
    "\n",
    "def get_accuracy(test_y_pred, test_y):\n",
    "    acc =  float(np.sum(test_y_pred == test_y)) / float(len(test_y))\n",
    "    return acc\n",
    "    \n",
    "    \n",
    "def predict_classes(prediction):\n",
    "    return prediction.argmax(axis=-1)\n",
    "\n",
    "\n",
    "def get_PRF1_semeval(y_pred, answer_key):\n",
    "    \n",
    "    f_out = open(proposed_ans, 'w')\n",
    "    for i in range(len(y_pred)):\n",
    "        f_out.write(str(i+1) + \"\\t\" + int_to_label[y_pred[i]] + \"\\n\" )\n",
    "    f_out.close()\n",
    "    \n",
    "    if os.name == 'nt': \n",
    "        os.system(perl_exe_path + \" \" + scorer_perl_script + \" \" + proposed_ans + \" \" + answer_key + \" > \" + scorer_output)\n",
    "    else: \n",
    "        os.system(\"perl \" + \" \" + scorer_perl_script + \" \" + proposed_ans + \" \" + answer_key + \" > \" + scorer_output)\n",
    "        \n",
    "    f_in = open(scorer_output, 'r')\n",
    "    lines = f_in.readlines()\n",
    "    f_in.close()\n",
    "    \n",
    "    lines = [ l  for l in lines[-30:] if l.strip() != '']\n",
    "        \n",
    "    acc = float(lines[-17].strip().split()[-1][:-1]) / 100.0\n",
    "    PRF1 = lines[-2].strip().split()\n",
    "    P = float(PRF1[2][:-1]) / 100.0\n",
    "    R = float(PRF1[5][:-1]) / 100.0\n",
    "    macro_f1 = float(PRF1[8][:-1]) / 100.0\n",
    "    \n",
    "    return (acc, P, R, macro_f1)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training... \n",
      "\n",
      "Epoch:  1 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 37s 5ms/step - loss: 5.0217 - acc: 0.1977\n",
      "7208/7208 [==============================] - 5s 738us/step\n",
      "792/792 [==============================] - 1s 716us/step\n",
      "2717/2717 [==============================] - 2s 705us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.3430 (max: 0.3430)\n",
      "Val   Accuracy: 0.3510 (max: 0.3510)\n",
      "Test  Accuracy: 0.3419 (max: 0.3419)\n",
      "Train Macro F1 Semeval Official: 0.2704 (max: 0.2704)\n",
      "Val   Macro F1 Semeval Official: 0.2704 (max: 0.2704)\n",
      "Test  Macro F1 Semeval Official: 0.2630 (max: 0.2630)\n",
      "Test P 0.3215 | R 0.3258 | macro_F1: 0.2630\n",
      "Test test_max_f1_final: 0.2630 (max: 0.2630)\n",
      "\n",
      "Epoch:  2 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 4.4629 - acc: 0.3453\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 715us/step\n",
      "2717/2717 [==============================] - 2s 700us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.4452 (max: 0.4452)\n",
      "Val   Accuracy: 0.4280 (max: 0.4280)\n",
      "Test  Accuracy: 0.4428 (max: 0.4428)\n",
      "Train Macro F1 Semeval Official: 0.4818 (max: 0.4818)\n",
      "Val   Macro F1 Semeval Official: 0.4618 (max: 0.4618)\n",
      "Test  Macro F1 Semeval Official: 0.4767 (max: 0.4767)\n",
      "Test P 0.4724 | R 0.5319 | macro_F1: 0.4767\n",
      "Test test_max_f1_final: 0.4767 (max: 0.4767)\n",
      "\n",
      "Epoch:  3 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 4.1034 - acc: 0.4462\n",
      "7208/7208 [==============================] - 5s 725us/step\n",
      "792/792 [==============================] - 1s 687us/step\n",
      "2717/2717 [==============================] - 2s 690us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.5165 (max: 0.5165)\n",
      "Val   Accuracy: 0.4949 (max: 0.4949)\n",
      "Test  Accuracy: 0.5083 (max: 0.5083)\n",
      "Train Macro F1 Semeval Official: 0.5513 (max: 0.5513)\n",
      "Val   Macro F1 Semeval Official: 0.5332 (max: 0.5332)\n",
      "Test  Macro F1 Semeval Official: 0.5254 (max: 0.5254)\n",
      "Test P 0.5710 | R 0.5773 | macro_F1: 0.5254\n",
      "Test test_max_f1_final: 0.5254 (max: 0.5254)\n",
      "\n",
      "Epoch:  4 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.8464 - acc: 0.5233\n",
      "7208/7208 [==============================] - 5s 705us/step\n",
      "792/792 [==============================] - 1s 688us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.5536 (max: 0.5536)\n",
      "Val   Accuracy: 0.5341 (max: 0.5341)\n",
      "Test  Accuracy: 0.5646 (max: 0.5646)\n",
      "Train Macro F1 Semeval Official: 0.6088 (max: 0.6088)\n",
      "Val   Macro F1 Semeval Official: 0.5892 (max: 0.5892)\n",
      "Test  Macro F1 Semeval Official: 0.6026 (max: 0.6026)\n",
      "Test P 0.6221 | R 0.6488 | macro_F1: 0.6026\n",
      "Test test_max_f1_final: 0.6026 (max: 0.6026)\n",
      "\n",
      "Epoch:  5 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.6293 - acc: 0.5959\n",
      "7208/7208 [==============================] - 5s 710us/step\n",
      "792/792 [==============================] - 1s 698us/step\n",
      "2717/2717 [==============================] - 2s 688us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.6465 (max: 0.6465)\n",
      "Val   Accuracy: 0.6275 (max: 0.6275)\n",
      "Test  Accuracy: 0.6411 (max: 0.6411)\n",
      "Train Macro F1 Semeval Official: 0.6968 (max: 0.6968)\n",
      "Val   Macro F1 Semeval Official: 0.6815 (max: 0.6815)\n",
      "Test  Macro F1 Semeval Official: 0.6896 (max: 0.6896)\n",
      "Test P 0.6638 | R 0.7538 | macro_F1: 0.6896\n",
      "Test test_max_f1_final: 0.6896 (max: 0.6896)\n",
      "\n",
      "Epoch:  6 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.5460 - acc: 0.6211\n",
      "7208/7208 [==============================] - 5s 692us/step\n",
      "792/792 [==============================] - 1s 689us/step\n",
      "2717/2717 [==============================] - 2s 689us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.7013 (max: 0.7013)\n",
      "Val   Accuracy: 0.6705 (max: 0.6705)\n",
      "Test  Accuracy: 0.6897 (max: 0.6897)\n",
      "Train Macro F1 Semeval Official: 0.7461 (max: 0.7461)\n",
      "Val   Macro F1 Semeval Official: 0.7183 (max: 0.7183)\n",
      "Test  Macro F1 Semeval Official: 0.7280 (max: 0.7280)\n",
      "Test P 0.7098 | R 0.7604 | macro_F1: 0.7280\n",
      "Test test_max_f1_final: 0.7280 (max: 0.7280)\n",
      "\n",
      "Epoch:  7 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.4475 - acc: 0.6515\n",
      "7208/7208 [==============================] - 5s 696us/step\n",
      "792/792 [==============================] - 1s 686us/step\n",
      "2717/2717 [==============================] - 2s 692us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.7293 (max: 0.7293)\n",
      "Val   Accuracy: 0.6957 (max: 0.6957)\n",
      "Test  Accuracy: 0.6997 (max: 0.6997)\n",
      "Train Macro F1 Semeval Official: 0.7723 (max: 0.7723)\n",
      "Val   Macro F1 Semeval Official: 0.7361 (max: 0.7361)\n",
      "Test  Macro F1 Semeval Official: 0.7387 (max: 0.7387)\n",
      "Test P 0.7704 | R 0.7219 | macro_F1: 0.7387\n",
      "Test test_max_f1_final: 0.7387 (max: 0.7387)\n",
      "\n",
      "Epoch:  8 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.3727 - acc: 0.6729\n",
      "7208/7208 [==============================] - 5s 696us/step\n",
      "792/792 [==============================] - 1s 688us/step\n",
      "2717/2717 [==============================] - 2s 693us/step\n",
      "Train Accuracy: 0.7372 (max: 0.7372)\n",
      "Val   Accuracy: 0.6793 (max: 0.6957)\n",
      "Test  Accuracy: 0.7122 (max: 0.7122)\n",
      "Train Macro F1 Semeval Official: 0.7786 (max: 0.7786)\n",
      "Val   Macro F1 Semeval Official: 0.7171 (max: 0.7361)\n",
      "Test  Macro F1 Semeval Official: 0.7524 (max: 0.7524)\n",
      "Test P 0.7424 | R 0.7937 | macro_F1: 0.7524\n",
      "Test test_max_f1_final: 0.7387 (max: 0.7387)\n",
      "\n",
      "Epoch:  9 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.3205 - acc: 0.6891\n",
      "7208/7208 [==============================] - 5s 700us/step\n",
      "792/792 [==============================] - 1s 685us/step\n",
      "2717/2717 [==============================] - 2s 692us/step\n",
      "Train Accuracy: 0.7335 (max: 0.7372)\n",
      "Val   Accuracy: 0.6818 (max: 0.6957)\n",
      "Test  Accuracy: 0.7133 (max: 0.7133)\n",
      "Train Macro F1 Semeval Official: 0.7915 (max: 0.7915)\n",
      "Val   Macro F1 Semeval Official: 0.7345 (max: 0.7361)\n",
      "Test  Macro F1 Semeval Official: 0.7655 (max: 0.7655)\n",
      "Test P 0.7245 | R 0.8277 | macro_F1: 0.7655\n",
      "Test test_max_f1_final: 0.7387 (max: 0.7387)\n",
      "\n",
      "Epoch:  10 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.2617 - acc: 0.7052\n",
      "7208/7208 [==============================] - 5s 694us/step\n",
      "792/792 [==============================] - 1s 696us/step\n",
      "2717/2717 [==============================] - 2s 694us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.7852 (max: 0.7852)\n",
      "Val   Accuracy: 0.7513 (max: 0.7513)\n",
      "Test  Accuracy: 0.7457 (max: 0.7457)\n",
      "Train Macro F1 Semeval Official: 0.8291 (max: 0.8291)\n",
      "Val   Macro F1 Semeval Official: 0.7956 (max: 0.7956)\n",
      "Test  Macro F1 Semeval Official: 0.7896 (max: 0.7896)\n",
      "Test P 0.7819 | R 0.8081 | macro_F1: 0.7896\n",
      "Test test_max_f1_final: 0.7896 (max: 0.7896)\n",
      "\n",
      "Epoch:  11 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.2193 - acc: 0.7146\n",
      "7208/7208 [==============================] - 5s 694us/step\n",
      "792/792 [==============================] - 1s 700us/step\n",
      "2717/2717 [==============================] - 2s 694us/step\n",
      "Train Accuracy: 0.7973 (max: 0.7973)\n",
      "Val   Accuracy: 0.7475 (max: 0.7513)\n",
      "Test  Accuracy: 0.7600 (max: 0.7600)\n",
      "Train Macro F1 Semeval Official: 0.8384 (max: 0.8384)\n",
      "Val   Macro F1 Semeval Official: 0.7922 (max: 0.7956)\n",
      "Test  Macro F1 Semeval Official: 0.7995 (max: 0.7995)\n",
      "Test P 0.7749 | R 0.8347 | macro_F1: 0.7995\n",
      "Test test_max_f1_final: 0.7896 (max: 0.7896)\n",
      "\n",
      "Epoch:  12 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.1884 - acc: 0.7292\n",
      "7208/7208 [==============================] - 5s 697us/step\n",
      "792/792 [==============================] - 1s 701us/step\n",
      "2717/2717 [==============================] - 2s 696us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.8083 (max: 0.8083)\n",
      "Val   Accuracy: 0.7525 (max: 0.7525)\n",
      "Test  Accuracy: 0.7619 (max: 0.7619)\n",
      "Train Macro F1 Semeval Official: 0.8504 (max: 0.8504)\n",
      "Val   Macro F1 Semeval Official: 0.7972 (max: 0.7972)\n",
      "Test  Macro F1 Semeval Official: 0.8013 (max: 0.8013)\n",
      "Test P 0.7819 | R 0.8269 | macro_F1: 0.8013\n",
      "Test test_max_f1_final: 0.8013 (max: 0.8013)\n",
      "\n",
      "Epoch:  13 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.1513 - acc: 0.7356\n",
      "7208/7208 [==============================] - 5s 715us/step\n",
      "792/792 [==============================] - 1s 705us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.8094 (max: 0.8094)\n",
      "Val   Accuracy: 0.7702 (max: 0.7702)\n",
      "Test  Accuracy: 0.7597 (max: 0.7619)\n",
      "Train Macro F1 Semeval Official: 0.8491 (max: 0.8504)\n",
      "Val   Macro F1 Semeval Official: 0.8062 (max: 0.8062)\n",
      "Test  Macro F1 Semeval Official: 0.7985 (max: 0.8013)\n",
      "Test P 0.7763 | R 0.8305 | macro_F1: 0.7985\n",
      "Test test_max_f1_final: 0.7985 (max: 0.8013)\n",
      "\n",
      "Epoch:  14 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.1145 - acc: 0.7432\n",
      "7208/7208 [==============================] - 5s 698us/step\n",
      "792/792 [==============================] - 1s 685us/step\n",
      "2717/2717 [==============================] - 2s 692us/step\n",
      "Train Accuracy: 0.8123 (max: 0.8123)\n",
      "Val   Accuracy: 0.7576 (max: 0.7702)\n",
      "Test  Accuracy: 0.7626 (max: 0.7626)\n",
      "Train Macro F1 Semeval Official: 0.8565 (max: 0.8565)\n",
      "Val   Macro F1 Semeval Official: 0.8000 (max: 0.8062)\n",
      "Test  Macro F1 Semeval Official: 0.8093 (max: 0.8093)\n",
      "Test P 0.7651 | R 0.8629 | macro_F1: 0.8093\n",
      "Test test_max_f1_final: 0.7985 (max: 0.8013)\n",
      "\n",
      "Epoch:  15 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.1001 - acc: 0.7489\n",
      "7208/7208 [==============================] - 5s 705us/step\n",
      "792/792 [==============================] - 1s 701us/step\n",
      "2717/2717 [==============================] - 2s 699us/step\n",
      "Train Accuracy: 0.8158 (max: 0.8158)\n",
      "Val   Accuracy: 0.7487 (max: 0.7702)\n",
      "Test  Accuracy: 0.7497 (max: 0.7626)\n",
      "Train Macro F1 Semeval Official: 0.8534 (max: 0.8565)\n",
      "Val   Macro F1 Semeval Official: 0.7890 (max: 0.8062)\n",
      "Test  Macro F1 Semeval Official: 0.7917 (max: 0.8093)\n",
      "Test P 0.7636 | R 0.8397 | macro_F1: 0.7917\n",
      "Test test_max_f1_final: 0.7985 (max: 0.8013)\n",
      "\n",
      "Epoch:  16 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.0636 - acc: 0.7593\n",
      "7208/7208 [==============================] - 5s 710us/step\n",
      "792/792 [==============================] - 1s 695us/step\n",
      "2717/2717 [==============================] - 2s 700us/step\n",
      "Train Accuracy: 0.8327 (max: 0.8327)\n",
      "Val   Accuracy: 0.7626 (max: 0.7702)\n",
      "Test  Accuracy: 0.7608 (max: 0.7626)\n",
      "Train Macro F1 Semeval Official: 0.8744 (max: 0.8744)\n",
      "Val   Macro F1 Semeval Official: 0.8026 (max: 0.8062)\n",
      "Test  Macro F1 Semeval Official: 0.8047 (max: 0.8093)\n",
      "Test P 0.7764 | R 0.8431 | macro_F1: 0.8047\n",
      "Test test_max_f1_final: 0.7985 (max: 0.8013)\n",
      "\n",
      "Epoch:  17 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.0182 - acc: 0.7758\n",
      "7208/7208 [==============================] - 5s 704us/step\n",
      "792/792 [==============================] - 1s 703us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Train Accuracy: 0.8130 (max: 0.8327)\n",
      "Val   Accuracy: 0.7487 (max: 0.7702)\n",
      "Test  Accuracy: 0.7449 (max: 0.7626)\n",
      "Train Macro F1 Semeval Official: 0.8627 (max: 0.8744)\n",
      "Val   Macro F1 Semeval Official: 0.8000 (max: 0.8062)\n",
      "Test  Macro F1 Semeval Official: 0.7933 (max: 0.8093)\n",
      "Test P 0.7494 | R 0.8553 | macro_F1: 0.7933\n",
      "Test test_max_f1_final: 0.7985 (max: 0.8013)\n",
      "\n",
      "Epoch:  18 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 3.0091 - acc: 0.7776\n",
      "7208/7208 [==============================] - 5s 705us/step\n",
      "792/792 [==============================] - 1s 689us/step\n",
      "2717/2717 [==============================] - 2s 698us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.8538 (max: 0.8538)\n",
      "Val   Accuracy: 0.7778 (max: 0.7778)\n",
      "Test  Accuracy: 0.7766 (max: 0.7766)\n",
      "Train Macro F1 Semeval Official: 0.8902 (max: 0.8902)\n",
      "Val   Macro F1 Semeval Official: 0.8186 (max: 0.8186)\n",
      "Test  Macro F1 Semeval Official: 0.8170 (max: 0.8170)\n",
      "Test P 0.7953 | R 0.8445 | macro_F1: 0.8170\n",
      "Test test_max_f1_final: 0.8170 (max: 0.8170)\n",
      "\n",
      "Epoch:  19 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.9767 - acc: 0.7791\n",
      "7208/7208 [==============================] - 5s 703us/step\n",
      "792/792 [==============================] - 1s 706us/step\n",
      "2717/2717 [==============================] - 2s 702us/step\n",
      "Train Accuracy: 0.8346 (max: 0.8538)\n",
      "Val   Accuracy: 0.7639 (max: 0.7778)\n",
      "Test  Accuracy: 0.7644 (max: 0.7766)\n",
      "Train Macro F1 Semeval Official: 0.8812 (max: 0.8902)\n",
      "Val   Macro F1 Semeval Official: 0.8090 (max: 0.8186)\n",
      "Test  Macro F1 Semeval Official: 0.8130 (max: 0.8170)\n",
      "Test P 0.7723 | R 0.8634 | macro_F1: 0.8130\n",
      "Test test_max_f1_final: 0.8170 (max: 0.8170)\n",
      "\n",
      "Epoch:  20 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.9465 - acc: 0.7925\n",
      "7208/7208 [==============================] - 5s 701us/step\n",
      "792/792 [==============================] - 1s 713us/step\n",
      "2717/2717 [==============================] - 2s 698us/step\n",
      "Train Accuracy: 0.8645 (max: 0.8645)\n",
      "Val   Accuracy: 0.7702 (max: 0.7778)\n",
      "Test  Accuracy: 0.7795 (max: 0.7795)\n",
      "Train Macro F1 Semeval Official: 0.8987 (max: 0.8987)\n",
      "Val   Macro F1 Semeval Official: 0.8041 (max: 0.8186)\n",
      "Test  Macro F1 Semeval Official: 0.8199 (max: 0.8199)\n",
      "Test P 0.8005 | R 0.8491 | macro_F1: 0.8199\n",
      "Test test_max_f1_final: 0.8170 (max: 0.8170)\n",
      "\n",
      "Epoch:  21 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.9270 - acc: 0.7912\n",
      "7208/7208 [==============================] - 5s 700us/step\n",
      "792/792 [==============================] - 1s 705us/step\n",
      "2717/2717 [==============================] - 2s 704us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.8726 (max: 0.8726)\n",
      "Val   Accuracy: 0.7866 (max: 0.7866)\n",
      "Test  Accuracy: 0.7803 (max: 0.7803)\n",
      "Train Macro F1 Semeval Official: 0.9030 (max: 0.9030)\n",
      "Val   Macro F1 Semeval Official: 0.8260 (max: 0.8260)\n",
      "Test  Macro F1 Semeval Official: 0.8203 (max: 0.8203)\n",
      "Test P 0.7982 | R 0.8502 | macro_F1: 0.8203\n",
      "Test test_max_f1_final: 0.8203 (max: 0.8203)\n",
      "\n",
      "Epoch:  22 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8990 - acc: 0.8016\n",
      "7208/7208 [==============================] - 5s 701us/step\n",
      "792/792 [==============================] - 1s 711us/step\n",
      "2717/2717 [==============================] - 2s 696us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.8782 (max: 0.8782)\n",
      "Val   Accuracy: 0.7866 (max: 0.7866)\n",
      "Test  Accuracy: 0.7825 (max: 0.7825)\n",
      "Train Macro F1 Semeval Official: 0.9100 (max: 0.9100)\n",
      "Val   Macro F1 Semeval Official: 0.8317 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8240 (max: 0.8240)\n",
      "Test P 0.8040 | R 0.8526 | macro_F1: 0.8240\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  23 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8946 - acc: 0.8022\n",
      "7208/7208 [==============================] - 5s 704us/step\n",
      "792/792 [==============================] - 1s 690us/step\n",
      "2717/2717 [==============================] - 2s 697us/step\n",
      "Train Accuracy: 0.8725 (max: 0.8782)\n",
      "Val   Accuracy: 0.7778 (max: 0.7866)\n",
      "Test  Accuracy: 0.7759 (max: 0.7825)\n",
      "Train Macro F1 Semeval Official: 0.9086 (max: 0.9100)\n",
      "Val   Macro F1 Semeval Official: 0.8203 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8212 (max: 0.8240)\n",
      "Test P 0.7838 | R 0.8649 | macro_F1: 0.8212\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  24 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8708 - acc: 0.8072\n",
      "7208/7208 [==============================] - 5s 703us/step\n",
      "792/792 [==============================] - 1s 705us/step\n",
      "2717/2717 [==============================] - 2s 704us/step\n",
      "Train Accuracy: 0.8586 (max: 0.8782)\n",
      "Val   Accuracy: 0.7639 (max: 0.7866)\n",
      "Test  Accuracy: 0.7707 (max: 0.7825)\n",
      "Train Macro F1 Semeval Official: 0.9010 (max: 0.9100)\n",
      "Val   Macro F1 Semeval Official: 0.8068 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8166 (max: 0.8240)\n",
      "Test P 0.7755 | R 0.8686 | macro_F1: 0.8166\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  25 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8455 - acc: 0.8190\n",
      "7208/7208 [==============================] - 5s 697us/step\n",
      "792/792 [==============================] - 1s 701us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Train Accuracy: 0.8914 (max: 0.8914)\n",
      "Val   Accuracy: 0.7816 (max: 0.7866)\n",
      "Test  Accuracy: 0.7814 (max: 0.7825)\n",
      "Train Macro F1 Semeval Official: 0.9175 (max: 0.9175)\n",
      "Val   Macro F1 Semeval Official: 0.8176 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8236 (max: 0.8240)\n",
      "Test P 0.7964 | R 0.8592 | macro_F1: 0.8236\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  26 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8152 - acc: 0.8192\n",
      "7208/7208 [==============================] - 5s 702us/step\n",
      "792/792 [==============================] - 1s 693us/step\n",
      "2717/2717 [==============================] - 2s 701us/step\n",
      "Train Accuracy: 0.8753 (max: 0.8914)\n",
      "Val   Accuracy: 0.7816 (max: 0.7866)\n",
      "Test  Accuracy: 0.7729 (max: 0.7825)\n",
      "Train Macro F1 Semeval Official: 0.9125 (max: 0.9175)\n",
      "Val   Macro F1 Semeval Official: 0.8245 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8235 (max: 0.8240)\n",
      "Test P 0.7857 | R 0.8697 | macro_F1: 0.8235\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  27 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.8114 - acc: 0.8278\n",
      "7208/7208 [==============================] - 5s 712us/step\n",
      "792/792 [==============================] - 1s 699us/step\n",
      "2717/2717 [==============================] - 2s 702us/step\n",
      "Train Accuracy: 0.9112 (max: 0.9112)\n",
      "Val   Accuracy: 0.7942 (max: 0.7942)\n",
      "Test  Accuracy: 0.7840 (max: 0.7840)\n",
      "Train Macro F1 Semeval Official: 0.9363 (max: 0.9363)\n",
      "Val   Macro F1 Semeval Official: 0.8316 (max: 0.8317)\n",
      "Test  Macro F1 Semeval Official: 0.8296 (max: 0.8296)\n",
      "Test P 0.8119 | R 0.8532 | macro_F1: 0.8296\n",
      "Test test_max_f1_final: 0.8240 (max: 0.8240)\n",
      "\n",
      "Epoch:  28 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7987 - acc: 0.8248\n",
      "7208/7208 [==============================] - 5s 708us/step\n",
      "792/792 [==============================] - 1s 703us/step\n",
      "2717/2717 [==============================] - 2s 697us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.9022 (max: 0.9112)\n",
      "Val   Accuracy: 0.7955 (max: 0.7955)\n",
      "Test  Accuracy: 0.7946 (max: 0.7946)\n",
      "Train Macro F1 Semeval Official: 0.9316 (max: 0.9363)\n",
      "Val   Macro F1 Semeval Official: 0.8357 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8389 (max: 0.8389)\n",
      "Test P 0.8090 | R 0.8732 | macro_F1: 0.8389\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  29 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7804 - acc: 0.8276\n",
      "7208/7208 [==============================] - 5s 717us/step\n",
      "792/792 [==============================] - 1s 699us/step\n",
      "2717/2717 [==============================] - 2s 696us/step\n",
      "Train Accuracy: 0.9069 (max: 0.9112)\n",
      "Val   Accuracy: 0.7917 (max: 0.7955)\n",
      "Test  Accuracy: 0.7814 (max: 0.7946)\n",
      "Train Macro F1 Semeval Official: 0.9315 (max: 0.9363)\n",
      "Val   Macro F1 Semeval Official: 0.8312 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8270 (max: 0.8389)\n",
      "Test P 0.8086 | R 0.8487 | macro_F1: 0.8270\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  30 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7585 - acc: 0.8285\n",
      "7208/7208 [==============================] - 5s 708us/step\n",
      "792/792 [==============================] - 1s 700us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Train Accuracy: 0.9108 (max: 0.9112)\n",
      "Val   Accuracy: 0.7854 (max: 0.7955)\n",
      "Test  Accuracy: 0.7696 (max: 0.7946)\n",
      "Train Macro F1 Semeval Official: 0.9336 (max: 0.9363)\n",
      "Val   Macro F1 Semeval Official: 0.8279 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8134 (max: 0.8389)\n",
      "Test P 0.8384 | R 0.8013 | macro_F1: 0.8134\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  31 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7432 - acc: 0.8381\n",
      "7208/7208 [==============================] - 5s 707us/step\n",
      "792/792 [==============================] - 1s 706us/step\n",
      "2717/2717 [==============================] - 2s 699us/step\n",
      "Train Accuracy: 0.9123 (max: 0.9123)\n",
      "Val   Accuracy: 0.7841 (max: 0.7955)\n",
      "Test  Accuracy: 0.7913 (max: 0.7946)\n",
      "Train Macro F1 Semeval Official: 0.9364 (max: 0.9364)\n",
      "Val   Macro F1 Semeval Official: 0.8192 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8326 (max: 0.8389)\n",
      "Test P 0.8108 | R 0.8641 | macro_F1: 0.8326\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  32 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7196 - acc: 0.8467\n",
      "7208/7208 [==============================] - 5s 705us/step\n",
      "792/792 [==============================] - 1s 695us/step\n",
      "2717/2717 [==============================] - 2s 699us/step\n",
      "Train Accuracy: 0.9197 (max: 0.9197)\n",
      "Val   Accuracy: 0.7980 (max: 0.7980)\n",
      "Test  Accuracy: 0.8009 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9395 (max: 0.9395)\n",
      "Val   Macro F1 Semeval Official: 0.8349 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8374 (max: 0.8389)\n",
      "Test P 0.8080 | R 0.8727 | macro_F1: 0.8374\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  33 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.7076 - acc: 0.8453\n",
      "7208/7208 [==============================] - 5s 700us/step\n",
      "792/792 [==============================] - 1s 693us/step\n",
      "2717/2717 [==============================] - 2s 698us/step\n",
      "Train Accuracy: 0.9036 (max: 0.9197)\n",
      "Val   Accuracy: 0.7803 (max: 0.7980)\n",
      "Test  Accuracy: 0.7817 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9316 (max: 0.9395)\n",
      "Val   Macro F1 Semeval Official: 0.8203 (max: 0.8357)\n",
      "Test  Macro F1 Semeval Official: 0.8245 (max: 0.8389)\n",
      "Test P 0.7980 | R 0.8623 | macro_F1: 0.8245\n",
      "Test test_max_f1_final: 0.8389 (max: 0.8389)\n",
      "\n",
      "Epoch:  34 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6737 - acc: 0.8570\n",
      "7208/7208 [==============================] - 5s 701us/step\n",
      "792/792 [==============================] - 1s 691us/step\n",
      "2717/2717 [==============================] - 2s 698us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.9272 (max: 0.9272)\n",
      "Val   Accuracy: 0.8018 (max: 0.8018)\n",
      "Test  Accuracy: 0.8009 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9499 (max: 0.9499)\n",
      "Val   Macro F1 Semeval Official: 0.8397 (max: 0.8397)\n",
      "Test  Macro F1 Semeval Official: 0.8409 (max: 0.8409)\n",
      "Test P 0.8072 | R 0.8788 | macro_F1: 0.8409\n",
      "Test test_max_f1_final: 0.8409 (max: 0.8409)\n",
      "\n",
      "Epoch:  35 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6696 - acc: 0.8579\n",
      "7208/7208 [==============================] - 5s 702us/step\n",
      "792/792 [==============================] - 1s 695us/step\n",
      "2717/2717 [==============================] - 2s 701us/step\n",
      "Train Accuracy: 0.9226 (max: 0.9272)\n",
      "Val   Accuracy: 0.7904 (max: 0.8018)\n",
      "Test  Accuracy: 0.7939 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9473 (max: 0.9499)\n",
      "Val   Macro F1 Semeval Official: 0.8279 (max: 0.8397)\n",
      "Test  Macro F1 Semeval Official: 0.8379 (max: 0.8409)\n",
      "Test P 0.8031 | R 0.8790 | macro_F1: 0.8379\n",
      "Test test_max_f1_final: 0.8409 (max: 0.8409)\n",
      "\n",
      "Epoch:  36 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6541 - acc: 0.8572\n",
      "7208/7208 [==============================] - 5s 696us/step\n",
      "792/792 [==============================] - 1s 699us/step\n",
      "2717/2717 [==============================] - 2s 699us/step\n",
      "Model saved model.keras\n",
      "Train Accuracy: 0.9463 (max: 0.9463)\n",
      "Val   Accuracy: 0.8081 (max: 0.8081)\n",
      "Test  Accuracy: 0.7935 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9587 (max: 0.9587)\n",
      "Val   Macro F1 Semeval Official: 0.8466 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8352 (max: 0.8409)\n",
      "Test P 0.8262 | R 0.8492 | macro_F1: 0.8352\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  37 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6390 - acc: 0.8649\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 694us/step\n",
      "2717/2717 [==============================] - 2s 696us/step\n",
      "Train Accuracy: 0.8986 (max: 0.9463)\n",
      "Val   Accuracy: 0.7677 (max: 0.8081)\n",
      "Test  Accuracy: 0.7736 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9273 (max: 0.9587)\n",
      "Val   Macro F1 Semeval Official: 0.8101 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8215 (max: 0.8409)\n",
      "Test P 0.7839 | R 0.8707 | macro_F1: 0.8215\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  38 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6286 - acc: 0.8614\n",
      "7208/7208 [==============================] - 5s 712us/step\n",
      "792/792 [==============================] - 1s 703us/step\n",
      "2717/2717 [==============================] - 2s 702us/step\n",
      "Train Accuracy: 0.9377 (max: 0.9463)\n",
      "Val   Accuracy: 0.7828 (max: 0.8081)\n",
      "Test  Accuracy: 0.7898 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9545 (max: 0.9587)\n",
      "Val   Macro F1 Semeval Official: 0.8165 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8306 (max: 0.8409)\n",
      "Test P 0.8078 | R 0.8598 | macro_F1: 0.8306\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  39 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.6229 - acc: 0.8672\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 689us/step\n",
      "2717/2717 [==============================] - 2s 692us/step\n",
      "Train Accuracy: 0.9408 (max: 0.9463)\n",
      "Val   Accuracy: 0.7967 (max: 0.8081)\n",
      "Test  Accuracy: 0.7972 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9587 (max: 0.9587)\n",
      "Val   Macro F1 Semeval Official: 0.8288 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8343 (max: 0.8409)\n",
      "Test P 0.8136 | R 0.8632 | macro_F1: 0.8343\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  40 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5836 - acc: 0.8789\n",
      "7208/7208 [==============================] - 5s 702us/step\n",
      "792/792 [==============================] - 1s 684us/step\n",
      "2717/2717 [==============================] - 2s 701us/step\n",
      "Train Accuracy: 0.9538 (max: 0.9538)\n",
      "Val   Accuracy: 0.7955 (max: 0.8081)\n",
      "Test  Accuracy: 0.8001 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9675 (max: 0.9675)\n",
      "Val   Macro F1 Semeval Official: 0.8374 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8396 (max: 0.8409)\n",
      "Test P 0.8219 | R 0.8613 | macro_F1: 0.8396\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  41 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5900 - acc: 0.8758\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 706us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Train Accuracy: 0.9605 (max: 0.9605)\n",
      "Val   Accuracy: 0.7992 (max: 0.8081)\n",
      "Test  Accuracy: 0.7983 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9707 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8354 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8421 (max: 0.8421)\n",
      "Test P 0.8368 | R 0.8492 | macro_F1: 0.8421\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  42 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5829 - acc: 0.8742\n",
      "7208/7208 [==============================] - 5s 702us/step\n",
      "792/792 [==============================] - 1s 699us/step\n",
      "2717/2717 [==============================] - 2s 693us/step\n",
      "Train Accuracy: 0.9444 (max: 0.9605)\n",
      "Val   Accuracy: 0.7828 (max: 0.8081)\n",
      "Test  Accuracy: 0.7950 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9607 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8153 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8380 (max: 0.8421)\n",
      "Test P 0.8158 | R 0.8674 | macro_F1: 0.8380\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  43 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5712 - acc: 0.8728\n",
      "7208/7208 [==============================] - 5s 708us/step\n",
      "792/792 [==============================] - 1s 722us/step\n",
      "2717/2717 [==============================] - 2s 701us/step\n",
      "Train Accuracy: 0.9566 (max: 0.9605)\n",
      "Val   Accuracy: 0.7955 (max: 0.8081)\n",
      "Test  Accuracy: 0.7935 (max: 0.8009)\n",
      "Train Macro F1 Semeval Official: 0.9693 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8328 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8358 (max: 0.8421)\n",
      "Test P 0.8181 | R 0.8585 | macro_F1: 0.8358\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  44 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5415 - acc: 0.8862\n",
      "7208/7208 [==============================] - 5s 700us/step\n",
      "792/792 [==============================] - 1s 694us/step\n",
      "2717/2717 [==============================] - 2s 698us/step\n",
      "Train Accuracy: 0.9478 (max: 0.9605)\n",
      "Val   Accuracy: 0.7992 (max: 0.8081)\n",
      "Test  Accuracy: 0.8038 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9640 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8334 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8431 (max: 0.8431)\n",
      "Test P 0.8137 | R 0.8797 | macro_F1: 0.8431\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  45 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5324 - acc: 0.8901\n",
      "7208/7208 [==============================] - 5s 706us/step\n",
      "792/792 [==============================] - 1s 708us/step\n",
      "2717/2717 [==============================] - 2s 690us/step\n",
      "Train Accuracy: 0.9519 (max: 0.9605)\n",
      "Val   Accuracy: 0.7891 (max: 0.8081)\n",
      "Test  Accuracy: 0.7965 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9679 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8293 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8391 (max: 0.8431)\n",
      "Test P 0.8037 | R 0.8781 | macro_F1: 0.8391\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  46 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5256 - acc: 0.8822\n",
      "7208/7208 [==============================] - 5s 712us/step\n",
      "792/792 [==============================] - 1s 722us/step\n",
      "2717/2717 [==============================] - 2s 701us/step\n",
      "Train Accuracy: 0.9526 (max: 0.9605)\n",
      "Val   Accuracy: 0.7929 (max: 0.8081)\n",
      "Test  Accuracy: 0.7976 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9648 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8354 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8361 (max: 0.8431)\n",
      "Test P 0.8035 | R 0.8782 | macro_F1: 0.8361\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  47 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5144 - acc: 0.8868\n",
      "7208/7208 [==============================] - 5s 698us/step\n",
      "792/792 [==============================] - 1s 705us/step\n",
      "2717/2717 [==============================] - 2s 693us/step\n",
      "Train Accuracy: 0.9520 (max: 0.9605)\n",
      "Val   Accuracy: 0.8018 (max: 0.8081)\n",
      "Test  Accuracy: 0.7884 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9655 (max: 0.9707)\n",
      "Val   Macro F1 Semeval Official: 0.8392 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8270 (max: 0.8431)\n",
      "Test P 0.8002 | R 0.8615 | macro_F1: 0.8270\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  48 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.5002 - acc: 0.8910\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 703us/step\n",
      "2717/2717 [==============================] - 2s 696us/step\n",
      "Train Accuracy: 0.9711 (max: 0.9711)\n",
      "Val   Accuracy: 0.8043 (max: 0.8081)\n",
      "Test  Accuracy: 0.7957 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9780 (max: 0.9780)\n",
      "Val   Macro F1 Semeval Official: 0.8376 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8365 (max: 0.8431)\n",
      "Test P 0.8187 | R 0.8580 | macro_F1: 0.8365\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  49 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4934 - acc: 0.8925\n",
      "7208/7208 [==============================] - 5s 698us/step\n",
      "792/792 [==============================] - 1s 701us/step\n",
      "2717/2717 [==============================] - 2s 691us/step\n",
      "Train Accuracy: 0.9502 (max: 0.9711)\n",
      "Val   Accuracy: 0.7828 (max: 0.8081)\n",
      "Test  Accuracy: 0.7858 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9666 (max: 0.9780)\n",
      "Val   Macro F1 Semeval Official: 0.8232 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8280 (max: 0.8431)\n",
      "Test P 0.7945 | R 0.8686 | macro_F1: 0.8280\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  50 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4825 - acc: 0.8905\n",
      "7208/7208 [==============================] - 5s 694us/step\n",
      "792/792 [==============================] - 1s 693us/step\n",
      "2717/2717 [==============================] - 2s 689us/step\n",
      "Train Accuracy: 0.9575 (max: 0.9711)\n",
      "Val   Accuracy: 0.7992 (max: 0.8081)\n",
      "Test  Accuracy: 0.7898 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9706 (max: 0.9780)\n",
      "Val   Macro F1 Semeval Official: 0.8370 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8320 (max: 0.8431)\n",
      "Test P 0.7987 | R 0.8722 | macro_F1: 0.8320\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  51 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4676 - acc: 0.8973\n",
      "7208/7208 [==============================] - 5s 694us/step\n",
      "792/792 [==============================] - 1s 695us/step\n",
      "2717/2717 [==============================] - 2s 694us/step\n",
      "Train Accuracy: 0.9710 (max: 0.9711)\n",
      "Val   Accuracy: 0.7967 (max: 0.8081)\n",
      "Test  Accuracy: 0.7869 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9795 (max: 0.9795)\n",
      "Val   Macro F1 Semeval Official: 0.8295 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8278 (max: 0.8431)\n",
      "Test P 0.8061 | R 0.8551 | macro_F1: 0.8278\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  52 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4533 - acc: 0.9032\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 696us/step\n",
      "2717/2717 [==============================] - 2s 690us/step\n",
      "Train Accuracy: 0.9741 (max: 0.9741)\n",
      "Val   Accuracy: 0.7929 (max: 0.8081)\n",
      "Test  Accuracy: 0.7880 (max: 0.8038)\n",
      "Train Macro F1 Semeval Official: 0.9801 (max: 0.9801)\n",
      "Val   Macro F1 Semeval Official: 0.8280 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8299 (max: 0.8431)\n",
      "Test P 0.8112 | R 0.8541 | macro_F1: 0.8299\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  53 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4542 - acc: 0.9018\n",
      "7208/7208 [==============================] - 5s 699us/step\n",
      "792/792 [==============================] - 1s 704us/step\n",
      "2717/2717 [==============================] - 2s 694us/step\n",
      "Train Accuracy: 0.9764 (max: 0.9764)\n",
      "Val   Accuracy: 0.8018 (max: 0.8081)\n",
      "Test  Accuracy: 0.8053 (max: 0.8053)\n",
      "Train Macro F1 Semeval Official: 0.9833 (max: 0.9833)\n",
      "Val   Macro F1 Semeval Official: 0.8382 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8449 (max: 0.8449)\n",
      "Test P 0.8184 | R 0.8744 | macro_F1: 0.8449\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  54 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4398 - acc: 0.9023\n",
      "7208/7208 [==============================] - 5s 698us/step\n",
      "792/792 [==============================] - 1s 705us/step\n",
      "2717/2717 [==============================] - 2s 695us/step\n",
      "Train Accuracy: 0.9581 (max: 0.9764)\n",
      "Val   Accuracy: 0.7891 (max: 0.8081)\n",
      "Test  Accuracy: 0.7858 (max: 0.8053)\n",
      "Train Macro F1 Semeval Official: 0.9708 (max: 0.9833)\n",
      "Val   Macro F1 Semeval Official: 0.8247 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8304 (max: 0.8449)\n",
      "Test P 0.8039 | R 0.8646 | macro_F1: 0.8304\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  55 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4231 - acc: 0.9061\n",
      "7208/7208 [==============================] - 5s 700us/step\n",
      "792/792 [==============================] - 1s 701us/step\n",
      "2717/2717 [==============================] - 2s 697us/step\n",
      "Train Accuracy: 0.9727 (max: 0.9764)\n",
      "Val   Accuracy: 0.8081 (max: 0.8081)\n",
      "Test  Accuracy: 0.7965 (max: 0.8053)\n",
      "Train Macro F1 Semeval Official: 0.9808 (max: 0.9833)\n",
      "Val   Macro F1 Semeval Official: 0.8441 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8382 (max: 0.8449)\n",
      "Test P 0.8111 | R 0.8695 | macro_F1: 0.8382\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  56 / 256\n",
      "Epoch 1/1\n",
      "7208/7208 [==============================] - 21s 3ms/step - loss: 2.4150 - acc: 0.9064\n",
      "7208/7208 [==============================] - 5s 697us/step\n",
      "792/792 [==============================] - 1s 700us/step\n",
      "2717/2717 [==============================] - 2s 694us/step\n",
      "Train Accuracy: 0.9825 (max: 0.9825)\n",
      "Val   Accuracy: 0.8030 (max: 0.8081)\n",
      "Test  Accuracy: 0.7968 (max: 0.8053)\n",
      "Train Macro F1 Semeval Official: 0.9872 (max: 0.9872)\n",
      "Val   Macro F1 Semeval Official: 0.8374 (max: 0.8466)\n",
      "Test  Macro F1 Semeval Official: 0.8362 (max: 0.8449)\n",
      "Test P 0.8213 | R 0.8563 | macro_F1: 0.8362\n",
      "Test test_max_f1_final: 0.8352 (max: 0.8409)\n",
      "\n",
      "Epoch:  57 / 256\n",
      "Early Stopping...\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# In[15]:\n",
    "\n",
    "print(\"Start training... \\n\")\n",
    "\n",
    "# train_x = train_x[:200]\n",
    "# train_y = train_y[:200]\n",
    "# train_x_copy = train_x_copy[:200]\n",
    "# train_y_copy = train_y_copy[:200]\n",
    "# val_x = val_x[:200]\n",
    "# val_y = val_y[:200]\n",
    "# test_x = test_x[:200]\n",
    "# test_y = test_y[:200]\n",
    "\n",
    "\n",
    "train_max_acc = 0\n",
    "train_max_f1 = 0\n",
    "val_max_acc = 0\n",
    "val_max_f1 = 0\n",
    "test_max_acc = 0\n",
    "test_max_f1 = 0\n",
    "\n",
    "test_f1_final = 0\n",
    "test_f1_final_max = 0\n",
    "\n",
    "es_epoch = 0 \n",
    "\n",
    "for epoch in range(nb_epoch):\n",
    "    print(\"Epoch: \", epoch+1, \"/\", nb_epoch)\n",
    "    es_epoch += 1\n",
    "    if es_epoch > es_epoch_stop:\n",
    "        print(\"Early Stopping...\") \n",
    "        break\n",
    "    \n",
    "    index = np.arange(len(train_x))\n",
    "    np.random.shuffle(index)\n",
    "    train_x = train_x[index]\n",
    "    train_x_mask = train_x_mask[index]\n",
    "    train_y = train_y[index]\n",
    "    \n",
    "    # Input lists for fit and predict\n",
    "    #\n",
    "    train_input_list = [train_x, train_x_mask]\n",
    "    train_input_list_copy = [train_x_copy, train_x_mask_copy]\n",
    "    val_input_list = [val_x, val_x_mask]\n",
    "    test_input_list = [test_x, test_x_mask]\n",
    "    #train_input_list = [train_x]\n",
    "    #train_input_list_copy = [train_x_copy]\n",
    "    #val_input_list = [val_x]\n",
    "    #test_input_list = [test_x]\n",
    "\n",
    "    \n",
    "    model.fit(train_input_list, train_y, batch_size=batch_size, verbose=1, epochs=1) \n",
    "    \n",
    "    train_y_pred = predict_classes(model.predict(train_input_list_copy, verbose=1))\n",
    "    val_y_pred = predict_classes(model.predict(val_input_list, verbose=1))\n",
    "    test_y_pred = predict_classes(model.predict(test_input_list, verbose=1))\n",
    "    \n",
    "    train_PRF1 = get_PRF1_semeval(train_y_pred, train_answer_key)\n",
    "    val_PRF1 = get_PRF1_semeval(val_y_pred, val_answer_key)\n",
    "    test_PRF1 = get_PRF1_semeval(test_y_pred, test_answer_key)\n",
    "\n",
    "    train_max_f1 = max(train_max_f1, train_PRF1[3])\n",
    "    val_max_f1 = max(val_max_f1, val_PRF1[3])\n",
    "    test_max_f1 = max(test_max_f1, test_PRF1[3])\n",
    "\n",
    "    train_max_acc = max(train_max_acc, train_PRF1[0])\n",
    "    val_max_acc = max(val_max_acc, val_PRF1[0])\n",
    "    test_max_acc = max(test_max_acc, test_PRF1[0])\n",
    "    \n",
    "    if val_max_f1 == val_PRF1[3]: \n",
    "        test_f1_final = test_PRF1[3]\n",
    "        test_f1_final_max = max(test_f1_final, test_f1_final_max)\n",
    "        if save_model : \n",
    "            model_file_name = \"model.keras\"\n",
    "            model.save('./model/' + model_file_name)\n",
    "            print(\"Model saved\", model_file_name)\n",
    "        es_epoch = 0\n",
    "    \n",
    "    print(\"Train Accuracy: %.4f (max: %.4f)\" % (train_PRF1[0], train_max_acc))\n",
    "    print(\"Val   Accuracy: %.4f (max: %.4f)\" % (val_PRF1[0], val_max_acc))\n",
    "    print(\"Test  Accuracy: %.4f (max: %.4f)\" % (test_PRF1[0], test_max_acc))\n",
    "    \n",
    "    print(\"Train Macro F1 Semeval Official: %.4f (max: %.4f)\" % (train_PRF1[3], train_max_f1))\n",
    "    print(\"Val   Macro F1 Semeval Official: %.4f (max: %.4f)\" % (val_PRF1[3], val_max_f1))\n",
    "    print(\"Test  Macro F1 Semeval Official: %.4f (max: %.4f)\" % (test_PRF1[3], test_max_f1))\n",
    "    print(\"Test P %.4f | R %.4f | macro_F1: %.4f\" % test_PRF1[1:] )\n",
    "    print(\"Test test_max_f1_final: %.4f (max: %.4f)\" % (test_f1_final, test_f1_final_max) )\n",
    "        \n",
    "    print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# # In[ ]:\n",
    "\n",
    "# from keras_visualize_activations_master import read_activations\n",
    "\n",
    "# def create_heatmap_file(model, layer_name, out_file_path=\"./model/test_heatmap.txt\", batch_size=500): \n",
    "    \n",
    "#     print(\"Loading data...\")\n",
    "    \n",
    "#     data_npy_path = './data/data_all.npy'\n",
    "#     train_file = \"./files/train_attn_sp.txt\"\n",
    "#     val_file = \"./files/val_attn_sp.txt\"\n",
    "#     test_file = \"./files/test_attn_sp.txt\"\n",
    "\n",
    "#     train_set, val_set, test_set, embedding, label_to_int, int_to_label = np.load(data_npy_path)\n",
    "#     train_x, train_y = train_set\n",
    "#     val_x, val_y = val_set\n",
    "#     test_x, test_y = test_set\n",
    "#     train_x_copy = train_x\n",
    "#     train_y_copy = train_y\n",
    "#     max_sentence_len = train_x.shape[1]\n",
    "#     n_out = len(label_to_int)\n",
    "    \n",
    "#     l2_val = 0.00001\n",
    "#     embeddings_reg = regularizers.l2(l2_val)\n",
    "#     kernel_reg = regularizers.l2(l2_val)\n",
    "#     recurrent_reg = regularizers.l2(l2_val)\n",
    "\n",
    "#     # # Getting activations \n",
    "    \n",
    "#     print(\"Getting activations...\")\n",
    "\n",
    "#     def get_activation(data, model, batch_size=100):\n",
    "#         ret = None\n",
    "#         for i in range(0, data.shape[0], batch_size): \n",
    "#             start = i \n",
    "#             endd = min(i+batch_size, data.shape[0]) \n",
    "#             act = read_activations.get_activations(model, data[start:endd], layer_name=layer_name, print_shape_only=True)\n",
    "#             if i == 0: \n",
    "#                 ret = act[0] \n",
    "#             else: \n",
    "#                 ret = np.concatenate((ret, act[0]))\n",
    "#         return ret \n",
    "\n",
    "#     test_act = get_activation(test_x, model, batch_size)\n",
    "#     print(\"Activatons shape test_act.shape\", test_act.shape)\n",
    "\n",
    "#     def softmax(x):\n",
    "#         e_x = np.exp(x - np.max(x))\n",
    "#         return e_x / e_x.sum(axis=0)    \n",
    "    \n",
    "#     print(\"Creating heatmap file...\")\n",
    "    \n",
    "#     def write_heatmap_file(act, in_file_name, out_file_path):\n",
    "#         f = open(in_file_name, 'r')\n",
    "#         lines = f.readlines()\n",
    "#         f.close()\n",
    "\n",
    "#         num_labels = [ l.strip().split()[:2] for l in lines ]\n",
    "#         lines = [ l.strip().split()[2:] for l in lines ]\n",
    "\n",
    "#         print(\"act\", act.shape)\n",
    "#         act = np.sum(act, axis=-2)\n",
    "#         print(\"act\", act.shape)\n",
    "\n",
    "#         for j in range(act.shape[0]):\n",
    "#             act[j] = softmax(act[j])\n",
    "            \n",
    "#         print(\"np.sum(act[0])\", np.sum(act[0]))\n",
    "\n",
    "#         f = open(out_file_path, \"w\")\n",
    "\n",
    "#         for i in range(len(lines)): \n",
    "#             f.write(str(num_labels[i][1]) + \"\\n\")\n",
    "#             for j in range(len(lines[i])):\n",
    "#                 tmp = ( \"%.6f \" % (act[i][j]*100.0)  )\n",
    "#                 f.write(lines[i][j] + \"\\t\" + tmp + \"\\n\")\n",
    "#             f.write(\"\\n\")\n",
    "\n",
    "#         f.close()\n",
    "        \n",
    "#         print(\"Created\", out_file_path)\n",
    "    \n",
    "#     write_heatmap_file(test_act, test_file, out_file_path)\n",
    "#     return test_act\n",
    "    \n",
    "\n",
    "# model_keras_path = \"./model/model.keras\"\n",
    "# model = load_model(model_keras_path, custom_objects={\"tf\": tf, \"MaskMaxPoolingLayer\": MaskMaxPoolingLayer})\n",
    "# model.summary()\n",
    "\n",
    "# layer_name = \"attn_softmax\"\n",
    "# act = create_heatmap_file(model, layer_name, out_file_path=\"./model/test_heatmap.txt\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}