-
Notifications
You must be signed in to change notification settings - Fork 1
/
utils.py
408 lines (333 loc) · 15.4 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
import numpy as np
import tensorflow as tf
from keras import backend as K
import keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.models import Sequential, load_model, Model
from keras.layers import concatenate, Activation, GlobalAveragePooling1D, GlobalMaxPooling1D, Layer, Dense, Embedding, LSTM, GRU, Dropout, SpatialDropout1D, Input, Average, Bidirectional, BatchNormalization
from keras.callbacks import Callback
from keras import initializers, regularizers, constraints, optimizers, layers
import gensim.models as gsm
import re
import json, argparse, os
import re
import io
import sys
print("Loading utils module")
label2emotion = {0:"others", 1:"happy", 2: "sad", 3:"angry"}
emotion2label = {"others":0, "happy":1, "sad":2, "angry":3}
def get_sswe_embeddings(wordIndex):
oov = []
count, total = 0, 0
embeddingMatrix = np.zeros((len(wordIndex) + 1, 150))
embeddingsIndex = {}
def get_start_index(lis):
for i, j in enumerate(lis):
try:
_ = float(j)
return i
except:
continue
faulty_count = 0
with open('sswe.tsv', 'r', errors='ignore') as f:
for i, line in enumerate(f):
stuff = line.rstrip('\n').split('\t')
breakoff_point = get_start_index(stuff)
word = "".join(stuff[:breakoff_point])
try:
embeddingsIndex[word] = np.asarray(stuff[breakoff_point:breakoff_point + 150], dtype='float32')
except:
faulty_count += 1
continue
print('Found unreadable %d word vectors' % faulty_count)
print('Found %s word vectors.' % len(embeddingsIndex))
for word, i in wordIndex.items():
embeddingVector = embeddingsIndex.get(word)
total += 1
if embeddingVector is not None:
# words not found in embedding index will be all-zeros.
embeddingMatrix[i] = embeddingVector
count += 1
else:
oov.append(word)
print("Found embedding for", str((100 * count) / total), "% embeddings")
return embeddingMatrix, oov
def split_into_three(texts, tknzr):
middle, left, right = [], [], []
for text in texts:
l, m, r = text.split(' <eos> ')
middle.append(m)
left.append(l)
right.append(r)
tokenize = lambda x: tknzr.texts_to_sequences(x)
return (tokenize(left), tokenize(middle), tokenize(right))
def add_emoji_embedding(wordIndex, existingEmbedding):
e2v = gsm.KeyedVectors.load_word2vec_format('./emoji2vec/pre-trained/emoji2vec.bin', binary=True)
count, total= 0, 0
for word, i in wordIndex.items():
total += 1
try:
embeddingVector = e2v.get_vector(word)
existingEmbedding[i] = embeddingVector
count += 1
except:
continue
print("Found embedding for", str((100 * count) / total), "% embeddings")
return existingEmbedding
def getEmbeddingMatrix(wordIndex, EMBEDDING_DIM):
embeddingsIndex = {}
# Load the embedding vectors from ther GloVe file
with io.open(os.path.join('./glove.6B.%dd.txt' % EMBEDDING_DIM), encoding="utf8") as f:
for line in f:
values = line.split()
word = values[0]
embeddingVector = np.asarray(values[1:], dtype='float32')
embeddingsIndex[word] = embeddingVector
print('Found %s word vectors.' % len(embeddingsIndex))
oov = []
oov_indices = []
# Minimum word index of any word is 1.
embeddingMatrix = np.zeros((len(wordIndex) + 1, EMBEDDING_DIM))
count, total= 0, 0
for word, i in wordIndex.items():
embeddingVector = embeddingsIndex.get(word)
total += 1
if embeddingVector is not None:
# words not found in embedding index will be all-zeros.
embeddingMatrix[i] = embeddingVector
count += 1
else:
oov_indices.append(wordIndex.get(word))
oov.append(word)
print("Found embedding for", str((100 * count) / total), "% embeddings")
return embeddingMatrix, oov, oov_indices
# https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate/code
class CyclicLR(Callback):
"""This callback implements a cyclical learning rate policy (CLR).
The method cycles the learning rate between two boundaries with
some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).
The amplitude of the cycle can be scaled on a per-iteration or
per-cycle basis.
This class has three built-in policies, as put forth in the paper.
"triangular":
A basic triangular cycle w/ no amplitude scaling.
"triangular2":
A basic triangular cycle that scales initial amplitude by half each cycle.
"exp_range":
A cycle that scales initial amplitude by gamma**(cycle iterations) at each
cycle iteration.
For more detail, please see paper.
# Example
```python
clr = CyclicLR(base_lr=0.001, max_lr=0.006,
step_size=2000., mode='triangular')
model.fit(X_train, Y_train, callbacks=[clr])
```
Class also supports custom scaling functions:
```python
clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))
clr = CyclicLR(base_lr=0.001, max_lr=0.006,
step_size=2000., scale_fn=clr_fn,
scale_mode='cycle')
model.fit(X_train, Y_train, callbacks=[clr])
```
# Arguments
base_lr: initial learning rate which is the
lower boundary in the cycle.
max_lr: upper boundary in the cycle. Functionally,
it defines the cycle amplitude (max_lr - base_lr).
The lr at any cycle is the sum of base_lr
and some scaling of the amplitude; therefore
max_lr may not actually be reached depending on
scaling function.
step_size: number of training iterations per
half cycle. Authors suggest setting step_size
2-8 x training iterations in epoch.
mode: one of {triangular, triangular2, exp_range}.
Default 'triangular'.
Values correspond to policies detailed above.
If scale_fn is not None, this argument is ignored.
gamma: constant in 'exp_range' scaling function:
gamma**(cycle iterations)
scale_fn: Custom scaling policy defined by a single
argument lambda function, where
0 <= scale_fn(x) <= 1 for all x >= 0.
mode paramater is ignored
scale_mode: {'cycle', 'iterations'}.
Defines whether scale_fn is evaluated on
cycle number or cycle iterations (training
iterations since start of cycle). Default is 'cycle'.
"""
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
self.trn_iterations = 0.
self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):
"""Resets cycle iterations.
Optional boundary/step size adjustment.
"""
if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
if self.scale_mode == 'cycle':
return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)
else:
return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)
def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
else:
K.set_value(self.model.optimizer.lr, self.clr())
def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', []).append(self.trn_iterations)
for k, v in logs.items():
self.history.setdefault(k, []).append(v)
K.set_value(self.model.optimizer.lr, self.clr())
def getMetrics(predictions, ground):
discretePredictions = to_categorical(predictions.argmax(axis=1), 4)
truePositives = np.sum(discretePredictions*ground, axis=0)
falsePositives = np.sum(np.clip(discretePredictions - ground, 0, 1), axis=0)
falseNegatives = np.sum(np.clip(ground-discretePredictions, 0, 1), axis=0)
print("True Positives per class : ", truePositives)
print("False Positives per class : ", falsePositives)
print("False Negatives per class : ", falseNegatives)
# ------------- Macro level calculation ---------------
macroPrecision = 0
macroRecall = 0
# We ignore the "Others" class during the calculation of Precision, Recall and F1
for c in range(1, 4):
precision = truePositives[c] / (truePositives[c] + falsePositives[c])
macroPrecision += precision
recall = truePositives[c] / (truePositives[c] + falseNegatives[c])
macroRecall += recall
f1 = ( 2 * recall * precision ) / (precision + recall) if (precision+recall) > 0 else 0
print("Class %s : Precision : %.3f, Recall : %.3f, F1 : %.3f" % (label2emotion[c], precision, recall, f1))
macroPrecision /= 3
macroRecall /= 3
macroF1 = (2 * macroRecall * macroPrecision ) / (macroPrecision + macroRecall) if (macroPrecision+macroRecall) > 0 else 0
print("Ignoring the Others class, Macro Precision : %.4f, Macro Recall : %.4f, Macro F1 : %.4f" % (macroPrecision, macroRecall, macroF1))
# ------------- Micro level calculation ---------------
truePositives = truePositives[1:].sum()
falsePositives = falsePositives[1:].sum()
falseNegatives = falseNegatives[1:].sum()
print("Ignoring the Others class, Micro TP : %d, FP : %d, FN : %d" % (truePositives, falsePositives, falseNegatives))
microPrecision = truePositives / (truePositives + falsePositives)
microRecall = truePositives / (truePositives + falseNegatives)
microF1 = ( 2 * microRecall * microPrecision ) / (microPrecision + microRecall) if (microPrecision+microRecall) > 0 else 0
# -----------------------------------------------------
predictions = predictions.argmax(axis=1)
ground = ground.argmax(axis=1)
accuracy = np.mean(predictions==ground)
print("Accuracy : %.4f, Micro Precision : %.4f, Micro Recall : %.4f, Micro F1 : %.4f" % (accuracy, microPrecision, microRecall, microF1))
return accuracy, microPrecision, microRecall, microF1
# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
self.W_regularizer = regularizers.get(W_regularizer)
self.b_regularizer = regularizers.get(b_regularizer)
self.W_constraint = constraints.get(W_constraint)
self.b_constraint = constraints.get(b_constraint)
self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self).__init__(**kwargs)
def build(self, input_shape):
assert len(input_shape) == 3
self.W = self.add_weight((input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight((input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
K.reshape(self.W, (features_dim, 1))), (-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
a = K.exp(eij)
if mask is not None:
a *= K.cast(mask, K.floatx())
a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
a = K.expand_dims(a)
weighted_input = x * a
return K.sum(weighted_input, axis=1)
def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def microF1Loss(ground, predictions):
discretePredictions = K.one_hot(K.argmax(predictions, axis=1), 4)
truePositives = K.sum(discretePredictions*ground, axis=0)
falsePositives = K.sum(K.clip(discretePredictions - ground, 0, 1), axis=0)
falseNegatives = K.sum(K.clip(ground-discretePredictions, 0, 1), axis=0)
macroPrecision = 0
macroRecall = 0
for c in range(1, 4):
precision = truePositives[c] / (truePositives[c] + falsePositives[c])
macroPrecision += precision
recall = truePositives[c] / (truePositives[c] + falseNegatives[c])
macroRecall += recall
f1 = ( 2 * recall * precision ) / (precision + recall + K.epsilon())
macroPrecision /= 3
macroRecall /= 3
macroF1 = (2 * macroRecall * macroPrecision ) / (macroPrecision + macroRecall + K.epsilon())
truePositives = K.sum(truePositives[1:])
falsePositives = K.sum(falsePositives[1:])
falseNegatives = K.sum(falseNegatives[1:])
microPrecision = truePositives / (truePositives + falsePositives)
microRecall = truePositives / (truePositives + falseNegatives)
microF1 = ( 2 * microRecall * microPrecision ) / (microPrecision + microRecall + + K.epsilon())
return microF1