Train.py

# -*- coding: utf-8 -*-
"""SentimentalAnalysisColab.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1hhniR3o_P-rUdGdu0U67mKu7UmpHOFso
"""

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import time
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
import bz2

save = False
"""# File reading"""

trainfile = bz2.BZ2File('./Dataset/train.ft.txt.bz2','r')
lines = trainfile.readlines()

docSentimentList=[]
def getDocumentSentimentList(docs,splitStr='__label__'):
    for i in range(len(docs)):
        #print('Processing doc ',i,' of ',len(docs))
        text=str(lines[i])
        #print(text)
        splitText=text.split(splitStr)
        secHalf=splitText[1]
        text=secHalf[2:len(secHalf)-1]
        sentiment=secHalf[0]
        docSentimentList.append([text,sentiment])
    print('Done!!')
    return docSentimentList

docSentimentList=getDocumentSentimentList(lines[:1000000],splitStr='__label__')

train_df = pd.DataFrame(docSentimentList,columns=['Text','Sentiment'])

train_df['Sentiment'][train_df['Sentiment']=='1'] = 0
train_df['Sentiment'][train_df['Sentiment']=='2'] = 1

train_df['word_count'] = train_df['Text'].str.lower().str.split().apply(len)


"""# Text preprocessing
"""

import string 
import re
def remove_punc(s):
    table = str.maketrans({key: None for key in string.punctuation})
    return s.translate(table)
def remove_url(text):
     url=re.compile(r"https?://\S+|www\.\S+")
     return url.sub(r" ",text)

def remove_html(text):
  cleanr = re.compile('<.*?>')
  return cleanr.sub(r" ",text)

def remove_num(text):
   output = re.sub(r'\d+', '', text)
   return output

train_df['Text'] = train_df['Text'].apply(remove_punc)
train_df['Text'] = train_df['Text'].apply(remove_html)
train_df['Text'] = train_df['Text'].apply(remove_url)
train_df['Text'] = train_df['Text'].apply(remove_num)


train_df1 = train_df[:][train_df['word_count']<=25]

from sklearn.feature_extraction import text
from sklearn.feature_extraction.text import CountVectorizer
st_wd = text.ENGLISH_STOP_WORDS
c_vector = CountVectorizer(stop_words = st_wd,min_df=.0001,lowercase=1)
c_vector.fit(train_df1['Text'].values)

word_list = list(c_vector.vocabulary_.keys())
stop_words = list(c_vector.stop_words)

len(stop_words),len(word_list)

def remove_words(raw_sen,stop_words):
    sen = [w for w in raw_sen if w not in stop_words]
    return sen

def reviewEdit(raw_sen_list,stop_words):
    sen_list = []
    for i in range(len(raw_sen_list)):
        raw_sen = raw_sen_list[i].split()
        sen_list.append(remove_words(raw_sen,stop_words))
    return sen_list

sen_list = reviewEdit(list(train_df1['Text']),stop_words)

#BERT TEST
#!pip install transformers
#from transformers import AutoTokenizer, TFAutoModel, AutoConfig, AutoModel, AutoModelForMaskedLM
#config = AutoConfig.from_pretrained('bert-base-uncased', hidden_size=100)
#config 
#tokenizer = AutoTokenizer.from_pretrained("distilroberta-base", add_prefix_space=True)
#model = TFAutoModel.from_pretrained("distilroberta-base")
#inputs = tokenizer(sen_list, max_length=25, truncation=True, padding=True, is_split_into_words=True, return_tensors="tf")
#outputs = model(**inputs)

from gensim.models import fasttext
ft_model = fasttext.FastText(sen_list,size=100)

from sklearn.model_selection import train_test_split
def getEmbedding3D(sen_list,wv_model):
    word_set = set(wv_model.wv.index2word)
    X = np.zeros([len(sen_list),25,100])
    c = 0
    for sen in sen_list:
        nw=24
        for w in list(reversed(sen)):
            if w in word_set:
                X[c,nw] = wv_model[w]
                nw=nw-1
        c=c+1
    return X

X = getEmbedding3D(sen_list,ft_model)

y = train_df1['Sentiment'].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42)
y_train = y_train.astype('bool')
y_test = y_test.astype('bool')

if save:
    np.save("./Dataset/test_X_ANN", X_test, allow_pickle=True, fix_imports=True)
    np.save("./Dataset/y_test_ANN", y_test, allow_pickle=True, fix_imports=True)

def get_embedding2D(sen_list,wv_model):
    word_set = set(wv_model.wv.index2word)
    X = np.zeros([len(sen_list),100])
    c = 0
    for sen in sen_list:
        nw=24
        for w in list(reversed(sen)):
            if w in word_set:
                X[c] = wv_model[w]
                nw=nw-1
        c=c+1
    return X

X = get_embedding2D(sen_list,ft_model)

y = train_df1['Sentiment'].values
X_trainBaseline, X_testBaseline, y_trainBaseline, y_testBaseline = train_test_split(X, y, test_size=0.1, random_state=42)
y_trainBaseline = y_trainBaseline.astype('bool')
y_testBaseline = y_testBaseline.astype('bool')


if save:
    np.save("/Dataset/test_X_baseline", X_testBaseline, allow_pickle=True, fix_imports=True)
    np.save("/Dataset/y_test_baseline", y_testBaseline, allow_pickle=True, fix_imports=True)

"""# Baseline models"""

from sklearn import svm
start_time = time.time()
svm = svm.SVC()
svm.fit(X_trainBaseline, y_trainBaseline)
print("Training time: %s seconds" % (time.time() - start_time))

from sklearn.metrics import accuracy_score

start_time = time.time()
y_pred = svm.predict(X_testBaseline)
print("Prediction time for the test set: %s seconds" % (time.time() - start_time))

accuracy_score(y_test, y_pred)

from sklearn.ensemble import RandomForestClassifier
start_time = time.time()
RF = RandomForestClassifier(max_depth=500, random_state=0)
RF.fit(X_trainBaseline, y_trainBaseline)
print("Training time: %s s" % (time.time() - start_time))

start_time = time.time()
y_pred = RF.predict(X_testBaseline)
print("Prediction time for the test set: %s" % (time.time() - start_time))

accuracy_score(y_test, y_pred)

from joblib import dump
dump(svm, './Models/svm.joblib')
dump(RF, './Models/randomforest.joblib')

"""# BDRNN train"""

from keras.models import Model
from keras.layers import Dense, Activation,LSTM ,GRU , Bidirectional,Input
import matplotlib.pyplot as plt


def plot_graphs(history, string):
  plt.plot(history.history[string])
  plt.plot(history.history['val_'+string])
  plt.xlabel("Epochs")
  plt.ylabel(string)
  plt.legend([string, 'val_'+string])
  plt.show()

"""## BDNN LSTM"""

input_st  = Input(shape=(25,100))
lstm1 = Bidirectional(LSTM(200,input_shape=(25,100),activation='relu',return_sequences=True),merge_mode='mul')(input_st)
lstm2 = Bidirectional(LSTM(1,input_shape=(25,100),activation='relu',return_sequences=True),merge_mode='mul')(lstm1)
lstm2 = Activation('sigmoid')(lstm2)

dense = Dense(100,activation='relu')(lstm2)
output = Dense(1,activation='sigmoid')(dense)

model = Model(inputs=input_st, outputs=output)
print(model.summary())

start_time = time.time()
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])
hist = model.fit(X_train,y_train,validation_split=0.1,
          epochs=10, batch_size=512)
print("Training time BDRNN LSTM: %s seconds" % (time.time() - start_time))

y_test = y_test.astype('bool')
model.evaluate(X_test, y_test, batch_size=64)

plot_graphs(hist, "accuracy")

plot_graphs(hist, "loss")

if save:
    model.save("/Dataset/BidirectionalLSTM")

del model

"""##BDNN GRU"""

input_st  = Input(shape=(25,100))
lstm1 = Bidirectional(GRU(200,input_shape=(25,100),activation='relu',return_sequences=True),merge_mode='mul')(input_st)
lstm2 = Bidirectional(GRU(1,input_shape=(25,100),activation='relu',return_sequences=True),merge_mode='mul')(lstm1)
dense = Dense(100,activation='relu')(lstm2)
output = Dense(1,activation='sigmoid')(dense)

model = Model(inputs=input_st, outputs=output)
print(model.summary())

start_time = time.time()
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])
hist = model.fit(X_train,y_train,validation_split=0.1,
          epochs=10, batch_size=512)
print("Training time BDRNN GRU: %s seconds" % (time.time() - start_time))

model.evaluate(X_test, y_test, batch_size=64)

plot_graphs(hist, "accuracy")

plot_graphs(hist, "loss")

if save:
    model.save("./Models")

del model