Twitter Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('data/tweetset.csv', encoding=\"windows-1254\")\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('data/TurkishTweets.csv')\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare & Concatenate Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop the unnecessary columns in 1. dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1.drop(['Unnamed: 2', 'Unnamed: 3', 'Unnamed: 4', 'Unnamed: 5'], axis = 1)\n",
    "df1.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2.dropna()\n",
    "df2.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Change labels in datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2['Etiket'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2[\"Etiket\"].replace({\"kızgın\": \"Negative\", \"korku\": \"Negative\", \"mutlu\": \"Positive\", \n",
    "                        \"surpriz\": \"Positive\", \"üzgün\": \"Negative\"}, inplace=True)\n",
    "df2['Etiket'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.columns = ['Tweets', 'Sentiment']\n",
    "df2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1[\"Tip\"].replace({\"Pozitif\": \"Positive\", \" Negatif\": \"Negative\", \"Negatif\": \"Negative\" }, inplace=True)\n",
    "df1['Tip'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.isnull().sum()\n",
    "df1.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df1.reindex(columns=['Paylaşım','Tip'])\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.columns = ['Tweets', 'Sentiment']\n",
    "df1.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Concatenate datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df1.append(df2, ignore_index=True)\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert to lower case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = [token.lower() for token in df['Tweets']]\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove @ mentions and hyperlinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "found = df[df['Tweets'].str.contains('@')]\n",
    "found.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = df['Tweets'].replace('@[A-Za-z0-9]+', '', regex=True).replace('@[A-Za-z0-9]+', '', regex=True)\n",
    "found = df[df['Tweets'].str.contains('@')]\n",
    "found.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "found = df[df['Tweets'].str.contains('http')]\n",
    "found.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = df['Tweets'].replace(r'http\\S+', '', regex=True).replace(r'www\\S+', '', regex=True)\n",
    "found = df[df['Tweets'].str.contains('http')]\n",
    "found.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove Punctations & Emojies & Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = df['Tweets'].copy()\n",
    "new_sent = []\n",
    "i = 0\n",
    "for sentence in sentences:\n",
    "    new_sentence = re.sub('[0-9]+', '', sentence)\n",
    "    new_sent.append(new_sentence)\n",
    "    i += 1\n",
    "    \n",
    "df['Tweets'] = new_sent\n",
    "df['Tweets'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "\n",
    "table = str.maketrans('', '', string.punctuation)\n",
    "sentences = df['Tweets'].copy()\n",
    "new_sent = []\n",
    "for sentence in sentences:\n",
    "    words = sentence.split()\n",
    "    stripped = [w.translate(table) for w in words]\n",
    "    new_sent.append(stripped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = new_sent\n",
    "df['Tweets'].head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zemberek-NLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import logging\n",
    "\n",
    "from zemberek import (\n",
    "    TurkishSpellChecker,\n",
    "    TurkishSentenceNormalizer,\n",
    "    TurkishSentenceExtractor,\n",
    "    TurkishMorphology,\n",
    "    TurkishTokenizer\n",
    ")\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "morphology = TurkishMorphology.create_with_defaults()\n",
    "normalizer = TurkishSentenceNormalizer(morphology)\n",
    "extractor = TurkishSentenceExtractor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentence Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_long_text(text):\n",
    "    normalized_sentences = [normalizer.normalize(word) for word in text]\n",
    "    normalized_text = \" \".join(normalized_sentences)\n",
    "    return normalized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = df['Tweets'].copy()\n",
    "new_sent = []\n",
    "start = time.time()\n",
    "\n",
    "for token in sentences:   \n",
    "    if token.count('') > 0:\n",
    "        token = list(filter(('').__ne__, token))\n",
    "    new_token = normalize_long_text(token)\n",
    "    new_sent.append(new_token)\n",
    "\n",
    "logger.info(f\"Sentences normalized in: {time.time() - start} s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "stops = set(stopwords.words('turkish'))\n",
    "print(stops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "splitted_words = []\n",
    "for sent in new_sent:\n",
    "    words = sent.split()\n",
    "    splitted_words.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_sent = []\n",
    "for sentence in splitted_words:\n",
    "    new_sentence = [w for w in sentence if w not in stops]\n",
    "    clean_sent.append(new_sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for token in clean_sent:\n",
    "    j = 0\n",
    "    for word in token:\n",
    "        new_word = word.replace('\"', '').replace(\"’\", '').replace(\"'\", '').replace(\"”\", '')\n",
    "        token[j] = new_word\n",
    "        j += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zeyrek\n",
    "\n",
    "analyzer = zeyrek.MorphAnalyzer()\n",
    "lem_sent = []\n",
    "for sent in clean_sent:\n",
    "    normalized_sent = []\n",
    "    for word in sent:\n",
    "        if word == '':\n",
    "            continue\n",
    "        else:\n",
    "            lem_word = analyzer.lemmatize(word)\n",
    "            normalized_sent.append(lem_word[0][1][0])\n",
    "    lem_sent.append(normalized_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = lem_sent.copy()\n",
    "for sent in x:\n",
    "    i = 0\n",
    "    for token in sent:\n",
    "        sent[i] = token.lower()\n",
    "        i += 1\n",
    "lem_sent = x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lem_sent = list(filter(('').__ne__, lem_sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = lem_sent\n",
    "df['Tweets'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = df.Tweets.apply(' '.join)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove Rare Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq = pd.Series(' '.join(df['Tweets']).split()).value_counts()\n",
    "less_freq = list(freq[freq == 1].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Tweets'] = df['Tweets'].apply(lambda x: \" \".join(x for x in x.split() if x not in less_freq))\n",
    "df['Tweets'].head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Visualization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Positive Negative Balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "import tweepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_list = df['Tweets']\n",
    "negative_list = df[df['Sentiment'] == \"Negative\"]\n",
    "positive_list = df[df['Sentiment'] == \"Positive\"]\n",
    "\n",
    "print(\"total number: \",len(tweet_list))\n",
    "print(\"positive number: \",len(positive_list))\n",
    "print(\"negative number: \", len(negative_list))\n",
    "\n",
    "print()\n",
    "\n",
    "print(\"% of positive: \",100*len(positive_list)/len(tweet_list))\n",
    "print(\"% of negative: \",100*len(negative_list)/len(tweet_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "positive = int(100*len(positive_list)/len(tweet_list))\n",
    "negative = int(100*len(negative_list)/len(tweet_list))\n",
    "\n",
    "labels = ['Positive ['+str(positive)+'%]','Negative ['+str(negative)+'%]']\n",
    "sizes = [positive, negative]\n",
    "colors = ['yellowgreen', 'blue']\n",
    "patches, texts = plt.pie(sizes,colors=colors, startangle=90, radius=50)\n",
    "plt.style.use('default')\n",
    "plt.legend(labels)\n",
    "plt.title(\"Sentiment Analysis Result\")\n",
    "plt.axis(\"equal\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing Top Words\n",
    "\n",
    "The distribution of top unigrams after removing stop words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def get_top_n_words(corpus, n=None):\n",
    "    vec = CountVectorizer().fit(corpus)\n",
    "    bag_of_words = vec.transform(corpus)\n",
    "    sum_words = bag_of_words.sum(axis=0)\n",
    "    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n",
    "    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n",
    "    return words_freq[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_words = get_top_n_words(df['Tweets'], 20)\n",
    "common_df = pd.DataFrame(common_words, columns = ['Tweets', 'count'])\n",
    "common_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_df.groupby('Tweets').sum()['count'].sort_values(ascending=False).plot(\n",
    "    kind='bar',\n",
    "    figsize=(8, 6),\n",
    "    xlabel = \"Top Words\",\n",
    "    ylabel = \"Count\",\n",
    "    title = \"Bar Chart of Top Words Frequency\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing N-Grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top Bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_n_bigram(corpus, n=None):\n",
    "    vec = CountVectorizer(ngram_range=(2,2)).fit(corpus)\n",
    "    bag_of_words = vec.transform(corpus)\n",
    "    sum_words = bag_of_words.sum(axis=0)\n",
    "    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n",
    "    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n",
    "    return words_freq[:n]\n",
    "\n",
    "common_words2 = get_top_n_bigram(df['Tweets'], 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_bigram = pd.DataFrame(common_words2, columns=['Tweets', \"Count\"])\n",
    "top_bigram.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_bigram.groupby('Tweets').sum()['Count'].sort_values(ascending=False).plot(\n",
    "    kind='bar',\n",
    "    figsize=(8,6),\n",
    "    xlabel = \"Bigram Words\",\n",
    "    ylabel = \"Count\",\n",
    "    title = \"Bar chart of Bigrams Frequency\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top Trigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_n_trigram(corpus, n=None):\n",
    "    vec = CountVectorizer(ngram_range=(3, 3)).fit(corpus)\n",
    "    bag_of_words = vec.transform(corpus)\n",
    "    sum_words = bag_of_words.sum(axis=0)\n",
    "    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n",
    "    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n",
    "    return words_freq[:n]\n",
    "\n",
    "common_words3 = get_top_n_trigram(df['Tweets'], 30)\n",
    "top_trigram = pd.DataFrame(common_words3, columns = ['Tweets' , 'Count'])\n",
    "top_trigram.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_trigram.groupby('Tweets').sum()['Count'].sort_values(ascending=False).plot(\n",
    "    kind='bar',\n",
    "    figsize=(8,6),\n",
    "    xlabel = \"Trigram Words\",\n",
    "    ylabel = \"Count\",\n",
    "    title = \"Bar chart of Trigrams Frequency\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WordCloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud, STOPWORDS\n",
    "\n",
    "def creat_wordcloud(tweets):\n",
    "    comment_words = ''\n",
    "    stopwords = set(STOPWORDS)\n",
    "    \n",
    "    # iterate through the csv file\n",
    "    for val in tweets:\n",
    "\n",
    "        # typecaste each val to string\n",
    "        val = str(val)\n",
    "\n",
    "        # split the value\n",
    "        tokens = val.split()\n",
    "\n",
    "        # Converts each token into lowercase\n",
    "        for i in range(len(tokens)):\n",
    "            tokens[i] = tokens[i].lower()\n",
    "\n",
    "        comment_words += \" \".join(tokens)+\" \"\n",
    "\n",
    "    wordcloud = WordCloud(width = 1200, height = 800,\n",
    "                    background_color ='white',\n",
    "                    max_words=3000,\n",
    "                    stopwords = stopwords,\n",
    "                    min_font_size = 10,\n",
    "                    repeat = True).generate(comment_words)\n",
    "\n",
    "    # plot the WordCloud image                       \n",
    "    plt.figure(figsize = (8, 8), facecolor = None)\n",
    "    plt.imshow(wordcloud)\n",
    "    plt.axis(\"off\")\n",
    "    plt.tight_layout(pad = 0)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All Tweets\n",
    "creat_wordcloud(df['Tweets'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Positive Tweets\n",
    "creat_wordcloud(df[df['Sentiment'] == 'Positive'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Negative Tweets\n",
    "creat_wordcloud(df[df['Sentiment'] == 'Negative'].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "# Packages for data preparation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Packages for modeling\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_WORDS = 10000  # Parameter indicating the number of words we'll put in the dictionary\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(df.Tweets, df.Sentiment, test_size=0.1, random_state=37)\n",
    "print('# Train data samples:', X_train.shape[0])\n",
    "print('# Test data samples:', X_test.shape[0])\n",
    "assert X_train.shape[0] == y_train.shape[0]\n",
    "assert X_test.shape[0] == y_test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tk = Tokenizer()\n",
    "tk.fit_on_texts(X_train)\n",
    "\n",
    "print('Fitted tokenizer on {} documents'.format(tk.document_count))\n",
    "print('{} words in dictionary'.format(tk.num_words))\n",
    "print('Top 5 most common words are:', collections.Counter(tk.word_counts).most_common(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_seq = tk.texts_to_sequences(X_train)\n",
    "X_test_seq = tk.texts_to_sequences(X_test)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(X_train[0], X_train_seq[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_seq(seqs, nb_features = NB_WORDS):\n",
    "    ohs = np.zeros((len(seqs), nb_features))\n",
    "    for i, s in enumerate(seqs):\n",
    "        ohs[i, s] = 1.\n",
    "    return ohs\n",
    "\n",
    "X_train_oh = one_hot_seq(X_train_seq)\n",
    "X_test_oh = one_hot_seq(X_test_seq)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(X_train_seq[0], X_train_oh[0]))\n",
    "print('For this example we have {} features with a value of 1.'.format(X_train_oh[0].sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "y_train_le = le.fit_transform(y_train)\n",
    "y_test_le = le.transform(y_test)\n",
    "y_train_oh = to_categorical(y_train_le)\n",
    "y_test_oh = to_categorical(y_test_le)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(y_train[0], y_train_le[0]))\n",
    "print('\"{}\" is converted into {}'.format(y_train_le[0], y_train_oh[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_rest, X_valid, y_train_rest, y_valid = train_test_split(X_train_oh, y_train_oh, test_size=0.1, random_state=37)\n",
    "\n",
    "assert X_valid.shape[0] == y_valid.shape[0]\n",
    "assert X_train_rest.shape[0] == y_train_rest.shape[0]\n",
    "\n",
    "print('Shape of validation set:',X_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = models.Sequential()\n",
    "base_model.add(layers.Dense(64, activation='relu', input_shape=(NB_WORDS,)))\n",
    "base_model.add(layers.Dense(64, activation='relu'))\n",
    "base_model.add(layers.Dense(2, activation='softmax'))\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_START_EPOCHS = 20  # Number of epochs we usually start to train with\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent\n",
    "\n",
    "def deep_model(model):\n",
    "    model.compile(optimizer='rmsprop'\n",
    "                  , loss='categorical_crossentropy'\n",
    "                  , metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(X_train_rest\n",
    "                       , y_train_rest\n",
    "                       , epochs=NB_START_EPOCHS\n",
    "                       , batch_size=BATCH_SIZE\n",
    "                       , validation_data=(X_valid, y_valid)\n",
    "                       , verbose=0)\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_history = deep_model(base_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_metric(history, metric_name):\n",
    "    metric = history.history[metric_name]\n",
    "    val_metric = history.history['val_' + metric_name]\n",
    "\n",
    "    e = range(1, NB_START_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, metric, 'bo', label='Train ' + metric_name)\n",
    "    plt.plot(e, val_metric, 'b', label='Validation ' + metric_name)\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(base_history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(base_history, 'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling overfitting\n",
    "\n",
    "* Option 1: reduce the network's size by removing layers or reducing the number of hidden elements in the layers\n",
    "* Option 2: add regularization, which comes down to adding a cost to the loss function for large weights\n",
    "* Option 3: adding dropout layers, which will randomly remove certain features by setting them to zero"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reducing network's size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_model = models.Sequential()\n",
    "reduced_model.add(layers.Dense(32, activation='relu', input_shape=(NB_WORDS,)))\n",
    "reduced_model.add(layers.Dense(2, activation='softmax'))\n",
    "reduced_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_history = deep_model(reduced_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_loss_with_baseline(h, model_name):\n",
    "    loss_base_model = base_history.history['val_loss']\n",
    "    loss_model = h.history['val_loss']\n",
    "\n",
    "    e = range(1, NB_START_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, loss_base_model, 'bo', label='Validation Loss Baseline Model')\n",
    "    plt.plot(e, loss_model, 'b', label='Validation Loss ' + model_name)\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(reduced_history, 'Reduced Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_model = models.Sequential()\n",
    "reg_model.add(layers.Dense(64, kernel_regularizer=regularizers.l2(0.001), activation='relu', input_shape=(NB_WORDS,)))\n",
    "reg_model.add(layers.Dense(64, kernel_regularizer=regularizers.l2(0.001), activation='relu'))\n",
    "reg_model.add(layers.Dense(2, activation='softmax'))\n",
    "reg_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_history = deep_model(reg_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(reg_history, 'Regularized Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding dropout layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_model = models.Sequential()\n",
    "drop_model.add(layers.Dense(64, activation='relu', input_shape=(NB_WORDS,)))\n",
    "drop_model.add(layers.Dropout(0.5))\n",
    "drop_model.add(layers.Dense(64, activation='relu'))\n",
    "drop_model.add(layers.Dropout(0.5))\n",
    "drop_model.add(layers.Dense(2, activation='softmax'))\n",
    "drop_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_history = deep_model(drop_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(drop_history, 'Dropout Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training on the full train data and evaluation on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model, epoch_stop):\n",
    "    model.fit(X_train_oh\n",
    "              , y_train_oh\n",
    "              , epochs=epoch_stop\n",
    "              , batch_size=BATCH_SIZE\n",
    "              , verbose=0)\n",
    "    results = model.evaluate(X_test_oh, y_test_oh)\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_results = test_model(base_model, 4)\n",
    "print('/n')\n",
    "print('Test accuracy of baseline model: {0:.2f}%'.format(base_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_results = test_model(reduced_model, 10)\n",
    "print('/n')\n",
    "print('Test accuracy of reduced model: {0:.2f}%'.format(reduced_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_results = test_model(reg_model, 5)\n",
    "print('/n')\n",
    "print('Test accuracy of regularized model: {0:.2f}%'.format(reg_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_results = test_model(drop_model, 6)\n",
    "print('/n')\n",
    "print('Test accuracy of dropout model: {0:.2f}%'.format(drop_results[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_WORDS = 10000  # Parameter indicating the number of words we'll put in the dictionary\n",
    "VAL_SIZE = 1000  # Size of the validation set\n",
    "NB_START_EPOCHS = 20  # Number of epochs we usually start to train with\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent\n",
    "MAX_LEN = 24  # Maximum number of words in a sequence\n",
    "GLOVE_DIM = 50  # Number of dimensions of the GloVe word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deep_model(model, X_train, y_train, X_valid, y_valid):\n",
    "    model.compile(optimizer='rmsprop'\n",
    "                  , loss='categorical_crossentropy'\n",
    "                  , metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(X_train\n",
    "                       , y_train\n",
    "                       , epochs=NB_START_EPOCHS\n",
    "                       , batch_size=BATCH_SIZE\n",
    "                       , validation_data=(X_valid, y_valid)\n",
    "                       , verbose=1)\n",
    "    return history\n",
    "\n",
    "def test_model(model, X_train, y_train, X_test, y_test, epoch_stop):\n",
    "    model.fit(X_train\n",
    "              , y_train\n",
    "              , epochs=epoch_stop\n",
    "              , batch_size=BATCH_SIZE\n",
    "              , verbose=0)\n",
    "    results = model.evaluate(X_test, y_test) \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(df.Tweets, df.Sentiment, test_size=0.1, random_state=37)\n",
    "print('# Train data samples:', X_train.shape[0])\n",
    "print('# Test data samples:', X_test.shape[0])\n",
    "assert X_train.shape[0] == y_train.shape[0]\n",
    "assert X_test.shape[0] == y_test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tk = Tokenizer(num_words=NB_WORDS)\n",
    "tk.fit_on_texts(X_train)\n",
    "\n",
    "X_train_seq = tk.texts_to_sequences(X_train)\n",
    "X_test_seq = tk.texts_to_sequences(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_lengths = X_train.apply(lambda x: len(x.split(' ')))\n",
    "seq_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_seq_trunc = pad_sequences(X_train_seq, maxlen=MAX_LEN)\n",
    "X_test_seq_trunc = pad_sequences(X_test_seq, maxlen=MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "y_train_le = le.fit_transform(y_train)\n",
    "y_test_le = le.transform(y_test)\n",
    "y_train_oh = to_categorical(y_train_le)\n",
    "y_test_oh = to_categorical(y_test_le)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_emb, X_valid_emb, y_train_emb, y_valid_emb = train_test_split(X_train_seq_trunc, y_train_oh, test_size=0.1, random_state=37)\n",
    "\n",
    "assert X_valid_emb.shape[0] == y_valid_emb.shape[0]\n",
    "assert X_train_emb.shape[0] == y_train_emb.shape[0]\n",
    "\n",
    "print('Shape of validation set:',X_valid_emb.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_model = models.Sequential()\n",
    "emb_model.add(layers.Embedding(NB_WORDS, 8, input_length=MAX_LEN))\n",
    "emb_model.add(layers.Flatten())\n",
    "emb_model.add(layers.Dense(2, activation='softmax'))\n",
    "emb_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_history = deep_model(emb_model, X_train_emb, y_train_emb, X_valid_emb, y_valid_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(emb_history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(emb_history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_results = test_model(emb_model, X_train_seq_trunc, y_train_oh, X_test_seq_trunc, y_test_oh, 6)\n",
    "print('/n')\n",
    "print('Test accuracy of word embeddings model: {0:.2f}%'.format(emb_results[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-trained Word Embedding: GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## !wget http://nlp.stanford.edu/data/glove.twitter.27B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## !unzip glove.twitter.27B.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_file = 'glove.twitter.27B.' + str(GLOVE_DIM) + 'd.txt'\n",
    "emb_dict = {}\n",
    "glove = open(glove_file)\n",
    "for line in glove:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    vector = np.asarray(values[1:], dtype='float32')\n",
    "    emb_dict[word] = vector\n",
    "glove.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_words = ['ben', 'koltuk', 'kraliçe', 'masa']\n",
    "for w in some_words:\n",
    "    if w in emb_dict.keys():\n",
    "        print('Found the word {} in the dictionary'.format(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_matrix = np.zeros((NB_WORDS, GLOVE_DIM))\n",
    "\n",
    "for w, i in tk.word_index.items():\n",
    "    if i < NB_WORDS:\n",
    "        vect = emb_dict.get(w)\n",
    "        if vect is not None:\n",
    "            emb_matrix[i] = vect\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_model = models.Sequential()\n",
    "glove_model.add(layers.Embedding(NB_WORDS, GLOVE_DIM, input_length=MAX_LEN))\n",
    "glove_model.add(layers.Flatten())\n",
    "glove_model.add(layers.Dense(2, activation='softmax'))\n",
    "glove_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_model.layers[0].set_weights([emb_matrix])\n",
    "glove_model.layers[0].trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_history = deep_model(glove_model, X_train_emb, y_train_emb, X_valid_emb, y_valid_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(glove_history, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(glove_history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_results = test_model(glove_model, X_train_seq_trunc, y_train_oh, X_test_seq_trunc, y_test_oh, 3)\n",
    "print('/n')\n",
    "print('Test accuracy of word glove model: {0:.2f}%'.format(glove_results[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train w/ more dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_model2 = models.Sequential()\n",
    "emb_model2.add(layers.Embedding(NB_WORDS, GLOVE_DIM, input_length=MAX_LEN))\n",
    "emb_model2.add(layers.Flatten())\n",
    "emb_model2.add(layers.Dense(2, activation='softmax'))\n",
    "emb_model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_history2 = deep_model(emb_model2, X_train_emb, y_train_emb, X_valid_emb, y_valid_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(emb_history2, 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(emb_history2, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_results2 = test_model(emb_model2, X_train_seq_trunc, y_train_oh, X_test_seq_trunc, y_test_oh, 3)\n",
    "print('/n')\n",
    "print('Test accuracy of word embedding model 2: {0:.2f}%'.format(emb_results2[1]*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}