hate_speech.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    },
    "colab": {
      "name": "Copy of submission_0.782_test.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "yjair9aPICXR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MkFW1k3Vvcwf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import f1_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nx-8GFjsvcwt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bc18jdwfJ5cU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip3 install tweet-preprocessor\n",
        "!pip3 install wordninja"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EC0wBt1bvkl3",
        "colab_type": "text"
      },
      "source": [
        "# Loading train data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WTsMHOKlvcw7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = pd.read_csv('/content/drive/My Drive/SMAI_Final_Assignment/Q1/final_train.csv')\n",
        "X = data['text']\n",
        "y = data['labels']\n",
        "X = X.to_numpy()\n",
        "y = y.to_numpy()\n",
        "# print(X)\n",
        "# print(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I2anDGh4vrYQ",
        "colab_type": "text"
      },
      "source": [
        "# The sections below show different methods for preprocessing of data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TeAuST7i288x",
        "colab_type": "text"
      },
      "source": [
        "## In this section we are removing the special characters and extra spaces from the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YfGc_-71vcxC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "no_space = re.compile(\"(&(\\w*))|(@(\\w*))|(\\;)|(\\')|(#)|(\\.)|(\\;)|(\\:)|(\\!)|(\\*)|(\\')|(\\?)|(\\,)|(\\\")|(\\()|(\\))|(\\[)|(\\])\")\n",
        "space = re.compile(\"(\\-)|(\\/)\")\n",
        "single_digits = re.compile(r\"\\b[A-Za-z]\\b\")\n",
        "digits = re.compile(\"\\d+\")\n",
        "extra_spaces = re.compile(r'\\s+')\n",
        "backslash = re.compile(r'\\\\')\n",
        "\n",
        "def preprocess_reviews(tweet):\n",
        "    \n",
        "    tweet = [no_space.sub(\"\", line.lower()) for line in tweet]\n",
        "    tweet = [space.sub(\" \", line) for line in tweet]\n",
        "    tweet = [single_digits.sub(\" \", line) for line in tweet]\n",
        "    tweet = [digits.sub(\" \", line) for line in tweet]\n",
        "    tweet = [backslash.sub(\" \", line) for line in tweet]\n",
        "    tweet = [extra_spaces.sub(\" \", line) for line in tweet]\n",
        "    \n",
        "    return tweet"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d0v-umqt3th2",
        "colab_type": "text"
      },
      "source": [
        "## Performing lemmatization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntpAmRr-vcxT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "def get_lemmatized_text(corpus):\n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    corpus = [' '.join([lemmatizer.lemmatize(word) for word in line.split()]) for line in corpus]\n",
        "    return corpus"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1hhT51w3xoR",
        "colab_type": "text"
      },
      "source": [
        "## In this section we are removing all URLs, emojis, numbers and mention tags from the data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fTkqYD5ivcxb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import preprocessor as p\n",
        "p.set_options(p.OPT.URL, p.OPT.EMOJI, p.OPT.SMILEY, p.OPT.NUMBER, p.OPT.MENTION)\n",
        "i = 0\n",
        "for line in X:\n",
        "    cleaned_line = p.clean(line)\n",
        "    X[i] = cleaned_line\n",
        "    i = i+1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0-Idlnch3_wB",
        "colab_type": "text"
      },
      "source": [
        "## Using the wordninja module, we are splitting the data into seperate words."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KmERAtKgvcxg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import wordninja\n",
        "i = 0\n",
        "for text in X:\n",
        "    arr = wordninja.split(text)\n",
        "    arr1 = \"\"\n",
        "    for j in arr:\n",
        "        arr1 = arr1+\" \"+j\n",
        "    X[i] = arr1\n",
        "    i = i+1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vz3CXAJKvcxl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = preprocess_reviews(X)\n",
        "# X"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "PXgwcePQvcxr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# X = remove_stop_words(X)\n",
        "# X"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xIYVNZfovcxx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = get_lemmatized_text(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ABpEBx6Kvcx3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# vectorizer = TfidfVectorizer(stop_words = 'english', ngram_range=(1, 3))\n",
        "vectorizer = CountVectorizer(stop_words = 'english', ngram_range=(1, 2))\n",
        "X = vectorizer.fit_transform(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeojwGVHvcx-",
        "colab_type": "code",
        "outputId": "c3ea3915-16e9-437a-89a9-dfb4ae738400",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "X_train = X\n",
        "y_train = y\n",
        "X_train"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<5266x58165 sparse matrix of type '<class 'numpy.int64'>'\n",
              "\twith 147176 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6oS1kE_ly9Ix",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X , y, train_size = 0.80)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKN8N5GA2D61",
        "colab_type": "text"
      },
      "source": [
        "# Logistic Regression model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vQBE-2MvcyE",
        "colab_type": "code",
        "outputId": "68d720ef-501e-48a5-d955-18787574e701",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "c = [0.001, 0.01, 0.05, 0.5, 0.1]\n",
        "for i in c:\n",
        "    model_lr = LogisticRegression(C=i)\n",
        "    model_lr.fit(X_train, y_train)\n",
        "    y_pred = model_lr.predict(X_test)\n",
        "    accuracy_lr = accuracy_score(y_test, y_pred)\n",
        "    f1_lr = f1_score(y_test, y_pred)\n",
        "    print (\"C = \", i, \" Accuracy: \", accuracy_lr)\n",
        "    print (\"C = \", i, \" F1 score: \", f1_lr)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C =  0.001  Accuracy:  0.627134724857685\n",
            "C =  0.001  F1 score:  0.7667655786350149\n",
            "C =  0.01  Accuracy:  0.6574952561669829\n",
            "C =  0.01  F1 score:  0.7705022250476795\n",
            "C =  0.05  Accuracy:  0.6698292220113852\n",
            "C =  0.05  F1 score:  0.7661290322580644\n",
            "C =  0.5  Accuracy:  0.6650853889943074\n",
            "C =  0.5  F1 score:  0.7494677075940384\n",
            "C =  0.1  Accuracy:  0.674573055028463\n",
            "C =  0.1  F1 score:  0.7658703071672355\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3deAgYpU2H9c",
        "colab_type": "text"
      },
      "source": [
        "# Linear SVC model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EbIOPVPDvcyK",
        "colab_type": "code",
        "outputId": "341873f1-8342-40e4-c04c-411b080d6948",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "from sklearn.svm import LinearSVC\n",
        "\n",
        "c = [0.001, 0.01, 0.05, 0.5, 0.1]\n",
        "for i in c:\n",
        "    model_svm = LinearSVC(C=i)\n",
        "    model_svm.fit(X_train, y_train)\n",
        "    y_pred = model_svm.predict(X_test)\n",
        "    accuracy_svm = accuracy_score(y_test, y_pred)\n",
        "    f1_svm = f1_score(y_test, y_pred)\n",
        "    print (\"C = \", i, \" Accuracy: \", accuracy_svm)\n",
        "    print (\"C = \", i, \" F1 score: \", f1_svm)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C =  0.001  Accuracy:  0.6584440227703985\n",
            "C =  0.001  F1 score:  0.7692307692307693\n",
            "C =  0.01  Accuracy:  0.6726755218216319\n",
            "C =  0.01  F1 score:  0.765146358066712\n",
            "C =  0.05  Accuracy:  0.6669829222011385\n",
            "C =  0.05  F1 score:  0.7505330490405118\n",
            "C =  0.5  Accuracy:  0.6527514231499051\n",
            "C =  0.5  F1 score:  0.7332361516034985\n",
            "C =  0.1  Accuracy:  0.6565464895635673\n",
            "C =  0.1  F1 score:  0.7391930835734869\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eykYsj-w275N",
        "colab_type": "text"
      },
      "source": [
        "# SVM with 'linear' kernel"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFhxh12GvcyV",
        "colab_type": "code",
        "outputId": "35620274-0d1a-4d51-a351-c13ae2037006",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "svm = SVC(kernel = 'linear')\n",
        "svm.fit(X_train, y_train)\n",
        "y_pred = svm.predict(X_test)\n",
        "accuracy_svm = accuracy_score(y_test, y_pred)\n",
        "f1_svm = f1_score(y_test, y_pred)\n",
        "print (\"Accuracy: \", accuracy_svm)\n",
        "print (\" F1 score: \", f1_svm)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.6480075901328273\n",
            " F1 score:  0.7278063096111519\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zN7FIoi32-_N",
        "colab_type": "text"
      },
      "source": [
        "# SVM with 'poly' kernel"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HPUg_Lw52yBc",
        "colab_type": "code",
        "outputId": "5f8da3fb-d696-4375-ee7c-4274c2d44a02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "svm = SVC(kernel = 'poly')\n",
        "svm.fit(X_train, y_train)\n",
        "y_pred = svm.predict(X_test)\n",
        "accuracy_svm = accuracy_score(y_test, y_pred)\n",
        "f1_svm = f1_score(y_test, y_pred)\n",
        "print (\"Accuracy: \", accuracy_svm)\n",
        "print (\" F1 score: \", f1_svm)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.6166982922201139\n",
            " F1 score:  0.7566265060240964\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fv5EMVFW3BSH",
        "colab_type": "text"
      },
      "source": [
        "# SVM with 'rbf' kernel"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XPwR0INtvcyQ",
        "colab_type": "code",
        "outputId": "afbf58fa-0840-4f85-9e16-7ec5d7161769",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "svm = SVC(kernel = 'rbf')\n",
        "svm.fit(X_train, y_train)\n",
        "y_pred = svm.predict(X_test)\n",
        "accuracy_svm = accuracy_score(y_test, y_pred)\n",
        "f1_svm = f1_score(y_test, y_pred)\n",
        "print (\"Accuracy: \", accuracy_svm)\n",
        "print (\" F1 score: \", f1_svm)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.6669829222011385\n",
            " F1 score:  0.772225827384815\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e39H3KbMG5m2",
        "colab_type": "text"
      },
      "source": [
        "# Test data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n49v9-PZvcy3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_test = pd.read_csv('/content/drive/My Drive/SMAI_Final_Assignment/Q1/final_test.csv')\n",
        "X_test = X_test['text']\n",
        "X_test = X_test.to_numpy()\n",
        "X_test1 = X_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JN-1Ucj0vcy8",
        "colab_type": "code",
        "outputId": "1a12bb94-a0f4-4c4e-a6be-a462a6a536cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "X_test = X_test1\n",
        "\n",
        "p.set_options(p.OPT.URL, p.OPT.EMOJI, p.OPT.SMILEY, p.OPT.NUMBER, p.OPT.MENTION)\n",
        "i = 0\n",
        "for line in X_test:\n",
        "#     print(line)\n",
        "    cleaned_tweet = p.clean(line)\n",
        "    X_test[i] = cleaned_tweet\n",
        "    i = i+1\n",
        "\n",
        "\n",
        "i = 0\n",
        "for text in X_test:\n",
        "    arr = wordninja.split(text)\n",
        "    arr1 = \"\"\n",
        "    for j in arr:\n",
        "        arr1 = arr1+\" \"+j\n",
        "    X_test[i] = arr1\n",
        "    i = i+1\n",
        "\n",
        "X_test = preprocess_reviews(X_test)\n",
        "\n",
        "# X_test = remove_stop_words(X_test)\n",
        "\n",
        "X_test = get_lemmatized_text(X_test)\n",
        "X_test = vectorizer.transform(X_test)\n",
        "X_test"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<1153x58165 sparse matrix of type '<class 'numpy.int64'>'\n",
              "\twith 16256 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cnE6--98vczA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "svm = SVC(kernel = 'rbf')\n",
        "svm.fit(X_train, y_train)\n",
        "y_pred = svm.predict(X_test)\n",
        "# y_pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7nP8Nwk7HJ1h",
        "colab_type": "text"
      },
      "source": [
        "# Saving predictions to csv file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gPwdKBYvczN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame()\n",
        "df['labels'] = y_pred.tolist()\n",
        "df.to_csv(\"submissionq1.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rOn3lysNvczQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp submissionq1.csv \"drive/My Drive/smai\""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}