Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import praw\n",
    "import math\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "import urllib.request\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from textblob import TextBlob\n",
    "from spellchecker import SpellChecker\n",
    "from collections import defaultdict\n",
    "from praw.models import MoreComments, Redditor, Submission\n",
    "from prawcore.exceptions import *\n",
    "from praw.reddit import models\n",
    "from typing import List, Dict\n",
    "from pandas import DataFrame as Df\n",
    "from collections import Counter\n",
    "from Analysis.FeatureBuilder import *\n",
    "from Ingest.pushShift import *\n",
    "from Ingest.Reddit import *\n",
    "from Analysis import Common\n",
    "import plotly.graph_objs as go\n",
    "from tqdm import tqdm\n",
    "import plotly.plotly as py\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "from plotly import offline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.decomposition import KernelPCA\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost import plot_tree\n",
    "from matplotlib.colors import ListedColormap\n",
    "import matplotlib.pyplot as plt\n",
    "init_notebook_mode(connected=True)\n",
    "%load_ext autoreload\n",
    "%autoreload\n",
    "tqdm.pandas()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "post_id = 'combined'\n",
    "post_dataset = Common.load_dataset_for_post(post_id)\n",
    "print(len(post_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data_features = ['comment_id', 'post_id', 'golds', 'comment_char_count',\n",
    "       'comment_text_polarity',\n",
    "       'network_comment_thread_max_depth',\n",
    "       'network_user_total_comment_count', 'news_subreddit_comment_karma',\n",
    "       'politics_subreddit_post_karma', 'left_subreddit_comment_count',\n",
    "       'news_subreddit_comment_count', 'comment_has_user_ref',\n",
    "       'user_email_verified', 'user_total_comment_count',\n",
    "       'comment_spelling_error_count', 'right_subreddit_comment_count',\n",
    "       'network_user_thread_comment_count', 'user_account_age_seconds',\n",
    "       'center_subreddit_comment_karma', 'post_comment_timedelta_seconds',\n",
    "       'right_subreddit_post_karma', 'comment_url_refer_count',\n",
    "       'center_subreddit_comment_count', 'comment_has_citation',\n",
    "       'center_subreddit_post_karma', 'politics_subreddit_comment_count',\n",
    "       'politics_subreddit_post_count', 'user_total_post_karma',\n",
    "       'politics_subreddit_comment_karma',\n",
    "       'left_subreddit_comment_karma', 'user_total_post_count',\n",
    "       'comment_text_profanity', 'network_comment_thread_size',\n",
    "       'user_total_comment_karma', 'network_comment_thread_top_level_count',\n",
    "       'network_user_top_level_comment_count', 'left_subreddit_post_count',\n",
    "       'news_subreddit_post_count', 'right_subreddit_comment_karma',\n",
    "       'left_subreddit_post_karma', 'comment_text_subjectivity',\n",
    "       'right_subreddit_post_count', 'center_subreddit_post_count',\n",
    "       'news_subreddit_post_karma']\n",
    "def gen_model_for(dataset: Df):\n",
    "    dataset.sort_values(by=['score'], inplace=True)\n",
    "    top = dataset[dataset.score < -10]\n",
    "    top['misinformation'] = 1             # Dependent variable\n",
    "    print(len(top))\n",
    "    bottom = dataset[dataset.score > 30]\n",
    "    bottom['misinformation'] = 0          # Dependent variable\n",
    "    print(len(bottom))\n",
    "    return pd.concat([top, bottom]).reset_index(drop=True)[training_data_features + ['misinformation']]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting the Dataset and Feature Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_frame  = gen_model_for(post_dataset)\n",
    "X = X_frame.iloc[:, 2:-1].values\n",
    "y = X_frame.iloc[:, -1].values\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "print(f'Training dataset is: {len(X_train)} rows')\n",
    "train_X, test_X = X_train, X_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_lda(train_X, train_Y, test_X):\n",
    "    lda = LinearDiscriminantAnalysis(n_components=2)\n",
    "    train_X = lda.fit_transform(train_X, train_Y)\n",
    "    test_X = lda.transform(test_X)\n",
    "    return train_X, test_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components=None)  \n",
    "train_X = pca.fit_transform(X_train)\n",
    "test_X = pca.transform(X_test)\n",
    "# cumsum = np.cumsum(pca.explained_variance_ratio_)\n",
    "# trace = go.Scatter(x=[i for i in range(len(cumsum))], y=cumsum,\n",
    "#                      marker=dict(color='rgb(150, 25, 120)'))\n",
    "# fig = go.Figure(data=[trace])\n",
    "# plot(fig, filename='cdf-dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kpca = KernelPCA(kernel='rbf', n_components=3)  \n",
    "train_X = kpca.fit_transform(X_train)\n",
    "test_X = kpca.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing the Principal Components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_principal_axis(X, Y, title):\n",
    "    trace0_x, trace0_y, trace1_x, trace1_y = [], [], [], []\n",
    "    for x_val, y_val in zip(X, Y):\n",
    "        if y_val == 0:\n",
    "            trace0_x.append(x_val[0])\n",
    "            trace0_y.append(x_val[1])\n",
    "        else:\n",
    "            trace1_x.append(x_val[0])\n",
    "            trace1_y.append(x_val[1])\n",
    "    trace0 = go.Scatter(\n",
    "        x = trace0_x,\n",
    "        y = trace0_y,\n",
    "        mode = 'markers',\n",
    "        name=\"Not Credible\"\n",
    "    )\n",
    "    trace1 = go.Scatter(\n",
    "        x = trace1_x,\n",
    "        y = trace1_y,\n",
    "        mode = 'markers',\n",
    "        name=\"Credible\"\n",
    "    )\n",
    "    layout = dict(\n",
    "        title=title,\n",
    "    )\n",
    "    return dict(data=[trace0, trace1], layout=layout)\n",
    "\n",
    "fig = visualize_principal_axis(X_test, y_train, \"PCA\")\n",
    "plot(fig, filename='pca')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performance Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_metrics(estimator, trainX, testX, trainY, testY):\n",
    "    # Performance Evaluation - Confusion Matrix Test Evaluation\n",
    "    pred_test_y = estimator.predict(testX)\n",
    "    cm = confusion_matrix(testY, pred_test_y)\n",
    "    print(f'Test Data Confusion Matrix: {cm}')\n",
    "    # Performance Evaluation - Confusion Matrix Train Evaluation\n",
    "    pred_train_y = estimator.predict(trainX)\n",
    "    cm = confusion_matrix(trainY, pred_train_y)\n",
    "    print(f'Train Data Confusion Matrix: {cm}')\n",
    "    # Performance Evaluation - K Means\n",
    "    accuracies = cross_val_score(estimator = estimator, X=trainX, y=trainY, cv=10)\n",
    "    print(f'K-Means Accuracy Average: {accuracies.mean()}')\n",
    "    print(f'K-Means Variance: {accuracies.std()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM Testbench"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "classifier = SVC(kernel = 'rbf', C=2.1, gamma= 0.04, probability=True)\n",
    "classifier.fit(X_train, y_train)\n",
    "run_metrics(classifier, X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyper parameter Tuning Grid Search: SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C_vals = [0.1 * i+1 for i in range(20)]\n",
    "def num_generator(start_num, intervals, interval_size):\n",
    "    nums = [start_num]\n",
    "    while intervals > 0:\n",
    "        nums.append(nums[-1] + interval_size)\n",
    "        intervals-=1\n",
    "    return nums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C_vals = num_generator(0.1, 20, 0.1)\n",
    "gamma = num_generator(0, 10, 0.01)\n",
    "parameters = [\n",
    "#     {\n",
    "#         'C': C_vals,\n",
    "#         'kernel': ['linear']\n",
    "#     },\n",
    "    {\n",
    "        'C': C_vals,\n",
    "        'kernel': [\n",
    "            'rbf',\n",
    "            'poly',\n",
    "            'sigmoid',\n",
    "        ],\n",
    "        'gamma': gamma\n",
    "    }\n",
    "]\n",
    "grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)\n",
    "grid_search = grid_search.fit(X_train, y_train)\n",
    "print(grid_search.best_score_, grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyper Parameter Tuning RandomForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = [\n",
    "    {\n",
    "        'n_estimators': [i for i in range(1, 100)],\n",
    "        'criterion': ['entropy']\n",
    "    }\n",
    "]\n",
    "classifier = RandomForestClassifier()\n",
    "grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)\n",
    "grid_search = grid_search.fit(X_train, y_train)\n",
    "best_accuracy = grid_search.best_score_\n",
    "print(grid_search.best_score_, grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = RandomForestClassifier(n_estimators = 47, criterion = 'entropy', random_state = 0)\n",
    "classifier.fit(X_train, y_train)\n",
    "run_metrics(classifier, X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = XGBClassifier(learning_rate=0.21, max_depth=4, n_estimators=138)\n",
    "classifier.fit(X_train, y_train)\n",
    "run_metrics(classifier, X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = XGBClassifier()\n",
    "classifier.fit(X_train, y_train)\n",
    "run_metrics(classifier, X_train, X_test, y_train, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importances = Df({'importance': classifier.feature_importances_, 'col_names': list(X_frame.columns[2:-1])})\n",
    "importances.sort_values(by=['importance'], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize Boosted Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tree(classifier)\n",
    "fig = plt.gcf()\n",
    "fig.set_size_inches(40, 40)\n",
    "fig.savefig('tree.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importances in Boosted Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importances = Df({'columns': X_frame.columns[2:-1], 'importances': classifier.feature_importances_})\n",
    "feature_importances.sort_values(by=['importances'], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyper Parameter Tuning XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rates = num_generator(0.1, 20, 0.01)\n",
    "num_estimators = num_generator(90, 20, 3)\n",
    "learning_rates, num_estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = [\n",
    "    {\n",
    "        'max_depth': [i for i in range(5)],\n",
    "        'learning_rate': learning_rates,\n",
    "        'n_estimators': num_estimators,        \n",
    "    }\n",
    "]\n",
    "classifier = XGBClassifier()\n",
    "grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)\n",
    "grid_search = grid_search.fit(X_train, y_train)\n",
    "best_accuracy = grid_search.best_score_\n",
    "print(grid_search.best_score_, grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CAP Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_xgb = XGBClassifier(learning_rate=0.21, max_depth=4, n_estimators=138)\n",
    "best_svm = SVC(kernel = 'rbf', C=2.1, gamma= 0.04, probability=True)\n",
    "best_random_forest = RandomForestClassifier(n_estimators = 47, criterion = 'entropy', random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_for_classifier(classifier, name):\n",
    "    classifier.fit(X_train, y_train)\n",
    "    test_data = Df(X_test)\n",
    "    total = len(test_data)\n",
    "    test_data['actual'] = y_test\n",
    "    test_data['predicted'] = classifier.predict_proba(X_test)[:, 1]\n",
    "    prob_sorted = test_data.sort_values(by=['predicted'], ascending=False)\n",
    "    model_y = list(prob_sorted.actual) \n",
    "    return go.Scatter(\n",
    "        x=np.arange(0, total + 1),\n",
    "        y=np.append([0], np.cumsum(model_y)),\n",
    "        name=name,\n",
    "        mode='lines'\n",
    "    )\n",
    "\n",
    "\n",
    "def get_model_random():\n",
    "    unique, counts = np.unique(y_test, return_counts=True)\n",
    "    class_1_count = dict(zip(unique, counts))[1]    \n",
    "    return {\n",
    "        \"type\": \"line\",\n",
    "        \"x0\": 0,\n",
    "        \"x1\": len(y_test),\n",
    "        \"y0\": 0,\n",
    "        \"y1\": class_1_count,\n",
    "        \"line\": {\n",
    "            \"color\": 'rgb(256, 0, 0)',\n",
    "            \"width\": 4,\n",
    "            \"dash\": \"dot\"\n",
    "        }\n",
    "    }\n",
    "\n",
    "\n",
    "def get_model_perfect():\n",
    "    unique, counts = np.unique(y_test, return_counts=True)\n",
    "    class_1_count = dict(zip(unique, counts))[1]    \n",
    "    return go.Scatter(\n",
    "        x=[0, class_1_count, len(y_test)],\n",
    "        y= [0, class_1_count, class_1_count],\n",
    "        mode='lines',\n",
    "        name='Perfect Model'\n",
    "    )\n",
    "\n",
    "def do_plot():\n",
    "    plt.figure(figsize=(20, 12))\n",
    "\n",
    "\n",
    "    fig = {\n",
    "        \"data\": [\n",
    "            get_model_for_classifier(best_xgb, 'Tuned XGBoost'),\n",
    "            get_model_for_classifier(best_svm, 'Tuned SVM'),\n",
    "            get_model_for_classifier(best_random_forest, 'Tuned Random Forest'),\n",
    "            get_model_perfect()\n",
    "        ],\n",
    "        \"layout\": go.Layout(\n",
    "            title=go.layout.Title(text=\"CAP Curves\"),\n",
    "            xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text='Observations')),\n",
    "            yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text='Misinformative Observations')),\n",
    "            shapes=[get_model_random()]\n",
    "        )\n",
    "    }\n",
    "    plot(fig, filename='CAP Curves of Various Models')\n",
    "#     offline.plot(fig, image='png', image_filename='CAP Curves', output_type='file')\n",
    "do_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_error_rates():\n",
    "    misinfo_missed = [32.8, 27.14, 14.1, 7.8, 12, 9.2]\n",
    "    false_positives = [6.1, 4.83, 0.7, 0.2, 0.5, 0.5]\n",
    "    labels = ['Kernel SVM', 'Tuned SVM', 'XGB', 'Tuned XGB', 'Random Forest', 'Tuned Random Forest']\n",
    "    trace0 = go.Bar(\n",
    "        x=labels,\n",
    "        y=false_positives,\n",
    "        text=false_positives,\n",
    "        textposition = 'auto',\n",
    "        marker=dict(\n",
    "        color='rgb(158,202,225)',\n",
    "        line=dict(\n",
    "            color='rgb(8,48,107)',\n",
    "            width=1.5),\n",
    "        ),\n",
    "        name='False Positives / Type 1 Error'\n",
    "    )\n",
    "    trace1 = go.Bar(\n",
    "        x=labels,\n",
    "        y=misinfo_missed,\n",
    "        text=misinfo_missed,\n",
    "        textposition = 'auto',\n",
    "        name='False Negatives / Type 2 Error',\n",
    "        marker=dict(\n",
    "        color='rgb(58,200,225)',\n",
    "        line=dict(\n",
    "            color='rgb(8,48,107)',\n",
    "            width=1.5),\n",
    "        ),\n",
    "    )\n",
    "    layout = go.Layout(\n",
    "        title=go.layout.Title(text=\"Error Types and Counts across Models\"),\n",
    "        xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text='Model Type')),\n",
    "        yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text='Error %')),\n",
    "    )\n",
    "    fig = {'data': [trace0, trace1], 'layout': layout}\n",
    "    plot(fig, filename='Error Rate Bar Chart')\n",
    "#     offline.plot(fig, image='png', image_filename='error_rate_bar', output_type='file')\n",
    "plot_error_rates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_tree(best_xgb)\n",
    "fig = plt.gcf()\n",
    "fig.set_size_inches(40, 40)\n",
    "fig.savefig('tree.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importances = Df({'columns': X_frame.columns[2:-1], 'importances': best_xgb.feature_importances_})\n",
    "feature_importances.sort_values(by=['importances'], ascending=False).head"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}