tweet_cleaning_final_github.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Commonly used\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import string\n",
    "import time\n",
    "import csv\n",
    "from collections import Counter\n",
    "import re\n",
    "from datetime import datetime, timedelta\n",
    "from codecs import encode\n",
    "import pytz\n",
    "from wordcloud import STOPWORDS\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# A package for preprocessing(officially used in SemEval NLP competition)\n",
    "from ekphrasis.classes.preprocessor import TextPreProcessor\n",
    "from ekphrasis.classes.tokenizer import SocialTokenizer\n",
    "from ekphrasis.dicts.emoticons import emoticons\n",
    "\n",
    "# For machine translation\n",
    "from googletrans import Translator\n",
    "from google.cloud import storage\n",
    "from google.cloud import translate\n",
    "\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the credential environment in jupyter notebook\n",
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"]=\"XXXXX\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiates a client\n",
    "translate_client = translate.Client()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_zone_hk = pytz.timezone('Asia/Shanghai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = r'XXXXX'\n",
    "cross_sectional = r'XXXXX'\n",
    "desktop = r'XXXXX'\n",
    "before_and_after = r'XXXXX'\n",
    "tweet_combined_path = r'XXXXX'\n",
    "tweet_2018_path = r'XXXXX'\n",
    "tweet_2017_path = r'XXXXX'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading twitter - 1grams ...\n",
      "Reading twitter - 2grams ...\n",
      "Reading twitter - 1grams ...\n"
     ]
    }
   ],
   "source": [
    "def remove_u_plus(text):\n",
    "    result = re.sub(pattern=r'U\\+00', repl=r'', string=text)\n",
    "    return result\n",
    "\n",
    "\n",
    "def show_emoji_in_tweet(text, emoji_dictionary):\n",
    "    without_u = remove_u_plus(text)\n",
    "    old_text = without_u\n",
    "    old_text = old_text.encode('unicode_escape').decode('utf-8')\n",
    "    result1 = re.sub(pattern='\\\\\\\\r', repl='', string=old_text)\n",
    "    result2 = re.sub(pattern='\\\\\\\\n', repl='', string=result1)\n",
    "    result3 = re.sub(pattern='\\\\\\\\x([a-z0-9]{2})', repl = '<\\\\1>', string=result2)\n",
    "    old_text = result3\n",
    "    for _, row in emoji_dictionary.iterrows():\n",
    "        if row['R_Encoding'] in old_text:\n",
    "            new_text = re.sub(pattern=row['R_Encoding'], repl=row['emoji'], string=old_text)\n",
    "            old_text = new_text\n",
    "        else:\n",
    "            pass\n",
    "        if row['R_Encoding_lower'] in old_text:\n",
    "            new_text = re.sub(pattern=row['R_Encoding_lower'], repl=row['emoji'], string=old_text)\n",
    "            old_text = new_text\n",
    "        else:\n",
    "            pass\n",
    "    return old_text\n",
    "\n",
    "def show_chinese_step1(text, emoji_dataset):\n",
    "    result1 = re.sub('\\<u\\+', '\\\\'+'u', text.lower())\n",
    "    result2 = re.sub('\\>', '', result1)\n",
    "    all_chars = result2.split()\n",
    "    new_all_chars = []\n",
    "    for char in all_chars:\n",
    "        emoji_in_char = False\n",
    "        for emoji in list(emoji_dataset['emoji']):\n",
    "            if emoji in char:\n",
    "                emoji_in_char = True\n",
    "                new_char = char.encode('utf-8').decode('utf-8')\n",
    "                new_all_chars.append(new_char)\n",
    "            else:\n",
    "                pass\n",
    "        if not emoji_in_char:\n",
    "            new_char = char.encode('utf-8').decode('unicode_escape')\n",
    "            new_all_chars.append(new_char)\n",
    "    return \" \".join(new_all_chars)\n",
    "\n",
    "def show_chinese_step2(text):\n",
    "    result1 = re.sub('<', '\\\\x', text)\n",
    "    result2 = encode(result1.encode().decode('unicode_escape', 'ignore'), 'raw_unicode_escape')\n",
    "    result3 = result2.decode('utf-8', 'ignore')\n",
    "    return result3\n",
    "\n",
    "\n",
    "def show_chinese_step3(text):\n",
    "    patterns = re.findall(pattern='\\\\\\\\u[a-z0-9]{4}', string=text)\n",
    "    old_text = text\n",
    "    for pattern in patterns:\n",
    "        new_pattern = pattern.encode('utf-8').decode('unicode_escape', 'ignore')\n",
    "        new_text = re.sub(pattern='\\\\'+pattern, repl=new_pattern, string=old_text)\n",
    "        old_text = new_text\n",
    "    return old_text\n",
    "\n",
    "text_processor = TextPreProcessor(\n",
    "    # terms that will be normalized\n",
    "    normalize=['url', 'email', 'percent', 'money', 'phone', 'user',\n",
    "               'time', 'url', 'date', 'number'],\n",
    "    # terms that will be annotated\n",
    "    annotate={\"hashtag\", \"allcaps\", \"elongated\", \"repeated\",\n",
    "              'emphasis', 'censored'},\n",
    "    fix_html=True,  # fix HTML tokens\n",
    "\n",
    "    # corpus from which the word statistics are going to be used\n",
    "    # for word segmentation\n",
    "    segmenter=\"twitter\",\n",
    "\n",
    "    # corpus from which the word statistics are going to be used\n",
    "    # for spell correction\n",
    "    corrector=\"twitter\",\n",
    "\n",
    "    unpack_hashtags=True,  # perform word segmentation on hashtags\n",
    "    unpack_contractions=True,  # Unpack contractions (can't -> can not)\n",
    "    spell_correct_elong=False,  # spell correction for elongated words\n",
    "\n",
    "    # select a tokenizer. You can use SocialTokenizer, or pass your own\n",
    "    # the tokenizer, should take as input a string and return a list of tokens\n",
    "    tokenizer=SocialTokenizer(lowercase=True).tokenize,\n",
    "\n",
    "    # list of dictionaries, for replacing tokens extracted from the text,\n",
    "    # with other expressions. You can pass more than one dictionaries.\n",
    "    dicts=[emoticons]\n",
    ")\n",
    "\n",
    "def preprocessing_for_english(text_preprocessor, raw_text):\n",
    "    preprocessed_text = ' '.join(text_preprocessor.pre_process_doc(str(raw_text)))\n",
    "    # remove punctuations\n",
    "    result = re.sub(u'[{}]'.format(string.punctuation), u'', preprocessed_text)\n",
    "    return result\n",
    "\n",
    "\n",
    "def clean_english_tweet_for_review(text, emoji_dictionary):\n",
    "    text_with_emoji = show_emoji_in_tweet(text, emoji_dictionary)\n",
    "    processed_text = preprocessing_for_english(text_processor, text_with_emoji)\n",
    "    return processed_text\n",
    "\n",
    "\n",
    "def clean_chinese_tweet_for_review(text, emoji_dictionary):\n",
    "    tweet_with_emoji = show_emoji_in_tweet(text, emoji_dictionary)\n",
    "    step1 = show_chinese_step1(tweet_with_emoji, emoji_dictionary)\n",
    "    step2 = show_chinese_step2(step1)\n",
    "    step3 = show_chinese_step3(step2)\n",
    "    return step3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_translated_chinese(text):\n",
    "    result1 = re.sub('\\<u\\+', '\\\\' + 'u', text.lower())\n",
    "    result2 = re.sub('\\>', '', result1)\n",
    "    result3 = re.sub('\\<([a-z0-9]{2})*', '', result2)\n",
    "    all_chars = result3.split()\n",
    "    for index, char in enumerate(all_chars):\n",
    "        try:\n",
    "            if translate_client.detect_language(char)['language'][:2] != 'en':\n",
    "                all_chars[index] = translate_client.translate(char, target_language='en')['translatedText']\n",
    "            else:\n",
    "                all_chars[index] = char.encode('utf-8').decode('utf-8')\n",
    "        except:\n",
    "            pass\n",
    "    return ' '.join(all_chars)\n",
    "\n",
    "def preprocessing_for_chinese(text_preprocessor, raw_text):\n",
    "    preprocessed_text = ' '.join(text_preprocessor.pre_process_doc(str(raw_text)))\n",
    "    # remove punctuations\n",
    "    result1 = re.sub(u'[{}]'.format(string.punctuation), u'', preprocessed_text)\n",
    "    # remove hashtag\n",
    "    result2 = re.sub(r'hashtag', u'', result1)\n",
    "    # remove url\n",
    "    result3 = re.sub(r'url', '', result2)\n",
    "    # replace the multiple blanks to one blank\n",
    "    result4 = re.sub('\\\\s+', u' ', result3)\n",
    "    # remove the digits\n",
    "    result5 = re.sub(r'number', u'', result4)\n",
    "    return result5\n",
    "\n",
    "def get_translated_text(text_string):\n",
    "    # remove hashtag\n",
    "    result1 = re.sub('#', '', string=text_string)\n",
    "    # remove @\n",
    "    result2 = re.sub('@', '', string=result1)\n",
    "    result3 = show_translated_chinese(result2)\n",
    "    processed_text = preprocessing_for_chinese(text_processor, result3)\n",
    "    return processed_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_selected_tweet_dataframe(dataframe, english_or_not = True):\n",
    "    selected_columns = ['created_at', 'id_str', 'lang', 'lat', 'lon',\n",
    "       'place_id', 'place_lat', 'place_lon', 'place_name', 'text', 'time_zone',\n",
    "       'truncated', 'url', 'user_created_at', 'user_id_str', 'user_lang',\n",
    "       'user_url', 'verified', 'hk_time', 'year', 'month', 'month_plus_year',\n",
    "       'day', 'TPU_longitudinal', 'TPU_cross_sectional', 'cleaned_text']\n",
    "    \n",
    "    if english_or_not:\n",
    "        result_dataframe = dataframe[selected_columns]\n",
    "        return result_dataframe\n",
    "    else:\n",
    "        translated_text_list = list(dataframe['translated_text'])\n",
    "        filtered_dataframe = dataframe[selected_columns]\n",
    "        filtered_dataframe_copy = filtered_dataframe.copy()\n",
    "        filtered_dataframe_copy['cleaned_text'] = translated_text_list\n",
    "        return filtered_dataframe_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_string_time_to_datetime(string):\n",
    "    \"\"\"\n",
    "    :param string: the string which records the time of the posted tweets(this string's timezone is HK time)\n",
    "    :return: a datetime object which could get access to the year, month, day easily\n",
    "    \"\"\"\n",
    "    datetime_object = datetime.strptime(string, '%Y-%m-%d %H:%M:%S+08:00')\n",
    "    final_time_object = datetime_object.replace(tzinfo=time_zone_hk)\n",
    "    return final_time_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoji_dict = pd.read_pickle(os.path.join(tweet_2017_path, 'emoji.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_combined_dataframe = pd.read_csv(os.path.join(tweet_combined_path, 'tweet_combined_in_hk_withoutbot_step2.csv'), \n",
    "                                      encoding='utf-8', quoting=csv.QUOTE_NONNUMERIC, dtype='str', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id_str</th>\n",
       "      <th>lang</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>place_id</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>place_name</th>\n",
       "      <th>...</th>\n",
       "      <th>user_lang</th>\n",
       "      <th>user_url</th>\n",
       "      <th>verified</th>\n",
       "      <th>hk_time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>month_plus_year</th>\n",
       "      <th>day</th>\n",
       "      <th>TPU_longitudinal</th>\n",
       "      <th>TPU_cross_sectional</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>Sat May 07 06:18:59 +0000 2016</td>\n",
       "      <td>7.28831E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.2788499</td>\n",
       "      <td>114.18462</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.271674</td>\n",
       "      <td>114.185178</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>en</td>\n",
       "      <td>http://www.facebook.com/derekhysteric525</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 14:18:59+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>146</td>\n",
       "      <td>146 - 147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19</td>\n",
       "      <td>Sat May 07 07:02:19 +0000 2016</td>\n",
       "      <td>7.28842E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.31530176</td>\n",
       "      <td>113.9348316</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>en</td>\n",
       "      <td>http://manishmaurya89.blogspot.com/?m=1</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:02:19+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>951</td>\n",
       "      <td>950 - 951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>Sat May 07 07:02:34 +0000 2016</td>\n",
       "      <td>7.28842E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.27680815</td>\n",
       "      <td>113.9161873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>en</td>\n",
       "      <td>http://kotakitam.wordpress.com</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:02:34+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>943</td>\n",
       "      <td>941 - 943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>21</td>\n",
       "      <td>Sat May 07 07:03:11 +0000 2016</td>\n",
       "      <td>7.28843E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.27564274</td>\n",
       "      <td>114.1711743</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.271674</td>\n",
       "      <td>114.185178</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>en</td>\n",
       "      <td>http://www.denkipenki.vsco.co</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:03:11+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>131</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32</td>\n",
       "      <td>Sat May 07 07:05:37 +0000 2016</td>\n",
       "      <td>7.28843E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.30277224</td>\n",
       "      <td>114.0117169</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>en</td>\n",
       "      <td>http://about.me/natalia.segura</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:05:37+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>971</td>\n",
       "      <td>971 - 974</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Unnamed: 0.1                      created_at       id_str lang          lat  \\\n",
       "0           12  Sat May 07 06:18:59 +0000 2016  7.28831E+17   en   22.2788499   \n",
       "1           19  Sat May 07 07:02:19 +0000 2016  7.28842E+17   en  22.31530176   \n",
       "2           20  Sat May 07 07:02:34 +0000 2016  7.28842E+17   en  22.27680815   \n",
       "3           21  Sat May 07 07:03:11 +0000 2016  7.28843E+17   en  22.27564274   \n",
       "4           32  Sat May 07 07:05:37 +0000 2016  7.28843E+17   en  22.30277224   \n",
       "\n",
       "           lon place_id   place_lat   place_lon place_name  \\\n",
       "0    114.18462      NaN   22.271674  114.185178        NaN   \n",
       "1  113.9348316      NaN  22.2465325  114.064237        NaN   \n",
       "2  113.9161873      NaN  22.2465325  114.064237        NaN   \n",
       "3  114.1711743      NaN   22.271674  114.185178        NaN   \n",
       "4  114.0117169      NaN  22.2465325  114.064237        NaN   \n",
       "\n",
       "          ...         user_lang                                  user_url  \\\n",
       "0         ...                en  http://www.facebook.com/derekhysteric525   \n",
       "1         ...                en   http://manishmaurya89.blogspot.com/?m=1   \n",
       "2         ...                en            http://kotakitam.wordpress.com   \n",
       "3         ...                en             http://www.denkipenki.vsco.co   \n",
       "4         ...                en            http://about.me/natalia.segura   \n",
       "\n",
       "  verified                    hk_time  year month month_plus_year day  \\\n",
       "0    FALSE  2016-05-07 14:18:59+08:00  2016     5          2016_5   7   \n",
       "1    FALSE  2016-05-07 15:02:19+08:00  2016     5          2016_5   7   \n",
       "2    FALSE  2016-05-07 15:02:34+08:00  2016     5          2016_5   7   \n",
       "3    FALSE  2016-05-07 15:03:11+08:00  2016     5          2016_5   7   \n",
       "4    FALSE  2016-05-07 15:05:37+08:00  2016     5          2016_5   7   \n",
       "\n",
       "  TPU_longitudinal TPU_cross_sectional  \n",
       "0              146           146 - 147  \n",
       "1              951           950 - 951  \n",
       "2              943           941 - 943  \n",
       "3              131                 131  \n",
       "4              971           971 - 974  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_combined_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Get the English Tweets and the Chinese Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_tweets = tweet_combined_dataframe.loc[tweet_combined_dataframe['lang'] == 'en']\n",
    "chinese_tweets = tweet_combined_dataframe.loc[tweet_combined_dataframe['lang'] == 'zh']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Clean the English Tweets and Chinese Tweets\n",
    "\n",
    "### Clean the English tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 17h 19min 38s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\python\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "english_tweets['cleaned_text'] = english_tweets.apply(\n",
    "    lambda row: clean_english_tweet_for_review(row['text'], emoji_dict), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>414677</th>\n",
       "      <td>Here we go... typhoon signal no. 8 is up.\\r\\r\\...</td>\n",
       "      <td>here we go  repeated typhoon signal no  number...</td>\n",
       "      <td>https://t.co/w4JgKFOEEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321352</th>\n",
       "      <td>When I passed by here, came across four uncles...</td>\n",
       "      <td>when i passed by here  came across four uncles...</td>\n",
       "      <td>https://t.co/UOhrwPHpu9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403290</th>\n",
       "      <td>#cyberformulagpx #aoiogre #an21 at #tsimshatsu...</td>\n",
       "      <td>hashtag cyber formula gpx hashtag hashtag aoi ...</td>\n",
       "      <td>https://t.co/nG7JbAa8m4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234004</th>\n",
       "      <td>Could this be #santa car?\\r\\r\\r\\nOr his ex-gir...</td>\n",
       "      <td>could this be hashtag santa hashtag car  or hi...</td>\n",
       "      <td>https://t.co/3WCmB8FywL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297127</th>\n",
       "      <td>2500m swim session today with 4 different stro...</td>\n",
       "      <td>2 5 0 0 m swim session today with number diffe...</td>\n",
       "      <td>https://t.co/obJn5z17ef</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321824</th>\n",
       "      <td>Only chocolate í ¼í½« makes me happy during fi...</td>\n",
       "      <td>only chocolate 🍫 makes me happy during final 😭...</td>\n",
       "      <td>https://t.co/uKZRuRFBwy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42622</th>\n",
       "      <td>#sunset on my #trip to #hongkong #instagramhub...</td>\n",
       "      <td>hashtag sunset hashtag on my hashtag trip hash...</td>\n",
       "      <td>https://t.co/slVmpcYiyg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30595</th>\n",
       "      <td>#brexitin5words or one picture? @drjesimon #re...</td>\n",
       "      <td>hashtag brexit in5 words hashtag or one pictur...</td>\n",
       "      <td>https://t.co/8ZfdnAbAku</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264787</th>\n",
       "      <td>Just some #BioQueens enjoying themselves on a ...</td>\n",
       "      <td>just some hashtag bio queens hashtag enjoying ...</td>\n",
       "      <td>https://t.co/WW7r0vdGRP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3525</th>\n",
       "      <td>Pacific Coffee at night, Mui Wo ferry pier. #c...</td>\n",
       "      <td>pacific coffee at night  mui wo ferry pier  ha...</td>\n",
       "      <td>https://t.co/MgOzNU3eFm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "414677  Here we go... typhoon signal no. 8 is up.\\r\\r\\...   \n",
       "321352  When I passed by here, came across four uncles...   \n",
       "403290  #cyberformulagpx #aoiogre #an21 at #tsimshatsu...   \n",
       "234004  Could this be #santa car?\\r\\r\\r\\nOr his ex-gir...   \n",
       "297127  2500m swim session today with 4 different stro...   \n",
       "321824  Only chocolate í ¼í½« makes me happy during fi...   \n",
       "42622   #sunset on my #trip to #hongkong #instagramhub...   \n",
       "30595   #brexitin5words or one picture? @drjesimon #re...   \n",
       "264787  Just some #BioQueens enjoying themselves on a ...   \n",
       "3525    Pacific Coffee at night, Mui Wo ferry pier. #c...   \n",
       "\n",
       "                                             cleaned_text  \\\n",
       "414677  here we go  repeated typhoon signal no  number...   \n",
       "321352  when i passed by here  came across four uncles...   \n",
       "403290  hashtag cyber formula gpx hashtag hashtag aoi ...   \n",
       "234004  could this be hashtag santa hashtag car  or hi...   \n",
       "297127  2 5 0 0 m swim session today with number diffe...   \n",
       "321824  only chocolate 🍫 makes me happy during final 😭...   \n",
       "42622   hashtag sunset hashtag on my hashtag trip hash...   \n",
       "30595   hashtag brexit in5 words hashtag or one pictur...   \n",
       "264787  just some hashtag bio queens hashtag enjoying ...   \n",
       "3525    pacific coffee at night  mui wo ferry pier  ha...   \n",
       "\n",
       "                            url  \n",
       "414677  https://t.co/w4JgKFOEEE  \n",
       "321352  https://t.co/UOhrwPHpu9  \n",
       "403290  https://t.co/nG7JbAa8m4  \n",
       "234004  https://t.co/3WCmB8FywL  \n",
       "297127  https://t.co/obJn5z17ef  \n",
       "321824  https://t.co/uKZRuRFBwy  \n",
       "42622   https://t.co/slVmpcYiyg  \n",
       "30595   https://t.co/8ZfdnAbAku  \n",
       "264787  https://t.co/WW7r0vdGRP  \n",
       "3525    https://t.co/MgOzNU3eFm  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_tweets_sample = english_tweets[['text', 'cleaned_text', 'url']].sample(10)\n",
    "en_tweets_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_tweets.to_csv(os.path.join(tweet_combined_path, 'tweet_combined_english.csv'), encoding='utf-8', \n",
    "                      quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean the Chinese tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5h 42min 22s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\python\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "chinese_tweets['cleaned_text'] = chinese_tweets.apply(lambda row: clean_chinese_tweet_for_review(\n",
    "    row['text'], emoji_dict), axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>url</th>\n",
       "      <th>year</th>\n",
       "      <th>hk_time</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>286025</th>\n",
       "      <td>é ­å",
       "æè¬å",
       "äºï¼éå·¦å»å°±ç®ï¼åä½¿æ...</td>\n",
       "      <td>頭先所講嘅事，過左去就算，唔使放喺心，睇波！btw，今日夢想沙龍乜事？對南區成隊飲左redb...</td>\n",
       "      <td>https://t.co/i0ZWZCXFTd</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-10-01 19:08:25+08:00</td>\n",
       "      <td>Sun Oct 01 11:08:25 +0000 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384814</th>\n",
       "      <td>í ½í± @ Ocean Park Hong Kong é¦æ¸¯æµ·æ´å",
       "¬å...</td>\n",
       "      <td>👌 @ ocean park hong kong 香港海洋公園 https://t.co/y...</td>\n",
       "      <td>https://t.co/yfrQFSEblU</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-06-10 11:14:13+08:00</td>\n",
       "      <td>Sun Jun 10 03:14:13 +0000 2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159369</th>\n",
       "      <td>&lt;U+5EFA&gt;&lt;U+8A2D&gt;&lt;U+4E2D&gt;&lt;U+3002&gt;&lt;U+5C1A&gt;&lt;U+572...</td>\n",
       "      <td>建設中。尚在討論建設中。尚在先決定後咨詢建設中。尚在建設衰敗中...... @ west k...</td>\n",
       "      <td>https://t.co/9vuJN0AxkC</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-01-22 18:39:03+08:00</td>\n",
       "      <td>Sun Jan 22 10:39:03 +0000 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395975</th>\n",
       "      <td>I'm at è±è¨éèä» in Tsim Sha Tsui https:...</td>\n",
       "      <td>i'm at 豐記雞蛋仔 in tsim sha tsui https://t.co/krt...</td>\n",
       "      <td>https://t.co/KRtF9ijaxj</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-07-15 12:12:17+08:00</td>\n",
       "      <td>Sun Jul 15 04:12:17 +0000 2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422639</th>\n",
       "      <td>@n_t_lie @Dualman @miniyoshima @uituit @28481k...</td>\n",
       "      <td>@n_t_lie @dualman @miniyoshima @uituit @28481k...</td>\n",
       "      <td>https://t.co/VtYoWqvN8a</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-10-10 16:11:24+08:00</td>\n",
       "      <td>Wed Oct 10 08:11:24 +0000 2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>&lt;U+9EC3&gt;&lt;U+91D1&gt;&lt;U+6BD2&gt;&lt;U+8CE4&gt;&lt;U+86D9&gt; (@ Go...</td>\n",
       "      <td>黃金毒賤蛙 (@ golden computer arcade 黃金電腦商場 in sham...</td>\n",
       "      <td>https://t.co/KaeBx77XF2</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016-05-07 17:39:55+08:00</td>\n",
       "      <td>Sat May 07 09:39:55 +0000 2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80804</th>\n",
       "      <td>&lt;U+5C0D&gt;&lt;U+9762&gt;&lt;U+6709&gt;&lt;U+96BB&gt;&lt;U+9CE5&gt; @ &lt;U+...</td>\n",
       "      <td>對面有隻鳥 @ 元朗西鐵站 https://t.co/9j0h0ivzjl</td>\n",
       "      <td>https://t.co/9j0h0IVZJl</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016-09-21 14:40:03+08:00</td>\n",
       "      <td>Wed Sep 21 06:40:03 +0000 2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353194</th>\n",
       "      <td>é¢éåªçºä¸æ¬¡åè¨\\r\\r\\r\\nè½é¨å²å¢æ¿...</td>\n",
       "      <td>離開只為下次再臨落雨啲嘢濕哂要返屋企洗嘢😑#逃脫 @ 東龍 https://t.co/d1p...</td>\n",
       "      <td>https://t.co/D1pZJmcWz9</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-03-04 14:02:36+08:00</td>\n",
       "      <td>Sun Mar 04 06:02:36 +0000 2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116563</th>\n",
       "      <td>Today's lunch is &lt;U+8C6C&gt;&lt;U+6252&gt;&lt;U+5305&gt;&lt;U+54...</td>\n",
       "      <td>today's lunch is 豬扒包和奶茶。 @ 蘭芳園 (中環) https://t....</td>\n",
       "      <td>https://t.co/fTIofoCX0X</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016-11-19 11:16:50+08:00</td>\n",
       "      <td>Sat Nov 19 03:16:50 +0000 2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24345</th>\n",
       "      <td>I'm at Main St Station &lt;U+7F8E&gt;&lt;U+570B&gt;&lt;U+5C0F...</td>\n",
       "      <td>i'm at main st station 美國小鎮大街火車站 in lantau isl...</td>\n",
       "      <td>https://t.co/7voQvQ0NsJ</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016-06-16 09:53:48+08:00</td>\n",
       "      <td>Thu Jun 16 01:53:48 +0000 2016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "286025  é ­å\n",
       "æè¬å\n",
       "äºï¼éå·¦å»å°±ç®ï¼åä½¿æ...   \n",
       "384814  í ½í± @ Ocean Park Hong Kong é¦æ¸¯æµ·æ´å\n",
       "¬å...   \n",
       "159369  <U+5EFA><U+8A2D><U+4E2D><U+3002><U+5C1A><U+572...   \n",
       "395975  I'm at è±è¨éèä» in Tsim Sha Tsui https:...   \n",
       "422639  @n_t_lie @Dualman @miniyoshima @uituit @28481k...   \n",
       "166     <U+9EC3><U+91D1><U+6BD2><U+8CE4><U+86D9> (@ Go...   \n",
       "80804   <U+5C0D><U+9762><U+6709><U+96BB><U+9CE5> @ <U+...   \n",
       "353194  é¢éåªçºä¸æ¬¡åè¨\\r\\r\\r\\nè½é¨å²å¢æ¿...   \n",
       "116563  Today's lunch is <U+8C6C><U+6252><U+5305><U+54...   \n",
       "24345   I'm at Main St Station <U+7F8E><U+570B><U+5C0F...   \n",
       "\n",
       "                                             cleaned_text  \\\n",
       "286025  頭先所講嘅事，過左去就算，唔使放喺心，睇波！btw，今日夢想沙龍乜事？對南區成隊飲左redb...   \n",
       "384814  👌 @ ocean park hong kong 香港海洋公園 https://t.co/y...   \n",
       "159369  建設中。尚在討論建設中。尚在先決定後咨詢建設中。尚在建設衰敗中...... @ west k...   \n",
       "395975  i'm at 豐記雞蛋仔 in tsim sha tsui https://t.co/krt...   \n",
       "422639  @n_t_lie @dualman @miniyoshima @uituit @28481k...   \n",
       "166     黃金毒賤蛙 (@ golden computer arcade 黃金電腦商場 in sham...   \n",
       "80804               對面有隻鳥 @ 元朗西鐵站 https://t.co/9j0h0ivzjl   \n",
       "353194  離開只為下次再臨落雨啲嘢濕哂要返屋企洗嘢😑#逃脫 @ 東龍 https://t.co/d1p...   \n",
       "116563  today's lunch is 豬扒包和奶茶。 @ 蘭芳園 (中環) https://t....   \n",
       "24345   i'm at main st station 美國小鎮大街火車站 in lantau isl...   \n",
       "\n",
       "                            url  year                    hk_time  \\\n",
       "286025  https://t.co/i0ZWZCXFTd  2017  2017-10-01 19:08:25+08:00   \n",
       "384814  https://t.co/yfrQFSEblU  2018  2018-06-10 11:14:13+08:00   \n",
       "159369  https://t.co/9vuJN0AxkC  2017  2017-01-22 18:39:03+08:00   \n",
       "395975  https://t.co/KRtF9ijaxj  2018  2018-07-15 12:12:17+08:00   \n",
       "422639  https://t.co/VtYoWqvN8a  2018  2018-10-10 16:11:24+08:00   \n",
       "166     https://t.co/KaeBx77XF2  2016  2016-05-07 17:39:55+08:00   \n",
       "80804   https://t.co/9j0h0IVZJl  2016  2016-09-21 14:40:03+08:00   \n",
       "353194  https://t.co/D1pZJmcWz9  2018  2018-03-04 14:02:36+08:00   \n",
       "116563  https://t.co/fTIofoCX0X  2016  2016-11-19 11:16:50+08:00   \n",
       "24345   https://t.co/7voQvQ0NsJ  2016  2016-06-16 09:53:48+08:00   \n",
       "\n",
       "                            created_at  \n",
       "286025  Sun Oct 01 11:08:25 +0000 2017  \n",
       "384814  Sun Jun 10 03:14:13 +0000 2018  \n",
       "159369  Sun Jan 22 10:39:03 +0000 2017  \n",
       "395975  Sun Jul 15 04:12:17 +0000 2018  \n",
       "422639  Wed Oct 10 08:11:24 +0000 2018  \n",
       "166     Sat May 07 09:39:55 +0000 2016  \n",
       "80804   Wed Sep 21 06:40:03 +0000 2016  \n",
       "353194  Sun Mar 04 06:02:36 +0000 2018  \n",
       "116563  Sat Nov 19 03:16:50 +0000 2016  \n",
       "24345   Thu Jun 16 01:53:48 +0000 2016  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zh_tweets_sample = chinese_tweets[['text', 'cleaned_text', 'url', 'year', 'hk_time', 'created_at']].sample(10)\n",
    "zh_tweets_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "chinese_tweets.to_csv(os.path.join(tweet_combined_path, 'tweet_combined_chinese.csv'), encoding='utf-8', \n",
    "                      quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Translate the Chinese Tweets to English Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "chinese_tweets_copy = chinese_tweets.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "chinese_tweets_copy['translated_text'] = chinese_tweets_copy.apply(\n",
    "    lambda row: get_translated_text(row['cleaned_text']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "chinese_tweets_copy.to_csv(os.path.join(tweet_combined_path, 'tweets_combined_chinese_translated.csv'), encoding='utf-8', \n",
    "                          quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Combine the Processed Chinese Tweets and English Tweets\n",
    "\n",
    "In this section, before combining the English tweets and Chinese tweets together, we should consider the following steps:\n",
    "\n",
    "1. delete rows of which the ```translated_text``` column is None\n",
    "2. Use the ```translated_text``` as the ```cleaned_text``` to combine two tweet dataset together\n",
    "\n",
    "After finishing the above two steps, order the combined data again and sort it by time. Finally, save the combined dataframe to the local directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id_str</th>\n",
       "      <th>lang</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>place_id</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>place_name</th>\n",
       "      <th>...</th>\n",
       "      <th>verified</th>\n",
       "      <th>hk_time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>month_plus_year</th>\n",
       "      <th>day</th>\n",
       "      <th>TPU_longitudinal</th>\n",
       "      <th>TPU_cross_sectional</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>translated_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0.1, created_at, id_str, lang, lat, lon, place_id, place_lat, place_lon, place_name, text, time_zone, truncated, url, user_created_at, user_id_str, user_lang, user_url, verified, hk_time, year, month, month_plus_year, day, TPU_longitudinal, TPU_cross_sectional, cleaned_text, translated_text]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 28 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chinese_tweets_copy[chinese_tweets_copy['translated_text'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0.1', 'created_at', 'id_str', 'lang', 'lat', 'lon',\n",
       "       'place_id', 'place_lat', 'place_lon', 'place_name', 'text', 'time_zone',\n",
       "       'truncated', 'url', 'user_created_at', 'user_id_str', 'user_lang',\n",
       "       'user_url', 'verified', 'hk_time', 'year', 'month', 'month_plus_year',\n",
       "       'day', 'TPU_longitudinal', 'TPU_cross_sectional', 'cleaned_text'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_tweets.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_tweet_dataframe = build_selected_tweet_dataframe(english_tweets, english_or_not=True)\n",
    "chinese_tweet_dataframe = build_selected_tweet_dataframe(chinese_tweets_copy, english_or_not=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe = pd.concat([english_tweet_dataframe, chinese_tweet_dataframe], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(431074, 26)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_tweet_dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>id_str</th>\n",
       "      <th>lang</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>place_id</th>\n",
       "      <th>place_lat</th>\n",
       "      <th>place_lon</th>\n",
       "      <th>place_name</th>\n",
       "      <th>text</th>\n",
       "      <th>...</th>\n",
       "      <th>user_url</th>\n",
       "      <th>verified</th>\n",
       "      <th>hk_time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>month_plus_year</th>\n",
       "      <th>day</th>\n",
       "      <th>TPU_longitudinal</th>\n",
       "      <th>TPU_cross_sectional</th>\n",
       "      <th>cleaned_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sat May 07 06:18:59 +0000 2016</td>\n",
       "      <td>7.28831E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.2788499</td>\n",
       "      <td>114.18462</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.271674</td>\n",
       "      <td>114.185178</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#Working #Saturday #Afternoon! #Final #Touch i...</td>\n",
       "      <td>...</td>\n",
       "      <td>http://www.facebook.com/derekhysteric525</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 14:18:59+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>146</td>\n",
       "      <td>146 - 147</td>\n",
       "      <td>hashtag working hashtag hashtag saturday hasht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sat May 07 07:02:19 +0000 2016</td>\n",
       "      <td>7.28842E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.31530176</td>\n",
       "      <td>113.9348316</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I'm at Hong Kong International Airport &lt;U+9999...</td>\n",
       "      <td>...</td>\n",
       "      <td>http://manishmaurya89.blogspot.com/?m=1</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:02:19+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>951</td>\n",
       "      <td>950 - 951</td>\n",
       "      <td>i am at hong kong international airport  u  nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sat May 07 07:02:34 +0000 2016</td>\n",
       "      <td>7.28842E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.27680815</td>\n",
       "      <td>113.9161873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The cable car ride... #cablecar #mountain #360...</td>\n",
       "      <td>...</td>\n",
       "      <td>http://kotakitam.wordpress.com</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:02:34+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>943</td>\n",
       "      <td>941 - 943</td>\n",
       "      <td>the cable car ride  repeated hashtag cable car...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sat May 07 07:03:11 +0000 2016</td>\n",
       "      <td>7.28843E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.27564274</td>\n",
       "      <td>114.1711743</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.271674</td>\n",
       "      <td>114.185178</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Love Roses! &lt;ed&gt;&lt;U+00A0&gt;&lt;U+00BD&gt;&lt;ed&gt;&lt;U+00B8&gt;&lt;U...</td>\n",
       "      <td>...</td>\n",
       "      <td>http://www.denkipenki.vsco.co</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:03:11+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>131</td>\n",
       "      <td>131</td>\n",
       "      <td>love roses  😍 😍 😍 🌹 🌷 💐 hashtag flower hashtag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sat May 07 07:05:37 +0000 2016</td>\n",
       "      <td>7.28843E+17</td>\n",
       "      <td>en</td>\n",
       "      <td>22.30277224</td>\n",
       "      <td>114.0117169</td>\n",
       "      <td>NaN</td>\n",
       "      <td>22.2465325</td>\n",
       "      <td>114.064237</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Preparing for a photo shoot #glamourshots #bla...</td>\n",
       "      <td>...</td>\n",
       "      <td>http://about.me/natalia.segura</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>2016-05-07 15:05:37+08:00</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2016_5</td>\n",
       "      <td>7</td>\n",
       "      <td>971</td>\n",
       "      <td>971 - 974</td>\n",
       "      <td>preparing for a photo shoot hashtag glamour sh...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       created_at       id_str lang          lat          lon  \\\n",
       "0  Sat May 07 06:18:59 +0000 2016  7.28831E+17   en   22.2788499    114.18462   \n",
       "1  Sat May 07 07:02:19 +0000 2016  7.28842E+17   en  22.31530176  113.9348316   \n",
       "2  Sat May 07 07:02:34 +0000 2016  7.28842E+17   en  22.27680815  113.9161873   \n",
       "3  Sat May 07 07:03:11 +0000 2016  7.28843E+17   en  22.27564274  114.1711743   \n",
       "4  Sat May 07 07:05:37 +0000 2016  7.28843E+17   en  22.30277224  114.0117169   \n",
       "\n",
       "  place_id   place_lat   place_lon place_name  \\\n",
       "0      NaN   22.271674  114.185178        NaN   \n",
       "1      NaN  22.2465325  114.064237        NaN   \n",
       "2      NaN  22.2465325  114.064237        NaN   \n",
       "3      NaN   22.271674  114.185178        NaN   \n",
       "4      NaN  22.2465325  114.064237        NaN   \n",
       "\n",
       "                                                text  \\\n",
       "0  #Working #Saturday #Afternoon! #Final #Touch i...   \n",
       "1  I'm at Hong Kong International Airport <U+9999...   \n",
       "2  The cable car ride... #cablecar #mountain #360...   \n",
       "3  Love Roses! <ed><U+00A0><U+00BD><ed><U+00B8><U...   \n",
       "4  Preparing for a photo shoot #glamourshots #bla...   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "3                        ...                           \n",
       "4                        ...                           \n",
       "\n",
       "                                   user_url verified  \\\n",
       "0  http://www.facebook.com/derekhysteric525    FALSE   \n",
       "1   http://manishmaurya89.blogspot.com/?m=1    FALSE   \n",
       "2            http://kotakitam.wordpress.com    FALSE   \n",
       "3             http://www.denkipenki.vsco.co    FALSE   \n",
       "4            http://about.me/natalia.segura    FALSE   \n",
       "\n",
       "                     hk_time  year month month_plus_year day TPU_longitudinal  \\\n",
       "0  2016-05-07 14:18:59+08:00  2016     5          2016_5   7              146   \n",
       "1  2016-05-07 15:02:19+08:00  2016     5          2016_5   7              951   \n",
       "2  2016-05-07 15:02:34+08:00  2016     5          2016_5   7              943   \n",
       "3  2016-05-07 15:03:11+08:00  2016     5          2016_5   7              131   \n",
       "4  2016-05-07 15:05:37+08:00  2016     5          2016_5   7              971   \n",
       "\n",
       "  TPU_cross_sectional                                       cleaned_text  \n",
       "0           146 - 147  hashtag working hashtag hashtag saturday hasht...  \n",
       "1           950 - 951  i am at hong kong international airport  u  nu...  \n",
       "2           941 - 943  the cable car ride  repeated hashtag cable car...  \n",
       "3                 131  love roses  😍 😍 😍 🌹 🌷 💐 hashtag flower hashtag...  \n",
       "4           971 - 974  preparing for a photo shoot hashtag glamour sh...  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_tweet_dataframe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe_copy = combined_tweet_dataframe.copy()\n",
    "\n",
    "combined_tweet_dataframe_copy['hk_time'] = combined_tweet_dataframe_copy.apply(lambda row: transform_string_time_to_datetime(row['hk_time']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe_copy_sorted = combined_tweet_dataframe_copy.sort_values(by='hk_time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hk_time</th>\n",
       "      <th>text</th>\n",
       "      <th>cleaned_text</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-05-07 14:12:59+08:00</td>\n",
       "      <td>#Working #Saturday #Afternoon! #Final #Touch i...</td>\n",
       "      <td>hashtag working hashtag hashtag saturday hasht...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-05-07 14:56:19+08:00</td>\n",
       "      <td>I'm at Hong Kong International Airport &lt;U+9999...</td>\n",
       "      <td>i am at hong kong international airport  u  nu...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-05-07 14:56:34+08:00</td>\n",
       "      <td>The cable car ride... #cablecar #mountain #360...</td>\n",
       "      <td>the cable car ride  repeated hashtag cable car...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-05-07 14:57:11+08:00</td>\n",
       "      <td>Love Roses! &lt;ed&gt;&lt;U+00A0&gt;&lt;U+00BD&gt;&lt;ed&gt;&lt;U+00B8&gt;&lt;U...</td>\n",
       "      <td>love roses  😍 😍 😍 🌹 🌷 💐 hashtag flower hashtag...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-05-07 14:59:37+08:00</td>\n",
       "      <td>Preparing for a photo shoot #glamourshots #bla...</td>\n",
       "      <td>preparing for a photo shoot hashtag glamour sh...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    hk_time  \\\n",
       "0 2016-05-07 14:12:59+08:00   \n",
       "1 2016-05-07 14:56:19+08:00   \n",
       "2 2016-05-07 14:56:34+08:00   \n",
       "3 2016-05-07 14:57:11+08:00   \n",
       "4 2016-05-07 14:59:37+08:00   \n",
       "\n",
       "                                                text  \\\n",
       "0  #Working #Saturday #Afternoon! #Final #Touch i...   \n",
       "1  I'm at Hong Kong International Airport <U+9999...   \n",
       "2  The cable car ride... #cablecar #mountain #360...   \n",
       "3  Love Roses! <ed><U+00A0><U+00BD><ed><U+00B8><U...   \n",
       "4  Preparing for a photo shoot #glamourshots #bla...   \n",
       "\n",
       "                                        cleaned_text lang  \n",
       "0  hashtag working hashtag hashtag saturday hasht...   en  \n",
       "1  i am at hong kong international airport  u  nu...   en  \n",
       "2  the cable car ride  repeated hashtag cable car...   en  \n",
       "3  love roses  😍 😍 😍 🌹 🌷 💐 hashtag flower hashtag...   en  \n",
       "4  preparing for a photo shoot hashtag glamour sh...   en  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_tweet_dataframe_copy_sorted[['hk_time', 'text', 'cleaned_text', 'lang']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe_copy_sorted.to_csv(os.path.join(tweet_combined_path, 'tweet_combined_cleaned_translated.csv'), \n",
    "                                           encoding = 'utf-8', quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = list(set(STOPWORDS))\n",
    "strange_terms = ['allcaps', 'repeated', 'elongated', 'repeat', 'user', 'percent_c', 'hong kong', 'hong',\n",
    "                 'kong', 'u_u', 'u_u_number', 'u_u_u_u', 'u_number', 'elongate', 'u_number_u',\n",
    "                 'u', 'number', 'm', 'will', 'hp', 'grad', 'ed', 'boo', 'url', 'hashtag']\n",
    "unuseful_terms = stopwords + strange_terms\n",
    "unuseful_terms_set = set(unuseful_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_words = set(nltk.corpus.words.words())\n",
    "english_words_lower = set(word.lower() for word in english_words)\n",
    "# emoji_dict = pd.read_pickle(os.path.join(read_data.tweet_2017, 'emoji.pkl'))\n",
    "emoji_list = list(emoji_dict['emoji'])\n",
    "english_words_lower.update(emoji_list)\n",
    "# Add station names to the english word set\n",
    "station_location = pd.read_csv(os.path.join(tweet_2017_path, 'station_location.csv'))\n",
    "station_names_list = list(station_location['Name'])\n",
    "names_lower = [word_tokenize(name.lower()) for name in station_names_list]\n",
    "words = []\n",
    "for word_list in names_lower:\n",
    "    for word in word_list:\n",
    "        words.append(word)\n",
    "english_words_lower.update(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "235928"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(english_words_lower)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The first 10000 has been processed!\n",
      "The first 20000 has been processed!\n",
      "The first 30000 has been processed!\n",
      "The first 40000 has been processed!\n",
      "The first 50000 has been processed!\n",
      "The first 60000 has been processed!\n",
      "The first 70000 has been processed!\n",
      "The first 80000 has been processed!\n",
      "The first 90000 has been processed!\n",
      "The first 100000 has been processed!\n",
      "The first 110000 has been processed!\n",
      "The first 120000 has been processed!\n",
      "The first 130000 has been processed!\n",
      "The first 140000 has been processed!\n",
      "The first 150000 has been processed!\n",
      "The first 160000 has been processed!\n",
      "The first 170000 has been processed!\n",
      "The first 180000 has been processed!\n",
      "The first 190000 has been processed!\n",
      "The first 200000 has been processed!\n",
      "The first 210000 has been processed!\n",
      "The first 220000 has been processed!\n",
      "The first 230000 has been processed!\n",
      "The first 240000 has been processed!\n",
      "The first 250000 has been processed!\n",
      "The first 260000 has been processed!\n",
      "The first 270000 has been processed!\n",
      "The first 280000 has been processed!\n",
      "The first 290000 has been processed!\n",
      "The first 300000 has been processed!\n",
      "The first 310000 has been processed!\n",
      "The first 320000 has been processed!\n",
      "The first 330000 has been processed!\n",
      "The first 340000 has been processed!\n",
      "The first 350000 has been processed!\n",
      "The first 360000 has been processed!\n",
      "The first 370000 has been processed!\n",
      "The first 380000 has been processed!\n",
      "The first 390000 has been processed!\n",
      "The first 400000 has been processed!\n",
      "The first 410000 has been processed!\n",
      "The first 420000 has been processed!\n",
      "The first 430000 has been processed!\n"
     ]
    }
   ],
   "source": [
    "processed_list = []\n",
    "\n",
    "for index, text in enumerate(list(combined_tweet_dataframe_copy_sorted['cleaned_text'])):\n",
    "    text_list = text.split()\n",
    "    processed_text_list_step1 = [text_step1 for text_step1 in text_list if text_step1 not in unuseful_terms_set]\n",
    "    processed_text_list_step2 = [text_step2 for text_step2 in text_list if text_step2 in english_words_lower]\n",
    "    if len(processed_text_list_step2) != 0:\n",
    "        processed_strings = ' '.join(processed_text_list_step2)\n",
    "        processed_list.append(processed_strings)\n",
    "    else:\n",
    "        processed_list.append(text)\n",
    "        \n",
    "    if (index + 1) % 10000 == 0:\n",
    "        print(\"The first {} has been processed!\".format(index+1))\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe_copy_sorted['cleaned_text'] = processed_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_tweet_dataframe_copy_sorted.to_csv(os.path.join(tweet_combined_path, 'tweet_combined_cleaned_translated.csv'), \n",
    "                                           encoding = 'utf-8', quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we use the following codes to generate the tweet representation and make sentiment prediction:\n",
    "\n",
    "- [model_selection](https://github.com/bright1993ff66/Social-Media-Data-Analysis/blob/master/model_selection.py)\n",
    "\n",
    "Attach the sentiment labels to the tweet dataframe\n",
    "\n",
    "After getting the sentiment of each tweet, we notice from [tweet_filtering_process](https://github.com/bright1993ff66/Social-Media-Data-Analysis/blob/master/tweet_filtering_final_github.ipynb) that there are still some bots in our data. Hence, we consider delete these users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "deleted_users_in_step2 = np.load(os.path.join(tweet_combined_path, 'botometer_check', 'deleted_bot_60_threshold.npy'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_combined_with_sentiment = pd.read_csv(os.path.join(tweet_combined_path, 'tweet_combined_with_sentiment.csv'), \n",
    "                                            encoding='utf-8', quoting=csv.QUOTE_NONNUMERIC, index_col=0, dtype='str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_combined_sentiment_without_bot = tweet_combined_with_sentiment.loc[~tweet_combined_with_sentiment['user_id_str'].isin(deleted_users_in_step2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_combined_sentiment_without_bot.to_csv(os.path.join(tweet_combined_path, 'tweet_combined_sentiment_without_bots.csv'), \n",
    "                                            encoding='utf-8', quoting=csv.QUOTE_NONNUMERIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}