Twitter Robust scraping .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6739ee1",
   "metadata": {},
   "source": [
    "### INSTALLING THE PYTHON MODULE FOR TWITTER SCRAPING(SNSCRAPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3264b1f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: snscrape in c:\\users\\dell\\anaconda3\\lib\\site-packages (0.4.3.20220106)\n",
      "Requirement already satisfied: lxml in c:\\users\\dell\\anaconda3\\lib\\site-packages (from snscrape) (4.6.3)\n",
      "Requirement already satisfied: requests[socks] in c:\\users\\dell\\anaconda3\\lib\\site-packages (from snscrape) (2.26.0)\n",
      "Requirement already satisfied: filelock in c:\\users\\dell\\anaconda3\\lib\\site-packages (from snscrape) (3.3.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from snscrape) (4.10.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from beautifulsoup4->snscrape) (2.2.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests[socks]->snscrape) (3.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests[socks]->snscrape) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests[socks]->snscrape) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests[socks]->snscrape) (2.0.4)\n",
      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in c:\\users\\dell\\anaconda3\\lib\\site-packages (from requests[socks]->snscrape) (1.7.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install snscrape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "578a5426",
   "metadata": {},
   "source": [
    "### IMPORTING THE NECESSARY LIBRARIES TO CARRYOUT THIS TASK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "52f8345d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import snscrape.modules.twitter as sntwitter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3fb3288",
   "metadata": {},
   "source": [
    "#### COLLECTION OF THE TWEET HASHTAG OR USERNAME,DATE PERIOD,NUMBER OF DATA,DESIRED COLUMNS AND SAVING THE SCRAPED DATA AS A .CSV_FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "59873be7",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"(#gistlovers) until:2022-09-04 since:2022-09-03\"\n",
    "tweets = []\n",
    "limit = 100\n",
    "\n",
    "\n",
    "for tweet in sntwitter.TwitterHashtagScraper(query).get_items():\n",
    "    \n",
    "    if len(tweets) == limit:\n",
    "        break\n",
    "    else:\n",
    "        tweets.append([tweet.date, tweet.url, tweet.user.username, tweet.sourceLabel, tweet.user.location, tweet.content, tweet.likeCount, tweet.retweetCount,  tweet.quoteCount, tweet.replyCount])\n",
    "        \n",
    "df = pd.DataFrame(tweets, columns=['Date', 'TweetURL','User', 'Source', 'Location', 'Tweet', 'Likes_Count','Retweet_Count', 'Quote_Count', 'Reply_Count'])\n",
    "\n",
    "df.to_csv('gistlovers.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef4211ca",
   "metadata": {},
   "source": [
    "### OBSERVE THE TWEET SCRAPED RESULT WITH PANDAS DATA_FRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "94822c9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>TweetURL</th>\n",
       "      <th>User</th>\n",
       "      <th>Source</th>\n",
       "      <th>Location</th>\n",
       "      <th>Tweet</th>\n",
       "      <th>Likes_Count</th>\n",
       "      <th>Retweet_Count</th>\n",
       "      <th>Quote_Count</th>\n",
       "      <th>Reply_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-09-03 23:41:00+00:00</td>\n",
       "      <td>https://twitter.com/7GossipVilla/status/156620...</td>\n",
       "      <td>7GossipVilla</td>\n",
       "      <td>Twitter for Android</td>\n",
       "      <td>Lagos, Nigeria</td>\n",
       "      <td>Gistlover wants to shut everything down ooo.\\n...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-09-03 23:04:21+00:00</td>\n",
       "      <td>https://twitter.com/HenryIzick/status/15662004...</td>\n",
       "      <td>HenryIzick</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>Kaduna, Nigeria</td>\n",
       "      <td>What would you attempt to do in life, if the p...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-09-03 22:59:39+00:00</td>\n",
       "      <td>https://twitter.com/Duruanyanwu_b/status/15661...</td>\n",
       "      <td>Duruanyanwu_b</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>Lagos, Nigeria</td>\n",
       "      <td>Gistlover has posted the continuation \\nWho el...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-09-03 22:59:31+00:00</td>\n",
       "      <td>https://twitter.com/Awixy2/status/156619923879...</td>\n",
       "      <td>Awixy2</td>\n",
       "      <td>Twitter for Android</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Which one of them is dumber?🤭🤣😄\\n\\n#TheRingsOf...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-09-03 22:57:11+00:00</td>\n",
       "      <td>https://twitter.com/lawmantv_/status/156619865...</td>\n",
       "      <td>lawmantv_</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>Nigeria</td>\n",
       "      <td>Continuation, @ndlea_nigeria should please ans...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Date  \\\n",
       "0 2022-09-03 23:41:00+00:00   \n",
       "1 2022-09-03 23:04:21+00:00   \n",
       "2 2022-09-03 22:59:39+00:00   \n",
       "3 2022-09-03 22:59:31+00:00   \n",
       "4 2022-09-03 22:57:11+00:00   \n",
       "\n",
       "                                            TweetURL           User  \\\n",
       "0  https://twitter.com/7GossipVilla/status/156620...   7GossipVilla   \n",
       "1  https://twitter.com/HenryIzick/status/15662004...     HenryIzick   \n",
       "2  https://twitter.com/Duruanyanwu_b/status/15661...  Duruanyanwu_b   \n",
       "3  https://twitter.com/Awixy2/status/156619923879...         Awixy2   \n",
       "4  https://twitter.com/lawmantv_/status/156619865...      lawmantv_   \n",
       "\n",
       "                Source         Location  \\\n",
       "0  Twitter for Android   Lagos, Nigeria   \n",
       "1      Twitter Web App  Kaduna, Nigeria   \n",
       "2      Twitter Web App   Lagos, Nigeria   \n",
       "3  Twitter for Android          Nigeria   \n",
       "4   Twitter for iPhone          Nigeria   \n",
       "\n",
       "                                               Tweet  Likes_Count  \\\n",
       "0  Gistlover wants to shut everything down ooo.\\n...            3   \n",
       "1  What would you attempt to do in life, if the p...            1   \n",
       "2  Gistlover has posted the continuation \\nWho el...            2   \n",
       "3  Which one of them is dumber?🤭🤣😄\\n\\n#TheRingsOf...            0   \n",
       "4  Continuation, @ndlea_nigeria should please ans...            1   \n",
       "\n",
       "   Retweet_Count  Quote_Count  Reply_Count  \n",
       "0              0            0            0  \n",
       "1              1            0            1  \n",
       "2              0            0            0  \n",
       "3              0            0            1  \n",
       "4              3            0            1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8168214",
   "metadata": {},
   "source": [
    "### TWEET DESCRIPTION USING PANDAS DATA_FRAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "72836cad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Likes_Count</th>\n",
       "      <th>Retweet_Count</th>\n",
       "      <th>Quote_Count</th>\n",
       "      <th>Reply_Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100.00000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>18.22000</td>\n",
       "      <td>3.940000</td>\n",
       "      <td>0.160000</td>\n",
       "      <td>1.350000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>103.46654</td>\n",
       "      <td>13.629158</td>\n",
       "      <td>0.706678</td>\n",
       "      <td>3.929621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.50000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1028.00000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>33.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Likes_Count  Retweet_Count  Quote_Count  Reply_Count\n",
       "count    100.00000     100.000000   100.000000   100.000000\n",
       "mean      18.22000       3.940000     0.160000     1.350000\n",
       "std      103.46654      13.629158     0.706678     3.929621\n",
       "min        0.00000       0.000000     0.000000     0.000000\n",
       "25%        1.00000       0.000000     0.000000     0.000000\n",
       "50%        2.50000       0.000000     0.000000     0.000000\n",
       "75%        6.00000       2.000000     0.000000     1.000000\n",
       "max     1028.00000     100.000000     5.000000    33.000000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a131e871",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d2608f4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}