Web_Scrapping_Amazon_Reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b1e419a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c5b32099",
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = [\n",
    "    \"https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews\",\n",
    "    \"https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_arp_d_paging_btm_next_2?ie=UTF8&reviewerType=all_reviews&pageNumber=2\",\n",
    "    \"https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_getr_d_paging_btm_next_3?ie=UTF8&reviewerType=all_reviews&pageNumber=3\"\n",
    "]\n",
    "\n",
    "headers = {\n",
    "    'authority': 'www.amazon.in',\n",
    "    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "    'accept-language': 'en-US,en;q=0.9',\n",
    "    'cache-control': 'max-age=0',\n",
    "    # Requests sorts cookies= alphabetically\n",
    "    # 'cookie': 'session-id=259-3113978-6678618; i18n-prefs=INR; ubid-acbin=260-8554202-6973909; lc-acbin=en_IN; csm-hit=tb:BS866TA0AKH6X86N924E+sa-7XYTQAXQHJP5ADH88228-DY27HYE0CK5V9FW24GBD|1656009294944&t:1656009294945&adb:adblk_yes; session-token=Z1j175VoYxPr2Un/9ciL3Q6lKw+QtLYYIwSQ+GLxjT06952u8vOZromD4WcFE0bs+yrUyLPy8HmIn7mTjUt8qsx3n0meC7yWKFqqwDEm5iecYedklsrNwmDrQOiaMH9lpacbdB8kgUk5IbZdg1VyhrdnY4OZrk6r350ARDEXJExuu2GZr0sV4fpbwUes/V9fDrfASeMQhVEEzmEAAHWN2g==; session-id-time=2082758401l',\n",
    "    'device-memory': '8',\n",
    "    'downlink': '10',\n",
    "    'dpr': '0.8',\n",
    "    'ect': '4g',\n",
    "    'referer': 'https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/dp/B09WQY65HN/ref=sr_1_4?crid=1D99WHM86WX80&keywords=oneplus&qid=1656009113&sprefix=onep%2Caps%2C315&sr=8-4&th=1',\n",
    "    'rtt': '0',\n",
    "    'sec-ch-device-memory': '8',\n",
    "    'sec-ch-dpr': '0.8',\n",
    "    'sec-ch-ua': '\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"102\", \"Google Chrome\";v=\"102\"',\n",
    "    'sec-ch-ua-mobile': '?0',\n",
    "    'sec-ch-ua-platform': '\"Windows\"',\n",
    "    'sec-ch-viewport-width': '2400',\n",
    "    'sec-fetch-dest': 'document',\n",
    "    'sec-fetch-mode': 'navigate',\n",
    "    'sec-fetch-site': 'same-origin',\n",
    "    'sec-fetch-user': '?1',\n",
    "    'service-worker-navigation-preload': 'true',\n",
    "    'upgrade-insecure-requests': '1',\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36',\n",
    "    'viewport-width': '2400',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1855e4ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = requests.get(pages[0], headers=headers)\n",
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "35e0201a",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(resp.text, \"html.parser\") # html.parser, lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cf9c9769",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10 divs\n",
    "reviews = soup.find_all(\"div\", {\"class\": \"a-section review aok-relative\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "468d60d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_name(soup_obj: BeautifulSoup) -> str:\n",
    "    try:\n",
    "        user_name = soup_obj.find('span', {\"class\": \"a-profile-name\"}).get_text()\n",
    "        return user_name\n",
    "    except Exception as e:\n",
    "        return 'not_found'\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "774eaaca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_review_stars(soup_obj: BeautifulSoup) -> str:\n",
    "    try:\n",
    "        review_str = soup_obj.find('a', {\"class\": \"a-link-normal\"}).get_text()\n",
    "        return review_str\n",
    "    except Exception as e:\n",
    "        return 'not_found'\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "267eb6d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_review_title(soup_obj: BeautifulSoup) -> str:\n",
    "    try:\n",
    "        review_title = soup_obj.find('a', \n",
    "            {\"class\": \"a-size-base a-link-normal review-title a-color-base review-title-content a-text-bold\"}\n",
    "        ).get_text().strip()\n",
    "        return review_title\n",
    "    except Exception as e:\n",
    "        return 'not_found'\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e683a9dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_review_body(soup_obj: BeautifulSoup) -> str:\n",
    "    try:\n",
    "        review_body = soup_obj.find('span', \n",
    "            {\"class\": \"a-size-base review-text review-text-content\"}\n",
    "        ).get_text().strip()\n",
    "        return review_body\n",
    "    except Exception as e:\n",
    "        return 'not_found'\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b2b4d94d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_review_date(soup_obj: BeautifulSoup) -> str:\n",
    "    try:\n",
    "        review_date = soup_obj.find('span', \n",
    "            {\"class\": \"a-size-base a-color-secondary review-date\"}\n",
    "        ).get_text().strip()\n",
    "        return review_date\n",
    "    except Exception as e:\n",
    "        return 'not_found'\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e5331760",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_review_components(soup_obj: BeautifulSoup) -> dict:\n",
    "    return {\n",
    "        'User': get_user_name(soup_obj) , \n",
    "        'Rating': get_review_stars(soup_obj), \n",
    "        'Title': get_review_title(soup_obj),\n",
    "        'Review': get_review_body(soup_obj),\n",
    "        'Date': get_review_date(soup_obj)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "dfcb30cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KAMAL VIRANI</td>\n",
       "      <td>2.0 out of 5 stars</td>\n",
       "      <td>Worst mobile not satisfied</td>\n",
       "      <td>Worst phone ever seen. Not upto the mark. Retu...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Verified purchase</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good in budget, not best.</td>\n",
       "      <td>The fingerprint sensor does not response somet...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rudri srihari</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Good</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Reshu Rani</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good phone with all advanced features in best ...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 8 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ranjeet Kumar</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>My First Sasta Phone 😊</td>\n",
       "      <td>It seems good for now, let us explore all feat...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NASH</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Crap product.</td>\n",
       "      <td>Oxygen OS has watered down and OnePlus is no l...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ankit Koshti</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>A great phone for family entertainment</td>\n",
       "      <td>One plus phones have always amazed me by the t...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>r</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Don't go for it ...</td>\n",
       "      <td>I am unable to export or import contacts with ...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Debjyoti Mallick</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good battery Backup.Good Camera .Everything is...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>sri</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Strong value phone</td>\n",
       "      <td>Great battery. Only 40% down since a day (not ...</td>\n",
       "      <td>Reviewed in India on 6 May 2022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                User              Rating  \\\n",
       "0       KAMAL VIRANI  2.0 out of 5 stars   \n",
       "1  Verified purchase  4.0 out of 5 stars   \n",
       "2      rudri srihari  5.0 out of 5 stars   \n",
       "3         Reshu Rani  4.0 out of 5 stars   \n",
       "4      Ranjeet Kumar  4.0 out of 5 stars   \n",
       "5               NASH  1.0 out of 5 stars   \n",
       "6       Ankit Koshti  4.0 out of 5 stars   \n",
       "7                  r  1.0 out of 5 stars   \n",
       "8   Debjyoti Mallick  4.0 out of 5 stars   \n",
       "9                sri  5.0 out of 5 stars   \n",
       "\n",
       "                                               Title  \\\n",
       "0                         Worst mobile not satisfied   \n",
       "1                          Good in budget, not best.   \n",
       "2                                               Good   \n",
       "3  Good phone with all advanced features in best ...   \n",
       "4                             My First Sasta Phone 😊   \n",
       "5                                      Crap product.   \n",
       "6             A great phone for family entertainment   \n",
       "7                                Don't go for it ...   \n",
       "8  Good battery Backup.Good Camera .Everything is...   \n",
       "9                                 Strong value phone   \n",
       "\n",
       "                                              Review  \\\n",
       "0  Worst phone ever seen. Not upto the mark. Retu...   \n",
       "1  The fingerprint sensor does not response somet...   \n",
       "2  The media could not be loaded.\\n              ...   \n",
       "3  The media could not be loaded.\\n              ...   \n",
       "4  It seems good for now, let us explore all feat...   \n",
       "5  Oxygen OS has watered down and OnePlus is no l...   \n",
       "6  One plus phones have always amazed me by the t...   \n",
       "7  I am unable to export or import contacts with ...   \n",
       "8  The media could not be loaded.\\n              ...   \n",
       "9  Great battery. Only 40% down since a day (not ...   \n",
       "\n",
       "                              Date  \n",
       "0  Reviewed in India on 4 May 2022  \n",
       "1  Reviewed in India on 4 May 2022  \n",
       "2  Reviewed in India on 4 May 2022  \n",
       "3  Reviewed in India on 8 May 2022  \n",
       "4  Reviewed in India on 4 May 2022  \n",
       "5  Reviewed in India on 5 May 2022  \n",
       "6  Reviewed in India on 5 May 2022  \n",
       "7  Reviewed in India on 5 May 2022  \n",
       "8  Reviewed in India on 4 May 2022  \n",
       "9  Reviewed in India on 6 May 2022  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records = [get_review_components(rev) for rev in reviews]\n",
    "df = pd.DataFrame.from_records(records)\n",
    "#df.to_csv('amazon_reviews.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "90ec89f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews\n",
      "https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_arp_d_paging_btm_next_2?ie=UTF8&reviewerType=all_reviews&pageNumber=2\n",
      "https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/ref=cm_cr_getr_d_paging_btm_next_3?ie=UTF8&reviewerType=all_reviews&pageNumber=3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KAMAL VIRANI</td>\n",
       "      <td>2.0 out of 5 stars</td>\n",
       "      <td>Worst mobile not satisfied</td>\n",
       "      <td>Worst phone ever seen. Not upto the mark. Retu...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Verified purchase</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good in budget, not best.</td>\n",
       "      <td>The fingerprint sensor does not response somet...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rudri srihari</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Good</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Reshu Rani</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good phone with all advanced features in best ...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 8 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ranjeet Kumar</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>My First Sasta Phone 😊</td>\n",
       "      <td>It seems good for now, let us explore all feat...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NASH</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Crap product.</td>\n",
       "      <td>Oxygen OS has watered down and OnePlus is no l...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ankit Koshti</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>A great phone for family entertainment</td>\n",
       "      <td>One plus phones have always amazed me by the t...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>r</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Don't go for it ...</td>\n",
       "      <td>I am unable to export or import contacts with ...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Debjyoti Mallick</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good battery Backup.Good Camera .Everything is...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>sri</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Strong value phone</td>\n",
       "      <td>Great battery. Only 40% down since a day (not ...</td>\n",
       "      <td>Reviewed in India on 6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Amit</td>\n",
       "      <td>3.0 out of 5 stars</td>\n",
       "      <td>One thing is missing that is amoled display</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ANINDYA DEY</td>\n",
       "      <td>2.0 out of 5 stars</td>\n",
       "      <td>avg phone, with a so so camera output</td>\n",
       "      <td>The media could not be loaded.</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Surya  Prakash</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Good phone with less budget</td>\n",
       "      <td>After using for one week, this is my reviewCam...</td>\n",
       "      <td>Reviewed in India on 14 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Paresh Patil</td>\n",
       "      <td>2.0 out of 5 stars</td>\n",
       "      <td>Battery and charging</td>\n",
       "      <td>Battery draines out too fast</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Rajjo</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Best In the Segment</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Geetanjali</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Phone is getting heated and they don't replace...</td>\n",
       "      <td>Phine is getting heated and they are not offer...</td>\n",
       "      <td>Reviewed in India on 6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>thamij</td>\n",
       "      <td>3.0 out of 5 stars</td>\n",
       "      <td>Is it me or else</td>\n",
       "      <td>Ya... Good enough and happy with one plus... B...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>DS</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Perfect phone !</td>\n",
       "      <td>first of all FASTEST delivery from amazon !!ph...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Amazon Customer</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Honest review after 5 days ..</td>\n",
       "      <td>Fingerprint sensor is responsive .Camera quali...</td>\n",
       "      <td>Reviewed in India on 7 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Hmingthanpari</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Overall OnePlus is worth buying.</td>\n",
       "      <td>OnePlus phone has beautiful design and feature...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Arnab Nandi</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Best phone in this range</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 17 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Megha</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>After 48 hours of use</td>\n",
       "      <td>I saw many negative reviews but still went for...</td>\n",
       "      <td>Reviewed in India on 6 June 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C handan</td>\n",
       "      <td>3.0 out of 5 stars</td>\n",
       "      <td>By far the most average mobile of one plus</td>\n",
       "      <td>At this price point the camera is below averag...</td>\n",
       "      <td>Reviewed in India on 11 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>PRASHANT KUMAR</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Average 5g smartphone</td>\n",
       "      <td>On this price range OnePlus is better. Camera ...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Ragunath L.</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>One Plus Nord 5G</td>\n",
       "      <td>Excellent phone. Very good quality. Price is v...</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Harsha Shillin</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Mind-Blowing Purchase</td>\n",
       "      <td>I'm giving a geniune review of the product.Fir...</td>\n",
       "      <td>Reviewed in India on 24 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Geetanjali</td>\n",
       "      <td>4.0 out of 5 stars</td>\n",
       "      <td>Less battery life, ok design</td>\n",
       "      <td>Pros:Camera is Good. Front camera has very goo...</td>\n",
       "      <td>Reviewed in India on 6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Rajesh V.</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Value for money. Good camera, good processor, ...</td>\n",
       "      <td>Camera quality and battery backup</td>\n",
       "      <td>Reviewed in India on 5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Suyash</td>\n",
       "      <td>5.0 out of 5 stars</td>\n",
       "      <td>Camera queality is above the range.</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>Reviewed in India on 12 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Karthik</td>\n",
       "      <td>1.0 out of 5 stars</td>\n",
       "      <td>Product and Amazon service is very worst</td>\n",
       "      <td>Dont Waste Your Money,My product was arrived w...</td>\n",
       "      <td>Reviewed in India on 11 May 2022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 User              Rating  \\\n",
       "0        KAMAL VIRANI  2.0 out of 5 stars   \n",
       "1   Verified purchase  4.0 out of 5 stars   \n",
       "2       rudri srihari  5.0 out of 5 stars   \n",
       "3          Reshu Rani  4.0 out of 5 stars   \n",
       "4       Ranjeet Kumar  4.0 out of 5 stars   \n",
       "5                NASH  1.0 out of 5 stars   \n",
       "6        Ankit Koshti  4.0 out of 5 stars   \n",
       "7                   r  1.0 out of 5 stars   \n",
       "8    Debjyoti Mallick  4.0 out of 5 stars   \n",
       "9                 sri  5.0 out of 5 stars   \n",
       "10               Amit  3.0 out of 5 stars   \n",
       "11        ANINDYA DEY  2.0 out of 5 stars   \n",
       "12     Surya  Prakash  4.0 out of 5 stars   \n",
       "13       Paresh Patil  2.0 out of 5 stars   \n",
       "14              Rajjo  5.0 out of 5 stars   \n",
       "15         Geetanjali  1.0 out of 5 stars   \n",
       "16             thamij  3.0 out of 5 stars   \n",
       "17                 DS  5.0 out of 5 stars   \n",
       "18    Amazon Customer  4.0 out of 5 stars   \n",
       "19      Hmingthanpari  4.0 out of 5 stars   \n",
       "20        Arnab Nandi  5.0 out of 5 stars   \n",
       "21              Megha  4.0 out of 5 stars   \n",
       "22           C handan  3.0 out of 5 stars   \n",
       "23     PRASHANT KUMAR  4.0 out of 5 stars   \n",
       "24        Ragunath L.  4.0 out of 5 stars   \n",
       "25     Harsha Shillin  4.0 out of 5 stars   \n",
       "26         Geetanjali  4.0 out of 5 stars   \n",
       "27          Rajesh V.  5.0 out of 5 stars   \n",
       "28             Suyash  5.0 out of 5 stars   \n",
       "29            Karthik  1.0 out of 5 stars   \n",
       "\n",
       "                                                Title  \\\n",
       "0                          Worst mobile not satisfied   \n",
       "1                           Good in budget, not best.   \n",
       "2                                                Good   \n",
       "3   Good phone with all advanced features in best ...   \n",
       "4                              My First Sasta Phone 😊   \n",
       "5                                       Crap product.   \n",
       "6              A great phone for family entertainment   \n",
       "7                                 Don't go for it ...   \n",
       "8   Good battery Backup.Good Camera .Everything is...   \n",
       "9                                  Strong value phone   \n",
       "10        One thing is missing that is amoled display   \n",
       "11              avg phone, with a so so camera output   \n",
       "12                        Good phone with less budget   \n",
       "13                               Battery and charging   \n",
       "14                                Best In the Segment   \n",
       "15  Phone is getting heated and they don't replace...   \n",
       "16                                   Is it me or else   \n",
       "17                                    Perfect phone !   \n",
       "18                      Honest review after 5 days ..   \n",
       "19                   Overall OnePlus is worth buying.   \n",
       "20                           Best phone in this range   \n",
       "21                              After 48 hours of use   \n",
       "22         By far the most average mobile of one plus   \n",
       "23                              Average 5g smartphone   \n",
       "24                                   One Plus Nord 5G   \n",
       "25                              Mind-Blowing Purchase   \n",
       "26                       Less battery life, ok design   \n",
       "27  Value for money. Good camera, good processor, ...   \n",
       "28                Camera queality is above the range.   \n",
       "29           Product and Amazon service is very worst   \n",
       "\n",
       "                                               Review  \\\n",
       "0   Worst phone ever seen. Not upto the mark. Retu...   \n",
       "1   The fingerprint sensor does not response somet...   \n",
       "2   The media could not be loaded.\\n              ...   \n",
       "3   The media could not be loaded.\\n              ...   \n",
       "4   It seems good for now, let us explore all feat...   \n",
       "5   Oxygen OS has watered down and OnePlus is no l...   \n",
       "6   One plus phones have always amazed me by the t...   \n",
       "7   I am unable to export or import contacts with ...   \n",
       "8   The media could not be loaded.\\n              ...   \n",
       "9   Great battery. Only 40% down since a day (not ...   \n",
       "10  The media could not be loaded.\\n              ...   \n",
       "11                     The media could not be loaded.   \n",
       "12  After using for one week, this is my reviewCam...   \n",
       "13                       Battery draines out too fast   \n",
       "14  The media could not be loaded.\\n              ...   \n",
       "15  Phine is getting heated and they are not offer...   \n",
       "16  Ya... Good enough and happy with one plus... B...   \n",
       "17  first of all FASTEST delivery from amazon !!ph...   \n",
       "18  Fingerprint sensor is responsive .Camera quali...   \n",
       "19  OnePlus phone has beautiful design and feature...   \n",
       "20  The media could not be loaded.\\n              ...   \n",
       "21  I saw many negative reviews but still went for...   \n",
       "22  At this price point the camera is below averag...   \n",
       "23  On this price range OnePlus is better. Camera ...   \n",
       "24  Excellent phone. Very good quality. Price is v...   \n",
       "25  I'm giving a geniune review of the product.Fir...   \n",
       "26  Pros:Camera is Good. Front camera has very goo...   \n",
       "27                  Camera quality and battery backup   \n",
       "28  The media could not be loaded.\\n              ...   \n",
       "29  Dont Waste Your Money,My product was arrived w...   \n",
       "\n",
       "                                Date  \n",
       "0    Reviewed in India on 4 May 2022  \n",
       "1    Reviewed in India on 4 May 2022  \n",
       "2    Reviewed in India on 4 May 2022  \n",
       "3    Reviewed in India on 8 May 2022  \n",
       "4    Reviewed in India on 4 May 2022  \n",
       "5    Reviewed in India on 5 May 2022  \n",
       "6    Reviewed in India on 5 May 2022  \n",
       "7    Reviewed in India on 5 May 2022  \n",
       "8    Reviewed in India on 4 May 2022  \n",
       "9    Reviewed in India on 6 May 2022  \n",
       "10   Reviewed in India on 6 May 2022  \n",
       "11   Reviewed in India on 5 May 2022  \n",
       "12  Reviewed in India on 14 May 2022  \n",
       "13   Reviewed in India on 5 May 2022  \n",
       "14   Reviewed in India on 5 May 2022  \n",
       "15   Reviewed in India on 6 May 2022  \n",
       "16   Reviewed in India on 5 May 2022  \n",
       "17   Reviewed in India on 5 May 2022  \n",
       "18   Reviewed in India on 7 May 2022  \n",
       "19   Reviewed in India on 5 May 2022  \n",
       "20  Reviewed in India on 17 May 2022  \n",
       "21  Reviewed in India on 6 June 2022  \n",
       "22  Reviewed in India on 11 May 2022  \n",
       "23   Reviewed in India on 5 May 2022  \n",
       "24   Reviewed in India on 5 May 2022  \n",
       "25  Reviewed in India on 24 May 2022  \n",
       "26   Reviewed in India on 6 May 2022  \n",
       "27   Reviewed in India on 5 May 2022  \n",
       "28  Reviewed in India on 12 May 2022  \n",
       "29  Reviewed in India on 11 May 2022  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ALL_REVIEWS = []\n",
    "\n",
    "for u in pages:\n",
    "    print(u)\n",
    "    resp = requests.get(u, headers=headers)\n",
    "    soup = BeautifulSoup(resp.text, \"html.parser\")\n",
    "    reviews = soup.find_all(\"div\", {\"class\": \"a-section review aok-relative\"})\n",
    "    records = [get_review_components(rev) for rev in reviews]\n",
    "    ALL_REVIEWS.extend(records) \n",
    "    \n",
    "    \n",
    "out = pd.DataFrame.from_records(ALL_REVIEWS)\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a161e0",
   "metadata": {},
   "source": [
    "## Looping through multiple pages\n",
    "One of the easiest methods to scrape multiple pages is to modify the base URL to accept a page variable that increments as needed.\n",
    "\n",
    "Try for yourself! See how the URL changes as you go through multiple pages.\n",
    "\n",
    "For Amazon product reviews, the only thing that seems to change is the number indicating which page it is."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dcc2e08",
   "metadata": {},
   "source": [
    "#### Okay, now let’s put this to work in a function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8ca0d1f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_soup(url):\n",
    "    #r = requests.get('http://localhost:8050/render.html', \n",
    "    # Run this instead if you haven't setup Splash & Docker\n",
    "    r = requests.get(url, headers=headers,\n",
    "    params={'url': url, 'wait': 2})\n",
    "    soup = BeautifulSoup(r.text, 'html.parser')\n",
    "    return soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cc95f3ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize list to store reviews data later on\n",
    "reviewlist = []\n",
    "\n",
    "# Function 2: look for web-tags in our soup, then append our data to reviewList\n",
    "def get_reviews(soup):\n",
    "    reviews = soup.find_all('div', {'data-hook': 'review'})\n",
    "    try:\n",
    "        for item in reviews:\n",
    "            review = {\n",
    "            'product': soup.title.text.replace('Amazon.in:Customer reviews: ', '').strip(),    \n",
    "            'date': item.find('span', {'data-hook': 'review-date'}).text.replace('Reviewed in India on', '').strip(),\n",
    "            'title': item.find('a', {'data-hook': 'review-title'}).text.strip(),\n",
    "            'rating':  float(item.find('i', {'data-hook': 'review-star-rating'}).text.replace('out of 5 stars', '').strip()),\n",
    "            'body': item.find('span', {'data-hook': 'review-body'}).text.strip(),\n",
    "            }\n",
    "            reviewlist.append(review)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c4ee2299",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize list to store reviews data later on\n",
    "reviewlist = []\n",
    "\n",
    "# Function 2: look for web-tags in our soup, then append our data to reviewList\n",
    "def get_reviews(soup):\n",
    "    reviews = soup.find_all('div', {'data-hook': 'review'})\n",
    "    try:\n",
    "        for item in reviews:\n",
    "            review = {\n",
    "            'Rating':  float(item.find('i', {'data-hook': 'review-star-rating'}).text.replace('out of 5 stars', '').strip()),\n",
    "            'Title': item.find('a', {'data-hook': 'review-title'}).text.strip(),\n",
    "            'Review': item.find('span', {'data-hook': 'review-body'}).text.strip(),\n",
    "            'Review_Date': item.find('span', {'data-hook': 'review-date'}).text.replace('Reviewed in India on', '').strip(),\n",
    "            }\n",
    "            reviewlist.append(review)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eb7c89ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting page: 1\n",
      "10\n",
      "Getting page: 2\n",
      "20\n",
      "Getting page: 3\n",
      "30\n",
      "Getting page: 4\n",
      "40\n"
     ]
    }
   ],
   "source": [
    "# loop through 1:x many pages, or until the css selector found only on the last page is found (when the next page button is greyed)\n",
    "for x in range(1,5):\n",
    "    soup = get_soup(f'https://www.amazon.in/OnePlus-Nord-Black-128GB-Storage/product-reviews/B09WQY65HN/\\\n",
    "ref=cm_cr_arp_d_paging_btm_next_2?ie=UTF8&reviewerType=all_reviews&pageNumber={x}')\n",
    "    print(f'Getting page: {x}')\n",
    "    get_reviews(soup)\n",
    "    print(len(reviewlist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "62a53ee4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rating</th>\n",
       "      <th>Title</th>\n",
       "      <th>Review</th>\n",
       "      <th>Review_Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>Worst mobile not satisfied</td>\n",
       "      <td>Worst phone ever seen. Not upto the mark. Retu...</td>\n",
       "      <td>4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Good in budget, not best.</td>\n",
       "      <td>The fingerprint sensor does not response somet...</td>\n",
       "      <td>4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Good</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Good phone with all advanced features in best ...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>8 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>My First Sasta Phone 😊</td>\n",
       "      <td>It seems good for now, let us explore all feat...</td>\n",
       "      <td>4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Crap product.</td>\n",
       "      <td>Oxygen OS has watered down and OnePlus is no l...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4.0</td>\n",
       "      <td>A great phone for family entertainment</td>\n",
       "      <td>One plus phones have always amazed me by the t...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Don't go for it ...</td>\n",
       "      <td>I am unable to export or import contacts with ...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Good battery Backup.Good Camera .Everything is...</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>4 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Strong value phone</td>\n",
       "      <td>Great battery. Only 40% down since a day (not ...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.0</td>\n",
       "      <td>One thing is missing that is amoled display</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.0</td>\n",
       "      <td>avg phone, with a so so camera output</td>\n",
       "      <td>The media could not be loaded.</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Good phone with less budget</td>\n",
       "      <td>After using for one week, this is my reviewCam...</td>\n",
       "      <td>14 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.0</td>\n",
       "      <td>Battery and charging</td>\n",
       "      <td>Battery draines out too fast</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Best In the Segment</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Phone is getting heated and they don't replace...</td>\n",
       "      <td>Phine is getting heated and they are not offer...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3.0</td>\n",
       "      <td>Is it me or else</td>\n",
       "      <td>Ya... Good enough and happy with one plus... B...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Perfect phone !</td>\n",
       "      <td>first of all FASTEST delivery from amazon !!ph...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Honest review after 5 days ..</td>\n",
       "      <td>Fingerprint sensor is responsive .Camera quali...</td>\n",
       "      <td>7 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Overall OnePlus is worth buying.</td>\n",
       "      <td>OnePlus phone has beautiful design and feature...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Best phone in this range</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>17 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>4.0</td>\n",
       "      <td>After 48 hours of use</td>\n",
       "      <td>I saw many negative reviews but still went for...</td>\n",
       "      <td>6 June 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3.0</td>\n",
       "      <td>By far the most average mobile of one plus</td>\n",
       "      <td>At this price point the camera is below averag...</td>\n",
       "      <td>11 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Average 5g smartphone</td>\n",
       "      <td>On this price range OnePlus is better. Camera ...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>4.0</td>\n",
       "      <td>One Plus Nord 5G</td>\n",
       "      <td>Excellent phone. Very good quality. Price is v...</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Mind-Blowing Purchase</td>\n",
       "      <td>I'm giving a geniune review of the product.Fir...</td>\n",
       "      <td>24 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4.0</td>\n",
       "      <td>Less battery life, ok design</td>\n",
       "      <td>Pros:Camera is Good. Front camera has very goo...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Value for money. Good camera, good processor, ...</td>\n",
       "      <td>Camera quality and battery backup</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Camera queality is above the range.</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>12 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Product and Amazon service is very worst</td>\n",
       "      <td>Dont Waste Your Money,My product was arrived w...</td>\n",
       "      <td>11 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Super 👌</td>\n",
       "      <td>Super 👌</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1.0</td>\n",
       "      <td>Poor quality..not expected from OnePlus</td>\n",
       "      <td>Very poor display quality..It's LCD display, n...</td>\n",
       "      <td>9 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1.0</td>\n",
       "      <td>No charge connect in 15 day on purchasing in t...</td>\n",
       "      <td>This my phone sume time change is connected so...</td>\n",
       "      <td>24 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Best mobile within ur budget</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>25 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Fast and impressive</td>\n",
       "      <td>The phone boasts of a sleek and lightweight de...</td>\n",
       "      <td>17 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Brand of OnePlus for all</td>\n",
       "      <td>Good Phone to buyAndroid 12 is awesomeSpeedy p...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Mind-blowing</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>31 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Very Good and Best product #</td>\n",
       "      <td>The media could not be loaded.\\n              ...</td>\n",
       "      <td>29 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>3.0</td>\n",
       "      <td>Not worth for money</td>\n",
       "      <td>Camera 6/10Overall 6/10</td>\n",
       "      <td>5 May 2022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Good phone,</td>\n",
       "      <td>The display is  good, smooth ui, primary camer...</td>\n",
       "      <td>6 May 2022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Rating                                              Title  \\\n",
       "0      2.0                         Worst mobile not satisfied   \n",
       "1      4.0                          Good in budget, not best.   \n",
       "2      5.0                                               Good   \n",
       "3      4.0  Good phone with all advanced features in best ...   \n",
       "4      4.0                             My First Sasta Phone 😊   \n",
       "5      1.0                                      Crap product.   \n",
       "6      4.0             A great phone for family entertainment   \n",
       "7      1.0                                Don't go for it ...   \n",
       "8      4.0  Good battery Backup.Good Camera .Everything is...   \n",
       "9      5.0                                 Strong value phone   \n",
       "10     3.0        One thing is missing that is amoled display   \n",
       "11     2.0              avg phone, with a so so camera output   \n",
       "12     4.0                        Good phone with less budget   \n",
       "13     2.0                               Battery and charging   \n",
       "14     5.0                                Best In the Segment   \n",
       "15     1.0  Phone is getting heated and they don't replace...   \n",
       "16     3.0                                   Is it me or else   \n",
       "17     5.0                                    Perfect phone !   \n",
       "18     4.0                      Honest review after 5 days ..   \n",
       "19     4.0                   Overall OnePlus is worth buying.   \n",
       "20     5.0                           Best phone in this range   \n",
       "21     4.0                              After 48 hours of use   \n",
       "22     3.0         By far the most average mobile of one plus   \n",
       "23     4.0                              Average 5g smartphone   \n",
       "24     4.0                                   One Plus Nord 5G   \n",
       "25     4.0                              Mind-Blowing Purchase   \n",
       "26     4.0                       Less battery life, ok design   \n",
       "27     5.0  Value for money. Good camera, good processor, ...   \n",
       "28     5.0                Camera queality is above the range.   \n",
       "29     1.0           Product and Amazon service is very worst   \n",
       "30     5.0                                            Super 👌   \n",
       "31     1.0            Poor quality..not expected from OnePlus   \n",
       "32     1.0  No charge connect in 15 day on purchasing in t...   \n",
       "33     5.0                       Best mobile within ur budget   \n",
       "34     5.0                                Fast and impressive   \n",
       "35     5.0                           Brand of OnePlus for all   \n",
       "36     5.0                                       Mind-blowing   \n",
       "37     5.0                       Very Good and Best product #   \n",
       "38     3.0                                Not worth for money   \n",
       "39     5.0                                        Good phone,   \n",
       "\n",
       "                                               Review  Review_Date  \n",
       "0   Worst phone ever seen. Not upto the mark. Retu...   4 May 2022  \n",
       "1   The fingerprint sensor does not response somet...   4 May 2022  \n",
       "2   The media could not be loaded.\\n              ...   4 May 2022  \n",
       "3   The media could not be loaded.\\n              ...   8 May 2022  \n",
       "4   It seems good for now, let us explore all feat...   4 May 2022  \n",
       "5   Oxygen OS has watered down and OnePlus is no l...   5 May 2022  \n",
       "6   One plus phones have always amazed me by the t...   5 May 2022  \n",
       "7   I am unable to export or import contacts with ...   5 May 2022  \n",
       "8   The media could not be loaded.\\n              ...   4 May 2022  \n",
       "9   Great battery. Only 40% down since a day (not ...   6 May 2022  \n",
       "10  The media could not be loaded.\\n              ...   6 May 2022  \n",
       "11                     The media could not be loaded.   5 May 2022  \n",
       "12  After using for one week, this is my reviewCam...  14 May 2022  \n",
       "13                       Battery draines out too fast   5 May 2022  \n",
       "14  The media could not be loaded.\\n              ...   5 May 2022  \n",
       "15  Phine is getting heated and they are not offer...   6 May 2022  \n",
       "16  Ya... Good enough and happy with one plus... B...   5 May 2022  \n",
       "17  first of all FASTEST delivery from amazon !!ph...   5 May 2022  \n",
       "18  Fingerprint sensor is responsive .Camera quali...   7 May 2022  \n",
       "19  OnePlus phone has beautiful design and feature...   5 May 2022  \n",
       "20  The media could not be loaded.\\n              ...  17 May 2022  \n",
       "21  I saw many negative reviews but still went for...  6 June 2022  \n",
       "22  At this price point the camera is below averag...  11 May 2022  \n",
       "23  On this price range OnePlus is better. Camera ...   5 May 2022  \n",
       "24  Excellent phone. Very good quality. Price is v...   5 May 2022  \n",
       "25  I'm giving a geniune review of the product.Fir...  24 May 2022  \n",
       "26  Pros:Camera is Good. Front camera has very goo...   6 May 2022  \n",
       "27                  Camera quality and battery backup   5 May 2022  \n",
       "28  The media could not be loaded.\\n              ...  12 May 2022  \n",
       "29  Dont Waste Your Money,My product was arrived w...  11 May 2022  \n",
       "30                                            Super 👌   5 May 2022  \n",
       "31  Very poor display quality..It's LCD display, n...   9 May 2022  \n",
       "32  This my phone sume time change is connected so...  24 May 2022  \n",
       "33  The media could not be loaded.\\n              ...  25 May 2022  \n",
       "34  The phone boasts of a sleek and lightweight de...  17 May 2022  \n",
       "35  Good Phone to buyAndroid 12 is awesomeSpeedy p...   6 May 2022  \n",
       "36  The media could not be loaded.\\n              ...  31 May 2022  \n",
       "37  The media could not be loaded.\\n              ...  29 May 2022  \n",
       "38                            Camera 6/10Overall 6/10   5 May 2022  \n",
       "39  The display is  good, smooth ui, primary camer...   6 May 2022  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save results to a dataframe, then export as CSV\n",
    "df = pd.DataFrame(reviewlist)\n",
    "#df.to_csv(r'sony-headphones.csv', index=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8afebe63",
   "metadata": {},
   "source": [
    "#### We can even add in a stop condition. For this one, we can tell Python to look for a greyed out “Next Page” button. To identify this element, use the element inspector.\n",
    "\n",
    "Add this to the bottom of the function above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07435f4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loop through 1:x many pages, or until the css selector found only on the last page is found (when the next page button is greyed)\n",
    "for x in range(1,20):\n",
    "    soup = get_soup(f'https://www.amazon.ca/Sony-WF-1000XM3-Industry-Canceling-Wireless/product-reviews/B07T81554H/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews&pageNumber={x}')\n",
    "    print(f'Getting page: {x}')\n",
    "    get_reviews(soup)\n",
    "    print(len(reviewlist))\n",
    "    if not soup.find('li', {'class': 'a-disabled a-last'}):\n",
    "        pass\n",
    "    else:\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}