image_scrapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "97bfc1c5",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import urllib\n",
    "import os\n",
    "import requests\n",
    "import io\n",
    "from PIL import *\n",
    "from PIL import Image\n",
    "import hashlib\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b74092b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is the path I use\n",
    "# DRIVER_PATH = '.../Desktop/Scraping/chromedriver 2'\n",
    "# Put the path for your ChromeDriver here\n",
    "# from selenium.webdriver.chrome.service import Service\n",
    "# # from webdriver_manager.chrome import ChromeDriverManager\n",
    "# # s=Service(ChromeDriverManager().install())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "454c48cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "DRIVER_PATH = r'C:\\Users\\tapen\\.wdm\\drivers\\chromedriver\\win32\\96.0.4664.45\\chromedriver.exe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d027f407",
   "metadata": {},
   "outputs": [],
   "source": [
    "# wd = webdriver.Chrome(executable_path=DRIVER_PATH)\n",
    "# wd.get('https://www.google.co.in/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9135d1d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_image_urls(query:str, max_links_to_fetch:int, wd:webdriver, sleep_between_interactions:int=1):\n",
    "    def scroll_to_end(wd):\n",
    "        wd.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        time.sleep(sleep_between_interactions)    \n",
    "    \n",
    "    # build the google query\n",
    "    search_url = \"https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img\"\n",
    "\n",
    "    # load the page\n",
    "    wd.get(search_url.format(q=query))\n",
    "\n",
    "    image_urls = set()\n",
    "    image_count = 0\n",
    "    results_start = 0\n",
    "    while image_count < max_links_to_fetch:\n",
    "        scroll_to_end(wd)\n",
    "\n",
    "        # get all image thumbnail results\n",
    "        thumbnail_results = wd.find_elements_by_css_selector(\"img.Q4LuWd\")\n",
    "        number_results = len(thumbnail_results)\n",
    "        \n",
    "        print(f\"Found: {number_results} search results. Extracting links from {results_start}:{number_results}\")\n",
    "        \n",
    "        for img in thumbnail_results[results_start:number_results]:\n",
    "            # try to click every thumbnail such that we can get the real image behind it\n",
    "            try:\n",
    "                img.click()\n",
    "                time.sleep(sleep_between_interactions)\n",
    "            except Exception:\n",
    "                continue\n",
    "\n",
    "            # extract image urls    \n",
    "            actual_images = wd.find_elements_by_css_selector('img.n3VNCb')\n",
    "            for actual_image in actual_images:\n",
    "                if actual_image.get_attribute('src') and 'http' in actual_image.get_attribute('src'):\n",
    "                    image_urls.add(actual_image.get_attribute('src'))\n",
    "\n",
    "            image_count = len(image_urls)\n",
    "\n",
    "            if len(image_urls) >= max_links_to_fetch:\n",
    "                print(f\"Found: {len(image_urls)} image links, done!\")\n",
    "                break\n",
    "        else:\n",
    "            print(\"Found:\", len(image_urls), \"image links, looking for more ...\")\n",
    "            time.sleep(30)\n",
    "            return\n",
    "            load_more_button = wd.find_element_by_css_selector(\".mye4qd\")\n",
    "            if load_more_button:\n",
    "                wd.execute_script(\"document.querySelector('.mye4qd').click();\")\n",
    "\n",
    "        # move the result startpoint further down\n",
    "        results_start = len(thumbnail_results)\n",
    "\n",
    "    return image_urls\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8aca0dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def persist_image(folder_path:str,url:str):\n",
    "    try:\n",
    "        image_content = requests.get(url).content\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"ERROR - Could not download {url} - {e}\")\n",
    "\n",
    "    try:\n",
    "        image_file = io.BytesIO(image_content)\n",
    "        image = Image.open(image_file).convert('RGB')\n",
    "        file_path = os.path.join(folder_path,hashlib.sha1(image_content).hexdigest()[:10] + '.jpg')\n",
    "        with open(file_path, 'wb') as f:\n",
    "            image.save(f, \"JPEG\", quality=85)\n",
    "        print(f\"SUCCESS - saved {url} - as {file_path}\")\n",
    "    except Exception as e:\n",
    "        print(f\"ERROR - Could not save {url} - {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e04034da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_and_download(search_term:str,driver_path:str,target_path='./images',number_images=50):\n",
    "    target_folder = os.path.join(target_path,'_'.join(search_term.lower().split(' ')))\n",
    "\n",
    "    if not os.path.exists(target_folder):\n",
    "        os.makedirs(target_folder)\n",
    "\n",
    "    with webdriver.Chrome(executable_path=driver_path, ) as wd:\n",
    "        res = fetch_image_urls(search_term, number_images, wd=wd, sleep_between_interactions=0.5)\n",
    "        \n",
    "    for elem in res:\n",
    "        persist_image(target_folder,elem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "86416ad6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-7-4c9fafe6fe71>:7: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  with webdriver.Chrome(executable_path=driver_path, ) as wd:\n",
      "<ipython-input-5-6fae6806c786>:19: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  thumbnail_results = wd.find_elements_by_css_selector(\"img.Q4LuWd\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found: 100 search results. Extracting links from 0:100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-5-6fae6806c786>:33: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
      "  actual_images = wd.find_elements_by_css_selector('img.n3VNCb')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found: 51 image links, done!\n",
      "SUCCESS - saved https://image.shutterstock.com/image-photo/chiangmai-thailand-dec-19-2016-260nw-539077027.jpg - as ./images\\indian_trucks_with_number_plate\\e54663e3fc.jpg\n",
      "SUCCESS - saved https://static.abplive.com/wp-content/uploads/2019/09/12125708/Truck-ss.jpg?impolicy=abp_images&imwidth=720 - as ./images\\indian_trucks_with_number_plate\\652a50f196.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSzTBc96FPPvmVHfY9bbAiqoUCNwL72GfTDmg&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\8943180197.jpg\n",
      "SUCCESS - saved https://assets.telegraphindia.com/telegraph/234ad152-9656-408f-83ce-0902e7b723ad.jpg - as ./images\\indian_trucks_with_number_plate\\4e2a44b04a.jpg\n",
      "SUCCESS - saved https://static.toiimg.com/thumb/msid-70636212,imgsize-600814,width-400,resizemode-4/70636212.jpg - as ./images\\indian_trucks_with_number_plate\\93fd26acff.jpg\n",
      "SUCCESS - saved https://compote.slate.com/images/4c0abd86-75f9-4e37-9b39-70dd6f0b0026.jpg - as ./images\\indian_trucks_with_number_plate\\6f8a499ab9.jpg\n",
      "SUCCESS - saved https://cdn.cnn.com/cnnnext/dam/assets/150728162730-truck-art-india-montage-1-exlarge-169.jpg - as ./images\\indian_trucks_with_number_plate\\3d7fd8225a.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSH_1xkwJD5lZUN1Gsi6P1yIVR6F00MIJWWIQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\14a9be5a41.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR6kGnqG0cCMtSZm6GsgSTqNlOcvNq3eW5urw&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\17fd2c8759.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT51T72TLoqy1LIyeHy-aarcDb1KkrfQKF6xA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\1f16c9a067.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRfloRWOkmaPsT8D2PVBrdeghBJ6ncmfX3udQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\b337c01b04.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSZ7sHQP2bd8am-opaOuCCmLOIxrJH924H2KA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\1066dc2b8f.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRvNQyrhTIAu5O6HLPOsC8lW5sX30hgGJN6tA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\69702bc3f5.jpg\n",
      "SUCCESS - saved https://5.imimg.com/data5/YP/OW/OT/SELLER-96448555/20190525-122517-jpg-500x500.jpg - as ./images\\indian_trucks_with_number_plate\\34b5d6d105.jpg\n",
      "SUCCESS - saved https://media-cdn.tripadvisor.com/media/photo-s/05/ff/73/29/potli-an-indian-market.jpg - as ./images\\indian_trucks_with_number_plate\\eb1b0e2d4f.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTIdNt-qFOcBo5ZRBSt8rpLNccIcODDJt0lbQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\c96bc85dc6.jpg\n",
      "SUCCESS - saved https://static.toiimg.com/thumb/msid-61148165,width-1070,height-580,imgsize-336217,resizemode-6,overlay-toi_sw,pt-32,y_pad-40/photo.jpg - as ./images\\indian_trucks_with_number_plate\\5cb41585b3.jpg\n",
      "SUCCESS - saved https://i1.wp.com/gomechanic.in/blog/wp-content/uploads/2019/05/typesofcarnumberplates-02-01.jpg?resize=828%2C552&ssl=1 - as ./images\\indian_trucks_with_number_plate\\f42c2abc29.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTXtpyCUECVYb71ntlMSn7MYt_dkSd8s4sq6w&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\606d38d910.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQZZFRjzNfPfPC2u9Cght5_LqkrgsT-yw2ArA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\fc743a906d.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSC0TXLCJSrEPhQnabEDfCc716XZuhF80_Nig&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\f46d328fd8.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ07wbiromet_jH6Eh2gszuFUN9RNSfL0htyQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\384f7c6a3e.jpg\n",
      "SUCCESS - saved https://thumbs.dreamstime.com/b/decorated-truck-india-west-bengal-sikkim-border-decorations-extensively-used-north-pakistan-to-63498830.jpg - as ./images\\indian_trucks_with_number_plate\\8c60c4a57c.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQrH8pSs73MaSoiynFuSF0NU988VcNuE7CAmQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\714c22bb27.jpg\n",
      "SUCCESS - saved https://www.team-bhp.com/forum/attachments/street-experiences/1064630d1363947597-take-look-number-plate-img544.jpg - as ./images\\indian_trucks_with_number_plate\\04304dead7.jpg\n",
      "SUCCESS - saved https://totalgyan.com/wp-content/uploads/2017/12/White-2BPlate.jpeg - as ./images\\indian_trucks_with_number_plate\\9cc40232fb.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSONUTVh5gq6ha4vHn1CgT1CftdNlJxUclLWw&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\c524e01f77.jpg\n",
      "SUCCESS - saved https://img.dtnext.in/Articles/2019/Aug/201908031640572224_Unnao-truck-collision-Why-was-the-number-plate-blackened_SECVPF.gif - as ./images\\indian_trucks_with_number_plate\\97fa687b4f.jpg\n",
      "SUCCESS - saved https://2.imimg.com/data2/BB/GI/MY-3050396/wooden-cabin-truck-500x500.jpg - as ./images\\indian_trucks_with_number_plate\\a1b42e985b.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR57dcbpLC7Wla9OLoRDro2NbcGT_Rt3lr45g&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\39e6c5c5e3.jpg\n",
      "SUCCESS - saved https://platerecognizer.com/wp-content/uploads/2020/07/India-ANPR-License-Plates.jpg - as ./images\\indian_trucks_with_number_plate\\235274dd0d.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT6UUPlXF_LSmY5dgf4MSJFkZi2Hb5knvLjIA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\29476c9dc4.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTI4tAKe7gnzkGhBs6HtrBs6qBlObCzTIrmLg&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\21c76de74e.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQBpnAE41pbOX7RisXtuMSjYTq9-L5E4YhY7Q&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\dd7658d0f4.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS-C-CvRKKzc_c19Ce3ulzGhFkRF227gQgZRA&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\322a5ef9bb.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQTylkRAiH-6s8Oy-KSCJyXxbPou9qlF1f0QQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\f3a378c117.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQo1pnL1evRaOX-BDHJ6oSSFaJBY1PlP8cyww&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\ba6111a1ee.jpg\n",
      "SUCCESS - saved https://qph.fs.quoracdn.net/main-qimg-67156c7bdfeac6c485182f8ab69ab57b-lq - as ./images\\indian_trucks_with_number_plate\\1e1dd8ad7c.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQ62li8sAvwxx_apPd2xsvCWqfM-z70wYJJiw&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\8418cc3e88.jpg\n",
      "SUCCESS - saved https://thumbs.dreamstime.com/b/decorated-truck-india-west-bengal-sikkim-border-decorations-extensively-used-north-pakistan-to-63498760.jpg - as ./images\\indian_trucks_with_number_plate\\e3948c6a00.jpg\n",
      "SUCCESS - saved https://thumbs.dreamstime.com/b/colorful-cargo-truck-rich-decorative-paintings-typical-trucks-india-rishikesh-circa-april-161978676.jpg - as ./images\\indian_trucks_with_number_plate\\bb61314beb.jpg\n",
      "SUCCESS - saved https://qph.fs.quoracdn.net/main-qimg-949b75738c84fd2fad1defc44095c0f4-c - as ./images\\indian_trucks_with_number_plate\\4084fc1851.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQfxNq1sVw2-99HAgd24S9M_Yjddil71jVmMQ&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\4b1670d805.jpg\n",
      "SUCCESS - saved https://c.ndtvimg.com/2019-07/j9f1ap7_unnao-accident-truck-_625x300_28_July_19.jpg - as ./images\\indian_trucks_with_number_plate\\5cffd8b9f4.jpg\n",
      "SUCCESS - saved https://m.economictimes.com/photo/35765351.cms - as ./images\\indian_trucks_with_number_plate\\15c2cd2585.jpg\n",
      "SUCCESS - saved https://img.jagranjosh.com/imported/images/E/Articles/Why-IND-is-written-on-vehicle-number-plates.jpg - as ./images\\indian_trucks_with_number_plate\\553cd2ea3a.jpg\n",
      "SUCCESS - saved https://startupnama.files.wordpress.com/2015/06/indian_truck.jpg - as ./images\\indian_trucks_with_number_plate\\9926d6d6e7.jpg\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SUCCESS - saved https://static.toiimg.com/thumb/msid-72048725,width-400,resizemode-4/72048725.jpg - as ./images\\indian_trucks_with_number_plate\\0929eaa595.jpg\n",
      "SUCCESS - saved https://upload.wikimedia.org/wikipedia/commons/b/b2/Back_of_an_Indian_truck.jpg - as ./images\\indian_trucks_with_number_plate\\520adddfd2.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT4zISxTXDOGxlPhxcXD3bvlh_SDcGH8G5KPw&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\100bc8ba94.jpg\n",
      "SUCCESS - saved https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSroSwQvW9qNjXe3gGBsC48SAL693gMOCCf_g&usqp=CAU - as ./images\\indian_trucks_with_number_plate\\da7c764966.jpg\n"
     ]
    }
   ],
   "source": [
    "serch_term = 'indian trucks with number plate'\n",
    "search_and_download(search_term=serch_term, driver_path=DRIVER_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e9c928f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd490fdd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}