관상어 크롤링.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "439e052e",
   "metadata": {},
   "source": [
    "# 수집 - 관상어 이름, 특징, 환경"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c459dbd4",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (4.1.3)\n",
      "Requirement already satisfied: trio~=0.17 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from selenium) (0.20.0)\n",
      "Requirement already satisfied: urllib3[secure,socks]~=1.26 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from selenium) (1.26.7)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from selenium) (0.9.2)\n",
      "Requirement already satisfied: sniffio in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: idna in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (3.2)\n",
      "Requirement already satisfied: async-generator>=1.9 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.10)\n",
      "Requirement already satisfied: outcome in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.1.0)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (1.14.6)\n",
      "Requirement already satisfied: attrs>=19.2.0 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio~=0.17->selenium) (21.2.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.20)\n",
      "Requirement already satisfied: wsproto>=0.14 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.1.0)\n",
      "Requirement already satisfied: pyOpenSSL>=0.14 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (21.0.0)\n",
      "Requirement already satisfied: cryptography>=1.3.4 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (3.4.8)\n",
      "Requirement already satisfied: certifi in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (2021.10.8)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: six>=1.5.2 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from pyOpenSSL>=0.14->urllib3[secure,socks]~=1.26->selenium) (1.16.0)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\smhrd\\anaconda3\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.13.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e13e1785",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup as bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49965c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "fish = pd.read_csv('fish_name_2.csv',\n",
    "                      encoding='euc-kr',index_col='관상어 목록')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9d973cdc",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:13: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for i in tq(range(len(fish))) :\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53274fa427ff4c9eae04105b1576a99e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:15: DeprecationWarning: find_element_by_css_selector is deprecated. Please use find_element(by=By.CSS_SELECTOR, value=css_selector) instead\n",
      "  btn = driver.find_element_by_css_selector(\"div > form > div > input\")\n",
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:21: DeprecationWarning: find_element_by_css_selector is deprecated. Please use find_element(by=By.CSS_SELECTOR, value=css_selector) instead\n",
      "  btn1 = driver.find_element_by_css_selector(\"div > a > span.searchmatch\")\n",
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:24: DeprecationWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
      "  name = driver.find_element_by_id(\"firstHeading\")\n",
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:28: DeprecationWarning: find_element_by_css_selector is deprecated. Please use find_element(by=By.CSS_SELECTOR, value=css_selector) instead\n",
      "  ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p \")\n",
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:33: DeprecationWarning: find_element_by_css_selector is deprecated. Please use find_element(by=By.CSS_SELECTOR, value=css_selector) instead\n",
      "  ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p~p \")\n",
      "C:\\Users\\smhrd\\AppData\\Local\\Temp/ipykernel_17008/1026846326.py:39: DeprecationWarning: find_element_by_css_selector is deprecated. Please use find_element(by=By.CSS_SELECTOR, value=css_selector) instead\n",
      "  ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p~p~p \")\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver as wb\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from tqdm import tqdm_notebook as tq\n",
    "import time\n",
    "\n",
    "# 물고기 수집\n",
    "\n",
    "driver = wb.Chrome()\n",
    "driver.set_window_size(1800, 1400)\n",
    "driver.get(\"https://ko.wikipedia.org/wiki/%EC%9C%84%ED%82%A4%EB%B0%B1%EA%B3%BC\")\n",
    "time.sleep(1)\n",
    "\n",
    "# 물고기 정보 담을 리스트 생성\n",
    "nameList=[]\n",
    "featureList=[]\n",
    "featureList2=[]\n",
    "featureList3=[]\n",
    "\n",
    "# 반복문을 통해서 정보 가지고 오기\n",
    "for i in tq(range(len(fish))) :\n",
    "    \n",
    "    btn = driver.find_element_by_css_selector(\"div > form > div > input\")\n",
    "    btn.click()\n",
    "    btn.send_keys(fish.index[i])\n",
    "    btn.send_keys(Keys.ENTER)\n",
    "    \n",
    "    time.sleep(1)\n",
    "    btn1 = driver.find_element_by_css_selector(\"div > a > span.searchmatch\")\n",
    "    btn1.click()\n",
    "    \n",
    "    name = driver.find_element_by_id(\"firstHeading\")\n",
    "    name.text\n",
    "    nameList.append(name.text)\n",
    "    \n",
    "    ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p \")\n",
    "    ft.text\n",
    "    featureList.append(ft.text)\n",
    "    time.sleep(1)\n",
    "    \n",
    "    ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p~p \")\n",
    "    ft.text\n",
    "    featureList2.append(ft.text)\n",
    "    time.sleep(1)\n",
    "    \n",
    "    try : \n",
    "        ft = driver.find_element_by_css_selector(\"div > div.mw-parser-output >  p~p~p~p \")\n",
    "        ft.text\n",
    "        featureList3.append(ft.text)\n",
    "        time.sleep(1)\n",
    "    except : \n",
    "        featureList3.append(\"\")\n",
    "        time.sleep(1)\n",
    "    \n",
    "    # 자꾸 크롬드라이버가 멈춰서 close하고 다시 열기 윈도우 사이즈까지 맞춰서\n",
    "    # 윈도우 사이즈에 따라 안보이는 태그가 존재해서 윈도우 사이즈 정해놓기\n",
    "    driver.close()\n",
    "    driver = wb.Chrome()\n",
    "    driver.set_window_size(1800, 1400)\n",
    "    driver.get(\"https://ko.wikipedia.org/wiki/%EC%9C%84%ED%82%A4%EB%B0%B1%EA%B3%BC\")\n",
    "    time.sleep(1)\n",
    "    #i += 1\n",
    "    \n",
    "\n",
    "fish_info = {\"관상어 이름\" : nameList, \"특징\" : featureList, \"특징2\": featureList2, \"특징3\" : featureList3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6aaab46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 엑셀 파일에 저장\n",
    "crw_fish = pd.DataFrame(fish_info)\n",
    "crw_fish\n",
    "crw_fish.to_csv(\"crw_fish3.csv\",encoding=\"euc-kr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9cd6c3e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1520d387",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}