Parser_news_LENTA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.1"
    },
    "colab": {
      "name": "Parser_news_LENTA.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "386782d834a749cea90df80e95d2340d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_409aefc0265149bca2b31deec7835db8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c0dde1dc2a024d40b26d64f5614b50a7",
              "IPY_MODEL_05ef7c0a9c91484d8bd72878c00545db"
            ]
          }
        },
        "409aefc0265149bca2b31deec7835db8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c0dde1dc2a024d40b26d64f5614b50a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4c463ce3ee3445b093184914e07353b0",
            "_dom_classes": [],
            "description": "  0%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 12,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 0,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b1ff020a00d74843ba0937172fef90f6"
          }
        },
        "05ef7c0a9c91484d8bd72878c00545db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6a8aba0142bb4d8c8ea7aec3300670cf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0/12 [00:00&lt;?, ?it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_df5f6c41aa90410ca0c987e8a6bd1f5e"
          }
        },
        "4c463ce3ee3445b093184914e07353b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b1ff020a00d74843ba0937172fef90f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6a8aba0142bb4d8c8ea7aec3300670cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "df5f6c41aa90410ca0c987e8a6bd1f5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "85d8c8cd0d714b38b779c08d973472ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d3885ec4bafb468d9ab4ae4c757f4359",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_562f87750da94614b1fdfba62cb9bfe3",
              "IPY_MODEL_46a18eb6ebf844fab67f024123bef2ac"
            ]
          }
        },
        "d3885ec4bafb468d9ab4ae4c757f4359": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "562f87750da94614b1fdfba62cb9bfe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b6a80c1477ec45d78de2c493b535bc0e",
            "_dom_classes": [],
            "description": " 19%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 31,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 6,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_295e4a6b10ec424a9fb22c1ff36489eb"
          }
        },
        "46a18eb6ebf844fab67f024123bef2ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_471839ac5f134669b8990a19b6b1e03e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6/31 [00:12&lt;00:51,  2.04s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d6db1f0690a403b88287d5f5f67063e"
          }
        },
        "b6a80c1477ec45d78de2c493b535bc0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "295e4a6b10ec424a9fb22c1ff36489eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "471839ac5f134669b8990a19b6b1e03e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d6db1f0690a403b88287d5f5f67063e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/andreytsimbalov/News_Classification_and_Vectorization/blob/main/Parser_news_LENTA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9rucQF6vOq1_"
      },
      "source": [
        "# Парсер с Lenta.ru новости\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X8eNNb9L6hwi"
      },
      "source": [
        "Данная программа парсит новости с https://lenta.ru/search за определенный период"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CddEe3nF-agt"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "md = '/content/drive/My Drive/Colab Notebooks/Diplom/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGUc0eFnOq2X"
      },
      "source": [
        "import requests\n",
        "import threading\n",
        "from bs4 import BeautifulSoup\n",
        "import numpy as np\n",
        "from tqdm.notebook import tqdm\n",
        "import pandas as pd\n",
        "import time\n",
        "import random\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0y9xyZVWjIg"
      },
      "source": [
        "days_in_month_dict ={\n",
        "    1:31,\n",
        "    2:29,\n",
        "    3:31,\n",
        "    4:30,\n",
        "    5:31,\n",
        "    6:30,\n",
        "    7:31,\n",
        "    8:31,\n",
        "    9:30,\n",
        "    10:31,\n",
        "    11:30,\n",
        "    12:31\n",
        "}\n",
        "\n",
        "number_of_month_dict ={\n",
        "    1:'01',\n",
        "    2:'02',\n",
        "    3:'03',\n",
        "    4:'04',\n",
        "    5:'05',\n",
        "    6:'06',\n",
        "    7:'07',\n",
        "    8:'08',\n",
        "    9:'09',\n",
        "    10:'10',\n",
        "    11:'11',\n",
        "    12:'12'\n",
        "}"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CfjbisRF0eRH"
      },
      "source": [
        "def soup_item_to_dict(s_item):\n",
        "  times = s_item.find('span' ,class_=\"g-date item__date\").text\n",
        "  item_dict = {}\n",
        "  item_dict['time'] = times[:5]\n",
        "  if times[8:]==' ':\n",
        "    item_dict['date'] = times[9:]\n",
        "  else:\n",
        "    item_dict['date'] = times[8:]\n",
        "  item_dict['link'] = 'https://lenta.ru' + s_item.find('a').get('href')\n",
        "  item_dict['headline'] = s_item.find('div', class_=\"titles\").find('span').text.replace('\\xa0', ' ')\n",
        "  return item_dict\n",
        "\n",
        "# soup_item_to_dict(topic_news[0])"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imRSlRRI7Yyh"
      },
      "source": [
        "lock = threading.Lock()\n",
        "\n",
        "def take_text_and_tag_news(id, now_link):\n",
        "  time_delay = time.time()\n",
        "  \n",
        "  tag =''\n",
        "  st=''\n",
        "  flag = True\n",
        "  iterator_0 = 0\n",
        "  while flag and iterator_0<10:\n",
        "    iterator_0+=1\n",
        "\n",
        "    r = requests.get(now_link)\n",
        "    soup = BeautifulSoup(r.text, 'html.parser')\n",
        "    \n",
        "    try:\n",
        "      tag = soup.find('a', class_ = 'b-header__block').text\n",
        "      sp = soup.find('div', class_ = 'b-text')\n",
        "      st = ''\n",
        "      for s_ in sp.find_all('p'):\n",
        "        st+=s_.text + ' '\n",
        "    except:\n",
        "      pass\n",
        "\n",
        "    if tag!='':\n",
        "      flag = False\n",
        "    else:\n",
        "      time.sleep(0.1)\n",
        "\n",
        "  if tag=='':\n",
        "    print('tag NOT defined (((')\n",
        "  time_delay = time.time() - time_delay\n",
        "  global max_time_delay\n",
        "  global news\n",
        "  with lock:\n",
        "      max_time_delay = max(max_time_delay, time_delay)\n",
        "      news.extend([(id,tag, st)])\n",
        "\n",
        "  if id%500==0:\n",
        "    print(f'{id} {tag}')\n",
        "  return tag, st\n",
        "\n",
        "# take_text_and_tag_news(0, 'https://lenta.ru/news/2020/12/01/nervy/')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Dwshlnb6zgP"
      },
      "source": [
        ""
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HbzaE0GN60f_"
      },
      "source": [
        "# Сбор новостей"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 152,
          "referenced_widgets": [
            "386782d834a749cea90df80e95d2340d",
            "409aefc0265149bca2b31deec7835db8",
            "c0dde1dc2a024d40b26d64f5614b50a7",
            "05ef7c0a9c91484d8bd72878c00545db",
            "4c463ce3ee3445b093184914e07353b0",
            "b1ff020a00d74843ba0937172fef90f6",
            "6a8aba0142bb4d8c8ea7aec3300670cf",
            "df5f6c41aa90410ca0c987e8a6bd1f5e",
            "85d8c8cd0d714b38b779c08d973472ea",
            "d3885ec4bafb468d9ab4ae4c757f4359",
            "562f87750da94614b1fdfba62cb9bfe3",
            "46a18eb6ebf844fab67f024123bef2ac",
            "b6a80c1477ec45d78de2c493b535bc0e",
            "295e4a6b10ec424a9fb22c1ff36489eb",
            "471839ac5f134669b8990a19b6b1e03e",
            "1d6db1f0690a403b88287d5f5f67063e"
          ]
        },
        "id": "HNZ4rl5F6zqB",
        "outputId": "c086d8ae-5354-4af7-a2ac-9f2c7e439133"
      },
      "source": [
        "# range(1, 13) for full year\n",
        "for mount_id in tqdm(range(1, 13)):\n",
        "\n",
        "  topic_news = []\n",
        "  month = mount_id\n",
        "\n",
        "  base_url = 'https://lenta.ru/news/2020/0' + str(month) + '/'\n",
        "  days_in_month = days_in_month_dict[month]\n",
        "  print(f'{month} - {days_in_month}')\n",
        "  for day_id in tqdm(range(1, days_in_month+1)):\n",
        "    now_day_str = str(day_id)\n",
        "    if len(now_day_str)==1:\n",
        "      now_day_str='0'+now_day_str\n",
        "    now_day_str = base_url+now_day_str\n",
        "    try:\n",
        "      r = requests.get(now_day_str)\n",
        "      soup = BeautifulSoup(r.text, 'html.parser')\n",
        "      now_day_items = soup.find_all('div', class_ =  'item news b-tabloid__topic_news')\n",
        "      topic_news.extend(now_day_items)\n",
        "\n",
        "      if len(now_day_items)==0:\n",
        "        print('now_day_items is NOT define')\n",
        "\n",
        "\n",
        "    except:\n",
        "      print(str(day_id)+' error')\n",
        "\n",
        "  len(topic_news)\n",
        "  topic_news_dict = [soup_item_to_dict(tn) for tn in topic_news]\n",
        "  print(f'total len: {len(topic_news_dict)}')\n",
        "\n",
        "\n",
        "  news = []\n",
        "  max_time_delay=0\n",
        "  max_thread_number = 50\n",
        "  threads_index = set([])\n",
        "\n",
        "\n",
        "  arr_news_of_time = time.time()\n",
        "\n",
        "  thrs_main = [threading.Thread(target=take_text_and_tag_news, args= (i, tnd['link'],)) \n",
        "                for i, tnd in enumerate(topic_news_dict)]\n",
        "\n",
        "  now_threads = set([])\n",
        "\n",
        "  iterator = 0\n",
        "\n",
        "  while iterator<len(thrs_main):\n",
        "    finished_threads=set([])\n",
        "    for th in now_threads:\n",
        "      if not th.isAlive():\n",
        "        finished_threads.add(th)\n",
        "    now_threads-=finished_threads\n",
        "    \n",
        "    while len(now_threads)<max_thread_number:\n",
        "      try:\n",
        "        now_threads.add(thrs_main[iterator])\n",
        "        thrs_main[iterator].start()\n",
        "        iterator+=1\n",
        "        time.sleep(0.01)\n",
        "      except:\n",
        "        print(f'iterator: {iterator} total len: {len(thrs_main)}')\n",
        "        break\n",
        "    time.sleep(0)\n",
        "\n",
        "\n",
        "  for ths in thrs_main:\n",
        "    ths.join()\n",
        "\n",
        "  print(time.time()-arr_news_of_time)\n",
        "  print(f'max_time_delay {max_time_delay}')\n",
        "  len(news)\n",
        "\n",
        "  for n in news:\n",
        "    topic_news_dict[n[0]]['tag'] = n[1]\n",
        "    topic_news_dict[n[0]]['text'] = n[2]\n",
        "\n",
        "\n",
        "  data = pd.DataFrame(topic_news_dict)\n",
        "  data.head()\n",
        "\n",
        "\n",
        "  data.to_pickle(md+\"data/news_lenta_\"+ number_of_month_dict[mount_id]+ \"_2020\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "386782d834a749cea90df80e95d2340d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=12.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "1 - 31\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "85d8c8cd0d714b38b779c08d973472ea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=31.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "3 error\n",
            "4 error\n",
            "5 error\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rUfAUBxf8EhT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}