Multilabel_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Multilabel Classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qcinuz4lChuy"
      },
      "source": [
        "# Data loading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovrecEvRu0M_"
      },
      "source": [
        "import numpy as np\r\n",
        "import pandas as pd\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "pd.options.display.max_columns = 200\r\n",
        "plt.style.use('ggplot')\r\n",
        "np.warnings.filterwarnings('ignore')"
      ],
      "execution_count": 338,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rOCeRQubu7JF",
        "outputId": "f9b38ec1-b121-4ac0-d821-23430c221db5"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/drive')\r\n",
        "%cd /content/drive/My Drive/DL project"
      ],
      "execution_count": 339,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/My Drive/DL project\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pnHZ-DfwvDDJ"
      },
      "source": [
        "#загружаем тренировочную, тестовую выборки, а также пример предсказаний\r\n",
        "train = pd.read_csv('train.csv')\r\n",
        "test = pd.read_csv('valid.csv')"
      ],
      "execution_count": 340,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "kIUVu5CjvqY9",
        "outputId": "0b6dcf31-ed70-4fa1-be51-511c525e1ebe"
      },
      "source": [
        "train.tail(2)"
      ],
      "execution_count": 341,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>Title</th>\n",
              "      <th>Body</th>\n",
              "      <th>Tags</th>\n",
              "      <th>CreationDate</th>\n",
              "      <th>Y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>44998</th>\n",
              "      <td>60465318</td>\n",
              "      <td>how to implement fill in the blank in Swift</td>\n",
              "      <td>&lt;p&gt;\"I _____ any questions.\"&lt;/p&gt;\\n\\n&lt;p&gt;I want t...</td>\n",
              "      <td>&lt;ios&gt;&lt;swift&gt;</td>\n",
              "      <td>2020-02-29 12:50:43</td>\n",
              "      <td>LQ_CLOSE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44999</th>\n",
              "      <td>60468018</td>\n",
              "      <td>How can I make a c# application outside of vis...</td>\n",
              "      <td>&lt;p&gt;I'm very new to programming and I'm teachin...</td>\n",
              "      <td>&lt;c#&gt;&lt;visual-studio&gt;</td>\n",
              "      <td>2020-02-29 17:55:56</td>\n",
              "      <td>LQ_CLOSE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             Id                                              Title  \\\n",
              "44998  60465318        how to implement fill in the blank in Swift   \n",
              "44999  60468018  How can I make a c# application outside of vis...   \n",
              "\n",
              "                                                    Body                 Tags  \\\n",
              "44998  <p>\"I _____ any questions.\"</p>\\n\\n<p>I want t...         <ios><swift>   \n",
              "44999  <p>I'm very new to programming and I'm teachin...  <c#><visual-studio>   \n",
              "\n",
              "              CreationDate         Y  \n",
              "44998  2020-02-29 12:50:43  LQ_CLOSE  \n",
              "44999  2020-02-29 17:55:56  LQ_CLOSE  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 341
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4Yc7HxQv7_b"
      },
      "source": [
        "X_train, y_train = train['Title'].values, train['Tags'].values\r\n",
        "X_test, y_test = test['Title'].values, test['Tags'].values"
      ],
      "execution_count": 342,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xKXR3MVECuMM"
      },
      "source": [
        "# Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sRXtmKv7ytXM",
        "outputId": "7d19e99c-f2d4-4efb-8780-f5052d99fad0"
      },
      "source": [
        "import nltk\r\n",
        "from nltk.corpus import stopwords\r\n",
        "import re\r\n",
        "nltk.download('stopwords')\r\n",
        "\r\n",
        "STOPWORDS = set(stopwords.words('english'))\r\n",
        "REPLACE_BY_SPACE_RE = re.compile('[/(){}\\[\\]\\|@,;]')\r\n",
        "BAD_SYMBOLS_RE = re.compile('[^0-9a-z #+_]')"
      ],
      "execution_count": 343,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kZHZKwArDm_E"
      },
      "source": [
        "Уберем лишнее из исходних текстов:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "voeJWEY6xVTE"
      },
      "source": [
        "def text_prepare(text):\r\n",
        "    \"\"\"\r\n",
        "        text: a string\r\n",
        "        return: modified initial string\r\n",
        "    \"\"\"\r\n",
        "    text = text.lower()\r\n",
        "    text = REPLACE_BY_SPACE_RE.sub(\" \", text)\r\n",
        "    text = BAD_SYMBOLS_RE.sub(\"\", text)\r\n",
        "    text = [word for word in text.split() if not word in STOPWORDS]\r\n",
        "    \r\n",
        "    \r\n",
        "    return (' '.join(text))"
      ],
      "execution_count": 344,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hql-F8LHy-zP"
      },
      "source": [
        "X_train = [text_prepare(x) for x in X_train]\r\n",
        "X_test = [text_prepare(x) for x in X_test]"
      ],
      "execution_count": 345,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Y-2yU9uzR60",
        "outputId": "20f51ba0-b388-4b37-fe81-7eb6a3e3fb2e"
      },
      "source": [
        "X_train[:5]"
      ],
      "execution_count": 346,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['java repeat task every random seconds',\n",
              " 'java optionals immutable',\n",
              " 'text overlay image darkened opacity react native',\n",
              " 'ternary operator swift picky',\n",
              " 'hide show fab scale animation']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 346
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CBcSU8jzD0NA"
      },
      "source": [
        "Преобразуем тэги в список:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S7LVy7gf0qhb"
      },
      "source": [
        "def tags_prepare(tags):\r\n",
        "    \"\"\"\r\n",
        "        tags: a string\r\n",
        "        return: list of tags\r\n",
        "    \"\"\"\r\n",
        "    tags = tags.replace(\"><\", \",\")\r\n",
        "    tags = tags.replace(\"<\", \"\")\r\n",
        "    tags = tags.replace(\">\", \"\")\r\n",
        "    tags = tags.split(',')\r\n",
        "    \r\n",
        "    return(tags)"
      ],
      "execution_count": 347,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgfkO97axP_M"
      },
      "source": [
        "y_train = [tags_prepare(x) for x in y_train]\r\n",
        "y_test = [tags_prepare(x) for x in y_test]"
      ],
      "execution_count": 348,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FlfZTk-m2545",
        "outputId": "633df0ee-39f7-4097-8075-c86e1ad350ca"
      },
      "source": [
        "y_train[:5]"
      ],
      "execution_count": 349,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['java', 'repeat'],\n",
              " ['java', 'optional'],\n",
              " ['javascript', 'image', 'overlay', 'react-native', 'opacity'],\n",
              " ['swift', 'operators', 'whitespace', 'ternary-operator', 'optional'],\n",
              " ['android', 'material-design', 'floating-action-button']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 349
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dyRzCgaCElMt"
      },
      "source": [
        "Создадим словари, а также посмотрим на самые популярные слова в текстах и теги:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8yXl3EDbzlcx"
      },
      "source": [
        "tokenizer = nltk.tokenize.TreebankWordTokenizer()\r\n",
        "\r\n",
        "train_X = [tokenizer.tokenize(text) for text in X_train]"
      ],
      "execution_count": 350,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G4BN1Dsj0Etm"
      },
      "source": [
        "# Dictionary of all tags from train corpus with their counts.\r\n",
        "tags_counts = {}\r\n",
        "# Dictionary of all words from train corpus with their counts.\r\n",
        "words_counts = {}\r\n",
        "\r\n",
        "for text in y_train:   \r\n",
        "    for word in text:\r\n",
        "        if word in tags_counts:\r\n",
        "            tags_counts[word] += 1\r\n",
        "        else:\r\n",
        "            tags_counts[word] = 1\r\n",
        "\r\n",
        "for text in train_X:    \r\n",
        "    for word in text:\r\n",
        "        if word in words_counts:\r\n",
        "            words_counts[word] += 1\r\n",
        "        else:\r\n",
        "            words_counts[word] = 1"
      ],
      "execution_count": 358,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VVNmv96dGPxK",
        "outputId": "d9e1902e-3114-42bb-acfa-096877f4f3f0"
      },
      "source": [
        "len(tags_counts)"
      ],
      "execution_count": 359,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9336"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 359
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hNCMLOBLJMx1"
      },
      "source": [
        "Так как в словаре имеется слишком много редких тегов(усложняются вычисления + зачастую они не имеют смысла), то для дальнейших предсказаний отсавим те, которые встречаются чаще всего(более n раз):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ddmvb70EJAa8",
        "outputId": "094156a8-61ca-4692-dc31-aea7c121da50"
      },
      "source": [
        "n = 4\r\n",
        "tags_counts = {k: v for k, v in tags_counts.items() if v > n}\r\n",
        "len(tags_counts)"
      ],
      "execution_count": 360,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2229"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 360
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_b2M_vg40Hzw"
      },
      "source": [
        "most_common_tags = sorted(tags_counts.items(), key=lambda x: x[1], reverse=True)[:3]\r\n",
        "most_common_words = sorted(words_counts.items(), key=lambda x: x[1], reverse=True)[:3]"
      ],
      "execution_count": 310,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Pl61Sbe0J4a",
        "outputId": "f8e5a791-40e8-4dd8-9ac9-a733e0b3de31"
      },
      "source": [
        "most_common_tags, most_common_words"
      ],
      "execution_count": 229,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([('javascript', 5385), ('python', 5041), ('java', 4677)],\n",
              " [('using', 3045), ('python', 2383), ('error', 2273)])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 229
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lY0qLLXGc8Jo"
      },
      "source": [
        "# MultiLogisticRegression"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dz_9Ly1vDbhU"
      },
      "source": [
        "## Data vectorization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bhTKf0QCErGY"
      },
      "source": [
        "Векторизуем предобработанные текста с помощью TfidfVectorizer для дальнейше подачи на вход модели:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dzu2Jz3d3FWF"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\r\n",
        "\r\n",
        "\r\n",
        "tfidf_vectorizer = TfidfVectorizer(min_df=5, max_df=0.9, ngram_range=(1,2), token_pattern='(\\S+)', max_features=5000)\r\n",
        "tfidf_vectorizer.fit(X_train)\r\n",
        "\r\n",
        "X_train_tfidf = tfidf_vectorizer.transform(X_train)\r\n",
        "X_test_tfidf = tfidf_vectorizer.transform(X_test)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GvIDUEQGE0Ar"
      },
      "source": [
        "Бинаризуем наши теги для подачи на вход модели:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUBh_pDs_C_k"
      },
      "source": [
        "from sklearn.preprocessing import MultiLabelBinarizer\r\n",
        "\r\n",
        "mlb = MultiLabelBinarizer(classes=sorted(tags_counts.keys()))\r\n",
        "y_train_mlb = mlb.fit_transform(y_train)\r\n",
        "y_test_mlb = mlb.fit_transform(y_test)"
      ],
      "execution_count": 361,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "czyNY5ZXK9Lq",
        "outputId": "95c053c1-c939-4b9f-df7d-c34260953d50"
      },
      "source": [
        "len(y_test)"
      ],
      "execution_count": 231,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 231
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gplM30RIBO8-"
      },
      "source": [
        "## Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VZclMmN7E_Jm"
      },
      "source": [
        "Создадим и обучим классификатор на основе подхода OneVsRest, внутри которого будем строить k штук LogisticRegression, где k - число различных тэгов:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z37MzW_rjMva"
      },
      "source": [
        "from sklearn.multiclass import OneVsRestClassifier\r\n",
        "from sklearn.linear_model import LogisticRegression, RidgeClassifier"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpkntQ8d_eTF",
        "outputId": "1401e06b-e673-4745-ee86-4263ccd7c785"
      },
      "source": [
        "%%time\r\n",
        "\r\n",
        "multi_lr = OneVsRestClassifier(LogisticRegression(random_state=42))\r\n",
        "multi_lr.fit(X_train_tfidf, y_train_mlb)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 5min 48s, sys: 3.84 s, total: 5min 52s\n",
            "Wall time: 5min 53s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AJRasO3SsZX"
      },
      "source": [
        "y_train_predicted = multi_lr.predict(X_train_tfidf)\r\n",
        "y_test_predicted = multi_lr.predict(X_test_tfidf)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-woky_6Z_-zR"
      },
      "source": [
        "y_test_predicted_inv = mlb.inverse_transform(y_test_predicted)\r\n",
        "y_test_inv = mlb.inverse_transform(y_test_mlb)\r\n",
        "\r\n",
        "for i in range(10, 15):\r\n",
        "    print('Title:\\t{}\\nTrue labels:\\t{}\\nPredicted labels:\\t{}\\n\\n'.format(\r\n",
        "        X_test[i],\r\n",
        "        ','.join(y_test_inv[i]),\r\n",
        "        ','.join(y_test_predicted_inv[i])\r\n",
        "    ))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EaqR_9KYAxs3"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\r\n",
        "\r\n",
        "def print_evaluation_scores(y_val, predicted):\r\n",
        "  \r\n",
        "    print('accuracy ' + str(accuracy_score(y_val, predicted)),\r\n",
        "          'precision ' + str(precision_score(y_val, predicted, average='micro')),\r\n",
        "          'recall ' + str(recall_score(y_val, predicted, average='micro')),\r\n",
        "          'f1 ' + str(f1_score(y_val, predicted, average='micro')))"
      ],
      "execution_count": 377,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_kyg-euLSou0",
        "outputId": "758ccb90-826f-4ea0-f7e4-17fe3dac2356"
      },
      "source": [
        "print_evaluation_scores(y_test_mlb, y_test_predicted)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy 0.12633333333333333 precision 0.8603867988068893 recall 0.24806502621577384 f1 0.3369575634475602\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZZ33-h84dI8r"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SP9eA7RRdJPK"
      },
      "source": [
        "Протеститируем работу для нового текста:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GTnauSvAAx1k",
        "outputId": "4cecb419-0f4c-4481-d664-0cf3627de6c5"
      },
      "source": [
        "text = \"I have a problem with python and html\"\r\n",
        "text = text_prepare(text)\r\n",
        "\r\n",
        "text = tfidf_vectorizer.transform([text])\r\n",
        "test_predictions = classifier_tfidf.predict(text)\r\n",
        "test_pred_inversed = mlb.inverse_transform(test_predictions)\r\n",
        "test_pred_inversed"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('html', 'python')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SlfgBF74YDib"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WS5ubLZzy-ZO"
      },
      "source": [
        "# Neural network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K9KdBdQXbVT5"
      },
      "source": [
        "## Without Pretrained Word Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVAblPmDOo4A"
      },
      "source": [
        "from numpy import array\r\n",
        "from keras.preprocessing.text import one_hot\r\n",
        "from keras.preprocessing.sequence import pad_sequences\r\n",
        "from keras.models import Sequential\r\n",
        "from keras.layers.core import Activation, Dropout, Dense\r\n",
        "from keras.layers import Flatten, LSTM\r\n",
        "from keras.layers import GlobalMaxPooling1D\r\n",
        "from keras.models import Model\r\n",
        "from keras.layers.embeddings import Embedding\r\n",
        "from sklearn.model_selection import train_test_split\r\n",
        "from keras.preprocessing.text import Tokenizer\r\n",
        "from keras.layers import Input\r\n",
        "from keras.layers.merge import Concatenate\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "from tensorflow import keras\r\n",
        "from tensorflow.keras.layers import Dense, Flatten, Input\r\n",
        "from keras.callbacks import EarlyStopping"
      ],
      "execution_count": 363,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pnnLd2bNY-4l"
      },
      "source": [
        "from nltk.tokenize import word_tokenize\r\n",
        "\r\n",
        "all_words = []\r\n",
        "for sent in X_train:\r\n",
        "    tokenize_word = word_tokenize(sent)\r\n",
        "    for word in tokenize_word:\r\n",
        "        all_words.append(word)"
      ],
      "execution_count": 364,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t3wIuiZgZDwx",
        "outputId": "c340a065-0861-45e0-de1b-fcf2cd3d0937"
      },
      "source": [
        "unique_words = set(all_words)\r\n",
        "print(len(unique_words))"
      ],
      "execution_count": 365,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "23501\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lyOuo97MB0pI"
      },
      "source": [
        "### Data vectorization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htzhGeXdKZsp"
      },
      "source": [
        "tokenizer = Tokenizer(num_words=5000)\r\n",
        "tokenizer.fit_on_texts(X_train)\r\n",
        "\r\n",
        "vocab_size = len(tokenizer.word_index) + 1\r\n",
        "\r\n",
        "X_train_num = tokenizer.texts_to_sequences(X_train)\r\n",
        "X_test_num = tokenizer.texts_to_sequences(X_test)"
      ],
      "execution_count": 366,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAyB3BcGYti3",
        "outputId": "d2d26ed8-4afb-4f80-a2c8-1a1622ff48a1"
      },
      "source": [
        "vocab_size"
      ],
      "execution_count": 367,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "22839"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 367
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQG8zoaESRZK"
      },
      "source": [
        "from nltk.tokenize import word_tokenize\r\n",
        "\r\n",
        "\r\n",
        "word_count = lambda sentence: len(word_tokenize(sentence))\r\n",
        "longest_sentence = max(X_train, key=word_count)\r\n",
        "length_long_sentence = len(word_tokenize(longest_sentence))"
      ],
      "execution_count": 368,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMWT_bsmTTkm",
        "outputId": "f30b47c0-b718-4f74-8b73-390a38af87df"
      },
      "source": [
        "length_long_sentence"
      ],
      "execution_count": 369,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 369
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z87dM5E1SSmz"
      },
      "source": [
        "X_train_num = pad_sequences(X_train_num, length_long_sentence, padding='post')\r\n",
        "X_test_num = pad_sequences(X_test_num, length_long_sentence, padding='post')"
      ],
      "execution_count": 370,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aTB0ClOGPzVs",
        "outputId": "4f47d359-817d-41bf-b75b-8598441945a4"
      },
      "source": [
        "X_train_num[3]"
      ],
      "execution_count": 182,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1519,  253,   61,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 182
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M2esqadmz9k4",
        "outputId": "39ffadb7-326d-4718-daf2-7556ed5a7cc5"
      },
      "source": [
        "len(y_train[:3])"
      ],
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8FYT9W72B3Nn"
      },
      "source": [
        "### Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XQj5n0DJGdV9"
      },
      "source": [
        "import keras.backend as K\r\n",
        "\r\n",
        "def get_f1(y_true, y_pred): #taken from old keras source code\r\n",
        "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\r\n",
        "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\r\n",
        "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\r\n",
        "    precision = true_positives / (predicted_positives + K.epsilon())\r\n",
        "    recall = true_positives / (possible_positives + K.epsilon())\r\n",
        "    f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())\r\n",
        "    return f1_val"
      ],
      "execution_count": 371,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "20s-wBP0OFb3",
        "outputId": "66b61c9a-3059-4411-9910-4a6edd4cb1af"
      },
      "source": [
        "model = Sequential()\r\n",
        "model.add(Embedding(vocab_size, 20, input_length=length_long_sentence))\r\n",
        "model.add(Flatten())\r\n",
        "model.add(Dense(600, activation='relu'))\r\n",
        "model.add(Dense(2229, activation='sigmoid'))\r\n",
        "\r\n",
        "\r\n",
        "model.compile(optimizer='adam',\r\n",
        "             loss='binary_crossentropy',\r\n",
        "             metrics=[get_f1])\r\n",
        "\r\n",
        "\r\n",
        "print(model.summary())      # вывод структуры НС в консоль"
      ],
      "execution_count": 387,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_18\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_18 (Embedding)     (None, 29, 20)            456780    \n",
            "_________________________________________________________________\n",
            "flatten_17 (Flatten)         (None, 580)               0         \n",
            "_________________________________________________________________\n",
            "dense_25 (Dense)             (None, 600)               348600    \n",
            "_________________________________________________________________\n",
            "dense_26 (Dense)             (None, 2229)              1339629   \n",
            "=================================================================\n",
            "Total params: 2,145,009\n",
            "Trainable params: 2,145,009\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R9S3OYOw1nEO",
        "outputId": "021d0967-4166-49a6-bc74-7c4810448e98"
      },
      "source": [
        "model.fit(X_train_num, y_train_mlb, batch_size=30, epochs=100, \r\n",
        "          validation_split=0.2, callbacks=EarlyStopping(monitor='val_get_f1', mode='max', patience = 5))"
      ],
      "execution_count": 388,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "1200/1200 [==============================] - 37s 30ms/step - loss: 0.0491 - get_f1: 4.6281e-05 - val_loss: 0.0062 - val_get_f1: 0.0000e+00\n",
            "Epoch 2/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0064 - get_f1: 0.0191 - val_loss: 0.0055 - val_get_f1: 0.1520\n",
            "Epoch 3/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0054 - get_f1: 0.1933 - val_loss: 0.0049 - val_get_f1: 0.2725\n",
            "Epoch 4/100\n",
            "1200/1200 [==============================] - 35s 30ms/step - loss: 0.0046 - get_f1: 0.3209 - val_loss: 0.0046 - val_get_f1: 0.3297\n",
            "Epoch 5/100\n",
            "1200/1200 [==============================] - 35s 30ms/step - loss: 0.0041 - get_f1: 0.3918 - val_loss: 0.0044 - val_get_f1: 0.3623\n",
            "Epoch 6/100\n",
            "1200/1200 [==============================] - 35s 30ms/step - loss: 0.0037 - get_f1: 0.4408 - val_loss: 0.0044 - val_get_f1: 0.3798\n",
            "Epoch 7/100\n",
            "1200/1200 [==============================] - 35s 29ms/step - loss: 0.0034 - get_f1: 0.4733 - val_loss: 0.0044 - val_get_f1: 0.3974\n",
            "Epoch 8/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0031 - get_f1: 0.5049 - val_loss: 0.0044 - val_get_f1: 0.3933\n",
            "Epoch 9/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0029 - get_f1: 0.5298 - val_loss: 0.0045 - val_get_f1: 0.4081\n",
            "Epoch 10/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0027 - get_f1: 0.5592 - val_loss: 0.0046 - val_get_f1: 0.4058\n",
            "Epoch 11/100\n",
            "1200/1200 [==============================] - 37s 31ms/step - loss: 0.0025 - get_f1: 0.5895 - val_loss: 0.0048 - val_get_f1: 0.4070\n",
            "Epoch 12/100\n",
            "1200/1200 [==============================] - 36s 30ms/step - loss: 0.0023 - get_f1: 0.6191 - val_loss: 0.0049 - val_get_f1: 0.4090\n",
            "Epoch 13/100\n",
            "1200/1200 [==============================] - 35s 29ms/step - loss: 0.0021 - get_f1: 0.6523 - val_loss: 0.0051 - val_get_f1: 0.4045\n",
            "Epoch 14/100\n",
            "1200/1200 [==============================] - 35s 30ms/step - loss: 0.0019 - get_f1: 0.6832 - val_loss: 0.0053 - val_get_f1: 0.4048\n",
            "Epoch 15/100\n",
            "1200/1200 [==============================] - 35s 29ms/step - loss: 0.0017 - get_f1: 0.7114 - val_loss: 0.0055 - val_get_f1: 0.4044\n",
            "Epoch 16/100\n",
            "1200/1200 [==============================] - 35s 29ms/step - loss: 0.0016 - get_f1: 0.7394 - val_loss: 0.0058 - val_get_f1: 0.3987\n",
            "Epoch 17/100\n",
            "1200/1200 [==============================] - 35s 29ms/step - loss: 0.0014 - get_f1: 0.7656 - val_loss: 0.0060 - val_get_f1: 0.3961\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fc6e03bcd90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 388
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dzFHRCjl1qmD",
        "outputId": "a5894a86-a326-4fe4-91ac-faeca8475259"
      },
      "source": [
        "model.evaluate(X_test_num, y_test_mlb)"
      ],
      "execution_count": 389,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "469/469 [==============================] - 4s 7ms/step - loss: 0.0057 - get_f1: 0.4222\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.005706890486180782, 0.42215412855148315]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 389
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YrXeejZm4Xvv"
      },
      "source": [
        "y_test_predicted = model.predict(X_test_num)\r\n",
        "y_test_predicted[y_test_predicted>=0.5] = 1\r\n",
        "y_test_predicted[y_test_predicted<0.5] = 0"
      ],
      "execution_count": 390,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CArgD6qNtufI",
        "outputId": "36293c2e-59d6-4ccf-a764-f99b9eddd7ea"
      },
      "source": [
        "print_evaluation_scores(y_test_mlb, y_test_predicted)"
      ],
      "execution_count": 391,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy 0.11026666666666667 precision 0.5828382195383267 recall 0.33201098565761367 f1 0.42303953624008056\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Le2lmv4e4mjT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7d061403-abbd-4880-bf2c-4f90c00c34f7"
      },
      "source": [
        "text = \"I have a problem with python and html\"\r\n",
        "text = text_prepare(text)\r\n",
        "\r\n",
        "text = tokenizer.texts_to_sequences([text])\r\n",
        "text = pad_sequences(text, length_long_sentence, padding='post')\r\n",
        "\r\n",
        "test_predictions = model.predict(text)\r\n",
        "test_predictions[test_predictions>=0.5] = 1\r\n",
        "test_predictions[test_predictions<0.5] = 0\r\n",
        "test_pred_inversed = mlb.inverse_transform(test_predictions)\r\n",
        "test_pred_inversed"
      ],
      "execution_count": 393,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('python',)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 393
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjVhCFD88K6I"
      },
      "source": [
        ""
      ],
      "execution_count": 257,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BMB_GA6wbPiw"
      },
      "source": [
        "## With Pretrained Word Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MOJg8ibIbT00"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}