2_TextCategorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:25.567887Z",
     "start_time": "2017-07-02T14:07:25.505089+02:00"
    },
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/gorosz/anaconda3/lib/python3.6/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.\n",
      "  \"`IPython.html.widgets` has moved to `ipywidgets`.\", ShimWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'scroll': True,\n",
       " 'slideNumber': True,\n",
       " 'start_slideshow_at': 'selected',\n",
       " 'theme': 'simple',\n",
       " 'transition': 'linear'}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.html.services.config import ConfigManager\n",
    "from IPython.paths import locate_profile\n",
    "cm = ConfigManager(profile_dir=locate_profile(get_ipython().profile))\n",
    "\n",
    "cm.update('notebook', {\"load_extensions\": {\"livereveal/main\": True}})\n",
    "cm.update('livereveal', {\n",
    "    'theme': 'simple',\n",
    "    'transition': 'linear',\n",
    "    'slideNumber': True,\n",
    "    'start_slideshow_at': 'selected',\n",
    "    'scroll': True,\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Text classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-26T20:42:01.236478Z",
     "start_time": "2017-06-26T22:42:01.092714+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "![](./img/categorization.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-17T19:11:46.142429Z",
     "start_time": "2017-06-17T21:11:46.137646+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "How can a computer learn to classify things?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-17T19:11:12.082892Z",
     "start_time": "2017-06-17T21:11:12.079153+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "![](./img/classification.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "But how documents are represented as vectors?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "![Document term matrix](./img/docterm.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "![Tfidf](./img/tfidf.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-17T20:25:05.228830Z",
     "start_time": "2017-06-17T22:25:05.205587+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Building a classification pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:49:04.999389Z",
     "start_time": "2017-06-23T23:49:04.996949+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### ...with `scikit-learn`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Notations:\n",
    "\n",
    "* `X` is a 2D matrix, rows represents data points, columns contains feature values\n",
    "* `y` is a 1D array containing the labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:25.578238Z",
     "start_time": "2017-07-02T14:07:25.569848+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "class SklearnPredictor:\n",
    "    def fit(self, X, y):\n",
    "        \"\"\"\n",
    "        Learning to classify from the data\n",
    "        \"\"\"\n",
    "        return self\n",
    "    \n",
    "    def predict(X):\n",
    "        \"\"\"\n",
    "        Predict a label for all examples in X\n",
    "        \"\"\"\n",
    "        return predictions\n",
    "        \n",
    "class SklearnTransformer:\n",
    "    def fit(self, X, y=None):\n",
    "        \"\"\"\n",
    "        Learning to transforming the data\n",
    "        \"\"\"\n",
    "        return self\n",
    "    \n",
    "    def transform(X):\n",
    "        \"\"\"\n",
    "        Transform all the data in X\n",
    "        \"\"\"\n",
    "        return transformed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:26.322045Z",
     "start_time": "2017-07-02T14:07:25.580665+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('transformer1', <__main__.SklearnTransformer object at 0x10a67fc18>), ('transformer2', <__main__.SklearnTransformer object at 0x10a67fba8>), ('mypredictor', <__main__.SklearnPredictor object at 0x10a67fc50>)])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "Pipeline([\n",
    "    (\"transformer1\", SklearnTransformer()),\n",
    "    (\"transformer2\", SklearnTransformer()), # ...\n",
    "    (\"mypredictor\", SklearnPredictor()),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Categorizing Hungarian news articles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-17T20:18:12.315404Z",
     "start_time": "2017-06-17T22:18:12.312439+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Have a look at the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:26.892654Z",
     "start_time": "2017-07-02T14:07:26.323773+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>category</th>\n",
       "      <th>url</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Lángoló kamion miatt zárták le az M5-öst Szege...</td>\n",
       "      <td>Teljes terjedelmében ég egy kamion szombat kor...</td>\n",
       "      <td>belfold</td>\n",
       "      <td>http://index.hu/belfold/2017/06/03/langolo_kam...</td>\n",
       "      <td>Lángoló kamion miatt zárták le az M5-öst Szege...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kigyulladt egy lakatlan épület a XII. kerületben</td>\n",
       "      <td>Kigyulladt egy kb. négyszáz négyzetméter alapt...</td>\n",
       "      <td>belfold</td>\n",
       "      <td>http://index.hu/belfold/2017/06/03/kigyulladt_...</td>\n",
       "      <td>Kigyulladt egy lakatlan épület a XII. kerületb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dubrovniknak elege lett a meztelen turistákból</td>\n",
       "      <td>A városi tanács döntése értelmében hamarosan p...</td>\n",
       "      <td>gazdasag</td>\n",
       "      <td>http://index.hu/gazdasag/2016/07/26/nincs_tobb...</td>\n",
       "      <td>Dubrovniknak elege lett a meztelen turistákból...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Húsz éve nem házasodtak ennyien Magyarországon</td>\n",
       "      <td>2016 januárja és novembere között a legmagasab...</td>\n",
       "      <td>gazdasag</td>\n",
       "      <td>http://index.hu/gazdasag/2017/02/14/husz_eve_n...</td>\n",
       "      <td>Húsz éve nem házasodtak ennyien Magyarországon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kútba esett egy asszony Nógrádban</td>\n",
       "      <td>Kútba esett egy asszony a Nógrád megyei Szurdo...</td>\n",
       "      <td>belfold</td>\n",
       "      <td>http://index.hu/belfold/hirek/2013/10/24/kutba...</td>\n",
       "      <td>Kútba esett egy asszony Nógrádban\\n\\nKútba ese...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  Lángoló kamion miatt zárták le az M5-öst Szege...   \n",
       "1   Kigyulladt egy lakatlan épület a XII. kerületben   \n",
       "2     Dubrovniknak elege lett a meztelen turistákból   \n",
       "3     Húsz éve nem házasodtak ennyien Magyarországon   \n",
       "4                  Kútba esett egy asszony Nógrádban   \n",
       "\n",
       "                                                Body  category  \\\n",
       "0  Teljes terjedelmében ég egy kamion szombat kor...   belfold   \n",
       "1  Kigyulladt egy kb. négyszáz négyzetméter alapt...   belfold   \n",
       "2  A városi tanács döntése értelmében hamarosan p...  gazdasag   \n",
       "3  2016 januárja és novembere között a legmagasab...  gazdasag   \n",
       "4  Kútba esett egy asszony a Nógrád megyei Szurdo...   belfold   \n",
       "\n",
       "                                                 url  \\\n",
       "0  http://index.hu/belfold/2017/06/03/langolo_kam...   \n",
       "1  http://index.hu/belfold/2017/06/03/kigyulladt_...   \n",
       "2  http://index.hu/gazdasag/2016/07/26/nincs_tobb...   \n",
       "3  http://index.hu/gazdasag/2017/02/14/husz_eve_n...   \n",
       "4  http://index.hu/belfold/hirek/2013/10/24/kutba...   \n",
       "\n",
       "                                                text  \n",
       "0  Lángoló kamion miatt zárták le az M5-öst Szege...  \n",
       "1  Kigyulladt egy lakatlan épület a XII. kerületb...  \n",
       "2  Dubrovniknak elege lett a meztelen turistákból...  \n",
       "3  Húsz éve nem házasodtak ennyien Magyarországon...  \n",
       "4  Kútba esett egy asszony Nógrádban\\n\\nKútba ese...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"./data/index_articles.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:26.898917Z",
     "start_time": "2017-07-02T14:07:26.894581+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4972"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:26.910558Z",
     "start_time": "2017-07-02T14:07:26.900592+02:00"
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "belfold     2440\n",
       "gazdasag    1573\n",
       "tech         622\n",
       "kultur       337\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.category.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "We usually split the data into training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:26.957966Z",
     "start_time": "2017-07-02T14:07:26.912813+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(df.text, df.category, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T20:29:36.862549Z",
     "start_time": "2017-06-25T22:29:36.847596+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Now we can build the preprocessing pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:55.825488Z",
     "start_time": "2017-07-02T14:07:26.959484+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hello', 'világ']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from spacy.hu import Hungarian\n",
    "\n",
    "nlp = Hungarian()\n",
    "\n",
    "def tokenize(text):\n",
    "    return [tok.text for tok in nlp(text) if tok.is_alpha and not tok.is_stop]\n",
    "\n",
    "tokenize(\"Hello világ. Itt vagyok.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "As `spaCy` does not support lemmatization currently, we rely on simple stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:55.867752Z",
     "start_time": "2017-07-02T14:07:55.828093+02:00"
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bagly'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import snowballstemmer\n",
    "\n",
    "stemmer = snowballstemmer.stemmer('hungarian')\n",
    "stemmer.stemWord(\"baglyom\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:07:55.875635Z",
     "start_time": "2017-07-02T14:07:55.869947+02:00"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hell', 'világ']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize_stem(text):\n",
    "    return [stemmer.stemWord(tok.text) for tok in nlp(text) if tok.is_alpha and not tok.is_stop]\n",
    "\n",
    "tokenize_stem(\"Hello világ. Itt vagyok.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:08:38.994950Z",
     "start_time": "2017-07-02T14:07:55.878125+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "        stop_words=None, strip_accents=None, sublinear_tf=False,\n",
       "        token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=<function tokenize_stem at 0x116de92f0>, use_idf=True,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(tokenizer=tokenize_stem, ngram_range=(1,2), lowercase=True)\n",
    "vectorizer.fit(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T20:44:34.199721Z",
     "start_time": "2017-06-25T22:44:34.194279+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Note that we used `ngram_range=(1,2)` that means we rely on words **and neighbouring word pairs** as features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:08:39.010280Z",
     "start_time": "2017-07-02T14:08:38.997040+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 0, 0, 0, 0, 0], dtype=int32),\n",
       " array([374579, 308404, 308327, 276716, 276542, 244104], dtype=int32))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_vector = vectorizer.transform([\"Az oktatási miniszter is levélben szögezte le: covfafa\"])\n",
    "example_vector.nonzero()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:08:39.412783Z",
     "start_time": "2017-07-02T14:08:39.012328+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "szögezt 0.45421955882361387\n",
      "oktatás miniszter 0.4848001115368416\n",
      "oktatás 0.2783541455813643\n",
      "miniszter levél 0.5695599453959549\n",
      "miniszter 0.23767731020511645\n",
      "levél 0.3166755100853776\n"
     ]
    }
   ],
   "source": [
    "doc_vector = example_vector[example_vector.nonzero()].tolist()[0]\n",
    "doc_features = vectorizer.inverse_transform(example_vector)[0]\n",
    "for feature, score in zip(doc_features, doc_vector):\n",
    "    print(feature, score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T20:38:19.429071Z",
     "start_time": "2017-06-25T22:38:19.425718+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Build a predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:09:07.867933Z",
     "start_time": "2017-07-02T14:08:39.415429+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "X_transformed = vectorizer.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:09:08.698344Z",
     "start_time": "2017-07-02T14:09:07.869875+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "classifier = LinearSVC()\n",
    "classifier.fit(X_transformed, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:09:08.714928Z",
     "start_time": "2017-07-02T14:09:08.701477+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['belfold'], dtype=object)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.predict(vectorizer.transform([\"Az oktatási minisztere is levélben szögezte le: covfafa\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### By building a pipeline we simplify the whole process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:09:45.372581Z",
     "start_time": "2017-07-02T14:09:08.717467+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tfidfvectorizer', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), norm='l2', preprocessor=None, smooth_i...ax_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0))])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "\n",
    "pipeline = make_pipeline(\n",
    "    TfidfVectorizer(tokenizer=tokenize_stem, ngram_range=(1,2)),\n",
    "    LinearSVC()\n",
    ")\n",
    "\n",
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "We used a simple linear SVM model, but good candidates could be:\n",
    "* Multinomial Naive Bayes\n",
    "* Logistic regression\n",
    "* Decision trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Evaluate the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Accuracy** is a common metric used in classification evaluation.\n",
    "\n",
    "$\\mathrm{Accuracy} = \\frac{\\mathrm{\\# correct\\ labels}}{N}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:10:29.037759Z",
     "start_time": "2017-07-02T14:09:45.375199+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 1.0\n",
      "Test accuracy : 0.890920170627666\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_tpred = pipeline.predict(X_train)\n",
    "y_pred = pipeline.predict(X_test)\n",
    "\n",
    "print(\"Train accuracy: {}\".format(accuracy_score(y_train, y_tpred)))\n",
    "print(\"Test accuracy : {}\".format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T20:48:53.275416Z",
     "start_time": "2017-06-25T22:48:53.272360+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Other metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-21T20:01:19.146950Z",
     "start_time": "2017-06-21T22:01:19.142719+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "![Precision, Recall](https://upload.wikimedia.org/wikipedia/commons/thumb/2/26/Precisionrecall.svg/440px-Precisionrecall.svg.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-21T20:05:27.790775Z",
     "start_time": "2017-06-21T22:05:27.787114+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "$\\mathrm{F\\ score} = 2 \\cdot \\frac{\\mathrm{Precision} \\cdot \\mathrm{Recall}}{ \\mathrm{Precision} + \\mathrm{Recall}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:10:44.251406Z",
     "start_time": "2017-07-02T14:10:29.040694+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "    belfold       0.88      0.94      0.91       778\n",
      "   gazdasag       0.89      0.88      0.88       523\n",
      "     kultur       0.93      0.55      0.69       118\n",
      "       tech       0.94      0.91      0.93       222\n",
      "\n",
      "avg / total       0.89      0.89      0.89      1641\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "#### So far, so good, but what are the common errors of the classifier?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:10:44.267965Z",
     "start_time": "2017-07-02T14:10:44.254285+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[733,  38,   3,   4],\n",
       "       [ 57, 462,   1,   3],\n",
       "       [ 36,  12,  65,   5],\n",
       "       [  9,  10,   1, 202]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Improve your classifier\n",
    "\n",
    "* Collect more data.\n",
    "* Collect even more data.\n",
    "* [Diagnose whether your classifier suffers from  bias or variance](https://www.coursera.org/learn/machine-learning/lecture/yCAup/diagnosing-bias-vs-variance)\n",
    "* Analyze the confusion matrix, and add new features if necessary.\n",
    "* Fine tune your model through optimizing your hyperparameters (`GridSearch`)\n",
    "* Try alternative methods such as [FastText](https://github.com/facebookresearch/fastText) (a neural network based text classification library from Facebook)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-21T20:23:40.755617Z",
     "start_time": "2017-06-21T22:23:40.752939+02:00"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Sentiment analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "![Sentiment scale](./img/sentiment.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Classification!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:10:44.304074Z",
     "start_time": "2017-07-02T14:10:44.270792+02:00"
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweeter_tweetid</th>\n",
       "      <th>tweet</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>_monana_626507788945125376</td>\n",
       "      <td>@kadarmatyas én élőben tettem, a munkatàrsaidat nem olyan könnyű leblokkolni</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aagitorok_620631032937775104</td>\n",
       "      <td>ez a Conanes GRRM videó iszonyat béna, de nagyon hangosan röhögtem végig az egészet :DD</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tocolade_559613056792399872</td>\n",
       "      <td>@KevinaZolvaso hihii elfelejtette behozni a dogát aztán meg jó akkor elhiszem hogy elolvastátok</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>niallphabetic_626856718690091009</td>\n",
       "      <td>persze amúgy meg nem kell vissza írni...........................................nem tényleg nem, elég hogy csak elolvastad jólvan akkor.....</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>batacinti_537844599465988097</td>\n",
       "      <td>Elindultam dolgozni. Otthonhagytam a cigim és a kajàm. Egyszerencsétlen vagyok. Büdösek az emberek. Jóreggelt.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    tweeter_tweetid  \\\n",
       "0        _monana_626507788945125376   \n",
       "1      aagitorok_620631032937775104   \n",
       "2       tocolade_559613056792399872   \n",
       "3  niallphabetic_626856718690091009   \n",
       "4      batacinti_537844599465988097   \n",
       "\n",
       "                                                                                                                                          tweet  \\\n",
       "0                                                                  @kadarmatyas én élőben tettem, a munkatàrsaidat nem olyan könnyű leblokkolni   \n",
       "1                                                       ez a Conanes GRRM videó iszonyat béna, de nagyon hangosan röhögtem végig az egészet :DD   \n",
       "2                                               @KevinaZolvaso hihii elfelejtette behozni a dogát aztán meg jó akkor elhiszem hogy elolvastátok   \n",
       "3  persze amúgy meg nem kell vissza írni...........................................nem tényleg nem, elég hogy csak elolvastad jólvan akkor.....   \n",
       "4                                Elindultam dolgozni. Otthonhagytam a cigim és a kajàm. Egyszerencsétlen vagyok. Büdösek az emberek. Jóreggelt.   \n",
       "\n",
       "   sentiment  \n",
       "0          3  \n",
       "1          5  \n",
       "2          4  \n",
       "3          2  \n",
       "4          2  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\"display.max_colwidth\", 500)\n",
    "\n",
    "sent_df = pd.read_csv(\"./data/twitter_emotion.csv\", index_col=None)\n",
    "sent_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:10:44.324195Z",
     "start_time": "2017-07-02T14:10:44.306994+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweeter_tweetid</th>\n",
       "      <th>tweet</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>_monana_626507788945125376</td>\n",
       "      <td>@kadarmatyas én élőben tettem, a munkatàrsaidat nem olyan könnyű leblokkolni</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aagitorok_620631032937775104</td>\n",
       "      <td>ez a Conanes GRRM videó iszonyat béna, de nagyon hangosan röhögtem végig az egészet :DD</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tocolade_559613056792399872</td>\n",
       "      <td>@KevinaZolvaso hihii elfelejtette behozni a dogát aztán meg jó akkor elhiszem hogy elolvastátok</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tweeter_tweetid  \\\n",
       "0    _monana_626507788945125376   \n",
       "1  aagitorok_620631032937775104   \n",
       "2   tocolade_559613056792399872   \n",
       "\n",
       "                                                                                             tweet  \\\n",
       "0                     @kadarmatyas én élőben tettem, a munkatàrsaidat nem olyan könnyű leblokkolni   \n",
       "1          ez a Conanes GRRM videó iszonyat béna, de nagyon hangosan röhögtem végig az egészet :DD   \n",
       "2  @KevinaZolvaso hihii elfelejtette behozni a dogát aztán meg jó akkor elhiszem hogy elolvastátok   \n",
       "\n",
       "   sentiment  \n",
       "0          0  \n",
       "1          1  \n",
       "2          1  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_df.sentiment = sent_df.sentiment.apply(lambda x: 1 if x >3 else -1 if x <3 else 0)\n",
    "sent_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T21:13:35.351038Z",
     "start_time": "2017-06-25T23:13:35.348058+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Tfidf pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:13.936584Z",
     "start_time": "2017-07-02T14:10:44.326985+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from spacy.hu import Hungarian\n",
    "nlp = Hungarian()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:13.954481Z",
     "start_time": "2017-07-02T14:11:13.938897+02:00"
    },
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Én', 'élő', 'tett', ':)', '!']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def is_useful_token(tok):\n",
    "    return not (tok.like_url or tok.like_num) and \"@\" not in tok.text and \"rt\" != tok.text.lower()\n",
    "    \n",
    "def sent_tokenize(text):\n",
    "    return [tok.text for tok in nlp(text) if is_useful_token(tok)]\n",
    "\n",
    "def sent_tokenize_stem(text):\n",
    "    return [stemmer.stemWord(tok) for tok in sent_tokenize(text)]\n",
    "\n",
    "\n",
    "sent_tokenize_stem(\"rt @kadarmatyas Én élőben tettem http://index.hu 89.2 21,3 :) !\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:13.964931Z",
     "start_time": "2017-07-02T14:11:13.957857+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "pipeline = make_pipeline(\n",
    "    TfidfVectorizer(\n",
    "        tokenizer=sent_tokenize_stem,\n",
    "        ngram_range=(1, 2),\n",
    "        lowercase=True,\n",
    "        token_pattern=\".*\"),\n",
    "    LinearSVC())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.854559Z",
     "start_time": "2017-07-02T14:11:13.967782+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.49      0.62      0.55      1129\n",
      "          0       0.53      0.36      0.43      1261\n",
      "          1       0.63      0.66      0.65      1610\n",
      "\n",
      "avg / total       0.56      0.56      0.55      4000\n",
      "\n",
      "[[ 704  199  226]\n",
      " [ 408  455  398]\n",
      " [ 335  205 1070]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.55725000000000002"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "X, y = sent_df.tweet, sent_df.sentiment\n",
    "\n",
    "random.seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "y_pred = cross_val_predict(pipeline, X, y, cv=5)\n",
    "print(classification_report(y, y_pred))\n",
    "\n",
    "cm = confusion_matrix(y, y_pred)\n",
    "print(cm)\n",
    "accuracy_score(y, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### Evaluation in context\n",
    "\n",
    "* A random classifier would result in 0.33 accuracy\n",
    "* Agreement rate between humans is below 0.8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Sentiment lexicons to the rescue?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.864045Z",
     "start_time": "2017-07-02T14:11:31.857055+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "pos_words = set(open(\"./data/PrecoSenti/PrecoPos.txt\").read().strip().split())\n",
    "neg_words = set(open(\"./data/PrecoSenti/PrecoNeg.txt\").read().strip().split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.873295Z",
     "start_time": "2017-07-02T14:11:31.865925+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['rajongó', 'mesteri', 'hivatástudat', 'professzionálisan', 'megfelelően']\n",
      "['vesztegető', 'vihar', 'agyonpuffant', 'nyűg', 'hazudós']\n"
     ]
    }
   ],
   "source": [
    "print(list(pos_words)[:5])\n",
    "print(list(neg_words)[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.881842Z",
     "start_time": "2017-07-02T14:11:31.875680+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# Taken from the Pattern library\n",
    "pos_emoticons = {\n",
    "    \"<3\", u\"♥\", u\"❤\", \">:D\", \":-D\", \":D\", \"=-D\", \"=D\", \"X-D\", \"x-D\", \"XD\",\n",
    "    \"xD\", \"8-D\", \">:P\", \":-P\", \":P\", \":-p\", \":p\", \":-b\", \":b\", \":c)\", \":o)\",\n",
    "    \":^)\", \">:)\", \":-)\", \":)\", \"=)\", \"=]\", \":]\", \":}\", \":>\", \":3\", \"8)\", \"8-)\",\n",
    "    \">;]\", \";-)\", \";)\", \";-]\", \";]\", \";D\", \";^)\", \"*-)\", \"*)\"\n",
    "}\n",
    "neg_emoticons = {\n",
    "    \">:/\", \":-/\", \":/\", \":\\\\\", \">:\\\\\", \":-.\", \":-s\", \":s\", \":S\", \":-S\", \">.>\",\n",
    "    \">:[\", \":-(\", \":(\", \"=(\", \":-[\", \":[\", \":{\", \":-<\", \":c\", \":-c\", \"=/\",\n",
    "    \":'(\", \":'''(\", \";'(\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.890134Z",
     "start_time": "2017-07-02T14:11:31.884192+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "pos_emojis = {\n",
    "    u\"❤️\", u\"💜\", u\"💚\", u\"💙\", u\"💛\", u\"💕\", u\"😀\", u\"😄\", u\"😃\", u\"😆\", u\"😅\", u\"😂\",\n",
    "    u\"😁\", u\"😻\", u\"😍\", u\"😈\", u\"👌\", u\"😛\", u\"😝\", u\"😜\", u\"😋\", u\"😇\", u\"😊\", u\"😌\",\n",
    "    u\"😏\", u\"😎\", u\"☺\", u\"👍\", u\"😉\"\n",
    "}\n",
    "neg_emojis = {\n",
    "    u\"😕\", u\"😬\", u\"😟\", u\"😒\", u\"😔\", u\"😞\", u\"😠\", u\"😩\", u\"😫\", u\"😡\", u\"👿\", u\"😢\",\n",
    "    u\"😥\", u\"😓\", u\"😪\", u\"😭\", u\"😿\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.897082Z",
     "start_time": "2017-07-02T14:11:31.892361+02:00"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "positives = pos_words | pos_emoticons | pos_emojis\n",
    "negatives = neg_words | neg_emoticons | neg_emojis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Can't we just count the ratio of sentiment tokens?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.909628Z",
     "start_time": "2017-07-02T14:11:31.899433+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-1, 0, 1, 0, 1, 0]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sentiment_polarizer(text):\n",
    "    words = sent_tokenize(text)\n",
    "    return [(1 if tok.lower() in positives else \n",
    "             (-1 if tok.lower()in negatives else 0)) \n",
    "            for tok in words]\n",
    "            \n",
    "sentiment_polarizer(\"Gagyi, :) nagyon jó!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:31.929181Z",
     "start_time": "2017-07-02T14:11:31.913093+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.base import ClassifierMixin, BaseEstimator\n",
    "from collections import Counter\n",
    "\n",
    "X, y = sent_df.tweet, sent_df.sentiment\n",
    "\n",
    "class SimpleSentiment(ClassifierMixin, BaseEstimator):\n",
    "    def __init__(self, ratio_threshold=1.5):\n",
    "        self.threshold = ratio_threshold\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "    \n",
    "    def _decide(self, x):\n",
    "        values = sentiment_polarizer(x) + [1, -1]\n",
    "        val_counts = Counter(values)\n",
    "        val_ratio = val_counts[1]/val_counts[-1]\n",
    "        if val_ratio > self.threshold:\n",
    "            return 1\n",
    "        elif val_ratio < 1 / (self.threshold):\n",
    "            return -1\n",
    "        else:\n",
    "            return 0\n",
    "                                   \n",
    "    def predict(self, X, *args):\n",
    "        return [self._decide(x) for x in X]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:11:32.911827Z",
     "start_time": "2017-07-02T14:11:31.932103+02:00"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.64      0.26      0.37      1129\n",
      "          0       0.38      0.76      0.51      1261\n",
      "          1       0.64      0.40      0.50      1610\n",
      "\n",
      "avg / total       0.56      0.48      0.46      4000\n",
      "\n",
      "[[295 682 152]\n",
      " [ 89 956 216]\n",
      " [ 79 880 651]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.47549999999999998"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = SimpleSentiment(1.5).predict(X)\n",
    "print(classification_report(y, y_pred))\n",
    "\n",
    "cm = confusion_matrix(y, y_pred)\n",
    "print(cm)\n",
    "accuracy_score(y, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Combining multiple information sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:13:53.304405Z",
     "start_time": "2017-07-02T14:13:53.295398+02:00"
    },
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_union\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import MaxAbsScaler\n",
    "\n",
    "random.seed(42)\n",
    "np.random.seed(42)\n",
    "\n",
    "pipeline = make_pipeline(\n",
    "    make_union(\n",
    "        TfidfVectorizer(\n",
    "            tokenizer=sent_tokenize,\n",
    "            ngram_range=(1, 2),\n",
    "            lowercase=True,\n",
    "            token_pattern=\".*\"),\n",
    "        make_pipeline(\n",
    "            CountVectorizer(tokenizer=sentiment_polarizer),\n",
    "            MaxAbsScaler()\n",
    "        )\n",
    "    ),\n",
    "    LinearSVC()\n",
    "\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-02T12:14:11.257137Z",
     "start_time": "2017-07-02T14:13:53.986308+02:00"
    },
    "scrolled": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "         -1       0.55      0.57      0.56      1129\n",
      "          0       0.51      0.48      0.50      1261\n",
      "          1       0.65      0.65      0.65      1610\n",
      "\n",
      "avg / total       0.58      0.58      0.58      4000\n",
      "\n",
      "[[ 648  265  216]\n",
      " [ 292  610  359]\n",
      " [ 242  318 1050]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.57699999999999996"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = cross_val_predict(pipeline, X, y, cv=10)\n",
    "print(classification_report(y, y_pred))\n",
    "\n",
    "cm = confusion_matrix(y, y_pred)\n",
    "print(cm)\n",
    "accuracy_score(y, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### What's next?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-21T21:31:24.327947Z",
     "start_time": "2017-06-21T23:31:24.322276+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "* Collect **more** data\n",
    "* Analyze the bias-variance of your model\n",
    "* PoS tagged, lemmatized sentiment lexicon\n",
    "* Use additional sentiment lexicons (such as [this emoji lexicon](http://kt.ijs.si/data/Emoji_sentiment_ranking/))\n",
    "* Be creative with the features (handing negation, irony, sarcasm)\n",
    "* Fine tune your model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-25T21:32:11.076516Z",
     "start_time": "2017-06-25T23:32:11.070941+02:00"
    },
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Do you really want to classify sentiments on tweets? How about analyzing \n",
    "* clause level sentiment or\n",
    "* entity level sentiment or\n",
    "* aspect level sentiment?"
   ]
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/77a4720be62840005ba7a11203f2be8b"
  },
  "celltoolbar": "Slideshow",
  "gist": {
   "data": {
    "description": "Text Categorization",
    "public": false
   },
   "id": "77a4720be62840005ba7a11203f2be8b"
  },
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "30px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": true,
   "toc_position": {
    "height": "601px",
    "left": "0px",
    "right": "1244px",
    "top": "106px",
    "width": "192px"
   },
   "toc_section_display": "none",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}