08_Stemming.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<small><small><i>\n",
    "All the IPython Notebooks in **[Python Natural Language Processing](https://github.com/milaan9/Python_Python_Natural_Language_Processing)** lecture series by **[Dr. Milaan Parmar](https://www.linkedin.com/in/milaanparmar/)** are available @ **[GitHub](https://github.com/milaan9)**\n",
    "</i></small></small>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/milaan9/Python_Python_Natural_Language_Processing/blob/main/08_Stemming.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 08 Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "id": "uROUhD6qDkcs",
    "outputId": "482bc406-753b-44bf-8371-63515e303b18"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I\n",
      "'m\n",
      "always\n",
      "here\n",
      "to\n",
      "help\n",
      "you\n",
      "all\n",
      "!\n",
      "Email:milaanparmar9@gmail.com\n",
      "or\n",
      "visit\n",
      "more\n",
      "at\n",
      "https://github.com/milaan9\n",
      "!\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "doc2 = nlp(u\"I'm always here to help you all! Email:milaanparmar9@gmail.com or visit more at https://github.com/milaan9!\")\n",
    "for t in doc2:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "id": "T9SuBFxJD60t",
    "outputId": "ba8e3005-92c2-45dd-8b35-2cbf45df14c0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A\n",
      "5\n",
      "km\n",
      "NYC\n",
      "cab\n",
      "ride\n",
      "costs\n",
      "$\n",
      "10.30\n"
     ]
    }
   ],
   "source": [
    "doc3 = nlp(u'A 5km NYC cab ride costs $10.30')\n",
    "for t in doc3:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "id": "W954E2p7EA06",
    "outputId": "7c7a5417-f259-49ab-d813-1022fc34b74a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let\n",
      "'s\n",
      "visit\n",
      "St.\n",
      "Louis\n",
      "in\n",
      "the\n",
      "U.S.\n",
      "next\n",
      "year\n",
      ".\n"
     ]
    }
   ],
   "source": [
    "doc4 = nlp(u\"Let's visit St. Louis in the U.S. next year.\")\n",
    "for t in doc4:\n",
    "    print(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MnVieHC_EBd6"
   },
   "source": [
    " ### ----   Porter Stemmer   ------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "32eUb4IHIxg-"
   },
   "outputs": [],
   "source": [
    "# Import the toolkit and the full Porter Stemmer library\n",
    "import nltk\n",
    "from nltk.stem.porter import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "Mqz3RhM5Ix9p"
   },
   "outputs": [],
   "source": [
    "p_stemmer = PorterStemmer()\n",
    "words = ['run','runner','running','ran','runs','easily','fairly']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "id": "3BtJibwLI1Ca",
    "outputId": "6a64b851-9353-4a52-cb12-a1f466f66bd7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run --> run\n",
      "runner --> runner\n",
      "running --> run\n",
      "ran --> ran\n",
      "runs --> run\n",
      "easily --> easili\n",
      "fairly --> fairli\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    print(word+' --> '+p_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "1YgYkshwI2jo"
   },
   "outputs": [],
   "source": [
    "#SnowballStemmer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "# The Snowball Stemmer requires that you pass a language parameter\n",
    "s_stemmer = SnowballStemmer(language='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "kDERoxYEI9R4"
   },
   "outputs": [],
   "source": [
    "words = ['run','runner','running','ran','runs','easily','fairly']\n",
    "# words = ['generous','generation','generously','generate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "id": "4M0epwwkI-yo",
    "outputId": "755b30e6-6940-42a7-aea4-3ddb52f85321"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run --> run\n",
      "runner --> runner\n",
      "running --> run\n",
      "ran --> ran\n",
      "runs --> run\n",
      "easily --> easili\n",
      "fairly --> fair\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    print(word+' --> '+s_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "HAhZxPlwJALZ"
   },
   "outputs": [],
   "source": [
    "# ----Do Some more practice -----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "pjnfD_FOJSx5"
   },
   "outputs": [],
   "source": [
    "words = ['consolingly']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "id": "WznSJQT6JUa6",
    "outputId": "a897f507-b64e-4a1c-de9e-886e90cc950a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Porter Stemmer:\n",
      "consolingly --> consolingli\n"
     ]
    }
   ],
   "source": [
    "print('Porter Stemmer:')\n",
    "for word in words:\n",
    "    print(word+' --> '+p_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "id": "kRWOhSIkJVvZ",
    "outputId": "c6ba8350-92b6-409f-ba39-eea293d607a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Porter2 Stemmer:\n",
      "consolingly --> consol\n"
     ]
    }
   ],
   "source": [
    "print('Porter2 Stemmer:')\n",
    "for word in words:\n",
    "    print(word+' --> '+s_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 153
    },
    "id": "CHjOtYJcJXLp",
    "outputId": "cc9a6cdc-1062-47f3-c523-b12b9a3815ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I --> i\n",
      "am --> am\n",
      "meeting --> meet\n",
      "him --> him\n",
      "tomorrow --> tomorrow\n",
      "at --> at\n",
      "the --> the\n",
      "meeting --> meet\n"
     ]
    }
   ],
   "source": [
    "# Stemming has its drawbacks. If given the token saw, stemming might always return saw, whereas lemmatization would likely return either\n",
    "# see or saw depending on whether the use of the token was as a verb or a noun. As an example, consider the following:\n",
    "\n",
    "phrase = 'I am meeting him tomorrow at the meeting'\n",
    "for word in phrase.split():\n",
    "    print(word+' --> '+p_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "euP5mgGRLzMP"
   },
   "outputs": [],
   "source": [
    "# Perform standard imports:\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "id": "u38F6ATiL3Np",
    "outputId": "da5dd4a5-1cd0-4f7e-ba22-cab74b306b86"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "John \t PROPN \t 11174346320140919546 \t John\n",
      "Adam \t PROPN \t 14264057329400597350 \t Adam\n",
      "is \t AUX \t 10382539506755952630 \t be\n",
      "one \t NUM \t 17454115351911680600 \t one\n",
      "the \t DET \t 7425985699627899538 \t the\n",
      "researcher \t NOUN \t 1317581537614213870 \t researcher\n",
      "who \t PRON \t 3876862883474502309 \t who\n",
      "invent \t VERB \t 5373681334090504585 \t invent\n",
      "the \t DET \t 7425985699627899538 \t the\n",
      "direction \t NOUN \t 895834437038626927 \t direction\n",
      "of \t ADP \t 886050111519832510 \t of\n",
      "way \t NOUN \t 6878210874361030284 \t way\n",
      "towards \t ADP \t 9315050841437086371 \t towards\n",
      "success \t NOUN \t 16089821935113899987 \t success\n",
      "! \t PUNCT \t 17494803046312582752 \t !\n"
     ]
    }
   ],
   "source": [
    "var1 = nlp(u\"John Adam is one the researcher who invent the direction of way towards success!\")\n",
    "\n",
    "for token in var1:\n",
    "    print(token.text, '\\t', token.pos_, '\\t', token.lemma, '\\t', token.lemma_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "15GTnATmL4h3"
   },
   "outputs": [],
   "source": [
    "def show_lemmas(text):\n",
    "    for token in text:\n",
    "        print(f'{token.text:{12}} {token.pos_:{6}} {token.lemma:<{22}} {token.lemma_}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "id": "kcR_PwbVL61X",
    "outputId": "44dcddcc-391f-477a-cd7b-ba0042e61a60"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "John         PROPN  11174346320140919546   John\n",
      "Adam         PROPN  14264057329400597350   Adam\n",
      "is           AUX    10382539506755952630   be\n",
      "one          NUM    17454115351911680600   one\n",
      "the          DET    7425985699627899538    the\n",
      "researcher   NOUN   1317581537614213870    researcher\n",
      "who          PRON   3876862883474502309    who\n",
      "invent       VERB   5373681334090504585    invent\n",
      "the          DET    7425985699627899538    the\n",
      "direction    NOUN   895834437038626927     direction\n",
      "of           ADP    886050111519832510     of\n",
      "way          NOUN   6878210874361030284    way\n",
      "towards      ADP    9315050841437086371    towards\n",
      "success      NOUN   16089821935113899987   success\n",
      "!            PUNCT  17494803046312582752   !\n"
     ]
    }
   ],
   "source": [
    "var2 = nlp(u\"John Adam is one the researcher who invent the direction of way towards success!\")\n",
    "show_lemmas(var2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "id": "u9HofN4pL8S3",
    "outputId": "fe8b7e71-7824-4d88-9d72-d0a4a9251d12"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I            PRON   4690420944186131903    I\n",
      "am           AUX    10382539506755952630   be\n",
      "meeting      VERB   6880656908171229526    meet\n",
      "him          PRON   1655312771067108281    he\n",
      "tomorrow     NOUN   3573583789758258062    tomorrow\n",
      "at           ADP    11667289587015813222   at\n",
      "the          DET    7425985699627899538    the\n",
      "meeting      NOUN   14798207169164081740   meeting\n",
      ".            PUNCT  12646065887601541794   .\n"
     ]
    }
   ],
   "source": [
    "var3 = nlp(u\"I am meeting him tomorrow at the meeting.\")\n",
    "show_lemmas(var3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 170
    },
    "id": "62Rj1pk3L-MH",
    "outputId": "8a271b77-0a39-4c8b-9e27-e62351dc3055"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "That         PRON   4380130941430378203    that\n",
      "'s           AUX    10382539506755952630   be\n",
      "of           ADP    886050111519832510     of\n",
      "the          DET    7425985699627899538    the\n",
      "greate       ADJ    4429768169814447593    greate\n",
      "person       NOUN   14800503047316267216   person\n",
      "in           ADP    3002984154512732771    in\n",
      "the          DET    7425985699627899538    the\n",
      "world        NOUN   1703489418272052182    world\n"
     ]
    }
   ],
   "source": [
    "var4 = nlp(u\"That's of the greate person in the world\")\n",
    "show_lemmas(var4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UGGTGXLfMtcu"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "8_Stemming.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}