NLP - Tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Akash_Shrivastava\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\numpy\\_distributor_init.py:32: UserWarning: loaded more than 1 DLL from .libs:\n",
      "C:\\Users\\Akash_Shrivastava\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.QVLO2T66WEPI7JZ63PS3HMOHFEY472BC.gfortran-win_amd64.dll\n",
      "C:\\Users\\Akash_Shrivastava\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\numpy\\.libs\\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll\n",
      "  stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "#!pip install nltk\n",
    "\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize, sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "Text = \"\"\" During the II world war in 1941,  it was a difficult time for our  family at Rameswaram. I was a ten \n",
    "year old boy then.  War had almost  reached our doors of Rameswaram since the cloud of war had already reached Colombo. \n",
    "Almost everything was a rarity from food articles to anything.  Ours was a large joint family. The size of our family was five sons and five daughters and three of whom had families \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize_word = nltk.word_tokenize(Text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['During',\n",
       " 'the',\n",
       " 'II',\n",
       " 'world',\n",
       " 'war',\n",
       " 'in',\n",
       " '1941',\n",
       " ',',\n",
       " 'it',\n",
       " 'was',\n",
       " 'a',\n",
       " 'difficult',\n",
       " 'time',\n",
       " 'for',\n",
       " 'our',\n",
       " 'family',\n",
       " 'at',\n",
       " 'Rameswaram',\n",
       " '.',\n",
       " 'I',\n",
       " 'was',\n",
       " 'a',\n",
       " 'ten',\n",
       " 'year',\n",
       " 'old',\n",
       " 'boy',\n",
       " 'then',\n",
       " '.',\n",
       " 'War',\n",
       " 'had',\n",
       " 'almost',\n",
       " 'reached',\n",
       " 'our',\n",
       " 'doors',\n",
       " 'of',\n",
       " 'Rameswaram',\n",
       " 'since',\n",
       " 'the',\n",
       " 'cloud',\n",
       " 'of',\n",
       " 'war',\n",
       " 'had',\n",
       " 'already',\n",
       " 'reached',\n",
       " 'Colombo',\n",
       " '.',\n",
       " 'Almost',\n",
       " 'everything',\n",
       " 'was',\n",
       " 'a',\n",
       " 'rarity',\n",
       " 'from',\n",
       " 'food',\n",
       " 'articles',\n",
       " 'to',\n",
       " 'anything',\n",
       " '.',\n",
       " 'Ours',\n",
       " 'was',\n",
       " 'a',\n",
       " 'large',\n",
       " 'joint',\n",
       " 'family',\n",
       " '.',\n",
       " 'The',\n",
       " 'size',\n",
       " 'of',\n",
       " 'our',\n",
       " 'family',\n",
       " 'was',\n",
       " 'five',\n",
       " 'sons',\n",
       " 'and',\n",
       " 'five',\n",
       " 'daughters',\n",
       " 'and',\n",
       " 'three',\n",
       " 'of',\n",
       " 'whom',\n",
       " 'had',\n",
       " 'families']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize_sent = nltk.sent_tokenize(Text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' During the II world war in 1941,  it was a difficult time for our  family at Rameswaram.',\n",
       " 'I was a ten \\nyear old boy then.',\n",
       " 'War had almost  reached our doors of Rameswaram since the cloud of war had already reached Colombo.',\n",
       " 'Almost everything was a rarity from food articles to anything.',\n",
       " 'Ours was a large joint family.',\n",
       " 'The size of our family was five sons and five daughters and three of whom had families']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}