03SentenceSegmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "03SentenceSegmentation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4NRuVTrqxZwY",
        "colab_type": "text"
      },
      "source": [
        "# 1) Sentence Segmentation Basics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjM9tFytxcL6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import spaCy\n",
        "import spacy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7z0r4rBoOMi5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# load the English language library\n",
        "nlp = spacy.load(name='en_core_web_sm')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usxPGG5NTra6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "doc = nlp(\"This is a sentence. This is second sentence. This is last sentence.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tOQnM_lNTs5K",
        "colab_type": "code",
        "outputId": "68b88cb0-daa6-4a7c-d7f6-7a953799e687",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "for sent in doc.sents:\n",
        "    print(sent.text)\n",
        "\n",
        "# note that doc.sents is a generator"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This is a sentence.\n",
            "This is second sentence.\n",
            "This is last sentence.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8xWqVdvUOSz",
        "colab_type": "code",
        "outputId": "7e849da2-e556-4b34-f3f0-ba9be9b74662",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "source": [
        "doc.sents[0]\n",
        "# if you try to grab individual sentences then you will get an error\n",
        "# TypeError: 'generator' object is not subscriptable\n",
        "# because this generates the sentences instead of holding them in memory"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-36abe73e7f01>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdoc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msents\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# if you try to grab individual sentences then you will get an error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# TypeError: 'generator' object is not subscriptable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# because this generates the sentences instead of holding them in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'generator' object is not subscriptable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6wOR4vDVUHL",
        "colab_type": "code",
        "outputId": "d9d9c8de-f715-4fb0-d11d-bf4503dc526a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "# grab a token\n",
        "doc[0]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "This"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fQUEqM_-U36j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# we can grab individual tokens because spacy knows the language library,\n",
        "# we can grab individual sentences with list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pQQVjBA4wcyj",
        "colab_type": "code",
        "outputId": "dc7c400d-fed1-4474-e78c-0ec98f78eeba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "list(doc.sents)[0]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "This is a sentence."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hPXzKiG7woGT",
        "colab_type": "code",
        "outputId": "7288c4f5-4085-40a7-918d-ba3de61e168e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "type(list(doc.sents)[0])\n",
        "# these are the spacy's span objects not normal strings"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "spacy.tokens.span.Span"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_AZTeqJtxVhZ",
        "colab_type": "text"
      },
      "source": [
        "# 2) Adding Rules"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pXGaLvBn4Q47",
        "colab_type": "text"
      },
      "source": [
        "We can add our own rules for sentence segmentation, but they have to be added before the creation of doc object\n",
        "\n",
        "becauseas that is where parsing of segment start"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBhq157L7cZo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def set_custom_boundaries(doc):\n",
        "    for token in doc[:-1]:   # we are going through every token except the last one\n",
        "      if token.text == ';':    # here we are adding a new Segmentation rule\n",
        "        doc[token.i+1].is_sent_start = True\n",
        "    return doc\n",
        "\n",
        "# after the semicolan, the next token is start of new sentence\n",
        "\n",
        "# we are passing all the tokens through for loop except the last one because,\n",
        "# we are taking \"token.i+1\" is the start of new sentence\n",
        "# that is \"token.i+1\" includes last token in document(doc_2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tBBvl6D3LXYp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp.add_pipe(set_custom_boundaries, before='parser')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PvbjPaGS98my",
        "colab_type": "code",
        "outputId": "52871c53-d1b8-4246-acc7-7a23c4d46364",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "nlp.pipe_names\n",
        "# new rule is added, that is 'set_custom_boundaries'\n",
        "# tagger, parser and ner are already there"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['tagger', 'set_custom_boundaries', 'parser', 'ner']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rn03CsaeMrKo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# new rule has to run before the document is parsed, hence define new document object witha same content \n",
        "doc_2 = nlp(\"This is a sentence; This is second sentence; This is last sentence.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_GcGIPXL_MN",
        "colab_type": "code",
        "outputId": "0eca6966-28d4-47f0-d47f-6c50f46ea498",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "for sent in doc_2.sents:\n",
        "    print(sent)\n",
        "\n",
        "# new rule is working fine here"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "This is a sentence;\n",
            "This is second sentence;\n",
            "This is last sentence.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}