Question Answering System Using Word2Vec Embedding Technique.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 25341,
     "status": "ok",
     "timestamp": 1619419552779,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "EY5dzvRHa1my",
    "outputId": "26dfc89c-c9f4-4367-e7b0-d4f2768efa44"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SkbI55yjfNYM"
   },
   "outputs": [],
   "source": [
    "import os \n",
    "os.chdir('gdrive/My Drive/Colab Notebooks')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 8442,
     "status": "ok",
     "timestamp": 1619419790081,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "DvP8WnJHfNVb",
    "outputId": "6b7dfde8-deb8-4e8a-aa4c-45465757db58"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pdfplumber\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/4d9768e9ed204c68bd5813a2a112d3d6af4912f0785d47080b5067cdce64/pdfplumber-0.5.27.tar.gz (44kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 3.8MB/s \n",
      "\u001b[?25hCollecting pdfminer.six==20200517\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b0/c0/ef1c8758bbd86edb10b5443700aac97d0ba27a9ca2e7696db8cd1fdbd5a8/pdfminer.six-20200517-py3-none-any.whl (5.6MB)\n",
      "\u001b[K     |████████████████████████████████| 5.6MB 10.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: Pillow>=7.0.0 in /usr/local/lib/python3.7/dist-packages (from pdfplumber) (7.1.2)\n",
      "Collecting Wand\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d7/f6/05f043c099639b9017b7244791048a4d146dfea45b41a199aed373246d50/Wand-0.6.6-py2.py3-none-any.whl (138kB)\n",
      "\u001b[K     |████████████████████████████████| 143kB 42.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: chardet; python_version > \"3.0\" in /usr/local/lib/python3.7/dist-packages (from pdfminer.six==20200517->pdfplumber) (3.0.4)\n",
      "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.7/dist-packages (from pdfminer.six==20200517->pdfplumber) (2.3.0)\n",
      "Collecting pycryptodome\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ad/16/9627ab0493894a11c68e46000dbcc82f578c8ff06bc2980dcd016aea9bd3/pycryptodome-3.10.1-cp35-abi3-manylinux2010_x86_64.whl (1.9MB)\n",
      "\u001b[K     |████████████████████████████████| 1.9MB 44.9MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: pdfplumber\n",
      "  Building wheel for pdfplumber (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pdfplumber: filename=pdfplumber-0.5.27-cp37-none-any.whl size=32071 sha256=97813ab99da0cbcdfbde3206fdec4106095462ef248356fa4123852fa42c9ad1\n",
      "  Stored in directory: /root/.cache/pip/wheels/b4/74/fc/f7b3a1a0732209027fb48a5f4392fc40d79970b11c2ba49e71\n",
      "Successfully built pdfplumber\n",
      "Installing collected packages: pycryptodome, pdfminer.six, Wand, pdfplumber\n",
      "Successfully installed Wand-0.6.6 pdfminer.six-20200517 pdfplumber-0.5.27 pycryptodome-3.10.1\n"
     ]
    }
   ],
   "source": [
    "pip install pdfplumber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2E2YFFZxfNSx"
   },
   "outputs": [],
   "source": [
    "import pdfplumber\n",
    "pdf = pdfplumber.open('lic_policy.pdf')\n",
    "page = pdf.pages[0]\n",
    "page1 = pdf.pages[1]\n",
    "pdf_txt = page.extract_text() + page1.extract_text()\n",
    "pdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2140,
     "status": "ok",
     "timestamp": 1619419793554,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "NW9vnIam7mwe",
    "outputId": "f60d2cf3-cc26-4318-92c6-6de00db83ed8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract. It\n",
      " is therefore, necessary that the terms and conditions of the agreement must be suitably documented in a manner that\n",
      " would make it clear that both parties to the contract are Ad- idem i.e., of the same mind. Ad-Idem means that both t\n",
      "he parties understand the same thing in the same sense or are of the same mind on the same subject. There must be c\n",
      "onsensus or Ad-Idem between the parties to the contract.  \n",
      "This is possible provided all the terms and conditions, rights and duties - privileges and obligations are properly doc\n",
      "umented in terms which can be clearly interpreted in a court of law. Between two human beings sometime silence m\n",
      "eans an acceptance. But as the insurer is a legal personality entitled to contract verbal discussion between parties to t\n",
      "he contract is not possible and hence there is a need for documentation.  \n",
      "Insurance is also a contract of utmost good faith and enforced only in the distant future. It is therefore necessary that \n",
      "the declarations made by both the parties should be put in black and white for future reference. Any suppression, wil\n",
      "lful and material shall make the contract void. The insured, therefore, has a duty to declare all that he knows about hi\n",
      "mself, his health, his financial status in answering questions contained in the proposal form and other ancillary docu\n",
      "ments which may be required by the insurer.  \n",
      "Age is an important factor in deciding the quantum of premium against a policy. The document proving the age, i.e. \n",
      "age proof must be reliable and the insured has to undertake as to its truthfulness.  \n",
      "Non-standard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with \n",
      "a pinch of salt. In other words the insurer takes certain precautions before accepting such age proofs as final.  \n",
      "Proof of income is the document may become necessary whenever the sum proposed is very high. Normally a sum p\n",
      "roposed which is seven to eight times of the declared income is acceptable for insurance. But proposals do come to t\n",
      "he insurer when the known source of income of the proposer is much less compared to the amount of insurance desir\n",
      "ed. A service holder normally does not face this problem as his sources of income are verifiable.  \n",
      "In case of business people, the assessed income is at times much less compared to what is a desirable income for the \n",
      "amount of insurance desired. In such cases the insurer at times calls for assessed income tax returns, or Chartered Ac\n",
      "countant’s certificate etc. Such precautions are necessary to eliminate the possibility of moral hazard.  \n",
      "Policy Contract is the policy document is a detailed document and it is the Evidence of the insurance contract which \n",
      "mentions all the terms and conditions of the insurance. The insured buys not the policy contract, but the right to the s\n",
      "um of money and its future delivery. The insurer on its part promises to pay a sum of money, provided of course the \n",
      "insured keeps its part of promise of paying the installments of premium as scheduled.  \n",
      "The pre-amble to the insurance contract makes the above statement clear and states that this policy is issued subject t\n",
      "o the conditions and privileges printed on the back of the policy. The endorsements placed on the policy shall also b\n",
      "e part of the policy and it also makes a reference to the proposal form saying that that the statements given in the pro\n",
      "posal form are the basis of the contract.  \n",
      "The schedule which is printed on the policy document identifies the office which has issued the policy. It states the n\n",
      "ame of the policyholder, the date of commencement of the policy, an identification number of the policy called polic\n",
      "y number. This number is extremely useful for making any reference to the insurer relating to this policy. This shall \n",
      "avoid needless delay.  \n",
      "Beneficiary’s name is also mentioned along with address.  \n",
      "It is necessary to check that it is correct and any mistake should be immediately pointed out for correction. A mistak\n",
      "e in the address may misdirect the premium notices and any other future correspondence. It also states the name of t\n",
      "he nominee and the date upto which premium has to be paid. The schedule goes on to mention, the type of policy, o\n",
      "n the happening of which, the sum assured is payable and to whom it is payable. It of course also mentions when an\n",
      "d how long the premium is to be paid.  \n",
      "The policy document is signed by an official of the insurer and dated and stamped as per the provision of the Stamp \n",
      "Act to make it a completely legally enforceable document.  \n",
      "An assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy\n",
      ". Re-assignment can also be done by a subsequent endorsement on the same policy.As a nomination is automatically\n",
      " cancelled due to an assignment, after re-assignment, it is necessary to make a fresh nomination.  \n",
      "In Duplicate policy a policy document is a valuable document and can be used for mortgage etc. Loss of policy docu\n",
      "ment does not absolve the insurer from the liability of payment of policy proceeds when the claim arises. The claim \n",
      "can be settled on the claimants, furnishing an indemnity bond jointly with one surety.  \n",
      "If a policy is irrevocably lost, a duplicate policy can be issued, after following a certain procedure. The insurer satisf\n",
      "ies itself of the circumstances leading to loss. Being so satisfied the insurer insists upon an advertisement in a news p\n",
      "aper, production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued. The duplicate policy is stamped “Duplicate Policy”.  \n",
      "Generally nomination is made at the time of taking a policy. In case it is not done, it is possible to make nomination \n",
      "subsequently by an endorsement on the policy. It is also possible to change a nomination subsequently by an endors\n",
      "ement. After marriage, such change in nomination is normally required.  \n",
      "Life insurance being a legally enforceable contract, needs to be documented with details of the rights and obligations\n",
      " of the parties to the contract. Proposal form duly filled in and signed by the proposer is the first document which for\n",
      "ms the basis of the contract.  \n",
      "Every time, the insured pays the premium, he receives a premium receipt. The premium needs to be paid in time, no\n",
      "n- payment of premium leads to policy-lapses. Re-instatement of the cover is called revival of the policy.  \n",
      "If the policy is not revived, the policy can become a paid up policy for a reduced sum assured under certain conditio\n",
      "ns.  \n",
      "The policy document mentions in detail all the rights and obligations of the policyholder. The agent is advised to ex\n",
      "plain the various provisions of the policy to the policyholder.  \n",
      "The wordings in the policy document are of technical nature and hence the need for explaining. If there are certain e\n",
      "ndorsements on the policy, that need to be explained too.  \n",
      "It needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowl\n",
      "edge of the close relatives.  \n",
      "  \n",
      "  \n"
     ]
    }
   ],
   "source": [
    "print(pdf_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3941,
     "status": "ok",
     "timestamp": 1619419796769,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "sTCcRsKkfNP3",
    "outputId": "2cb8206a-ce93-47be-9c27-7cdc969aecaa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (3.2.5)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2318,
     "status": "ok",
     "timestamp": 1619423225948,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "sQlQc2kiJPQJ",
    "outputId": "3f73525f-4678-42a1-b624-cb9de6fe5ada"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "Life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract. \n",
      "\n",
      "It is therefore, necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are Ad- idem i.e., of the same mind. \n",
      "\n",
      "Ad-Idem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject. \n",
      "\n",
      "There must be consensus or Ad-Idem between the parties to the contract. \n",
      "\n",
      "This is possible provided all the terms and conditions, rights and duties - privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law. \n",
      "\n",
      "Between two human beings sometime silence means an acceptance. \n",
      "\n",
      "But as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation. \n",
      "\n",
      "Insurance is also a contract of utmost good faith and enforced only in the distant future. \n",
      "\n",
      "It is therefore necessary that the declarations made by both the parties should be put in black and white for future reference. \n",
      "\n",
      "Any suppression, willful and material shall make the contract void. \n",
      "\n",
      "The insured, therefore, has a duty to declare all that he knows about himself, his health, his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer. \n",
      "\n",
      "Age is an important factor in deciding the quantum of premium against a policy. \n",
      "\n",
      "The document proving the age, i.e. \n",
      "\n",
      "age proof must be reliable and the insured has to undertake as to its truthfulness. \n",
      "\n",
      "Non-standard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt. \n",
      "\n",
      "In other words the insurer takes certain precautions before accepting such age proofs as final. \n",
      "\n",
      "Proof of income is the document may become necessary whenever the sum proposed is very high. \n",
      "\n",
      "Normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance. \n",
      "\n",
      "But proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired. \n",
      "\n",
      "A service holder normally does not face this problem as his sources of income are verifiable. \n",
      "\n",
      "In case of business people, the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired. \n",
      "\n",
      "In such cases the insurer at times calls for assessed income tax returns, or Chartered Accountant’s certificate etc. \n",
      "\n",
      "Such precautions are necessary to eliminate the possibility of moral hazard. \n",
      "\n",
      "Policy Contract is the policy document is a detailed document and it is the Evidence of the insurance contract which mentions all the terms and conditions of the insurance. \n",
      "\n",
      "The insured buys not the policy contract, but the right to the sum of money and its future delivery. \n",
      "\n",
      "The insurer on its part promises to pay a sum of money, provided of course the insured keeps its part of promise of paying the installments of premium as scheduled. \n",
      "\n",
      "The pre-amble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy. \n",
      "\n",
      "The endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract. \n",
      "\n",
      "The schedule which is printed on the policy document identifies the office which has issued the policy. \n",
      "\n",
      "It states the name of the policyholder, the date of commencement of the policy, an identification number of the policy called policy number. \n",
      "\n",
      "This number is extremely useful for making any reference to the insurer relating to this policy. \n",
      "\n",
      "This shall avoid needless delay. \n",
      "\n",
      "Beneficiary’s name is also mentioned along with address. \n",
      "\n",
      "It is necessary to check that it is correct and any mistake should be immediately pointed out for correction. \n",
      "\n",
      "A mistake in the address may misdirect the premium notices and any other future correspondence. \n",
      "\n",
      "It also states the name of the nominee and the date upto which premium has to be paid. \n",
      "\n",
      "The schedule goes on to mention, the type of policy, on the happening of which, the sum assured is payable and to whom it is payable. \n",
      "\n",
      "It of course also mentions when and how long the premium is to be paid. \n",
      "\n",
      "The policy document is signed by an official of the insurer and dated and stamped as per the provision of the Stamp Act to make it a completely legally enforceable document. \n",
      "\n",
      "An assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy. \n",
      "\n",
      "Re-assignment can also be done by a subsequent endorsement on the same policy.As a nomination is automatically cancelled due to an assignment, after re-assignment, it is necessary to make a fresh nomination. \n",
      "\n",
      "In Duplicate policy a policy document is a valuable document and can be used for mortgage etc. \n",
      "\n",
      "Loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises. \n",
      "\n",
      "The claim can be settled on the claimants, furnishing an indemnity bond jointly with one surety. \n",
      "\n",
      "If a policy is irrevocably lost, a duplicate policy can be issued, after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper, production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued. \n",
      "\n",
      "Generally nomination is made at the time of taking a policy. \n",
      "\n",
      "In case it is not done, it is possible to make nomination subsequently by an endorsement on the policy. \n",
      "\n",
      "It is also possible to change a nomination subsequently by an endorsement. \n",
      "\n",
      "After marriage, such change in nomination is normally required. \n",
      "\n",
      "Life insurance being a legally enforceable contract, needs to be documented with details of the rights and obligations of the parties to the contract. \n",
      "\n",
      "Proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract. \n",
      "\n",
      "Every time, the insured pays the premium, he receives a premium receipt. \n",
      "\n",
      "The premium needs to be paid in time, non- payment of premium leads to policy-lapses. \n",
      "\n",
      "Re-instatement of the cover is called revival of the policy. \n",
      "\n",
      "If the policy is not revived, the policy can become a paid up policy for a reduced sum assured under certain conditions. \n",
      "\n",
      "The policy document mentions in detail all the rights and obligations of the policyholder. \n",
      "\n",
      "The agent is advised to explain the various provisions of the policy to the policyholder. \n",
      "\n",
      "The wordings in the policy document are of technical nature and hence the need for explaining. \n",
      "\n",
      "If there are certain endorsements on the policy, that need to be explained too. \n",
      "\n",
      "It needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "pdf_txt = \"\"\"Life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract. It is therefore, necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are Ad- idem i.e., of the same mind. Ad-Idem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject. There must be consensus or Ad-Idem between the parties to the contract.\n",
    "This is possible provided all the terms and conditions, rights and duties - privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law. Between two human beings sometime silence means an acceptance. But as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation.\n",
    "Insurance is also a contract of utmost good faith and enforced only in the distant future. It is therefore necessary that the declarations made by both the parties should be put in black and white for future reference. Any suppression, willful and material shall make the contract void. The insured, therefore, has a duty to declare all that he knows about himself, his health, his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer.\n",
    "Age is an important factor in deciding the quantum of premium against a policy. The document proving the age, i.e. age proof must be reliable and the insured has to undertake as to its truthfulness.\n",
    "Non-standard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt. In other words the insurer takes certain precautions before accepting such age proofs as final.\n",
    "Proof of income is the document may become necessary whenever the sum proposed is very high. Normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance. But proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired. A service holder normally does not face this problem as his sources of income are verifiable.\n",
    "In case of business people, the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired. In such cases the insurer at times calls for assessed income tax returns, or Chartered Accountant’s certificate etc. Such precautions are necessary to eliminate the possibility of moral hazard.\n",
    "Policy Contract is the policy document is a detailed document and it is the Evidence of the insurance contract which mentions all the terms and conditions of the insurance. The insured buys not the policy contract, but the right to the sum of money and its future delivery. The insurer on its part promises to pay a sum of money, provided of course the insured keeps its part of promise of paying the installments of premium as scheduled.\n",
    "The pre-amble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy. The endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract.\n",
    "The schedule which is printed on the policy document identifies the office which has issued the policy. It states the name of the policyholder, the date of commencement of the policy, an identification number of the policy called policy number. This number is extremely useful for making any reference to the insurer relating to this policy. This shall avoid needless delay.\n",
    "Beneficiary’s name is also mentioned along with address.\n",
    "It is necessary to check that it is correct and any mistake should be immediately pointed out for correction. A mistake in the address may misdirect the premium notices and any other future correspondence. It also states the name of the nominee and the date upto which premium has to be paid. The schedule goes on to mention, the type of policy, on the happening of which, the sum assured is payable and to whom it is payable. It of course also mentions when and how long the premium is to be paid.\n",
    "The policy document is signed by an official of the insurer and dated and stamped as per the provision of the Stamp Act to make it a completely legally enforceable document.\n",
    "An assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy. Re-assignment can also be done by a subsequent endorsement on the same policy.As a nomination is automatically cancelled due to an assignment, after re-assignment, it is necessary to make a fresh nomination.\n",
    "In Duplicate policy a policy document is a valuable document and can be used for mortgage etc. Loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises. The claim can be settled on the claimants, furnishing an indemnity bond jointly with one surety.\n",
    "If a policy is irrevocably lost, a duplicate policy can be issued, after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper, production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued.\n",
    "Generally nomination is made at the time of taking a policy. In case it is not done, it is possible to make nomination subsequently by an endorsement on the policy. It is also possible to change a nomination subsequently by an endorsement. After marriage, such change in nomination is normally required.\n",
    "Life insurance being a legally enforceable contract, needs to be documented with details of the rights and obligations of the parties to the contract. Proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract.\n",
    "Every time, the insured pays the premium, he receives a premium receipt. The premium needs to be paid in time, non- payment of premium leads to policy-lapses. Re-instatement of the cover is called revival of the policy.\n",
    "If the policy is not revived, the policy can become a paid up policy for a reduced sum assured under certain conditions.\n",
    "The policy document mentions in detail all the rights and obligations of the policyholder. The agent is advised to explain the various provisions of the policy to the policyholder.\n",
    "The wordings in the policy document are of technical nature and hence the need for explaining. If there are certain endorsements on the policy, that need to be explained too.\n",
    "It needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives.\n",
    "\"\"\"\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "tokens = nltk.sent_tokenize(pdf_txt)\n",
    "for t in tokens:\n",
    "    print(t, \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6254,
     "status": "ok",
     "timestamp": 1619423235140,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "ko73Pu2FkKSL",
    "outputId": "a3aa237a-4250-422a-c2ec-932c7fc48a80"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: gensim in /usr/local/lib/python3.7/dist-packages (4.0.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.11.3 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.19.5)\n",
      "Requirement already satisfied, skipping upgrade: smart-open>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (5.0.0)\n",
      "Requirement already satisfied, skipping upgrade: scipy>=0.18.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.4.1)\n"
     ]
    }
   ],
   "source": [
    "pip install -U gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lwHObl-Sj-ZJ"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import gensim\n",
    "from gensim.parsing.preprocessing import remove_stopwords\n",
    "\n",
    "def clean_sentence(sentence, stopwords=False):\n",
    "  sentence = sentence.lower().strip()\n",
    "  sentence = re.sub(r'[^a-z0-9\\s]', '', sentence)\n",
    "  if stopwords:\n",
    "    sentence = remove_stopwords(sentence)\n",
    "  return sentence\n",
    "\n",
    "def get_cleaned_sentences(tokens, stopwords=False):\n",
    "  cleaned_sentences = []\n",
    "  for row in tokens:\n",
    "    cleaned = clean_sentence(row, stopwords)\n",
    "    cleaned_sentences.append(cleaned)\n",
    "  return cleaned_sentences\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1032,
     "status": "ok",
     "timestamp": 1619423240069,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "dFL9sfOfiZpI",
    "outputId": "335cd839-d4ff-4b5b-9be3-fb090292b51b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['life insurance legally enforceable contract parties legally qualified contract', 'necessary terms conditions agreement suitably documented manner clear parties contract ad idem mind', 'adidem means parties understand thing sense mind subject', 'consensus adidem parties contract', 'possible provided terms conditions rights duties privileges obligations properly documented terms clearly interpreted court law', 'human beings silence means acceptance', 'insurer legal personality entitled contract verbal discussion parties contract possible need documentation', 'insurance contract utmost good faith enforced distant future', 'necessary declarations parties black white future reference', 'suppression willful material shall contract void', 'insured duty declare knows health financial status answering questions contained proposal form ancillary documents required insurer', 'age important factor deciding quantum premium policy', 'document proving age', 'age proof reliable insured undertake truthfulness', 'nonstandard age proofs comparatively reliable insurer accepts pinch salt', 'words insurer takes certain precautions accepting age proofs final', 'proof income document necessary sum proposed high', 'normally sum proposed seven times declared income acceptable insurance', 'proposals come insurer known source income proposer compared insurance desired', 'service holder normally face problem sources income verifiable', 'case business people assessed income times compared desirable income insurance desired', 'cases insurer times calls assessed income tax returns chartered accountants certificate', 'precautions necessary eliminate possibility moral hazard', 'policy contract policy document detailed document evidence insurance contract mentions terms conditions insurance', 'insured buys policy contract right sum money future delivery', 'insurer promises pay sum money provided course insured keeps promise paying installments premium scheduled', 'preamble insurance contract makes statement clear states policy issued subject conditions privileges printed policy', 'endorsements placed policy shall policy makes reference proposal form saying statements given proposal form basis contract', 'schedule printed policy document identifies office issued policy', 'states policyholder date commencement policy identification number policy called policy number', 'number extremely useful making reference insurer relating policy', 'shall avoid needless delay', 'beneficiarys mentioned address', 'necessary check correct mistake immediately pointed correction', 'mistake address misdirect premium notices future correspondence', 'states nominee date upto premium paid', 'schedule goes mention type policy happening sum assured payable payable', 'course mentions long premium paid', 'policy document signed official insurer dated stamped provision stamp act completely legally enforceable document', 'assignment policy favour person institution effected endorsement policy', 'reassignment subsequent endorsement policyas nomination automatically cancelled assignment reassignment necessary fresh nomination', 'duplicate policy policy document valuable document mortgage', 'loss policy document absolve insurer liability payment policy proceeds claim arises', 'claim settled claimants furnishing indemnity bond jointly surety', 'policy irrevocably lost duplicate policy issued following certain procedure like insurer satisfies circumstances leading loss satisfied insurer insists advertisement news paper production indemnity bond payment policy preparation charges duplicate policy issued', 'generally nomination time taking policy', 'case possible nomination subsequently endorsement policy', 'possible change nomination subsequently endorsement', 'marriage change nomination normally required', 'life insurance legally enforceable contract needs documented details rights obligations parties contract', 'proposal form duly filled signed proposer document forms basis contract', 'time insured pays premium receives premium receipt', 'premium needs paid time non payment premium leads policylapses', 'reinstatement cover called revival policy', 'policy revived policy paid policy reduced sum assured certain conditions', 'policy document mentions rights obligations policyholder', 'agent advised explain provisions policy policyholder', 'wordings policy document technical nature need explaining', 'certain endorsements policy need explained', 'needs explained policy valuable document needs kept safe custody knowledge close relatives']\n",
      "['life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract', 'it is therefore necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are ad idem ie of the same mind', 'adidem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject', 'there must be consensus or adidem between the parties to the contract', 'this is possible provided all the terms and conditions rights and duties  privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law', 'between two human beings sometime silence means an acceptance', 'but as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation', 'insurance is also a contract of utmost good faith and enforced only in the distant future', 'it is therefore necessary that the declarations made by both the parties should be put in black and white for future reference', 'any suppression willful and material shall make the contract void', 'the insured therefore has a duty to declare all that he knows about himself his health his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer', 'age is an important factor in deciding the quantum of premium against a policy', 'the document proving the age ie', 'age proof must be reliable and the insured has to undertake as to its truthfulness', 'nonstandard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt', 'in other words the insurer takes certain precautions before accepting such age proofs as final', 'proof of income is the document may become necessary whenever the sum proposed is very high', 'normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance', 'but proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired', 'a service holder normally does not face this problem as his sources of income are verifiable', 'in case of business people the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired', 'in such cases the insurer at times calls for assessed income tax returns or chartered accountants certificate etc', 'such precautions are necessary to eliminate the possibility of moral hazard', 'policy contract is the policy document is a detailed document and it is the evidence of the insurance contract which mentions all the terms and conditions of the insurance', 'the insured buys not the policy contract but the right to the sum of money and its future delivery', 'the insurer on its part promises to pay a sum of money provided of course the insured keeps its part of promise of paying the installments of premium as scheduled', 'the preamble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy', 'the endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract', 'the schedule which is printed on the policy document identifies the office which has issued the policy', 'it states the name of the policyholder the date of commencement of the policy an identification number of the policy called policy number', 'this number is extremely useful for making any reference to the insurer relating to this policy', 'this shall avoid needless delay', 'beneficiarys name is also mentioned along with address', 'it is necessary to check that it is correct and any mistake should be immediately pointed out for correction', 'a mistake in the address may misdirect the premium notices and any other future correspondence', 'it also states the name of the nominee and the date upto which premium has to be paid', 'the schedule goes on to mention the type of policy on the happening of which the sum assured is payable and to whom it is payable', 'it of course also mentions when and how long the premium is to be paid', 'the policy document is signed by an official of the insurer and dated and stamped as per the provision of the stamp act to make it a completely legally enforceable document', 'an assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy', 'reassignment can also be done by a subsequent endorsement on the same policyas a nomination is automatically cancelled due to an assignment after reassignment it is necessary to make a fresh nomination', 'in duplicate policy a policy document is a valuable document and can be used for mortgage etc', 'loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises', 'the claim can be settled on the claimants furnishing an indemnity bond jointly with one surety', 'if a policy is irrevocably lost a duplicate policy can be issued after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued', 'generally nomination is made at the time of taking a policy', 'in case it is not done it is possible to make nomination subsequently by an endorsement on the policy', 'it is also possible to change a nomination subsequently by an endorsement', 'after marriage such change in nomination is normally required', 'life insurance being a legally enforceable contract needs to be documented with details of the rights and obligations of the parties to the contract', 'proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract', 'every time the insured pays the premium he receives a premium receipt', 'the premium needs to be paid in time non payment of premium leads to policylapses', 'reinstatement of the cover is called revival of the policy', 'if the policy is not revived the policy can become a paid up policy for a reduced sum assured under certain conditions', 'the policy document mentions in detail all the rights and obligations of the policyholder', 'the agent is advised to explain the various provisions of the policy to the policyholder', 'the wordings in the policy document are of technical nature and hence the need for explaining', 'if there are certain endorsements on the policy that need to be explained too', 'it needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives']\n"
     ]
    }
   ],
   "source": [
    "cleaned_sentences = get_cleaned_sentences(tokens, stopwords=True)\n",
    "cleaned_sentences_with_stopwords = get_cleaned_sentences(tokens, stopwords=False)\n",
    "print(cleaned_sentences)\n",
    "print(cleaned_sentences_with_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1565,
     "status": "ok",
     "timestamp": 1619423243453,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "6nf64qiNoNU2",
    "outputId": "9f1e3066-de5d-4664-9c08-a79dd2032ffe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  :  a\n",
      "1  :  are\n",
      "2  :  between\n",
      "3  :  both\n",
      "4  :  contract\n",
      "5  :  enforceable\n",
      "6  :  insurance\n",
      "7  :  is\n",
      "8  :  legally\n",
      "9  :  life\n",
      "10  :  of\n",
      "11  :  parties\n",
      "12  :  qualified\n",
      "13  :  to\n",
      "14  :  two\n",
      "15  :  whom\n",
      "16  :  ad\n",
      "17  :  agreement\n",
      "18  :  and\n",
      "19  :  be\n",
      "20  :  clear\n",
      "21  :  conditions\n",
      "22  :  documented\n",
      "23  :  idem\n",
      "24  :  ie\n",
      "25  :  in\n",
      "26  :  it\n",
      "27  :  make\n",
      "28  :  manner\n",
      "29  :  mind\n",
      "30  :  must\n",
      "31  :  necessary\n",
      "32  :  same\n",
      "33  :  suitably\n",
      "34  :  terms\n",
      "35  :  that\n",
      "36  :  the\n",
      "37  :  therefore\n",
      "38  :  would\n",
      "39  :  adidem\n",
      "40  :  means\n",
      "41  :  on\n",
      "42  :  or\n",
      "43  :  sense\n",
      "44  :  subject\n",
      "45  :  thing\n",
      "46  :  understand\n",
      "47  :  consensus\n",
      "48  :  there\n",
      "49  :  all\n",
      "50  :  can\n",
      "51  :  clearly\n",
      "52  :  court\n",
      "53  :  duties\n",
      "54  :  interpreted\n",
      "55  :  law\n",
      "56  :  obligations\n",
      "57  :  possible\n",
      "58  :  privileges\n",
      "59  :  properly\n",
      "60  :  provided\n",
      "61  :  rights\n",
      "62  :  this\n",
      "63  :  which\n",
      "64  :  acceptance\n",
      "65  :  an\n",
      "66  :  beings\n",
      "67  :  human\n",
      "68  :  silence\n",
      "69  :  sometime\n",
      "70  :  as\n",
      "71  :  but\n",
      "72  :  discussion\n",
      "73  :  documentation\n",
      "74  :  entitled\n",
      "75  :  for\n",
      "76  :  hence\n",
      "77  :  insurer\n",
      "78  :  legal\n",
      "79  :  need\n",
      "80  :  not\n",
      "81  :  personality\n",
      "82  :  verbal\n",
      "83  :  also\n",
      "84  :  distant\n",
      "85  :  enforced\n",
      "86  :  faith\n",
      "87  :  future\n",
      "88  :  good\n",
      "89  :  only\n",
      "90  :  utmost\n",
      "91  :  black\n",
      "92  :  by\n",
      "93  :  declarations\n",
      "94  :  made\n",
      "95  :  put\n",
      "96  :  reference\n",
      "97  :  should\n",
      "98  :  white\n",
      "99  :  any\n",
      "100  :  material\n",
      "101  :  shall\n",
      "102  :  suppression\n",
      "103  :  void\n",
      "104  :  willful\n",
      "105  :  about\n",
      "106  :  ancillary\n",
      "107  :  answering\n",
      "108  :  contained\n",
      "109  :  declare\n",
      "110  :  documents\n",
      "111  :  duty\n",
      "112  :  financial\n",
      "113  :  form\n",
      "114  :  has\n",
      "115  :  he\n",
      "116  :  health\n",
      "117  :  himself\n",
      "118  :  his\n",
      "119  :  insured\n",
      "120  :  knows\n",
      "121  :  may\n",
      "122  :  other\n",
      "123  :  proposal\n",
      "124  :  questions\n",
      "125  :  required\n",
      "126  :  status\n",
      "127  :  against\n",
      "128  :  age\n",
      "129  :  deciding\n",
      "130  :  factor\n",
      "131  :  important\n",
      "132  :  policy\n",
      "133  :  premium\n",
      "134  :  quantum\n",
      "135  :  document\n",
      "136  :  proving\n",
      "137  :  its\n",
      "138  :  proof\n",
      "139  :  reliable\n",
      "140  :  truthfulness\n",
      "141  :  undertake\n",
      "142  :  accepts\n",
      "143  :  comparatively\n",
      "144  :  less\n",
      "145  :  nonstandard\n",
      "146  :  pinch\n",
      "147  :  proofs\n",
      "148  :  salt\n",
      "149  :  them\n",
      "150  :  those\n",
      "151  :  with\n",
      "152  :  accepting\n",
      "153  :  before\n",
      "154  :  certain\n",
      "155  :  final\n",
      "156  :  precautions\n",
      "157  :  such\n",
      "158  :  takes\n",
      "159  :  words\n",
      "160  :  become\n",
      "161  :  high\n",
      "162  :  income\n",
      "163  :  proposed\n",
      "164  :  sum\n",
      "165  :  very\n",
      "166  :  whenever\n",
      "167  :  acceptable\n",
      "168  :  declared\n",
      "169  :  eight\n",
      "170  :  normally\n",
      "171  :  seven\n",
      "172  :  times\n",
      "173  :  amount\n",
      "174  :  come\n",
      "175  :  compared\n",
      "176  :  desired\n",
      "177  :  do\n",
      "178  :  known\n",
      "179  :  much\n",
      "180  :  proposals\n",
      "181  :  proposer\n",
      "182  :  source\n",
      "183  :  when\n",
      "184  :  does\n",
      "185  :  face\n",
      "186  :  holder\n",
      "187  :  problem\n",
      "188  :  service\n",
      "189  :  sources\n",
      "190  :  verifiable\n",
      "191  :  assessed\n",
      "192  :  at\n",
      "193  :  business\n",
      "194  :  case\n",
      "195  :  desirable\n",
      "196  :  people\n",
      "197  :  what\n",
      "198  :  accountants\n",
      "199  :  calls\n",
      "200  :  cases\n",
      "201  :  certificate\n",
      "202  :  chartered\n",
      "203  :  etc\n",
      "204  :  returns\n",
      "205  :  tax\n",
      "206  :  eliminate\n",
      "207  :  hazard\n",
      "208  :  moral\n",
      "209  :  possibility\n",
      "210  :  detailed\n",
      "211  :  evidence\n",
      "212  :  mentions\n",
      "213  :  buys\n",
      "214  :  delivery\n",
      "215  :  money\n",
      "216  :  right\n",
      "217  :  course\n",
      "218  :  installments\n",
      "219  :  keeps\n",
      "220  :  part\n",
      "221  :  pay\n",
      "222  :  paying\n",
      "223  :  promise\n",
      "224  :  promises\n",
      "225  :  scheduled\n",
      "226  :  above\n",
      "227  :  back\n",
      "228  :  issued\n",
      "229  :  makes\n",
      "230  :  preamble\n",
      "231  :  printed\n",
      "232  :  statement\n",
      "233  :  states\n",
      "234  :  basis\n",
      "235  :  endorsements\n",
      "236  :  given\n",
      "237  :  placed\n",
      "238  :  saying\n",
      "239  :  statements\n",
      "240  :  identifies\n",
      "241  :  office\n",
      "242  :  schedule\n",
      "243  :  called\n",
      "244  :  commencement\n",
      "245  :  date\n",
      "246  :  identification\n",
      "247  :  name\n",
      "248  :  number\n",
      "249  :  policyholder\n",
      "250  :  extremely\n",
      "251  :  making\n",
      "252  :  relating\n",
      "253  :  useful\n",
      "254  :  avoid\n",
      "255  :  delay\n",
      "256  :  needless\n",
      "257  :  address\n",
      "258  :  along\n",
      "259  :  beneficiarys\n",
      "260  :  mentioned\n",
      "261  :  check\n",
      "262  :  correct\n",
      "263  :  correction\n",
      "264  :  immediately\n",
      "265  :  mistake\n",
      "266  :  out\n",
      "267  :  pointed\n",
      "268  :  correspondence\n",
      "269  :  misdirect\n",
      "270  :  notices\n",
      "271  :  nominee\n",
      "272  :  paid\n",
      "273  :  upto\n",
      "274  :  assured\n",
      "275  :  goes\n",
      "276  :  happening\n",
      "277  :  mention\n",
      "278  :  payable\n",
      "279  :  type\n",
      "280  :  how\n",
      "281  :  long\n",
      "282  :  act\n",
      "283  :  completely\n",
      "284  :  dated\n",
      "285  :  official\n",
      "286  :  per\n",
      "287  :  provision\n",
      "288  :  signed\n",
      "289  :  stamp\n",
      "290  :  stamped\n",
      "291  :  another\n",
      "292  :  assignment\n",
      "293  :  effected\n",
      "294  :  endorsement\n",
      "295  :  favour\n",
      "296  :  institution\n",
      "297  :  person\n",
      "298  :  after\n",
      "299  :  automatically\n",
      "300  :  cancelled\n",
      "301  :  done\n",
      "302  :  due\n",
      "303  :  fresh\n",
      "304  :  nomination\n",
      "305  :  policyas\n",
      "306  :  reassignment\n",
      "307  :  subsequent\n",
      "308  :  duplicate\n",
      "309  :  mortgage\n",
      "310  :  used\n",
      "311  :  valuable\n",
      "312  :  absolve\n",
      "313  :  arises\n",
      "314  :  claim\n",
      "315  :  from\n",
      "316  :  liability\n",
      "317  :  loss\n",
      "318  :  payment\n",
      "319  :  proceeds\n",
      "320  :  bond\n",
      "321  :  claimants\n",
      "322  :  furnishing\n",
      "323  :  indemnity\n",
      "324  :  jointly\n",
      "325  :  one\n",
      "326  :  settled\n",
      "327  :  surety\n",
      "328  :  advertisement\n",
      "329  :  being\n",
      "330  :  charges\n",
      "331  :  circumstances\n",
      "332  :  following\n",
      "333  :  if\n",
      "334  :  insists\n",
      "335  :  irrevocably\n",
      "336  :  itself\n",
      "337  :  leading\n",
      "338  :  like\n",
      "339  :  lost\n",
      "340  :  news\n",
      "341  :  paper\n",
      "342  :  preparation\n",
      "343  :  procedure\n",
      "344  :  production\n",
      "345  :  satisfied\n",
      "346  :  satisfies\n",
      "347  :  so\n",
      "348  :  upon\n",
      "349  :  generally\n",
      "350  :  taking\n",
      "351  :  time\n",
      "352  :  subsequently\n",
      "353  :  change\n",
      "354  :  marriage\n",
      "355  :  details\n",
      "356  :  needs\n",
      "357  :  duly\n",
      "358  :  filled\n",
      "359  :  first\n",
      "360  :  forms\n",
      "361  :  every\n",
      "362  :  pays\n",
      "363  :  receipt\n",
      "364  :  receives\n",
      "365  :  leads\n",
      "366  :  non\n",
      "367  :  policylapses\n",
      "368  :  cover\n",
      "369  :  reinstatement\n",
      "370  :  revival\n",
      "371  :  reduced\n",
      "372  :  revived\n",
      "373  :  under\n",
      "374  :  up\n",
      "375  :  detail\n",
      "376  :  advised\n",
      "377  :  agent\n",
      "378  :  explain\n",
      "379  :  provisions\n",
      "380  :  various\n",
      "381  :  explaining\n",
      "382  :  nature\n",
      "383  :  technical\n",
      "384  :  wordings\n",
      "385  :  explained\n",
      "386  :  too\n",
      "387  :  close\n",
      "388  :  custody\n",
      "389  :  kept\n",
      "390  :  knowledge\n",
      "391  :  relatives\n",
      "392  :  safe\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "sentences = cleaned_sentences_with_stopwords\n",
    "sentence_words = [[word for word in document.split()]\n",
    "                  for document in sentences]\n",
    "\n",
    "from gensim import corpora\n",
    "dictionary = corpora.Dictionary(sentence_words)\n",
    "for key, value in dictionary.items():\n",
    "  print(key, ' : ', value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1062,
     "status": "ok",
     "timestamp": 1619423246792,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "nLkQY9FEoqlt",
    "outputId": "2420f0e8-8411-4813-8409-939eb56a5290"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract\n",
      "[(0, 1), (1, 1), (2, 1), (3, 1), (4, 2), (5, 1), (6, 1), (7, 1), (8, 2), (9, 1), (10, 1), (11, 1), (12, 1), (13, 1), (14, 1), (15, 1)]\n",
      "it is therefore necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are ad idem ie of the same mind\n",
      "[(0, 1), (1, 1), (3, 1), (4, 1), (7, 1), (10, 2), (11, 1), (13, 1), (16, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1), (22, 1), (23, 1), (24, 1), (25, 1), (26, 2), (27, 1), (28, 1), (29, 1), (30, 1), (31, 1), (32, 1), (33, 1), (34, 1), (35, 3), (36, 4), (37, 1), (38, 1)]\n",
      "adidem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject\n",
      "[(1, 1), (3, 1), (10, 1), (11, 1), (25, 1), (29, 1), (32, 4), (35, 1), (36, 5), (39, 1), (40, 1), (41, 1), (42, 1), (43, 1), (44, 1), (45, 1), (46, 1)]\n",
      "there must be consensus or adidem between the parties to the contract\n",
      "[(2, 1), (4, 1), (11, 1), (13, 1), (19, 1), (30, 1), (36, 2), (39, 1), (42, 1), (47, 1), (48, 1)]\n",
      "this is possible provided all the terms and conditions rights and duties  privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law\n",
      "[(0, 1), (1, 1), (7, 1), (10, 1), (18, 3), (19, 1), (21, 1), (22, 1), (25, 2), (34, 2), (36, 1), (49, 1), (50, 1), (51, 1), (52, 1), (53, 1), (54, 1), (55, 1), (56, 1), (57, 1), (58, 1), (59, 1), (60, 1), (61, 1), (62, 1), (63, 1)]\n",
      "between two human beings sometime silence means an acceptance\n",
      "[(2, 1), (14, 1), (40, 1), (64, 1), (65, 1), (66, 1), (67, 1), (68, 1), (69, 1)]\n",
      "but as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation\n",
      "[(0, 2), (2, 1), (4, 2), (7, 3), (11, 1), (13, 2), (18, 1), (36, 2), (48, 1), (57, 1), (70, 1), (71, 1), (72, 1), (73, 1), (74, 1), (75, 1), (76, 1), (77, 1), (78, 1), (79, 1), (80, 1), (81, 1), (82, 1)]\n",
      "insurance is also a contract of utmost good faith and enforced only in the distant future\n",
      "[(0, 1), (4, 1), (6, 1), (7, 1), (10, 1), (18, 1), (25, 1), (36, 1), (83, 1), (84, 1), (85, 1), (86, 1), (87, 1), (88, 1), (89, 1), (90, 1)]\n",
      "it is therefore necessary that the declarations made by both the parties should be put in black and white for future reference\n",
      "[(3, 1), (7, 1), (11, 1), (18, 1), (19, 1), (25, 1), (26, 1), (31, 1), (35, 1), (36, 2), (37, 1), (75, 1), (87, 1), (91, 1), (92, 1), (93, 1), (94, 1), (95, 1), (96, 1), (97, 1), (98, 1)]\n",
      "any suppression willful and material shall make the contract void\n",
      "[(4, 1), (18, 1), (27, 1), (36, 1), (99, 1), (100, 1), (101, 1), (102, 1), (103, 1), (104, 1)]\n",
      "the insured therefore has a duty to declare all that he knows about himself his health his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer\n",
      "[(0, 1), (13, 1), (18, 1), (19, 1), (25, 2), (35, 1), (36, 3), (37, 1), (49, 1), (63, 1), (77, 1), (92, 1), (105, 1), (106, 1), (107, 1), (108, 1), (109, 1), (110, 1), (111, 1), (112, 1), (113, 1), (114, 1), (115, 1), (116, 1), (117, 1), (118, 2), (119, 1), (120, 1), (121, 1), (122, 1), (123, 1), (124, 1), (125, 1), (126, 1)]\n",
      "age is an important factor in deciding the quantum of premium against a policy\n",
      "[(0, 1), (7, 1), (10, 1), (25, 1), (36, 1), (65, 1), (127, 1), (128, 1), (129, 1), (130, 1), (131, 1), (132, 1), (133, 1), (134, 1)]\n",
      "the document proving the age ie\n",
      "[(24, 1), (36, 2), (128, 1), (135, 1), (136, 1)]\n",
      "age proof must be reliable and the insured has to undertake as to its truthfulness\n",
      "[(13, 2), (18, 1), (19, 1), (30, 1), (36, 1), (70, 1), (114, 1), (119, 1), (128, 1), (137, 1), (138, 1), (139, 1), (140, 1), (141, 1)]\n",
      "nonstandard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt\n",
      "[(0, 1), (1, 2), (10, 1), (18, 1), (36, 1), (37, 1), (63, 1), (77, 1), (128, 1), (139, 1), (142, 1), (143, 1), (144, 1), (145, 1), (146, 1), (147, 1), (148, 1), (149, 1), (150, 1), (151, 1)]\n",
      "in other words the insurer takes certain precautions before accepting such age proofs as final\n",
      "[(25, 1), (36, 1), (70, 1), (77, 1), (122, 1), (128, 1), (147, 1), (152, 1), (153, 1), (154, 1), (155, 1), (156, 1), (157, 1), (158, 1), (159, 1)]\n",
      "proof of income is the document may become necessary whenever the sum proposed is very high\n",
      "[(7, 2), (10, 1), (31, 1), (36, 2), (121, 1), (135, 1), (138, 1), (160, 1), (161, 1), (162, 1), (163, 1), (164, 1), (165, 1), (166, 1)]\n",
      "normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance\n",
      "[(0, 1), (6, 1), (7, 2), (10, 1), (13, 1), (36, 1), (63, 1), (75, 1), (162, 1), (163, 1), (164, 1), (167, 1), (168, 1), (169, 1), (170, 1), (171, 1), (172, 1)]\n",
      "but proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired\n",
      "[(6, 1), (7, 1), (10, 3), (13, 2), (36, 4), (71, 1), (77, 1), (144, 1), (162, 1), (173, 1), (174, 1), (175, 1), (176, 1), (177, 1), (178, 1), (179, 1), (180, 1), (181, 1), (182, 1), (183, 1)]\n",
      "a service holder normally does not face this problem as his sources of income are verifiable\n",
      "[(0, 1), (1, 1), (10, 1), (62, 1), (70, 1), (80, 1), (118, 1), (162, 1), (170, 1), (184, 1), (185, 1), (186, 1), (187, 1), (188, 1), (189, 1), (190, 1)]\n",
      "in case of business people the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired\n",
      "[(0, 1), (6, 1), (7, 2), (10, 2), (13, 1), (25, 1), (36, 2), (75, 1), (144, 1), (162, 2), (172, 1), (173, 1), (175, 1), (176, 1), (179, 1), (191, 1), (192, 1), (193, 1), (194, 1), (195, 1), (196, 1), (197, 1)]\n",
      "in such cases the insurer at times calls for assessed income tax returns or chartered accountants certificate etc\n",
      "[(25, 1), (36, 1), (42, 1), (75, 1), (77, 1), (157, 1), (162, 1), (172, 1), (191, 1), (192, 1), (198, 1), (199, 1), (200, 1), (201, 1), (202, 1), (203, 1), (204, 1), (205, 1)]\n",
      "such precautions are necessary to eliminate the possibility of moral hazard\n",
      "[(1, 1), (10, 1), (13, 1), (31, 1), (36, 1), (156, 1), (157, 1), (206, 1), (207, 1), (208, 1), (209, 1)]\n",
      "policy contract is the policy document is a detailed document and it is the evidence of the insurance contract which mentions all the terms and conditions of the insurance\n",
      "[(0, 1), (4, 2), (6, 2), (7, 3), (10, 2), (18, 2), (21, 1), (26, 1), (34, 1), (36, 5), (49, 1), (63, 1), (132, 2), (135, 2), (210, 1), (211, 1), (212, 1)]\n",
      "the insured buys not the policy contract but the right to the sum of money and its future delivery\n",
      "[(4, 1), (10, 1), (13, 1), (18, 1), (36, 4), (71, 1), (80, 1), (87, 1), (119, 1), (132, 1), (137, 1), (164, 1), (213, 1), (214, 1), (215, 1), (216, 1)]\n",
      "the insurer on its part promises to pay a sum of money provided of course the insured keeps its part of promise of paying the installments of premium as scheduled\n",
      "[(0, 1), (10, 5), (13, 1), (36, 3), (41, 1), (60, 1), (70, 1), (77, 1), (119, 1), (133, 1), (137, 2), (164, 1), (215, 1), (217, 1), (218, 1), (219, 1), (220, 2), (221, 1), (222, 1), (223, 1), (224, 1), (225, 1)]\n",
      "the preamble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy\n",
      "[(4, 1), (6, 1), (7, 1), (10, 1), (13, 2), (18, 2), (20, 1), (21, 1), (35, 1), (36, 6), (41, 1), (44, 1), (58, 1), (62, 1), (132, 2), (226, 1), (227, 1), (228, 1), (229, 1), (230, 1), (231, 1), (232, 1), (233, 1)]\n",
      "the endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract\n",
      "[(0, 1), (1, 1), (4, 1), (10, 2), (13, 1), (18, 1), (19, 1), (25, 1), (26, 1), (35, 2), (36, 8), (41, 1), (83, 2), (96, 1), (101, 1), (113, 2), (123, 2), (132, 2), (220, 1), (229, 1), (234, 1), (235, 1), (236, 1), (237, 1), (238, 1), (239, 1)]\n",
      "the schedule which is printed on the policy document identifies the office which has issued the policy\n",
      "[(7, 1), (36, 4), (41, 1), (63, 2), (114, 1), (132, 2), (135, 1), (228, 1), (231, 1), (240, 1), (241, 1), (242, 1)]\n",
      "it states the name of the policyholder the date of commencement of the policy an identification number of the policy called policy number\n",
      "[(10, 4), (26, 1), (36, 5), (65, 1), (132, 3), (233, 1), (243, 1), (244, 1), (245, 1), (246, 1), (247, 1), (248, 2), (249, 1)]\n",
      "this number is extremely useful for making any reference to the insurer relating to this policy\n",
      "[(7, 1), (13, 2), (36, 1), (62, 2), (75, 1), (77, 1), (96, 1), (99, 1), (132, 1), (248, 1), (250, 1), (251, 1), (252, 1), (253, 1)]\n",
      "this shall avoid needless delay\n",
      "[(62, 1), (101, 1), (254, 1), (255, 1), (256, 1)]\n",
      "beneficiarys name is also mentioned along with address\n",
      "[(7, 1), (83, 1), (151, 1), (247, 1), (257, 1), (258, 1), (259, 1), (260, 1)]\n",
      "it is necessary to check that it is correct and any mistake should be immediately pointed out for correction\n",
      "[(7, 2), (13, 1), (18, 1), (19, 1), (26, 2), (31, 1), (35, 1), (75, 1), (97, 1), (99, 1), (261, 1), (262, 1), (263, 1), (264, 1), (265, 1), (266, 1), (267, 1)]\n",
      "a mistake in the address may misdirect the premium notices and any other future correspondence\n",
      "[(0, 1), (18, 1), (25, 1), (36, 2), (87, 1), (99, 1), (121, 1), (122, 1), (133, 1), (257, 1), (265, 1), (268, 1), (269, 1), (270, 1)]\n",
      "it also states the name of the nominee and the date upto which premium has to be paid\n",
      "[(10, 1), (13, 1), (18, 1), (19, 1), (26, 1), (36, 3), (63, 1), (83, 1), (114, 1), (133, 1), (233, 1), (245, 1), (247, 1), (271, 1), (272, 1), (273, 1)]\n",
      "the schedule goes on to mention the type of policy on the happening of which the sum assured is payable and to whom it is payable\n",
      "[(7, 2), (10, 2), (13, 2), (15, 1), (18, 1), (26, 1), (36, 4), (41, 2), (63, 1), (132, 1), (164, 1), (242, 1), (274, 1), (275, 1), (276, 1), (277, 1), (278, 2), (279, 1)]\n",
      "it of course also mentions when and how long the premium is to be paid\n",
      "[(7, 1), (10, 1), (13, 1), (18, 1), (19, 1), (26, 1), (36, 1), (83, 1), (133, 1), (183, 1), (212, 1), (217, 1), (272, 1), (280, 1), (281, 1)]\n",
      "the policy document is signed by an official of the insurer and dated and stamped as per the provision of the stamp act to make it a completely legally enforceable document\n",
      "[(0, 1), (5, 1), (7, 1), (8, 1), (10, 2), (13, 1), (18, 2), (26, 1), (27, 1), (36, 4), (65, 1), (70, 1), (77, 1), (92, 1), (132, 1), (135, 2), (282, 1), (283, 1), (284, 1), (285, 1), (286, 1), (287, 1), (288, 1), (289, 1), (290, 1)]\n",
      "an assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy\n",
      "[(0, 1), (10, 2), (19, 1), (25, 1), (36, 1), (41, 1), (42, 1), (50, 1), (65, 2), (92, 1), (132, 2), (291, 1), (292, 1), (293, 1), (294, 1), (295, 1), (296, 1), (297, 1)]\n",
      "reassignment can also be done by a subsequent endorsement on the same policyas a nomination is automatically cancelled due to an assignment after reassignment it is necessary to make a fresh nomination\n",
      "[(0, 3), (7, 2), (13, 2), (19, 1), (26, 1), (27, 1), (31, 1), (32, 1), (36, 1), (41, 1), (50, 1), (65, 1), (83, 1), (92, 1), (292, 1), (294, 1), (298, 1), (299, 1), (300, 1), (301, 1), (302, 1), (303, 1), (304, 2), (305, 1), (306, 2), (307, 1)]\n",
      "in duplicate policy a policy document is a valuable document and can be used for mortgage etc\n",
      "[(0, 2), (7, 1), (18, 1), (19, 1), (25, 1), (50, 1), (75, 1), (132, 2), (135, 2), (203, 1), (308, 1), (309, 1), (310, 1), (311, 1)]\n",
      "loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises\n",
      "[(10, 3), (36, 3), (77, 1), (80, 1), (132, 2), (135, 1), (183, 1), (184, 1), (312, 1), (313, 1), (314, 1), (315, 1), (316, 1), (317, 1), (318, 1), (319, 1)]\n",
      "the claim can be settled on the claimants furnishing an indemnity bond jointly with one surety\n",
      "[(19, 1), (36, 2), (41, 1), (50, 1), (65, 1), (151, 1), (314, 1), (320, 1), (321, 1), (322, 1), (323, 1), (324, 1), (325, 1), (326, 1), (327, 1)]\n",
      "if a policy is irrevocably lost a duplicate policy can be issued after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued\n",
      "[(0, 5), (7, 2), (10, 3), (13, 1), (18, 3), (19, 1), (25, 1), (36, 3), (48, 1), (50, 1), (65, 2), (77, 2), (132, 4), (154, 1), (228, 2), (298, 2), (308, 2), (317, 1), (318, 1), (320, 1), (323, 1), (328, 1), (329, 1), (330, 1), (331, 1), (332, 1), (333, 1), (334, 1), (335, 1), (336, 1), (337, 1), (338, 1), (339, 1), (340, 1), (341, 1), (342, 1), (343, 1), (344, 1), (345, 1), (346, 1), (347, 1), (348, 1)]\n",
      "generally nomination is made at the time of taking a policy\n",
      "[(0, 1), (7, 1), (10, 1), (36, 1), (94, 1), (132, 1), (192, 1), (304, 1), (349, 1), (350, 1), (351, 1)]\n",
      "in case it is not done it is possible to make nomination subsequently by an endorsement on the policy\n",
      "[(7, 2), (13, 1), (25, 1), (26, 2), (27, 1), (36, 1), (41, 1), (57, 1), (65, 1), (80, 1), (92, 1), (132, 1), (194, 1), (294, 1), (301, 1), (304, 1), (352, 1)]\n",
      "it is also possible to change a nomination subsequently by an endorsement\n",
      "[(0, 1), (7, 1), (13, 1), (26, 1), (57, 1), (65, 1), (83, 1), (92, 1), (294, 1), (304, 1), (352, 1), (353, 1)]\n",
      "after marriage such change in nomination is normally required\n",
      "[(7, 1), (25, 1), (125, 1), (157, 1), (170, 1), (298, 1), (304, 1), (353, 1), (354, 1)]\n",
      "life insurance being a legally enforceable contract needs to be documented with details of the rights and obligations of the parties to the contract\n",
      "[(0, 1), (4, 2), (5, 1), (6, 1), (8, 1), (9, 1), (10, 2), (11, 1), (13, 2), (18, 1), (19, 1), (22, 1), (36, 3), (56, 1), (61, 1), (151, 1), (329, 1), (355, 1), (356, 1)]\n",
      "proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract\n",
      "[(4, 1), (7, 1), (10, 1), (18, 1), (25, 1), (36, 4), (63, 1), (92, 1), (113, 1), (123, 1), (135, 1), (181, 1), (234, 1), (288, 1), (357, 1), (358, 1), (359, 1), (360, 1)]\n",
      "every time the insured pays the premium he receives a premium receipt\n",
      "[(0, 1), (36, 2), (115, 1), (119, 1), (133, 2), (351, 1), (361, 1), (362, 1), (363, 1), (364, 1)]\n",
      "the premium needs to be paid in time non payment of premium leads to policylapses\n",
      "[(10, 1), (13, 2), (19, 1), (25, 1), (36, 1), (133, 2), (272, 1), (318, 1), (351, 1), (356, 1), (365, 1), (366, 1), (367, 1)]\n",
      "reinstatement of the cover is called revival of the policy\n",
      "[(7, 1), (10, 2), (36, 2), (132, 1), (243, 1), (368, 1), (369, 1), (370, 1)]\n",
      "if the policy is not revived the policy can become a paid up policy for a reduced sum assured under certain conditions\n",
      "[(0, 2), (7, 1), (21, 1), (36, 2), (50, 1), (75, 1), (80, 1), (132, 3), (154, 1), (160, 1), (164, 1), (272, 1), (274, 1), (333, 1), (371, 1), (372, 1), (373, 1), (374, 1)]\n",
      "the policy document mentions in detail all the rights and obligations of the policyholder\n",
      "[(10, 1), (18, 1), (25, 1), (36, 3), (49, 1), (56, 1), (61, 1), (132, 1), (135, 1), (212, 1), (249, 1), (375, 1)]\n",
      "the agent is advised to explain the various provisions of the policy to the policyholder\n",
      "[(7, 1), (10, 1), (13, 2), (36, 4), (132, 1), (249, 1), (376, 1), (377, 1), (378, 1), (379, 1), (380, 1)]\n",
      "the wordings in the policy document are of technical nature and hence the need for explaining\n",
      "[(1, 1), (10, 1), (18, 1), (25, 1), (36, 3), (75, 1), (76, 1), (79, 1), (132, 1), (135, 1), (381, 1), (382, 1), (383, 1), (384, 1)]\n",
      "if there are certain endorsements on the policy that need to be explained too\n",
      "[(1, 1), (13, 1), (19, 1), (35, 1), (36, 1), (41, 1), (48, 1), (79, 1), (132, 1), (154, 1), (235, 1), (333, 1), (385, 1), (386, 1)]\n",
      "it needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives\n",
      "[(0, 1), (7, 1), (10, 1), (13, 2), (18, 2), (19, 2), (25, 2), (26, 1), (35, 1), (36, 3), (132, 1), (135, 1), (311, 1), (356, 2), (385, 1), (387, 1), (388, 1), (389, 1), (390, 1), (391, 1), (392, 1)]\n"
     ]
    }
   ],
   "source": [
    "import pprint\n",
    "bow_corpus = [dictionary.doc2bow(text) for text in sentence_words]\n",
    "for sent, embedding in zip(sentences, bow_corpus):\n",
    "  print(sent)\n",
    "  print(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1QdzdD8BpLOQ"
   },
   "outputs": [],
   "source": [
    "question = \"When can a duplicate policy be issued ?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1366,
     "status": "ok",
     "timestamp": 1619423272524,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "CDihOJGSpOr2",
    "outputId": "a244d9e2-ca85-40f1-be67-3efc9ed5504b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "When can a duplicate policy be issued ? \n",
      " [(0, 1), (19, 1), (50, 1), (132, 1), (183, 1), (228, 1), (308, 1)]\n"
     ]
    }
   ],
   "source": [
    "question = clean_sentence(question, stopwords=False)\n",
    "question_embedding = dictionary.doc2bow(question.split())\n",
    "print(question, \"\\n\", question_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1585,
     "status": "ok",
     "timestamp": 1619423276809,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "393ezGDGok6u",
    "outputId": "d9941a9f-58c1-4583-9526-5cc326bffa9b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1.0 life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract\n",
      "1 1.0 it is therefore necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are ad idem ie of the same mind\n",
      "2 0.7071067811865475 adidem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject\n",
      "3 0.4472135954999579 there must be consensus or adidem between the parties to the contract\n",
      "4 1.0 this is possible provided all the terms and conditions rights and duties  privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law\n",
      "5 0.4472135954999579 between two human beings sometime silence means an acceptance\n",
      "6 1.0 but as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation\n",
      "7 1.0 insurance is also a contract of utmost good faith and enforced only in the distant future\n",
      "8 0.31622776601683794 it is therefore necessary that the declarations made by both the parties should be put in black and white for future reference\n",
      "9 0.24253562503633297 any suppression willful and material shall make the contract void\n",
      "10 1.0 the insured therefore has a duty to declare all that he knows about himself his health his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer\n",
      "11 1.0 age is an important factor in deciding the quantum of premium against a policy\n",
      "12 0.04163054471218133 the document proving the age ie\n",
      "13 0.1520571842539411 age proof must be reliable and the insured has to undertake as to its truthfulness\n",
      "14 1.0 nonstandard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt\n",
      "15 0.039968038348871575 in other words the insurer takes certain precautions before accepting such age proofs as final\n",
      "16 0.27472112789737807 proof of income is the document may become necessary whenever the sum proposed is very high\n",
      "17 1.0 normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance\n",
      "18 0.1643989873053573 but proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired\n",
      "19 1.0 a service holder normally does not face this problem as his sources of income are verifiable\n",
      "20 1.0 in case of business people the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired\n",
      "21 0.039968038348871575 in such cases the insurer at times calls for assessed income tax returns or chartered accountants certificate etc\n",
      "22 0.7071067811865475 such precautions are necessary to eliminate the possibility of moral hazard\n",
      "23 1.0 policy contract is the policy document is a detailed document and it is the evidence of the insurance contract which mentions all the terms and conditions of the insurance\n",
      "24 0.24253562503633297 the insured buys not the policy contract but the right to the sum of money and its future delivery\n",
      "25 1.0 the insurer on its part promises to pay a sum of money provided of course the insured keeps its part of promise of paying the installments of premium as scheduled\n",
      "26 0.24253562503633297 the preamble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy\n",
      "27 1.0 the endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract\n",
      "28 0.1414213562373095 the schedule which is printed on the policy document identifies the office which has issued the policy\n",
      "29 0.3713906763541037 it states the name of the policyholder the date of commencement of the policy an identification number of the policy called policy number\n",
      "30 0.1414213562373095 this number is extremely useful for making any reference to the insurer relating to this policy\n",
      "31 0.016126934718260072 this shall avoid needless delay\n",
      "32 0.1414213562373095 beneficiarys name is also mentioned along with address\n",
      "33 0.27472112789737807 it is necessary to check that it is correct and any mistake should be immediately pointed out for correction\n",
      "34 1.0 a mistake in the address may misdirect the premium notices and any other future correspondence\n",
      "35 0.09950371902099892 it also states the name of the nominee and the date upto which premium has to be paid\n",
      "36 0.27472112789737807 the schedule goes on to mention the type of policy on the happening of which the sum assured is payable and to whom it is payable\n",
      "37 0.1414213562373095 it of course also mentions when and how long the premium is to be paid\n",
      "38 1.0 the policy document is signed by an official of the insurer and dated and stamped as per the provision of the stamp act to make it a completely legally enforceable document\n",
      "39 1.0 an assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy\n",
      "40 1.0 reassignment can also be done by a subsequent endorsement on the same policyas a nomination is automatically cancelled due to an assignment after reassignment it is necessary to make a fresh nomination\n",
      "41 1.0 in duplicate policy a policy document is a valuable document and can be used for mortgage etc\n",
      "42 0.2873478855663454 loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises\n",
      "43 0.052558833122763673 the claim can be settled on the claimants furnishing an indemnity bond jointly with one surety\n",
      "44 1.0 if a policy is irrevocably lost a duplicate policy can be issued after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued\n",
      "45 1.0 generally nomination is made at the time of taking a policy\n",
      "46 0.27472112789737807 in case it is not done it is possible to make nomination subsequently by an endorsement on the policy\n",
      "47 1.0 it is also possible to change a nomination subsequently by an endorsement\n",
      "48 0.1414213562373095 after marriage such change in nomination is normally required\n",
      "49 1.0 life insurance being a legally enforceable contract needs to be documented with details of the rights and obligations of the parties to the contract\n",
      "50 0.24253562503633297 proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract\n",
      "51 1.0 every time the insured pays the premium he receives a premium receipt\n",
      "52 0.09950371902099892 the premium needs to be paid in time non payment of premium leads to policylapses\n",
      "53 0.1414213562373095 reinstatement of the cover is called revival of the policy\n",
      "54 1.0 if the policy is not revived the policy can become a paid up policy for a reduced sum assured under certain conditions\n",
      "55 0.09950371902099892 the policy document mentions in detail all the rights and obligations of the policyholder\n",
      "56 0.1414213562373095 the agent is advised to explain the various provisions of the policy to the policyholder\n",
      "57 0.7071067811865475 the wordings in the policy document are of technical nature and hence the need for explaining\n",
      "58 0.7071067811865475 if there are certain endorsements on the policy that need to be explained too\n",
      "59 1.0 it needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives\n"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def retrieveAndPrintFAQAnswer(question_embedding, sentence_embeddings, sentences):\n",
    "  max_sim = -1\n",
    "  index_sim = -1\n",
    "  for index, embedding in enumerate(sentence_embeddings):\n",
    "    sim = cosine_similarity(embedding, question_embedding)[0][0]\n",
    "    print(index, sim, sentences[index])\n",
    "    if sim > max_sim:\n",
    "      max_sim = sim\n",
    "      index_sim = index\n",
    "  \n",
    "  return index_sim\n",
    "index = retrieveAndPrintFAQAnswer(question_embedding, bow_corpus, sentences)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1430,
     "status": "ok",
     "timestamp": 1619423281769,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "OJn2VJ24C9zK",
    "outputId": "ad4cecdd-a8c2-4e20-acd7-51a367462ced"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question:  when can a duplicate policy be issued \n",
      "Answer:  life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract\n"
     ]
    }
   ],
   "source": [
    "print(\"Question: \", question)\n",
    "print(\"Answer: \", sentences[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1PX0IqCPis1-"
   },
   "source": [
    "***Word Embedding Technique***\n",
    "\n",
    "\n",
    "\n",
    "1.   Word2Vec Embedding Technique\n",
    "2.   Glove Embedding Technique\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TjTGjdIOi5D-"
   },
   "source": [
    "***Using Word2Vec Model Approch***\n",
    "\n",
    "In these appproch we takes input of large corpus words and produces a vector space, typically of several hundreds dimensions, with each unique word in the corpus being assigned a corresponding vector in the space.\n",
    "\n",
    "Word vectors are positioned in the vector space such that words that share common contexts in the corpus are located in close proximity to one another in the space. Word2Vec is a particularly computationally-efficient predictive model for learning word embeddings from raw text.\n",
    "\n",
    "**Working**\n",
    "\n",
    "Word2Vec is a simple neural network with a single hidden layer, and like all neural networks, it has weights, and during training, its goal is to adjust those weights to reduce a loss function. However, Word2Vec is not going to be used for the task it was trained on, instead, we will just take its hidden weights, use them as our word embeddings, and toss the rest of the model (this came out a little bit evil).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 819078,
     "status": "ok",
     "timestamp": 1619421436799,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "2wtseJzQgz7w",
    "outputId": "17fc54ca-7e42-42a6-d12d-65b859f9ea52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[==================================================] 100.0% 1662.8/1662.8MB downloaded\n",
      "w2v Model Saved\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import gensim.downloader as api\n",
    "\n",
    "v2w_model = None\n",
    "try:\n",
    "  v2w_model = gensim.models.Keyedvectors.load('./w2vecmodel.mod')\n",
    "  print(\"w2v Model Successfully loaded\")\n",
    "except:\n",
    "  v2w_model = api.load('word2vec-google-news-300')\n",
    "  v2w_model.save(\"./w2vecmodel.mod\")\n",
    "  print(\"w2v Model Saved\")\n",
    "\n",
    "w2vec_embedding_size = len(v2w_model['pc'])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "swnbdWppgz4s"
   },
   "outputs": [],
   "source": [
    "def getWordVec(word, model):\n",
    "  samp = model['pc']\n",
    "  vec = [0]*len(samp)\n",
    "  try:\n",
    "    vec = model[word]\n",
    "  except:\n",
    "    vec = [0]*len(samp)\n",
    "  return (vec)\n",
    "\n",
    "\n",
    "def getPhraseEmbedding(phrase, embeddingmodel):\n",
    "  samp = getWordVec('computer', embeddingmodel)\n",
    "  vec = numpy.array([0]*len(samp))\n",
    "  den = 0;\n",
    "  for word in phrase.split():\n",
    "    den = den+1\n",
    "    vec = vec+numpy.array(getWordVec(word, embeddingmodel))\n",
    "  return vec.reshape(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1492,
     "status": "ok",
     "timestamp": 1619423304263,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "NTbclmsijIY4",
    "outputId": "e50aacb7-2c13-47cc-b3a4-7af7bb629f85"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0.5513437234038394 life insurance is a legally enforceable contract between two parties both of whom are legally qualified to contract\n",
      "1 0.7025787291556693 it is therefore necessary that the terms and conditions of the agreement must be suitably documented in a manner that would make it clear that both parties to the contract are ad idem ie of the same mind\n",
      "2 0.6070676295451412 adidem means that both the parties understand the same thing in the same sense or are of the same mind on the same subject\n",
      "3 0.6058007155185451 there must be consensus or adidem between the parties to the contract\n",
      "4 0.6729521680764039 this is possible provided all the terms and conditions rights and duties  privileges and obligations are properly documented in terms which can be clearly interpreted in a court of law\n",
      "5 0.430919800661395 between two human beings sometime silence means an acceptance\n",
      "6 0.6073519392489676 but as the insurer is a legal personality entitled to contract verbal discussion between parties to the contract is not possible and hence there is a need for documentation\n",
      "7 0.5664510941271083 insurance is also a contract of utmost good faith and enforced only in the distant future\n",
      "8 0.69478734559311 it is therefore necessary that the declarations made by both the parties should be put in black and white for future reference\n",
      "9 0.5198375172683362 any suppression willful and material shall make the contract void\n",
      "10 0.6611051312698848 the insured therefore has a duty to declare all that he knows about himself his health his financial status in answering questions contained in the proposal form and other ancillary documents which may be required by the insurer\n",
      "11 0.5488815994630494 age is an important factor in deciding the quantum of premium against a policy\n",
      "12 0.4924984808143421 the document proving the age ie\n",
      "13 0.5737018659197571 age proof must be reliable and the insured has to undertake as to its truthfulness\n",
      "14 0.5541358790129955 nonstandard age proofs are those which are comparatively less reliable and therefore the insurer accepts them with a pinch of salt\n",
      "15 0.5892738745304305 in other words the insurer takes certain precautions before accepting such age proofs as final\n",
      "16 0.6380306388761755 proof of income is the document may become necessary whenever the sum proposed is very high\n",
      "17 0.5844253616891335 normally a sum proposed which is seven to eight times of the declared income is acceptable for insurance\n",
      "18 0.5939455208747356 but proposals do come to the insurer when the known source of income of the proposer is much less compared to the amount of insurance desired\n",
      "19 0.6163914527266781 a service holder normally does not face this problem as his sources of income are verifiable\n",
      "20 0.5758529661170784 in case of business people the assessed income is at times much less compared to what is a desirable income for the amount of insurance desired\n",
      "21 0.5321177098723562 in such cases the insurer at times calls for assessed income tax returns or chartered accountants certificate etc\n",
      "22 0.5108475232749291 such precautions are necessary to eliminate the possibility of moral hazard\n",
      "23 0.6187452256409256 policy contract is the policy document is a detailed document and it is the evidence of the insurance contract which mentions all the terms and conditions of the insurance\n",
      "24 0.6078912050401158 the insured buys not the policy contract but the right to the sum of money and its future delivery\n",
      "25 0.5113160375428871 the insurer on its part promises to pay a sum of money provided of course the insured keeps its part of promise of paying the installments of premium as scheduled\n",
      "26 0.7062835520058147 the preamble to the insurance contract makes the above statement clear and states that this policy is issued subject to the conditions and privileges printed on the back of the policy\n",
      "27 0.7158689433626614 the endorsements placed on the policy shall also be part of the policy and it also makes a reference to the proposal form saying that that the statements given in the proposal form are the basis of the contract\n",
      "28 0.706824893174336 the schedule which is printed on the policy document identifies the office which has issued the policy\n",
      "29 0.6524455876351415 it states the name of the policyholder the date of commencement of the policy an identification number of the policy called policy number\n",
      "30 0.6456558511425788 this number is extremely useful for making any reference to the insurer relating to this policy\n",
      "31 0.4885339015459536 this shall avoid needless delay\n",
      "32 0.45063037248827803 beneficiarys name is also mentioned along with address\n",
      "33 0.6974006724824884 it is necessary to check that it is correct and any mistake should be immediately pointed out for correction\n",
      "34 0.6332707996414288 a mistake in the address may misdirect the premium notices and any other future correspondence\n",
      "35 0.6010562976675246 it also states the name of the nominee and the date upto which premium has to be paid\n",
      "36 0.6237862046101611 the schedule goes on to mention the type of policy on the happening of which the sum assured is payable and to whom it is payable\n",
      "37 0.6336198754598449 it of course also mentions when and how long the premium is to be paid\n",
      "38 0.7018142646666798 the policy document is signed by an official of the insurer and dated and stamped as per the provision of the stamp act to make it a completely legally enforceable document\n",
      "39 0.6902136621204256 an assignment of a policy in favour of another person or institution can be effected by an endorsement on the policy\n",
      "40 0.6468801409824072 reassignment can also be done by a subsequent endorsement on the same policyas a nomination is automatically cancelled due to an assignment after reassignment it is necessary to make a fresh nomination\n",
      "41 0.781584191920509 in duplicate policy a policy document is a valuable document and can be used for mortgage etc\n",
      "42 0.6386906511713176 loss of policy document does not absolve the insurer from the liability of payment of policy proceeds when the claim arises\n",
      "43 0.5318085695524406 the claim can be settled on the claimants furnishing an indemnity bond jointly with one surety\n",
      "44 0.7858182816467707 if a policy is irrevocably lost a duplicate policy can be issued after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued\n",
      "45 0.6146633961785062 generally nomination is made at the time of taking a policy\n",
      "46 0.6758885826306097 in case it is not done it is possible to make nomination subsequently by an endorsement on the policy\n",
      "47 0.5326113011888557 it is also possible to change a nomination subsequently by an endorsement\n",
      "48 0.5417891646891542 after marriage such change in nomination is normally required\n",
      "49 0.5734139667541358 life insurance being a legally enforceable contract needs to be documented with details of the rights and obligations of the parties to the contract\n",
      "50 0.5622228070547911 proposal form duly filled in and signed by the proposer is the first document which forms the basis of the contract\n",
      "51 0.48859572989447 every time the insured pays the premium he receives a premium receipt\n",
      "52 0.5232983062072266 the premium needs to be paid in time non payment of premium leads to policylapses\n",
      "53 0.5389555641509096 reinstatement of the cover is called revival of the policy\n",
      "54 0.7402111154032076 if the policy is not revived the policy can become a paid up policy for a reduced sum assured under certain conditions\n",
      "55 0.5827225356005116 the policy document mentions in detail all the rights and obligations of the policyholder\n",
      "56 0.6076604964630864 the agent is advised to explain the various provisions of the policy to the policyholder\n",
      "57 0.6647071279323338 the wordings in the policy document are of technical nature and hence the need for explaining\n",
      "58 0.7181611927293274 if there are certain endorsements on the policy that need to be explained too\n",
      "59 0.6695013538805216 it needs to be explained that the policy is a valuable document and needs to be kept in safe custody and in the knowledge of the close relatives\n"
     ]
    }
   ],
   "source": [
    "sent_embeddings = []\n",
    "for sent in sentences:\n",
    "  sent_embeddings.append(getPhraseEmbedding(sent, v2w_model))\n",
    "\n",
    "question_embedding = getPhraseEmbedding(question, v2w_model)\n",
    "index = retrieveAndPrintFAQAnswer(question_embedding, sent_embeddings, cleaned_sentences_with_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 935,
     "status": "ok",
     "timestamp": 1619423317678,
     "user": {
      "displayName": "SATYAM SONI",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhG4rcPYa7hnDqGvLTYPIBiXe9m-k4IPTmu9dWD=s64",
      "userId": "06127152709350492896"
     },
     "user_tz": -330
    },
    "id": "FnPvYc4-jIUK",
    "outputId": "f4a3118a-4c10-457a-d8f4-2b3cd17d0f43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question:  when can a duplicate policy be issued \n",
      "Answer:  if a policy is irrevocably lost a duplicate policy can be issued after following a certain procedure like the insurer satisfies itself of the circumstances leading to loss and being so satisfied the insurer insists upon an advertisement in a news paper production of an indemnity bond and payment of policy preparation charges and there after a duplicate policy is issued\n"
     ]
    }
   ],
   "source": [
    "print(\"Question: \", question)\n",
    "print(\"Answer: \", cleaned_sentences_with_stopwords[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "90Ua1NIogzrL"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyOmUnlVA5ba2hwPlk4wgGWT",
   "collapsed_sections": [],
   "name": "Question Answering System Using Word2Vec Embedding Technique.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}