TMRetrieval_weighted_Optimised.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Translation Memory Retrieval using Weighted N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import math\n",
    "from collections import Counter\n",
    "import string\n",
    "import numpy as np\n",
    "import json\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/khannatanmai/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting the M_ngrams and C_ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_M_ngrams(sentence):\n",
    "    ngrams_list_sent = []\n",
    "    M_ngrams = []\n",
    "    counter_ngrams = []\n",
    "    \n",
    "    ngrams = list(nltk.ngrams(sentence.split(), 2))\n",
    "    ngrams_list_sent.append(list(ngrams))\n",
    "    M_ngrams = [y for x in ngrams_list_sent for y in x]\n",
    "    \n",
    "    for ngrams in M_ngrams:\n",
    "        counter_ngrams.append(Counter(ngrams))\n",
    "        \n",
    "    return M_ngrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_C_ngrams(candidate_sentence):\n",
    "    ngrams_list_sent = []\n",
    "    C_ngrams = []\n",
    "    counter_ngrams = []\n",
    "    \n",
    "    ngrams = list(nltk.ngrams(candidate_sentence.split(), 2))\n",
    "    ngrams_list_sent.append(list(ngrams))\n",
    "    C_ngrams = [y for x in ngrams_list_sent for y in x]\n",
    "    ngrams_sents = []\n",
    "    ngrams_list_sent = []\n",
    "    \n",
    "    for ngrams in C_ngrams:\n",
    "        counter_ngrams.append(Counter(ngrams))\n",
    "    \n",
    "    return C_ngrams\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I request you to please remove the drive safely.\n",
      "['request', 'please', 'remove', 'drive', 'safely', '.']\n"
     ]
    }
   ],
   "source": [
    "input_line = input()\n",
    "\n",
    "#convert input to lowercase\n",
    "input_line = input_line.lower()\n",
    "\n",
    "#tokenise\n",
    "input_tokens = word_tokenize(input_line)\n",
    "\n",
    "content_words = [word for word in input_tokens if word not in stop_words] #Removing Stopwords\n",
    "print(content_words)\n",
    "\n",
    "# new_M_sentence = ' '.join(content_words)\n",
    "M_ngrams = get_M_ngrams(input_line)\n",
    "\n",
    "# sentence = \"I request you to please remove the drive safely.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighted N-Gram Precision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_tm_words = [] #Content Words in Source TM\n",
    "\n",
    "with open('tm_data/tm_src_2000_pp.txt') as src_tm:\n",
    "    line = src_tm.readline()\n",
    "    \n",
    "    while line:\n",
    "        line = line.rstrip() #Removing Trailing Whitespace\n",
    "        \n",
    "        words = line.split('\\t')\n",
    "        src_tm_words.append(words)\n",
    "        \n",
    "        line = src_tm.readline()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get sentences and IDF values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"tm_data/tm_src_2000_lower.txt\") as source_file:\n",
    "    sentences = source_file.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tm_data/idf_values_2000.json') as json_file:\n",
    "    idf_values_str = json.load(json_file)\n",
    "\n",
    "idf_values = ast.literal_eval(idf_values_str)\n",
    "#print(idf_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To compute numerator and denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngrams_intersection(candidate_sentence):\n",
    "    C_ngrams = get_C_ngrams(candidate_sentence)\n",
    "    \n",
    "    M_set = set(M_ngrams)\n",
    "    C_set = set(C_ngrams)\n",
    "    \n",
    "    return list(M_set & C_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_w_sum(ngrams_list):\n",
    "    w = 0\n",
    "    \n",
    "    \n",
    "    for ngram in ngrams_list:\n",
    "        for token in ngram:\n",
    "            if token in idf_values:\n",
    "                w += idf_values[token]\n",
    "#                 print(w)\n",
    "    \n",
    "    return w"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final score for each sentence wrt to input sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_wpn(candidate_sentence):\n",
    "    C_ngrams = get_C_ngrams(candidate_sentence)\n",
    "    intersection_ngrams = ngrams_intersection(candidate_sentence)\n",
    "        \n",
    "    Z = 0.75\n",
    "    \n",
    "    w_M_ngrams = compute_w_sum(M_ngrams)\n",
    "    w_C_ngrams = compute_w_sum(C_ngrams)\n",
    "    w_intersection_ngrams = compute_w_sum(intersection_ngrams)\n",
    "\n",
    "    \n",
    "    \n",
    "    wpn = w_intersection_ngrams / ((Z*w_M_ngrams) + ((1-Z)*w_C_ngrams))\n",
    "    \n",
    "    return wpn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running WNGP on 740 Candidates out of a possible 2000!\n",
      "\n",
      "[674] ['desert', 'rocky', 'terrain', 'small', 'hills', 'traversed', 'tanami', 'track', '.'] 0.0\n",
      "[680] ['brought', 'magistrate', 'fourteen', 'day', 'period', '.'] 0.0\n",
      "[2000] ['gdm', '(', 'gnome', 'display', 'manager', ')', 'running', '.'] 0.0\n",
      "[66] ['<', 'dfn', '>', '...', '<', '/dfn', '>', 'element', '(', 'html', 'definition', 'element', ')', 'allows', 'specify', 'introducing', 'special', 'term', '.'] 0.06429637723185896\n",
      "[208] ['safely', 'remove', 'selected', 'drive'] 0.12471474617158834\n"
     ]
    }
   ],
   "source": [
    "N = 5 #Top N matches returned\n",
    "\n",
    "wpn_all = []\n",
    "indices_all = []\n",
    "\n",
    "j = 0\n",
    "count = 0\n",
    "\n",
    "for i, candidate in enumerate(src_tm_words):\n",
    "    \n",
    "    #Check if Content Words present in Candidate\n",
    "    for word in content_words:\n",
    "        if(word in candidate):\n",
    "            count += 1\n",
    "            \n",
    "            wpn = compute_wpn(sentences[i])\n",
    "            \n",
    "            wpn_all.append(wpn)\n",
    "            indices_all.append(j)\n",
    "            \n",
    "            break\n",
    "    \n",
    "    j += 1\n",
    "    \n",
    "print('Running WNGP on ' + str(count) + ' Candidates out of a possible ' + str(j) + '!\\n')\n",
    "\n",
    "\n",
    "#Get top N results\n",
    "wpn_all = np.array(wpn_all)\n",
    "\n",
    "sorted_indices = np.argsort(wpn_all) #Sorts in ascending order and returns the indices of indices_all array\n",
    "least_N_indices = sorted_indices[-N:] \n",
    "\n",
    "for i in least_N_indices:\n",
    "    print([indices_all[i]+1], src_tm_words[indices_all[i]], wpn_all[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieval of Target from TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[674] रेगिस्तान में छोटी पहाड़ियों के साथ चट्टानी इलाका है और यहां के रास्ते को तनामी ट्रैक कहा जाता है।\n",
      "\n",
      "[680] आपको प्रत्येक चौदह दिन की अवधि के बाद\n",
      "\n",
      "[2000] जीडीएम (गनोम डिस्प्ले प्रबंधक) नहीं चल रहा है.\n",
      "[66] इसमें <dfn>...</dfn> एलमेंट\n",
      "\n",
      "[208] चयनित ड्राइव सुरक्षित रूप से निकालें\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tgt_tm_array = []\n",
    "\n",
    "with open('tm_data/tm_tgt_2000.txt') as tgt_tm:\n",
    "    line = tgt_tm.readline()\n",
    "    \n",
    "    while line:\n",
    "        tgt_tm_array.append(line)\n",
    "        line = tgt_tm.readline()\n",
    "        \n",
    "for i in least_N_indices:\n",
    "    print([indices_all[i]+1], tgt_tm_array[indices_all[i]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}