IBMmodel1_0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from nltk.tokenize import word_tokenize\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DATA\n",
    "file = open('./train.en', 'r')\n",
    "train_en = file.read()\n",
    "sentences_train_en = train_en.split(\"\\n\")\n",
    "\n",
    "file = open('./train.hi', 'r')\n",
    "train_hi = file.read()\n",
    "sentences_train_hi = train_hi.split(\"\\n\")\n",
    "\n",
    "file = open('./test.en', 'r')\n",
    "test_en = file.read()\n",
    "sentences_test_en = test_en.split(\"\\n\")\n",
    "\n",
    "file = open('./test.hi', 'r')\n",
    "test_hi = file.read()\n",
    "sentences_test_hi = test_hi.split(\"\\n\")\n",
    "\n",
    "file = open('./dev.en.txt', 'r')\n",
    "dev_en = file.read()\n",
    "sentences_dev_en = dev_en.split(\"\\n\")\n",
    "\n",
    "file = open('./dev.hi', 'r')\n",
    "dev_hi = file.read()\n",
    "sentences_dev_hi = dev_hi.split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_tokenised_sentences(dataset):\n",
    "    return_sentence_list = []\n",
    "    word_dictionary = {}\n",
    "    lang_order = 0\n",
    "    for sentence in dataset:\n",
    "        tokens = word_tokenize(sentence.lower())\n",
    "        produced_sentence = \"\"\n",
    "        for token in tokens:\n",
    "            if token not in word_dictionary:\n",
    "                word_dictionary[token] = lang_order\n",
    "                lang_order += 1\n",
    "            produced_sentence = produced_sentence + token + \" \"\n",
    "        produced_sentence = produced_sentence[:(len(produced_sentence) - 1)] # remove last empty\n",
    "        return_sentence_list.append(produced_sentence)\n",
    "\n",
    "    return return_sentence_list, word_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_converged(new, old):\n",
    "    epsilone = 0.00000001\n",
    "    new = new.values()\n",
    "    old = old.values()\n",
    "    \n",
    "    for i in range(len(new)):\n",
    "            if math.fabs(new[i]- old[i]) > epsilone: \n",
    "                return False\n",
    "            return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "def perform_EM(en_sentences, hi_sentences, en_wd, hi_wd):\n",
    "    translation_prob = defaultdict(float)\n",
    "    translation_prob_prev = defaultdict(float)\n",
    "    uni_ini = 1/len(hi_wd)\n",
    "    \n",
    "    while not is_converged(translation_prob, translation_prob_prev):\n",
    "        count = defaultdict(float)\n",
    "        total = defaultdict(float)\n",
    "        for index_sen, hin_sen in enumerate(hi_sentences):\n",
    "            #compute normalization\n",
    "            hin_sen_words = hin_sen.split(\" \")\n",
    "            s_total = defaultdict(float)\n",
    "            for hin_word in hin_sen_words:\n",
    "                s_total[hin_word] = 0\n",
    "                eng_sen_words = en_sentences[index_sen].split(\" \")\n",
    "                for eng_word in eng_sen_words:\n",
    "                    s_total[hin_word] += uni_ini\n",
    "                    translation_prob[(hin_word, eng_word)] = uni_ini\n",
    "            \n",
    "            #collect counts\n",
    "            for hin_word in hin_sen_words:\n",
    "                eng_sen_words = en_sentences[index_sen].split(\" \")\n",
    "                for eng_word in eng_sen_words:\n",
    "                    count[(hin_word, eng_word)] += uni_ini/s_total[hin_word]\n",
    "                    translation_prob[(hin_word, eng_word)] = uni_ini\n",
    "                    total[eng_word] += uni_ini/s_total[hin_word]\n",
    "        \n",
    "        #estimate probabilities\n",
    "        for eng_word in en_wd.keys():\n",
    "            for hin_word in hi_wd.keys():\n",
    "                translation_prob[(hin_word, eng_word)] = count[(hin_word, eng_word)]/total[eng_word]\n",
    "        \n",
    "        translation_prob_prev = translation_prob\n",
    "        \n",
    "    return translation_prob\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(sentences_en, sentences_hi, en_wd, hi_wd):\n",
    "    print(e_tokenised_sentences)\n",
    "    \n",
    "    translation_prob = perform_EM(e_tokenised_sentences, h_tokenised_sentences, en_wd, hi_wd)\n",
    "    np.save(\"./models/IBMmodel1_tef\", translation_prob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def test_model():\n",
    "#     tef = np.load('./models/t_e_f_mat_model1.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "e_tokenised_sentences, en_wd = generate_tokenised_sentences(sentences_train_en)\n",
    "h_tokenised_sentences, hi_wd = generate_tokenised_sentences(sentences_train_hi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-175-2eb3ab311438>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences_train_en\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentences_train_hi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0men_wd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhi_wd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-164-5c9f33d0b510>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(sentences_en, sentences_hi, en_wd, hi_wd)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences_en\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentences_hi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0men_wd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhi_wd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mtranslation_prob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mperform_EM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me_tokenised_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_tokenised_sentences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0men_wd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhi_wd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"./models/IBMmodel1_tef\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranslation_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-173-23cc433ee582>\u001b[0m in \u001b[0;36mperform_EM\u001b[0;34m(en_sentences, hi_sentences, en_wd, hi_wd)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0meng_word\u001b[0m \u001b[0;32min\u001b[0m \u001b[0men_wd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mhin_word\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhi_wd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m                 \u001b[0mtranslation_prob\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhin_word\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meng_word\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhin_word\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meng_word\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mtotal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0meng_word\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mtranslation_prob_prev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtranslation_prob\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train_model(sentences_train_en, sentences_train_hi, en_wd, hi_wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}