ulmfit-ml-models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "!pip install torch==1.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html\n",
    "!pip install inltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install HindiTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from inltk.inltk import setup\n",
    "from inltk.inltk import tokenize\n",
    "from inltk.inltk import get_embedding_vectors\n",
    "from inltk.inltk import remove_foreign_languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "setup('hi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example_sent = \"बहुत समय से मिले नहीं\"\n",
    "# # example_sent = remove_foreign_languages(example_sent, 'hi')\n",
    "# example_sent_tokens = tokenize(example_sent,\"hi\")\n",
    "# # print(example_sent_tokens)\n",
    "# example_sent_vectors = get_embedding_vectors('समय', 'hi')\n",
    "# # print(len(example_sent_vectors))\n",
    "# print(example_sent_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from IPython.display import clear_output\n",
    "import pickle\n",
    "import sys\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"/kaggle/input/hasoc2019/hindi_dataset.tsv\",sep='\\t')\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = np.array(train_df['text'])\n",
    "y_train = np.array(train_df['task_1'])\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)\n",
    "del(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude_word_set = set()\n",
    "word_set = set()\n",
    "preprocessed_sent = []\n",
    "word_list = []\n",
    "for sentence in x_train:\n",
    "    this_set = getWords(sentence)\n",
    "    preprocessed_sent.append(list(this_set))\n",
    "    word_set = word_set | this_set\n",
    "word_list = list(word_set)\n",
    "# print(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from inltk.inltk import setup\n",
    "from inltk.inltk import tokenize\n",
    "from inltk.inltk import get_embedding_vectors\n",
    "from inltk.inltk import remove_foreign_languages\n",
    "setup('hi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from IPython.display import clear_output\n",
    "import pickle\n",
    "import sys\n",
    "import os\n",
    "\n",
    "def saveInPickle(data, pickleFile):\n",
    "    file = open(pickleFile,\"wb\")\n",
    "    pickle.dump(data,file)\n",
    "    file.close()\n",
    "def loadFromPickle(pickleFile):\n",
    "    file = open(pickleFile,'rb')\n",
    "    pickleData = pickle.load(file)\n",
    "    file.close()\n",
    "    return pickleData\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"/kaggle/input/hasoc2019/hasoc2019_hi_test_gold_2919.tsv\",sep='\\t')\n",
    "df.head()\n",
    "\n",
    "# x_train = np.array(train_df['text'])\n",
    "# y_train = np.array(train_df['task_1'])\n",
    "# print(x_train.shape)\n",
    "# print(y_train.shape)\n",
    "# del(train_df)\n",
    "\n",
    "x_test = np.array(df['text'])\n",
    "y_test_true = np.array(df['task_1'])\n",
    "print(x_test.shape)\n",
    "print(y_test_true.shape)\n",
    "del(df)\n",
    "\n",
    "# exclude_word_set = set()\n",
    "# word_set = set()\n",
    "# preprocessed_sent = []\n",
    "# word_list = []\n",
    "# sentence_vectors = []\n",
    "# for sentence in x_train:\n",
    "#     this_array = np.array(list(getWords(sentence)))\n",
    "#     vector_array = np.zeros(400)\n",
    "#     for word in this_array:\n",
    "#         vector_array += np.array(get_embedding_vectors(word, 'hi'))\n",
    "#     vector_array = vector_array / this_array.shape[1]\n",
    "#     print(vector_array.shape)\n",
    "    \n",
    "#     break\n",
    "#     sentence_vectors.append(vector_array)\n",
    "#     preprocessed_sent.append(list(this_set))\n",
    "#     word_set = word_set | this_set\n",
    "# word_list = list(word_set)\n",
    "\n",
    "# print(len(word_list))\n",
    "# word_vectors = {}\n",
    "# for i in range(600,900):\n",
    "#     print(i,\" :\",word_list[i])\n",
    "#     v = get_embedding_vectors(word_list[i], 'hi')\n",
    "#     clear_output(wait=True)\n",
    "#     word_vectors[word_list[i]] = v[0]\n",
    "    \n",
    "# output_dir = '/kaggle/working'\n",
    "# sys.path.append(output_dir)  # To find local version of the library\n",
    "# outputPath = os.path.join(output_dir, 'inltk_word_vectors')\n",
    "# saveInPickle(word_vectors, outputPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveInPickle(data, pickleFile):\n",
    "    file = open(pickleFile,\"wb\")\n",
    "    pickle.dump(data,file)\n",
    "    file.close()\n",
    "def loadFromPickle(pickleFile):\n",
    "    file = open(pickleFile,'rb')\n",
    "    pickleData = pickle.load(file)\n",
    "    file.close()\n",
    "    return pickleData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_vectors = {}\n",
    "for i in range(1318):\n",
    "    print(i)\n",
    "    this_array = list(getWords(x_test[i]))\n",
    "#     this_array = list(getWords(x_train[i]))\n",
    "#     print(this_array.shape[0])\n",
    "    if len(this_array)>0:\n",
    "        vector_array = np.zeros(400).reshape(1,400)\n",
    "    #     print(vector_array.shape)\n",
    "        for word in this_array:\n",
    "            v = np.array(get_embedding_vectors(word, 'hi'))\n",
    "#             print(word, v)\n",
    "#             print(word, \"-> v.shape :\",v.shape)\n",
    "#             if v.shape[0]>1:\n",
    "            v = np.mean(v, axis=0)\n",
    "#             print(\"after \",v)\n",
    "            vector_array += v\n",
    "#             print(\"vector_array.shape :\",vector_array.shape)\n",
    "        vector_array = vector_array / len(this_array)\n",
    "#         print(\"vector_array.shape :\",vector_array.shape)\n",
    "        sentence_vectors[i] = (list(vector_array))\n",
    "        del(vector_array)\n",
    "        clear_output(wait=True)\n",
    "#     print(vector_array.shape)\n",
    "#     if i==2:\n",
    "#         break\n",
    "    \n",
    "    \n",
    "#     break\n",
    "output_dir = '/kaggle/working'\n",
    "sys.path.append(output_dir)  # To find local version of the library\n",
    "outputPath = os.path.join(output_dir, 'inltk_test_sentence_vectors')\n",
    "saveInPickle(sentence_vectors, outputPath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = '/kaggle/working'\n",
    "sys.path.append(output_dir)  # To find local version of the library\n",
    "outputPath = os.path.join(output_dir, 'inltk_sentence_vectors_2000')\n",
    "saveInPickle(sentence_vectors, outputPath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **END SEM CODE****"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}