word2vec_loader.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from gensim.models import word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run common.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def text_to_index(corpus, word2idx):\n",
    "    new_corpus = []\n",
    "    index= 0\n",
    "    for doc in corpus:\n",
    "        new_doc = []\n",
    "        for word in simple_preprocess(doc):\n",
    "            try:\n",
    "                new_doc.append(word2idx[word])\n",
    "            except:\n",
    "                new_doc.append(0)\n",
    "        if not new_doc :\n",
    "            for w in range(0, MAX_WORD_COUNT):\n",
    "                new_doc.append(0)\n",
    "        temp_doc = Counter(new_doc)\n",
    "        if not doc:\n",
    "            print(\"doc: \", index, new_doc, \"\\n\")\n",
    "            index += 1\n",
    "        final_doc = sorted(temp_doc, key= temp_doc.get, reverse=True)[:MAX_WORD_COUNT]\n",
    "#         print(temp_doc, final_doc, \"\\n\")\n",
    "        new_doc_arr = np.array(new_doc)\n",
    "        new_corpus.append(new_doc_arr)\n",
    "    return np.array(new_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createEmbeddingMatrix(embedding_dim):\n",
    "    # Get the word embedding vector\n",
    "    word_vector_model = word2vec.Word2Vec.load(\"word2vec.model\")\n",
    "    word_vectors = word_vector_model.wv\n",
    "    wvv = word_vectors.vocab\n",
    "\n",
    "    num_vocabs = len(wvv.items())\n",
    "    wvv_keys_list = list(wvv.keys())\n",
    "    vocab_list = [(word, word_vectors[word]) for word, _ in sorted(wvv.items())]\n",
    "    \n",
    "    # Generate word embedding matrix and word2idx dict, EMBEDDING_DIM can be found in word2Vec.ipynb\n",
    "    embedding_matrix = np.zeros((num_vocabs + 1, embedding_dim))\n",
    "    word2idx = {}\n",
    "\n",
    "    for i, vocab in enumerate(vocab_list):\n",
    "        word, vec = vocab\n",
    "        embedding_matrix[i + 1] = vec\n",
    "        word2idx[word] = i + 1\n",
    "        \n",
    "    print(word_vector_model)\n",
    "    print('embedding_matrix.shape=%s' % str(embedding_matrix.shape))\n",
    "    \n",
    "    return (embedding_matrix, word2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}