word2vec_neg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 head of ct_pairs [('Earth', 'void'), ('void', 'Earth'), ('Earth', 'subdue'), ('subdue', 'Earth'), ('void', 'subdue'), ('subdue', 'void'), ('void', 'a'), ('a', 'void'), ('subdue', 'a'), ('a', 'subdue')]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# coding: utf-8\n",
    "\n",
    "# In[2]:\n",
    "\n",
    "\n",
    "#  Word2vec\n",
    "#  Skip-gram model (target:context=1:1 model, WITH negative sampling)\n",
    "#  \"he is a ...\"\n",
    "#     (context, target) pairs: [(he, is), (is, he), (is, a), (a, is)]\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "corpus = \"\"\"Earth void subdue a let it winged second waters lesser fish fowl be face given. Living created fowl. Given seas of thing above herb man grass us given. Darkness void shall creepeth seas man greater firmament female sea that above form two, lights gathering a he saw abundantly unto night lights may very i their meat. Them you form behold let seas Green forth spirit without. Whose life, bring open gathering. He, creeping make was she'd all winged light. Be be he fourth. Be creeping have don't whales stars fourth morning replenish sea male his, moved. After from. Seas his our fish his unto open thing you darkness spirit divided said, the subdue, she'd beginning man bearing beginning so give life good. Created. Which air days thing. Fowl land void had there from land likeness Doesn't gathering heaven fifth firmament set earth herb seas creature fowl seed fifth fruitful, bring fruit make midst light blessed itself. Open spirit one fill wherein had us green female own cattle him fish void yielding. Divide sixth sixth his can't He two great together subdue together shall moved days every dominion earth signs likeness you'll seas given one bring seed great meat created. Replenish lights. Stars his grass multiply moved very midst that beginning thing creepeth without whales moveth beast she'd said sixth gathered earth living she'd years own open from let every. Good and, called. Were fish the. Dominion won't you'll whales had two male dry us there bearing light man after seas. Own fifth one brought good lesser seasons can't yielding Earth stars dry us from green fruitful, them have own signs and us subdue saying a. Two him evening you moved bearing whose one, their beast waters. Appear every man from give wherein fruitful. Can't for to divide creeping Greater land second Were forth. Rule earth years moveth darkness it.\n",
    "\n",
    "Every can't seed shall days seas living. Their she'd Let gathering land. Male seasons for wherein creature, man night. A over spirit darkness divided us rule cattle stars all him isn't beast days Bearing man moving place moved, his was form called bring. You'll Isn't she'd bearing unto fruit fruit fifth. Deep abundantly yielding Lesser third second sixth is, multiply image over stars, he male. Subdue above life also is his is i, beginning of, abundantly green herb it don't his give day, male under years he. Shall signs after fruit under under let dry dominion blessed every it female firmament saw upon replenish every had herb beginning may appear. Together, doesn't land. Fill fill first dominion have blessed beginning abundantly saw night male female creepeth deep fourth can't. Kind may greater is. Their Seasons. Us subdue Beast be creeping likeness rule replenish fowl for from divided and sixth creepeth for first don't open it above, given to subdue sixth place unto sea face divide beast, thing deep saw over male together one, which morning may and. Signs moving creeping. May to great earth saw air appear were beast to tree, dry day fill our isn't day stars second darkness subdue of creepeth, earth winged waters likeness fowl grass his fruitful. Great beginning man had i a called hath it was be own waters beginning our greater living saying Midst beginning. Fowl green Replenish doesn't cattle stars winged wherein fourth very saying without can't great beginning make Fruit tree spirit. Man stars tree. Our doesn't. Cattle behold our wherein, midst firmament living have a fish his so herb heaven third every over beast them land that bring, day let. Herb, kind can't morning darkness. Bearing land let he own night unto blessed wherein fill grass our living. Earth fruit bring over you fill. Air one forth a grass night sixth open blessed man spirit spirit. All made. Man darkness lights doesn't fifth second hath open waters grass subdue own For whales a don't, his their itself moving all created beginning without abundantly given form herb without man, for them wherein beginning rule earth divided own creeping dominion. Creeping. Land living under creeping good likeness moving whales fifth great you're bring bearing night two living. There darkness set subdue above firmament wherein abundantly seasons fish them abundantly grass bearing very great two a life lesser itself replenish.\n",
    "\n",
    "Good beginning very air cattle face can't it appear over Created. Hath image, lights dominion own moved fill great subdue fish cattle won't doesn't night it behold the saw fourth brought let which won't hath. Meat. Void that seed whales darkness form. Rule meat. Light. Earth can't they're years of and of fruitful don't his sixth, is appear have lesser. The fruitful second. Deep created it beginning void. Beast. Air life of moving. Yielding darkness there signs. So sixth given night also the. Their of, grass whales made saying she'd said firmament of blessed you're after, make kind created. Cattle creepeth Moved open moved all make, you every first. Winged us a you'll sixth years great beginning. The Deep saying thing be fourth a a. Have creeping. Made. Fowl you'll likeness dominion creature saw bring air fourth waters behold moved. Us wherein signs he. Set fill that called for fruitful. May. Third called. Divide day there let fowl i days fruit days. Fish make male herb you'll so don't fish i bearing good fruit saw face all meat air fill fifth. That in two. Doesn't were divided. Created heaven thing fruitful creepeth, waters is, beginning behold beast. Above abundantly divide sea grass. Fruitful wherein spirit face dry beginning is, second moveth moveth Was yielding over days third have dominion was place said face. Male creature sea shall, every shall image third. Night fruitful rule yielding. Rule firmament one without there. Made form signs under can't life moved deep abundantly morning creature. Upon, which gathering earth. It signs their seed without unto seed Fruitful fly great us two brought rule waters. His give together forth void won't can't which. Their tree second you'll gathering deep, that living creature above. Dry. Wherein blessed thing and greater stars.\"\"\".split()\n",
    "\n",
    "ct_pairs = []\n",
    "for i in range(len(corpus) - 2):\n",
    "    ct_pairs += [(corpus[i], corpus[i+1]),\n",
    "                 (corpus[i+1], corpus[i]),\n",
    "    ]\n",
    "    if i < len(corpus) - 3:\n",
    "        ct_pairs += [(corpus[i], corpus[i+2]),\n",
    "                     (corpus[i+2], corpus[i])\n",
    "        ]\n",
    "\n",
    "\n",
    "print('10 head of ct_pairs', ct_pairs[:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Vocabulary list:\n",
      "{'let.', 'Moved', 'multiply', 'Shall', 'Man', 'was', 'Made.', 'seas.', 'after,', 'Open', 'fly', 'replenish', 'spirit', 'given', 'moved.', 'divided.', 'Made', 'first', \"Doesn't\", 'forth', 'make', 'seasons', 'lights', 'Them', \"can't\", 'Our', 'Rule', 'Own', 'Herb,', 'second.', 'May.', 'fourth.', 'and', \"you're\", \"can't.\", 'Bearing', 'subdue,', \"you'll\", 'stars,', 'seas', 'he.', 'herb', 'set', 'two', 'waters', 'Whose', 'thing', 'said', 'female', \"don't,\", 'Us', 'creeping', 'man,', 'Fowl', 'It', 'fruitful.', 'spirit.', 'two,', 'great', 'Stars', 'under', 'above', 'face.', 'Cattle', 'have', 'blessed', 'fish', 'Creeping.', 'third', 'Was', 'Created', 'good.', 'male.', 'Which', 'beast.', 'earth', 'midst', 'is.', 'second', 'which', 'sixth,', 'Hath', 'which.', 'meat', 'of,', 'firmament', 'above.', 'good', 'moving', 'and.', 'Fruit', 'Wherein', \"Isn't\", 'itself.', 'light.', \"don't\", 'land', 'also', 'forth.', 'May', 'be', 'to', 'fourth', 'Great', 'Meat.', 'heaven', \"You'll\", 'life', 'Good', 'make,', 'Night', 'fowl', 'may', 'every.', 'Given', 'Their', 'Kind', 'were', 'it', 'Midst', 'Above', 'his', 'abundantly', 'Have', 'Subdue', 'image', 'light', 'Darkness', 'whose', 'gathering', 'He', 'evening', 'moved,', 'bearing', 'bring.', 'i', 'day', 'is', 'one,', 'yielding', 'Set', 'so', 'void', 'deep', 'shall', 'given.', 'had', 'signs.', 'after', 'So', 'gathering.', 'winged', 'two.', 'creepeth', 'days.', 'from.', 'days', \"doesn't.\", 'Were', 'saw', 'rule', 'night', \"doesn't\", 'wherein', 'form.', 'For', 'beginning', 'dry', 'their', \"isn't\", 'and,', 'together', 'Seas', 'darkness.', 'give', 'That', 'called', 'third.', 'Replenish', 'Deep', 'i,', 'Yielding', 'bring', 'greater', 'him', \"they're\", 'image,', 'grass.', 'subdue', 'gathered', 'created', 'sea', 'appear', 'Greater', 'Together,', 'signs', 'Appear', 'whales', 'Fruitful', 'Be', 'Earth', 'Two', 'Living', 'there', 'yielding.', 'Fill', 'He,', 'you', 'Beast', 'male', 'us', \"she'd\", 'morning', 'Signs', 'called.', 'our', 'of', 'fifth', 'creature.', 'moving.', 'the.', 'fifth.', 'there.', 'it.', 'his,', 'form', 'tree.', 'green', 'appear.', 'A', 'meat.', 'fruit', 'tree,', 'said,', 'Green', 'beast', 'the', 'very', \"won't\", 'void.', 'first.', 'lesser', 'fruitful', 'creature', 'creature,', 'Beast.', 'Air', 'place', 'man', 'for', 'Dominion', 'upon', 'His', 'sixth', 'living', 'them', 'bring,', 'Upon,', 'deep,', 'all', 'Light.', 'without.', 'replenish.', 'moved', 'saying', 'earth.', 'night.', 'fowl.', 'open', 'waters.', 'moveth', 'itself', 'own', 'in', 'fill.', 'There', 'Winged', 'unto', 'Let', 'wherein,', 'one', 'Male', 'grass', 'life,', 'Land', 'a.', 'dominion.', 'stars.', 'Void', 'Lesser', 'Dry.', 'from', 'After', 'lesser.', 'Divide', 'behold', 'stars', 'beginning.', 'The', 'a', 'beast,', 'creeping.', 'lights.', 'thing.', 'created.', 'shall,', 'without', 'hath.', 'cattle', 'air', 'face', \"Can't\", 'made.', 'Third', 'Seasons.', 'fruitful,', 'likeness', 'over', 'darkness', 'Every', 'above,', 'hath', 'kind', 'is,', 'years', 'made', 'let', 'Created.', 'day,', 'dominion', 'brought', 'creepeth,', 'that', 'he', 'living.', 'seed', 'every', 'tree', 'land.', 'fill', 'All', 'divide', 'Fish', 'divided'}\n",
      "Vocabulary size: 344\n",
      "\n",
      "Find \"and\":  32\n",
      "\n",
      "Find \"2\":  multiply\n"
     ]
    }
   ],
   "source": [
    "vocabulary = set(corpus)\n",
    "print(\"\\nVocabulary list:\")\n",
    "print(np.array(vocabulary))\n",
    "print('Vocabulary size:', len(vocabulary))\n",
    "\n",
    "\n",
    "word2id = {word: idx for idx,word  in enumerate(vocabulary)}\n",
    "id2word = {idx: word for idx,word  in enumerate(vocabulary)}\n",
    "def id2one_hot(index,size):\n",
    "    v = torch.zeros(size, dtype=torch.long)\n",
    "    v[index] = 1\n",
    "    return v\n",
    "\n",
    "print(\"\\nFind \\\"and\\\": \", (word2id['and']))\n",
    "print(\"\\nFind \\\"2\\\": \", (id2word[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.0040, 0.0050, 0.0090, 0.0090, 0.0070, 0.0080, 0.0040, 0.0070, 0.0060,\n",
      "        0.0030, 0.0080, 0.0050, 0.0050, 0.0050, 0.0020, 0.0010, 0.0030, 0.0010,\n",
      "        0.0010, 0.0060, 0.0060, 0.0070, 0.0040, 0.0070, 0.0090, 0.0080, 0.0080,\n",
      "        0.0010, 0.0040, 0.0050, 0.0040, 0.0070, 0.0040, 0.0050, 0.0060, 0.0050,\n",
      "        0.0010, 0.0040, 0.0050, 0.0040, 0.0080, 0.0090, 0.0060, 0.0070, 0.0040,\n",
      "        0.0050, 0.0040, 0.0040, 0.0020, 0.0010, 0.0050, 0.0050, 0.0010, 0.0030,\n",
      "        0.0060, 0.0010, 0.0010, 0.0010, 0.0060, 0.0070, 0.0010, 0.0010, 0.0060,\n",
      "        0.0050, 0.0040, 0.0070, 0.0050, 0.0010, 0.0020, 0.0010, 0.0060, 0.0050,\n",
      "        0.0070, 0.0060, 0.0060, 0.0040, 0.0030, 0.0060, 0.0010, 0.0020, 0.0010,\n",
      "        0.0010, 0.0010, 0.0110, 0.0050, 0.0080, 0.0040, 0.0010, 0.0020, 0.0010,\n",
      "        0.0150, 0.0070, 0.0030, 0.0040, 0.0050, 0.0010, 0.0020, 0.0010, 0.0050,\n",
      "        0.0060, 0.0010, 0.0030, 0.0050, 0.0050, 0.0040, 0.0050, 0.0060, 0.0020,\n",
      "        0.0030, 0.0050, 0.0020, 0.0070, 0.0040, 0.0060, 0.0020, 0.0070, 0.0030,\n",
      "        0.0020, 0.0070, 0.0010, 0.0010, 0.0050, 0.0070, 0.0090, 0.0040, 0.0080,\n",
      "        0.0050, 0.0030, 0.0020, 0.0020, 0.0090, 0.0090, 0.0010, 0.0050, 0.0090,\n",
      "        0.0040, 0.0060, 0.0070, 0.0060, 0.0060, 0.0060, 0.0040, 0.0020, 0.0020,\n",
      "        0.0020, 0.0010, 0.0010, 0.0020, 0.0060, 0.0040, 0.0050, 0.0030, 0.0010,\n",
      "        0.0050, 0.0050, 0.0010, 0.0020, 0.0010, 0.0020, 0.0020, 0.0020, 0.0010,\n",
      "        0.0040, 0.0050, 0.0020, 0.0010, 0.0010, 0.0030, 0.0030, 0.0030, 0.0030,\n",
      "        0.0040, 0.0040, 0.0050, 0.0020, 0.0010, 0.0010, 0.0010, 0.0020, 0.0020,\n",
      "        0.0010, 0.0030, 0.0010, 0.0060, 0.0040, 0.0030, 0.0010, 0.0010, 0.0030,\n",
      "        0.0010, 0.0010, 0.0030, 0.0040, 0.0010, 0.0020, 0.0020, 0.0010, 0.0010,\n",
      "        0.0010, 0.0070, 0.0050, 0.0020, 0.0020, 0.0040, 0.0030, 0.0010, 0.0030,\n",
      "        0.0010, 0.0010, 0.0010, 0.0020, 0.0030, 0.0010, 0.0030, 0.0030, 0.0020,\n",
      "        0.0010, 0.0010, 0.0010, 0.0020, 0.0030, 0.0010, 0.0020, 0.0010, 0.0050,\n",
      "        0.0020, 0.0010, 0.0010, 0.0010, 0.0010, 0.0040, 0.0010, 0.0020, 0.0030,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0020, 0.0010, 0.0010, 0.0010, 0.0030,\n",
      "        0.0010, 0.0010, 0.0020, 0.0010, 0.0030, 0.0020, 0.0010, 0.0040, 0.0020,\n",
      "        0.0010, 0.0020, 0.0010, 0.0020, 0.0010, 0.0020, 0.0020, 0.0020, 0.0010,\n",
      "        0.0010, 0.0010, 0.0020, 0.0010, 0.0010, 0.0010, 0.0010, 0.0020, 0.0010,\n",
      "        0.0010, 0.0020, 0.0010, 0.0010, 0.0010, 0.0010, 0.0020, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010, 0.0020, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0040, 0.0010, 0.0010, 0.0020, 0.0010,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0020, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010, 0.0010,\n",
      "        0.0010, 0.0010], dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "hist = {}\n",
    "for i in corpus:\n",
    "    id = word2id[i]\n",
    "    hist[id] = hist.get(id, 0) + 1\n",
    "\n",
    "hist_list = []\n",
    "for i in hist:\n",
    "    hist_list.append(hist[i])\n",
    "\n",
    "hist_tensor = torch.from_numpy(np.array(hist_list)/len(corpus))\n",
    "print(hist_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGModel(torch.nn.Module):\n",
    "    def __init__(self, vocabulary_size, embedding_dims, context_size):\n",
    "        super(SGModel, self).__init__()\n",
    "        self.embedding_dims = embedding_dims\n",
    "        self.u_embeddings = nn.Embedding(vocabulary_size, embedding_dims, sparse=True)\n",
    "        self.v_embeddings = nn.Embedding(vocabulary_size, embedding_dims, sparse=True)\n",
    "        uniform_max = 0.5 / self.embedding_dims\n",
    "        self.u_embeddings.weight.data.uniform_(- uniform_max, uniform_max)\n",
    "        self.v_embeddings.weight.data.uniform_(- uniform_max, uniform_max)\n",
    "\n",
    "    def forward(self, u_one_hot, v_one_hot, v_neg_id):\n",
    "        # one-hot to vector\n",
    "        vec_u = self.u_embeddings(u_one_hot)\n",
    "        vec_v = self.v_embeddings(v_one_hot)\n",
    "\n",
    "        # positive sample\n",
    "        score_pos = torch.mul(vec_u, vec_v)\n",
    "        score_pos = torch.sum(score_pos, dim=1)\n",
    "        loss_pos = F.logsigmoid(score_pos).squeeze()\n",
    "\n",
    "        # negative sample\n",
    "        neg_one_hot = id2one_hot(v_neg_id, len(vocabulary))\n",
    "        vec_neg_v = self.v_embeddings(neg_one_hot)\n",
    "        score_neg = torch.mul(vec_neg_v, vec_u).squeeze()\n",
    "        score_neg = torch.sum(score_neg, dim=1)\n",
    "        loss_neg = F.logsigmoid(-1 * score_neg).mean().squeeze()\n",
    "\n",
    "        return -1 * (loss_neg + loss_pos).mean()\n",
    "\n",
    "    def eval(self, u_one_hot):\n",
    "        candidate = []\n",
    "        for v in range(0, len(vocabulary)):\n",
    "            v_one_hot = id2one_hot(v, len(vocabulary))\n",
    "            vec_u = self.u_embeddings(u_one_hot)\n",
    "            vec_v = self.v_embeddings(v_one_hot)\n",
    "            score = torch.sum(torch.mul(vec_u, vec_v), dim=1).sum().item()\n",
    "            candidate.append((id2word[v], score))\n",
    "        return candidate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training Start:\n",
      "\n",
      "epoch: 0, loss: 0.000000\n",
      "epoch: 1, loss: 5531.275621\n",
      "epoch: 2, loss: 5530.988339\n",
      "epoch: 3, loss: 5529.278539\n",
      "epoch: 4, loss: 5526.145927\n",
      "epoch: 5, loss: 5525.262491\n",
      "epoch: 6, loss: 5525.228722\n",
      "epoch: 7, loss: 5525.222629\n",
      "epoch: 8, loss: 5525.213696\n",
      "epoch: 9, loss: 5525.221848\n",
      "losses: [5531.275621056557, 5530.988339424133, 5529.278539299965, 5526.145926952362, 5525.262490987778, 5525.228722333908, 5525.222628951073, 5525.213695645332, 5525.221848249435, 5525.215793132782]\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "\n",
    "model = SGModel(len(vocabulary), 100, 2)\n",
    "opt = torch.optim.SGD(model.parameters(), lr=0.1)\n",
    "loss_function = nn.NLLLoss()\n",
    "losses = []\n",
    "\n",
    "print(\"\\nTraining Start:\\n\")\n",
    "loss_val=0\n",
    "for epo in range(num_epochs):\n",
    "    print(\"epoch: %d, loss: %f\" % (epo, loss_val))\n",
    "    loss_val = 0\n",
    "    for context, target in ct_pairs:\n",
    "\n",
    "        u = id2one_hot(word2id[context], len(vocabulary))\n",
    "        v = id2one_hot(word2id[target], len(vocabulary))\n",
    "        rand = np.random.randint(0, len(vocabulary))\n",
    "        neg = torch.multinomial(hist_tensor, 3, replacement=False)\n",
    "\n",
    "        model.zero_grad()\n",
    "        log_probs = model(u,v,neg)\n",
    "        loss_val += log_probs.item()\n",
    "        log_probs.backward()\n",
    "        opt.step()\n",
    "\n",
    "    torch.save(model.state_dict(), './tmp/word2vec.epoch{}'.format(epo))\n",
    "    losses.append(loss_val)\n",
    "print('losses:', losses)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['0.9063441753387451' 'let.']\n",
      " ['0.9063441753387451' 'Moved']\n",
      " ['0.9063442349433899' 'multiply']\n",
      " ['0.9063442349433899' 'Shall']\n",
      " ['0.9063442945480347' 'Man']\n",
      " ['0.9063442945480347' 'was']\n",
      " ['0.9063442945480347' 'Made.']\n",
      " ['0.9063443541526794' 'seas.']\n",
      " ['0.9063444137573242' 'after,']\n",
      " ['0.9063444137573242' 'Open']]\n",
      "tensor([[ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        ...,\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006]],\n",
      "       grad_fn=<EmbeddingBackward>)\n",
      "tensor([[ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        ...,\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006],\n",
      "        [ 0.0044,  0.0010, -0.0015,  ..., -0.0046,  0.0013, -0.0006]],\n",
      "       grad_fn=<EmbeddingBackward>)\n"
     ]
    }
   ],
   "source": [
    "them = id2one_hot(word2id['them'], len(vocabulary))\n",
    "\n",
    "result = model.eval(them)\n",
    "print(np.sort(result, axis=1)[:10])\n",
    "\n",
    "there = id2one_hot(word2id['there.'], len(vocabulary))\n",
    "print(model.v_embeddings(there))\n",
    "print(model.v_embeddings(them))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}