word2vec_restaurant_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 2087 labeled train reviews\n",
      "parsing sentences from training set\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file /anaconda/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n",
      "/anaconda/lib/python3.6/site-packages/bs4/__init__.py:219: UserWarning: \"b'.'\" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.\n",
      "  ' Beautiful Soup.' % markup)\n",
      "/anaconda/lib/python3.6/site-packages/bs4/__init__.py:219: UserWarning: \"b'..'\" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.\n",
      "  ' Beautiful Soup.' % markup)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "1018368\n",
      "parsing finished!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-02 16:49:15,172 : INFO : 'pattern' package not found; tag filters are not available for English\n",
      "2018-05-02 16:49:15,179 : INFO : collecting all words and their counts\n",
      "2018-05-02 16:49:15,180 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2018-05-02 16:49:15,223 : INFO : PROGRESS: at sentence #10000, processed 127734 words, keeping 7660 word types\n",
      "2018-05-02 16:49:15,258 : INFO : PROGRESS: at sentence #20000, processed 263109 words, keeping 11213 word types\n",
      "2018-05-02 16:49:15,296 : INFO : PROGRESS: at sentence #30000, processed 400444 words, keeping 13614 word types\n",
      "2018-05-02 16:49:15,326 : INFO : PROGRESS: at sentence #40000, processed 534494 words, keeping 15826 word types\n",
      "2018-05-02 16:49:15,363 : INFO : PROGRESS: at sentence #50000, processed 669076 words, keeping 17439 word types\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-02 16:49:15,399 : INFO : PROGRESS: at sentence #60000, processed 805023 words, keeping 18954 word types\n",
      "2018-05-02 16:49:15,457 : INFO : PROGRESS: at sentence #70000, processed 941028 words, keeping 20254 word types\n",
      "2018-05-02 16:49:15,520 : INFO : PROGRESS: at sentence #80000, processed 1076273 words, keeping 21541 word types\n",
      "2018-05-02 16:49:15,586 : INFO : PROGRESS: at sentence #90000, processed 1209192 words, keeping 22499 word types\n",
      "2018-05-02 16:49:15,633 : INFO : PROGRESS: at sentence #100000, processed 1340161 words, keeping 23537 word types\n",
      "2018-05-02 16:49:15,694 : INFO : PROGRESS: at sentence #110000, processed 1475649 words, keeping 24865 word types\n",
      "2018-05-02 16:49:15,755 : INFO : PROGRESS: at sentence #120000, processed 1613248 words, keeping 26035 word types\n",
      "2018-05-02 16:49:15,815 : INFO : PROGRESS: at sentence #130000, processed 1750338 words, keeping 27130 word types\n",
      "2018-05-02 16:49:15,879 : INFO : PROGRESS: at sentence #140000, processed 1885806 words, keeping 28096 word types\n",
      "2018-05-02 16:49:15,943 : INFO : PROGRESS: at sentence #150000, processed 2022807 words, keeping 29176 word types\n",
      "2018-05-02 16:49:15,999 : INFO : PROGRESS: at sentence #160000, processed 2157611 words, keeping 30134 word types\n",
      "2018-05-02 16:49:16,042 : INFO : PROGRESS: at sentence #170000, processed 2291257 words, keeping 30937 word types\n",
      "2018-05-02 16:49:16,095 : INFO : PROGRESS: at sentence #180000, processed 2423707 words, keeping 31643 word types\n",
      "2018-05-02 16:49:16,129 : INFO : PROGRESS: at sentence #190000, processed 2555613 words, keeping 32411 word types\n",
      "2018-05-02 16:49:16,179 : INFO : PROGRESS: at sentence #200000, processed 2692569 words, keeping 33259 word types\n",
      "2018-05-02 16:49:16,215 : INFO : PROGRESS: at sentence #210000, processed 2826588 words, keeping 33922 word types\n",
      "2018-05-02 16:49:16,252 : INFO : PROGRESS: at sentence #220000, processed 2959087 words, keeping 34586 word types\n",
      "2018-05-02 16:49:16,291 : INFO : PROGRESS: at sentence #230000, processed 3099506 words, keeping 35390 word types\n",
      "2018-05-02 16:49:16,322 : INFO : PROGRESS: at sentence #240000, processed 3236883 words, keeping 36134 word types\n",
      "2018-05-02 16:49:16,366 : INFO : PROGRESS: at sentence #250000, processed 3371788 words, keeping 36818 word types\n",
      "2018-05-02 16:49:16,398 : INFO : PROGRESS: at sentence #260000, processed 3507802 words, keeping 37503 word types\n",
      "2018-05-02 16:49:16,439 : INFO : PROGRESS: at sentence #270000, processed 3643375 words, keeping 38224 word types\n",
      "2018-05-02 16:49:16,473 : INFO : PROGRESS: at sentence #280000, processed 3780600 words, keeping 39060 word types\n",
      "2018-05-02 16:49:16,507 : INFO : PROGRESS: at sentence #290000, processed 3920292 words, keeping 39684 word types\n",
      "2018-05-02 16:49:16,538 : INFO : PROGRESS: at sentence #300000, processed 4057505 words, keeping 40389 word types\n",
      "2018-05-02 16:49:16,572 : INFO : PROGRESS: at sentence #310000, processed 4193062 words, keeping 40972 word types\n",
      "2018-05-02 16:49:16,608 : INFO : PROGRESS: at sentence #320000, processed 4327896 words, keeping 41590 word types\n",
      "2018-05-02 16:49:16,645 : INFO : PROGRESS: at sentence #330000, processed 4461957 words, keeping 42083 word types\n",
      "2018-05-02 16:49:16,675 : INFO : PROGRESS: at sentence #340000, processed 4586420 words, keeping 42613 word types\n",
      "2018-05-02 16:49:16,706 : INFO : PROGRESS: at sentence #350000, processed 4722402 words, keeping 43209 word types\n",
      "2018-05-02 16:49:16,752 : INFO : PROGRESS: at sentence #360000, processed 4856066 words, keeping 43795 word types\n",
      "2018-05-02 16:49:16,814 : INFO : PROGRESS: at sentence #370000, processed 4987338 words, keeping 44335 word types\n",
      "2018-05-02 16:49:16,844 : INFO : PROGRESS: at sentence #380000, processed 5120128 words, keeping 44875 word types\n",
      "2018-05-02 16:49:16,876 : INFO : PROGRESS: at sentence #390000, processed 5255736 words, keeping 45482 word types\n",
      "2018-05-02 16:49:16,915 : INFO : PROGRESS: at sentence #400000, processed 5394301 words, keeping 46035 word types\n",
      "2018-05-02 16:49:16,948 : INFO : PROGRESS: at sentence #410000, processed 5527853 words, keeping 46665 word types\n",
      "2018-05-02 16:49:16,981 : INFO : PROGRESS: at sentence #420000, processed 5665361 words, keeping 47189 word types\n",
      "2018-05-02 16:49:17,014 : INFO : PROGRESS: at sentence #430000, processed 5798939 words, keeping 47571 word types\n",
      "2018-05-02 16:49:17,053 : INFO : PROGRESS: at sentence #440000, processed 5938130 words, keeping 48062 word types\n",
      "2018-05-02 16:49:17,086 : INFO : PROGRESS: at sentence #450000, processed 6072083 words, keeping 48532 word types\n",
      "2018-05-02 16:49:17,119 : INFO : PROGRESS: at sentence #460000, processed 6209627 words, keeping 49093 word types\n",
      "2018-05-02 16:49:17,169 : INFO : PROGRESS: at sentence #470000, processed 6341509 words, keeping 49582 word types\n",
      "2018-05-02 16:49:17,209 : INFO : PROGRESS: at sentence #480000, processed 6479721 words, keeping 50188 word types\n",
      "2018-05-02 16:49:17,241 : INFO : PROGRESS: at sentence #490000, processed 6615931 words, keeping 50704 word types\n",
      "2018-05-02 16:49:17,275 : INFO : PROGRESS: at sentence #500000, processed 6753042 words, keeping 51262 word types\n",
      "2018-05-02 16:49:17,309 : INFO : PROGRESS: at sentence #510000, processed 6890537 words, keeping 51824 word types\n",
      "2018-05-02 16:49:17,342 : INFO : PROGRESS: at sentence #520000, processed 7022189 words, keeping 52385 word types\n",
      "2018-05-02 16:49:17,376 : INFO : PROGRESS: at sentence #530000, processed 7161671 words, keeping 52943 word types\n",
      "2018-05-02 16:49:17,410 : INFO : PROGRESS: at sentence #540000, processed 7301184 words, keeping 53450 word types\n",
      "2018-05-02 16:49:17,447 : INFO : PROGRESS: at sentence #550000, processed 7444569 words, keeping 53864 word types\n",
      "2018-05-02 16:49:17,476 : INFO : PROGRESS: at sentence #560000, processed 7583393 words, keeping 54289 word types\n",
      "2018-05-02 16:49:17,511 : INFO : PROGRESS: at sentence #570000, processed 7717437 words, keeping 54602 word types\n",
      "2018-05-02 16:49:17,551 : INFO : PROGRESS: at sentence #580000, processed 7852566 words, keeping 54995 word types\n",
      "2018-05-02 16:49:17,586 : INFO : PROGRESS: at sentence #590000, processed 7985708 words, keeping 55468 word types\n",
      "2018-05-02 16:49:17,621 : INFO : PROGRESS: at sentence #600000, processed 8133168 words, keeping 56048 word types\n",
      "2018-05-02 16:49:17,658 : INFO : PROGRESS: at sentence #610000, processed 8271446 words, keeping 56526 word types\n",
      "2018-05-02 16:49:17,689 : INFO : PROGRESS: at sentence #620000, processed 8402511 words, keeping 56892 word types\n",
      "2018-05-02 16:49:17,721 : INFO : PROGRESS: at sentence #630000, processed 8538892 words, keeping 57276 word types\n",
      "2018-05-02 16:49:17,757 : INFO : PROGRESS: at sentence #640000, processed 8678724 words, keeping 57690 word types\n",
      "2018-05-02 16:49:17,796 : INFO : PROGRESS: at sentence #650000, processed 8821530 words, keeping 58259 word types\n",
      "2018-05-02 16:49:17,832 : INFO : PROGRESS: at sentence #660000, processed 8965574 words, keeping 58950 word types\n",
      "2018-05-02 16:49:17,861 : INFO : PROGRESS: at sentence #670000, processed 9100876 words, keeping 59374 word types\n",
      "2018-05-02 16:49:17,898 : INFO : PROGRESS: at sentence #680000, processed 9235046 words, keeping 59784 word types\n",
      "2018-05-02 16:49:17,930 : INFO : PROGRESS: at sentence #690000, processed 9371554 words, keeping 60329 word types\n",
      "2018-05-02 16:49:17,963 : INFO : PROGRESS: at sentence #700000, processed 9509423 words, keeping 60664 word types\n",
      "2018-05-02 16:49:17,993 : INFO : PROGRESS: at sentence #710000, processed 9643240 words, keeping 61064 word types\n",
      "2018-05-02 16:49:18,027 : INFO : PROGRESS: at sentence #720000, processed 9776641 words, keeping 61402 word types\n",
      "2018-05-02 16:49:18,065 : INFO : PROGRESS: at sentence #730000, processed 9915550 words, keeping 61980 word types\n",
      "2018-05-02 16:49:18,098 : INFO : PROGRESS: at sentence #740000, processed 10043968 words, keeping 62272 word types\n",
      "2018-05-02 16:49:18,131 : INFO : PROGRESS: at sentence #750000, processed 10173423 words, keeping 62586 word types\n",
      "2018-05-02 16:49:18,161 : INFO : PROGRESS: at sentence #760000, processed 10307281 words, keeping 62977 word types\n",
      "2018-05-02 16:49:18,206 : INFO : PROGRESS: at sentence #770000, processed 10447109 words, keeping 63398 word types\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-02 16:49:18,242 : INFO : PROGRESS: at sentence #780000, processed 10581814 words, keeping 63774 word types\n",
      "2018-05-02 16:49:18,279 : INFO : PROGRESS: at sentence #790000, processed 10720625 words, keeping 64135 word types\n",
      "2018-05-02 16:49:18,312 : INFO : PROGRESS: at sentence #800000, processed 10856799 words, keeping 64519 word types\n",
      "2018-05-02 16:49:18,346 : INFO : PROGRESS: at sentence #810000, processed 10996193 words, keeping 64984 word types\n",
      "2018-05-02 16:49:18,378 : INFO : PROGRESS: at sentence #820000, processed 11129959 words, keeping 65420 word types\n",
      "2018-05-02 16:49:18,410 : INFO : PROGRESS: at sentence #830000, processed 11265782 words, keeping 65760 word types\n",
      "2018-05-02 16:49:18,446 : INFO : PROGRESS: at sentence #840000, processed 11401362 words, keeping 66109 word types\n",
      "2018-05-02 16:49:18,479 : INFO : PROGRESS: at sentence #850000, processed 11539208 words, keeping 66463 word types\n",
      "2018-05-02 16:49:18,512 : INFO : PROGRESS: at sentence #860000, processed 11674432 words, keeping 66797 word types\n",
      "2018-05-02 16:49:18,545 : INFO : PROGRESS: at sentence #870000, processed 11811438 words, keeping 67204 word types\n",
      "2018-05-02 16:49:18,577 : INFO : PROGRESS: at sentence #880000, processed 11943281 words, keeping 67478 word types\n",
      "2018-05-02 16:49:18,610 : INFO : PROGRESS: at sentence #890000, processed 12077361 words, keeping 67951 word types\n",
      "2018-05-02 16:49:18,645 : INFO : PROGRESS: at sentence #900000, processed 12211439 words, keeping 68252 word types\n",
      "2018-05-02 16:49:18,681 : INFO : PROGRESS: at sentence #910000, processed 12346878 words, keeping 68589 word types\n",
      "2018-05-02 16:49:18,715 : INFO : PROGRESS: at sentence #920000, processed 12483481 words, keeping 68966 word types\n",
      "2018-05-02 16:49:18,749 : INFO : PROGRESS: at sentence #930000, processed 12621195 words, keeping 69447 word types\n",
      "2018-05-02 16:49:18,779 : INFO : PROGRESS: at sentence #940000, processed 12758945 words, keeping 69753 word types\n",
      "2018-05-02 16:49:18,814 : INFO : PROGRESS: at sentence #950000, processed 12892466 words, keeping 70144 word types\n",
      "2018-05-02 16:49:18,852 : INFO : PROGRESS: at sentence #960000, processed 13030561 words, keeping 70569 word types\n",
      "2018-05-02 16:49:18,887 : INFO : PROGRESS: at sentence #970000, processed 13171036 words, keeping 70957 word types\n",
      "2018-05-02 16:49:18,921 : INFO : PROGRESS: at sentence #980000, processed 13308840 words, keeping 71232 word types\n",
      "2018-05-02 16:49:18,954 : INFO : PROGRESS: at sentence #990000, processed 13447136 words, keeping 71715 word types\n",
      "2018-05-02 16:49:18,988 : INFO : PROGRESS: at sentence #1000000, processed 13583516 words, keeping 72045 word types\n",
      "2018-05-02 16:49:19,027 : INFO : PROGRESS: at sentence #1010000, processed 13720458 words, keeping 72337 word types\n",
      "2018-05-02 16:49:19,055 : INFO : collected 72631 word types from a corpus of 13835145 raw words and 1018368 sentences\n",
      "2018-05-02 16:49:19,055 : INFO : Loading a fresh vocabulary\n",
      "2018-05-02 16:49:19,138 : INFO : min_count=3 retains 32058 unique words (44% of original 72631, drops 40573)\n",
      "2018-05-02 16:49:19,138 : INFO : min_count=3 leaves 13785437 word corpus (99% of original 13835145, drops 49708)\n",
      "2018-05-02 16:49:19,235 : INFO : deleting the raw counts dictionary of 72631 items\n",
      "2018-05-02 16:49:19,238 : INFO : sample=0.001 downsamples 51 most-common words\n",
      "2018-05-02 16:49:19,239 : INFO : downsampling leaves estimated 10137688 word corpus (73.5% of prior 13785437)\n",
      "2018-05-02 16:49:19,349 : INFO : estimated required memory for 32058 words and 300 dimensions: 92968200 bytes\n",
      "2018-05-02 16:49:19,350 : INFO : resetting layer weights\n",
      "2018-05-02 16:49:19,926 : INFO : training model with 4 workers on 32058 vocabulary and 300 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "2018-05-02 16:49:20,943 : INFO : EPOCH 1 - PROGRESS: at 7.14% examples, 714039 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:21,950 : INFO : EPOCH 1 - PROGRESS: at 13.47% examples, 672559 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:22,958 : INFO : EPOCH 1 - PROGRESS: at 19.61% examples, 651142 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:23,967 : INFO : EPOCH 1 - PROGRESS: at 27.25% examples, 680468 words/s, in_qsize 6, out_qsize 1\n",
      "2018-05-02 16:49:24,975 : INFO : EPOCH 1 - PROGRESS: at 35.13% examples, 701075 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:25,976 : INFO : EPOCH 1 - PROGRESS: at 42.98% examples, 715952 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:27,006 : INFO : EPOCH 1 - PROGRESS: at 49.80% examples, 709854 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:28,019 : INFO : EPOCH 1 - PROGRESS: at 56.51% examples, 705910 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:29,025 : INFO : EPOCH 1 - PROGRESS: at 63.07% examples, 701946 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:30,030 : INFO : EPOCH 1 - PROGRESS: at 70.52% examples, 707470 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:31,036 : INFO : EPOCH 1 - PROGRESS: at 78.21% examples, 713354 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:32,042 : INFO : EPOCH 1 - PROGRESS: at 85.04% examples, 711559 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:33,042 : INFO : EPOCH 1 - PROGRESS: at 91.95% examples, 710309 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:34,044 : INFO : EPOCH 1 - PROGRESS: at 98.01% examples, 704013 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:34,292 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-02 16:49:34,303 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-02 16:49:34,305 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-02 16:49:34,318 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-02 16:49:34,319 : INFO : EPOCH - 1 : training on 13835145 raw words (10135522 effective words) took 14.4s, 704772 effective words/s\n",
      "2018-05-02 16:49:35,334 : INFO : EPOCH 2 - PROGRESS: at 7.36% examples, 739450 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:36,346 : INFO : EPOCH 2 - PROGRESS: at 14.97% examples, 749165 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:37,347 : INFO : EPOCH 2 - PROGRESS: at 21.80% examples, 725371 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:38,348 : INFO : EPOCH 2 - PROGRESS: at 28.59% examples, 718008 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:39,349 : INFO : EPOCH 2 - PROGRESS: at 35.13% examples, 704309 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:40,354 : INFO : EPOCH 2 - PROGRESS: at 41.98% examples, 701267 words/s, in_qsize 7, out_qsize 1\n",
      "2018-05-02 16:49:41,360 : INFO : EPOCH 2 - PROGRESS: at 49.32% examples, 707089 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:42,367 : INFO : EPOCH 2 - PROGRESS: at 55.57% examples, 698437 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:43,380 : INFO : EPOCH 2 - PROGRESS: at 62.65% examples, 700339 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:44,391 : INFO : EPOCH 2 - PROGRESS: at 69.03% examples, 695517 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:45,395 : INFO : EPOCH 2 - PROGRESS: at 76.16% examples, 696625 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:46,401 : INFO : EPOCH 2 - PROGRESS: at 83.53% examples, 701059 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:47,404 : INFO : EPOCH 2 - PROGRESS: at 89.88% examples, 696005 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:48,419 : INFO : EPOCH 2 - PROGRESS: at 97.00% examples, 697838 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:48,944 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-02 16:49:48,956 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-02 16:49:48,967 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-02 16:49:48,969 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-02 16:49:48,969 : INFO : EPOCH - 2 : training on 13835145 raw words (10137549 effective words) took 14.6s, 692629 effective words/s\n",
      "2018-05-02 16:49:49,994 : INFO : EPOCH 3 - PROGRESS: at 7.21% examples, 720818 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:50,996 : INFO : EPOCH 3 - PROGRESS: at 14.76% examples, 739967 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:52,000 : INFO : EPOCH 3 - PROGRESS: at 22.36% examples, 745202 words/s, in_qsize 7, out_qsize 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-02 16:49:53,013 : INFO : EPOCH 3 - PROGRESS: at 28.97% examples, 725369 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:54,014 : INFO : EPOCH 3 - PROGRESS: at 36.24% examples, 725113 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:55,020 : INFO : EPOCH 3 - PROGRESS: at 43.11% examples, 719568 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:49:56,025 : INFO : EPOCH 3 - PROGRESS: at 50.03% examples, 716472 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:57,034 : INFO : EPOCH 3 - PROGRESS: at 56.88% examples, 713811 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:58,035 : INFO : EPOCH 3 - PROGRESS: at 63.28% examples, 707780 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:49:59,038 : INFO : EPOCH 3 - PROGRESS: at 70.37% examples, 709267 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:00,043 : INFO : EPOCH 3 - PROGRESS: at 78.00% examples, 714292 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:01,049 : INFO : EPOCH 3 - PROGRESS: at 84.62% examples, 710621 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:02,054 : INFO : EPOCH 3 - PROGRESS: at 91.87% examples, 711994 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:03,068 : INFO : EPOCH 3 - PROGRESS: at 98.14% examples, 706549 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:50:03,282 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-02 16:50:03,293 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-02 16:50:03,301 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-02 16:50:03,317 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-02 16:50:03,318 : INFO : EPOCH - 3 : training on 13835145 raw words (10139213 effective words) took 14.3s, 707505 effective words/s\n",
      "2018-05-02 16:50:04,339 : INFO : EPOCH 4 - PROGRESS: at 7.36% examples, 739759 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:05,344 : INFO : EPOCH 4 - PROGRESS: at 14.76% examples, 741349 words/s, in_qsize 8, out_qsize 1\n",
      "2018-05-02 16:50:06,364 : INFO : EPOCH 4 - PROGRESS: at 21.14% examples, 701318 words/s, in_qsize 6, out_qsize 1\n",
      "2018-05-02 16:50:07,375 : INFO : EPOCH 4 - PROGRESS: at 28.30% examples, 707296 words/s, in_qsize 6, out_qsize 1\n",
      "2018-05-02 16:50:08,379 : INFO : EPOCH 4 - PROGRESS: at 34.92% examples, 696926 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:09,382 : INFO : EPOCH 4 - PROGRESS: at 42.05% examples, 700066 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:10,395 : INFO : EPOCH 4 - PROGRESS: at 49.25% examples, 703274 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:50:11,402 : INFO : EPOCH 4 - PROGRESS: at 55.35% examples, 693354 words/s, in_qsize 6, out_qsize 1\n",
      "2018-05-02 16:50:12,409 : INFO : EPOCH 4 - PROGRESS: at 62.43% examples, 696304 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:13,411 : INFO : EPOCH 4 - PROGRESS: at 68.27% examples, 686690 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:14,413 : INFO : EPOCH 4 - PROGRESS: at 75.49% examples, 690001 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:15,413 : INFO : EPOCH 4 - PROGRESS: at 82.91% examples, 695284 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:16,417 : INFO : EPOCH 4 - PROGRESS: at 90.44% examples, 700107 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:17,430 : INFO : EPOCH 4 - PROGRESS: at 97.00% examples, 697599 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:18,005 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-02 16:50:18,013 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-02 16:50:18,027 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-02 16:50:18,028 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-02 16:50:18,029 : INFO : EPOCH - 4 : training on 13835145 raw words (10137214 effective words) took 14.7s, 690098 effective words/s\n",
      "2018-05-02 16:50:19,040 : INFO : EPOCH 5 - PROGRESS: at 7.07% examples, 708079 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:50:20,046 : INFO : EPOCH 5 - PROGRESS: at 14.34% examples, 717632 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:21,057 : INFO : EPOCH 5 - PROGRESS: at 21.94% examples, 728944 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:22,062 : INFO : EPOCH 5 - PROGRESS: at 29.46% examples, 738090 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:23,083 : INFO : EPOCH 5 - PROGRESS: at 36.40% examples, 725162 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:24,094 : INFO : EPOCH 5 - PROGRESS: at 43.25% examples, 718969 words/s, in_qsize 8, out_qsize 0\n",
      "2018-05-02 16:50:25,103 : INFO : EPOCH 5 - PROGRESS: at 50.34% examples, 717673 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:26,112 : INFO : EPOCH 5 - PROGRESS: at 57.18% examples, 715006 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:27,113 : INFO : EPOCH 5 - PROGRESS: at 64.58% examples, 720900 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:28,116 : INFO : EPOCH 5 - PROGRESS: at 71.36% examples, 717385 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:29,119 : INFO : EPOCH 5 - PROGRESS: at 78.07% examples, 713254 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:30,119 : INFO : EPOCH 5 - PROGRESS: at 85.49% examples, 716626 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:31,129 : INFO : EPOCH 5 - PROGRESS: at 92.67% examples, 716648 words/s, in_qsize 7, out_qsize 0\n",
      "2018-05-02 16:50:32,073 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-02 16:50:32,084 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-02 16:50:32,094 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-02 16:50:32,099 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-02 16:50:32,100 : INFO : EPOCH - 5 : training on 13835145 raw words (10137437 effective words) took 14.1s, 720836 effective words/s\n",
      "2018-05-02 16:50:32,102 : INFO : training on a 69175725 raw words (50686935 effective words) took 72.2s, 702299 effective words/s\n",
      "2018-05-02 16:50:32,103 : INFO : saving Word2Vec object under yelp_restaurant_300features_3minwords_5context, separately None\n",
      "2018-05-02 16:50:32,105 : INFO : not storing attribute vectors_norm\n",
      "2018-05-02 16:50:32,107 : INFO : not storing attribute cum_table\n",
      "2018-05-02 16:50:33,434 : INFO : saved yelp_restaurant_300features_3minwords_5context\n",
      "/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:135: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "2018-05-02 16:50:33,437 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('guy', 0.7689632773399353),\n",
       " ('woman', 0.7436196208000183),\n",
       " ('lady', 0.7430142164230347),\n",
       " ('gentleman', 0.7339794039726257),\n",
       " ('girl', 0.7078391909599304),\n",
       " ('gal', 0.6827062964439392),\n",
       " ('dude', 0.6572169065475464),\n",
       " ('gentlemen', 0.5664964914321899),\n",
       " ('boy', 0.544607400894165),\n",
       " ('cashier', 0.507962703704834)]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "'''\n",
    "train a model with all restaurant review\n",
    "'''\n",
    "import pandas as pd\n",
    "\n",
    "# read\n",
    "train = pd.read_table(\"/Users/luxiaopeng/notebook/inls613/closure_prediction/review_by_business.txt\")\n",
    "\n",
    "# Verify the number of reviews that were read (100,000 in total)\n",
    "print(\"Read %d labeled train reviews\" % (train[\"text\"].size))\n",
    "\n",
    "\n",
    "\n",
    "train.rename(columns={\"text\":\"review\"}, inplace=True)\n",
    "\n",
    "train = train[[\"business_id\",\"review\"]]\n",
    "\n",
    "len(train)\n",
    "\n",
    "# drop na\n",
    "train = train.dropna()\n",
    "\n",
    "len(train)\n",
    "\n",
    "\n",
    "\n",
    "# Import various modules for string cleaning\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "def review_to_wordlist(review, remove_stopwords = False):\n",
    "    # Function to convert a document to a sequence of words,\n",
    "    # optionally removing stop words.  Returns a list of words.\n",
    "    #\n",
    "    # 1. Remove HTML\n",
    "    review_text = BeautifulSoup(review).get_text()\n",
    "    #  \n",
    "    # 2. Remove non-letters\n",
    "    review_text = re.sub(\"[^a-zA-Z]\",\" \", review_text)\n",
    "    #\n",
    "    # 3. Convert words to lower case and split them\n",
    "    words = review_text.lower().split()\n",
    "    #\n",
    "    # 4. Optionally remove stop words (false by default)\n",
    "    if remove_stopwords:\n",
    "        stops = set(stopwords.words(\"english\"))\n",
    "        words = [w for w in words if not w in stops]\n",
    "    #\n",
    "    # 5. Return a list of words\n",
    "    return(words)\n",
    "\n",
    "'''\n",
    "Next, we want a specific input format. Word2Vec expects single sentences, \n",
    "each one as a list of words. In other words, the input format is a list of lists.\n",
    "\n",
    "It is not at all straightforward how to split a paragraph into sentences. \n",
    "There are all kinds of gotchas in natural language. English sentences can \n",
    "end with \"?\", \"!\", \"\"\", or \".\", among other things, and spacing and \n",
    "capitalization are not reliable guides either. For this reason, \n",
    "we'll use NLTK's punkt tokenizer for sentence splitting. In order to \n",
    "use this, you will need to install NLTK and use nltk.download() to \n",
    "download the relevant training file for punkt.\n",
    "'''\n",
    "# Download the punkt tokenizer for sentence splitting\n",
    "import nltk.data\n",
    "import nltk\n",
    "# nltk.download('punkt')\n",
    "# nltk.download('popular')\n",
    "# Load the punkt tokenizer\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "\n",
    "# Define a function to split a review into parsed sentences\n",
    "def review_to_sentences(review, tokenizer, remove_stopwords = False):\n",
    "    # Function to split a review into parsed sentences. Returns a \n",
    "    # list of sentences, where each sentence is a list of words\n",
    "    #\n",
    "    # 1. Use the NLTK tokenizer to split the paragraph into sentences\n",
    "    # raw_sentences = tokenizer.tokenize(review.decode('utf-8').strip())\n",
    "    raw_sentences = tokenizer.tokenize(review.strip())\n",
    "    #\n",
    "    # 2. Loop over each sentence\n",
    "    sentences = []\n",
    "    for raw_sentence in raw_sentences:\n",
    "        # If a sentence is empty, skip it\n",
    "        if len(raw_sentence) > 0:\n",
    "            # Otherwise, call review_to_wordlist to get a list of words\n",
    "            sentences.append(review_to_wordlist(raw_sentence, remove_stopwords))\n",
    "    #\n",
    "    # Return the list of sentences (each sentence is a list of words,\n",
    "    # so this returns a list of lists\n",
    "    return sentences\n",
    "\n",
    "sentences = []\n",
    "count = 0\n",
    "print('parsing sentences from training set')\n",
    "for review in train['review']:\n",
    "    sentences += review_to_sentences(review, tokenizer)\n",
    "    count += 1\n",
    "    if count % 1000 == 0:\n",
    "        print(count)\n",
    "\n",
    "# check\n",
    "print(len(sentences))\n",
    "print(\"parsing finished!\")\n",
    "\n",
    "# try:\n",
    "#     import pickle\n",
    "#     with open('sentences.pkl', 'wb') as f:\n",
    "#         pickle.dump(sentences, f)\n",
    "# except:\n",
    "#     print(\"cannot save as pkl\")\n",
    "#     pass\n",
    "\n",
    "\n",
    "# Import the built-in logging module and configure it so that Word2Vec \n",
    "# creates nice output messages\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\\\n",
    "    level=logging.INFO)\n",
    "# Set values for various parameters\n",
    "num_features = 300    # Word vector dimensionality                      \n",
    "min_word_count = 3   # Minimum word count                        \n",
    "num_workers = 4       # Number of threads to run in parallel\n",
    "context = 5          # Context window size                                                                                    \n",
    "# downsampling = 1e-3   # Downsample setting for frequent words\n",
    "\n",
    "# Initialize and train the model (this will take some time)\n",
    "from gensim.models import word2vec\n",
    "print('training model...')\n",
    "model = word2vec.Word2Vec(sentences, workers=num_workers, \\\n",
    "            size=num_features, min_count = min_word_count, \\\n",
    "            window = context)# , sample = downsampling\n",
    "model_name = \"yelp_restaurant_300features_3minwords_5context\"\n",
    "model.save(model_name)\n",
    "print(\"model saved!\")\n",
    "model.most_similar(\"man\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-d6fe38f9fd6d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msimilar_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmost_similar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpositive\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"great\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtopn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtupl\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msimilar_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"'\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtupl\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\"'\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\",\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "similar_list = model.most_similar(positive=[\"great\"],topn=20)\n",
    "for tupl in similar_list:\n",
    "    print(\"'\",tupl[0],\"'\",\",\",sep='',end='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "model = Word2Vec.load(\"yelp_restaurant_300features_3minwords_5context\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'desert','tiramisu','cheesecake','gelato','desserts','cannoli','baklava','flan','brownie','panna','sundae','nutella','smoothie','creme','torte','souffle','beignets','cupcakes','cobbler','starters',"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "similar_list = model.most_similar(positive=[\"dessert\"],topn=20)\n",
    "for tupl in similar_list:\n",
    "    print(\"'\",tupl[0],\"'\",\",\",sep='',end='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}