CommonGround.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User History\n",
    "\n",
    "Our appmakes a User/VenueHistory API call on behalf of both users to create a profile of the places they like. This will inform the recommendations later on.\n",
    "\n",
    "We accomplish this by using Watson's Natural Language Understanding API to find \"concepts\" among everything in each user's history.\n",
    "\n",
    "We use the Main Category insight learned from the Foursquare API tutorial to make the list relevant to the end goal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import dependencies\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "from os.path import join, dirname\n",
    "import os\n",
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import watson_developer_cloud\n",
    "import watson_developer_cloud.natural_language_understanding.features.v1 as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load credentials for Watson and Foursquare\n",
    "load_dotenv('.env')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from User import Users\n",
    "\n",
    "A = Users('a', 'Food')\n",
    "A.eval_user()\n",
    "A.concepts_for_all_relevant_visits()\n",
    "A.all_concepts()\n",
    "ra = A.pool_concepts()\n",
    "ma = A.mean_concepts(ra)\n",
    "\n",
    "B = Users('y', 'Food')\n",
    "B.eval_user()\n",
    "B.concepts_for_all_relevant_visits()\n",
    "B.all_concepts()\n",
    "rb = B.pool_concepts()\n",
    "mb = B.mean_concepts(rb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`ma` and `mb` are dicts with an aggregate topic analysis of all places user A and user B have visited."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flatbreads\t0.03442984210526316\n",
      "Apple\t0.04742078947368421\n",
      "Hamburger\t0.05144894736842105\n",
      "Iced coffee\t0.04958947368421053\n",
      "Mister Donut\t0.030257052631578945\n",
      "Kringle\t0.040217368421052634\n",
      "New Mexican cuisine\t0.039545473684210526\n",
      "Juice\t0.037463315789473685\n",
      "Mayonnaise\t0.041639368421052626\n",
      "Pretzel\t0.04419631578947368\n",
      "The Skinny\t0.025386789473684213\n",
      "Bartender\t0.026027894736842107\n",
      "Jack Daniel's\t0.025390105263157894\n",
      "Latte\t0.03423473684210526\n",
      "Barista\t0.02056557894736842\n",
      "Chicken\t0.043589368421052634\n",
      "Soft drink\t0.050869\n",
      "Coffeehouse\t0.095212\n",
      "Dunkin' Donuts\t0.045095052631578945\n",
      "German cuisine\t0.03911036842105263\n",
      "Maize\t0.04847305263157895\n",
      "Coors Brewing Company\t0.025368105263157893\n",
      "Fast food\t0.07426363157894737\n",
      "Appetizers\t0.04930021052631579\n",
      "Bagel\t0.102008\n",
      "Waiting staff\t0.026100473684210524\n",
      "Bag-In-Box\t0.038551894736842104\n",
      "Taste\t0.032445842105263155\n",
      "Meat\t0.04627768421052631\n",
      "Sandwich\t0.05175457894736842\n",
      "Ice cream\t0.02725284210526316\n",
      "Pie\t0.04557631578947369\n",
      "Debut albums\t0.053903\n",
      "Breakfast\t0.034487894736842106\n",
      "New York City\t0.05335578947368421\n",
      "Smoothie\t0.03555315789473684\n",
      "Block\t0.02600021052631579\n",
      "French fries\t0.04078621052631579\n",
      "Sugar\t0.034\n",
      "Tea\t0.07419473684210526\n",
      "Milkshake\t0.09707557894736842\n",
      "Toast\t0.031558105263157894\n",
      "Doughnut\t0.05020957894736842\n",
      "Full breakfast\t0.03141573684210526\n",
      "Customer service\t0.04412484210526316\n",
      "Garden strawberry\t0.024892210526315788\n",
      "English-language films\t0.13799373684210525\n",
      "Drink\t0.027132736842105265\n",
      "French onion soup\t0.04097278947368421\n",
      "Potato salad\t0.03235736842105263\n",
      "Albany, New York\t0.028263842105263157\n",
      "Stout\t0.04002647368421053\n",
      "Diet Coke\t0.04273321052631579\n",
      "Customer\t0.05142552631578947\n",
      "Drive-in\t0.049175315789473685\n",
      "Coffee culture\t0.05156984210526316\n",
      "Starbucks\t0.15132515789473686\n",
      "Service\t0.03869989473684211\n",
      "Bread\t0.06575463157894737\n",
      "Hors d'œuvre\t0.03084421052631579\n",
      "Milk\t0.049107368421052636\n",
      "Krispy Kreme\t0.033870578947368425\n",
      "Parking lot\t0.028689105263157898\n",
      "Egg salad\t0.03049278947368421\n",
      "2006 albums\t0.07026657894736843\n",
      "Coca-Cola Black Cherry Vanilla\t0.03817884210526316\n",
      "Good\t0.03181273684210526\n",
      "Coffee\t0.2663545263157895\n",
      "2005 albums\t0.024021526315789476\n",
      "Pudding\t0.03565094736842105\n",
      "Nutrition\t0.045871105263157894\n",
      "Hungarian cuisine\t0.051270526315789475\n",
      "Vending machine\t0.02937636842105263\n",
      "T.G.I. Friday's\t0.03527357894736842\n",
      "Internet\t0.020570473684210527\n",
      "American cuisine\t0.03288747368421053\n",
      "Salads\t0.04030547368421053\n",
      "Egg sandwich\t0.017184315789473686\n",
      "Salad\t0.05135705263157895\n",
      "American films\t0.027467631578947372\n",
      "Breakfast foods\t0.04515447368421053\n",
      "Vanilla\t0.02676136842105263\n",
      "Espresso\t0.18347963157894737\n",
      "Buffalo wings\t0.07243063157894737\n",
      "Drive-through\t0.08060647368421052\n",
      "Beef\t0.039538789473684215\n",
      "Bacon\t0.030486157894736843\n",
      "Coca-Cola\t0.0476398947368421\n",
      "Desserts\t0.042229526315789474\n",
      "Soda fountain\t0.03804994736842105\n",
      "Cashier\t0.025363315789473685\n",
      "Hot chocolate\t0.03217652631578947\n",
      "Soy milk\t0.02043984210526316\n",
      "Pork\t0.044053736842105264\n",
      "Meal\t0.08118757894736842\n",
      "McDonald's\t0.09565015789473684\n",
      "Alcoholic beverage\t0.03396836842105263\n",
      "Pennsylvania\t0.041971684210526314\n",
      "Diedrich Coffee\t0.023581894736842107\n",
      "\n",
      "Fast food\t0.733611\n",
      "American cuisine\t0.677592\n",
      "Potato\t0.490209\n",
      "Beer style\t0.484654\n",
      "Hamburger\t0.957893\n",
      "Onions\t0.57151\n",
      "Burger\t0.505451\n",
      "Debut albums\t0.553642\n"
     ]
    }
   ],
   "source": [
    "for i in ma:\n",
    "    print(\"{}\\t{}\".format(i, ma[i]))\n",
    "print()\n",
    "\n",
    "for i in mb:\n",
    "    print(\"{}\\t{}\".format(i, mb[i]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Midpoint and Return Venues\n",
    "\n",
    "Given the users address the midpoint function give find the centroid based on the latitude and longitude of users address.\n",
    "\n",
    "Around the midpoint, we querry the 30 closest results in a radius of 500m, filtering by interest (restaurant or bar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from geopy.geocoders import Nominatim # module to convert an address into latitude and longitude values\n",
    "import requests # library to handle requests\n",
    "import pandas as pd # library for data analsysis\n",
    "import numpy as np # library to handle data in a vectorized manner\n",
    "import random # library for random number generation\n",
    "import folium\n",
    "import matplotlib.cm as cm\n",
    "#from shapely.geometry import MultiPoint\n",
    "\n",
    "# tranforming json file into a pandas dataframe library\n",
    "from pandas.io.json import json_normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40.6555065 -74.0035751 40.7965898 -73.9476099084148\n"
     ]
    }
   ],
   "source": [
    "address_U1 = \"176 34th Street Brooklyn, NY\"\n",
    "address_U2 = \"1660 Madison Avenue New York, NY\"# Geolocator translate addrss into lat, lon\n",
    "geolocator = Nominatim()\n",
    "location_U1 = geolocator.geocode(address_U1)\n",
    "latitude_U1 = location_U1.latitude\n",
    "longitude_U1 = location_U1.longitude\n",
    "location_U2 = geolocator.geocode(address_U2)\n",
    "latitude_U2 = location_U2.latitude\n",
    "longitude_U2 = location_U2.longitude\n",
    "print (latitude_U1, longitude_U1, latitude_U2, longitude_U2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Multipoint has a function called centroid, that find the midpoint among any number of points listed\n",
    "points = MultiPoint([(latitude_U1, longitude_U1), (latitude_U2, longitude_U2)])\n",
    "latitude = points.centroid.x\n",
    "longitude = points.centroid.y\n",
    "print (points.centroid) #True centroid, not necessarily an existing point on map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "latitude = 40.72604815"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "longitude = -73.97559250420741"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"width:100%;\"><div style=\"position:relative;width:100%;height:0;padding-bottom:60%;\"><iframe src=\"data:text/html;charset=utf-8;base64,PCFET0NUWVBFIGh0bWw+CjxoZWFkPiAgICAKICAgIDxtZXRhIGh0dHAtZXF1aXY9ImNvbnRlbnQtdHlwZSIgY29udGVudD0idGV4dC9odG1sOyBjaGFyc2V0PVVURi04IiAvPgogICAgPHNjcmlwdD5MX1BSRUZFUl9DQU5WQVMgPSBmYWxzZTsgTF9OT19UT1VDSCA9IGZhbHNlOyBMX0RJU0FCTEVfM0QgPSBmYWxzZTs8L3NjcmlwdD4KICAgIDxzY3JpcHQgc3JjPSJodHRwczovL3VucGtnLmNvbS9sZWFmbGV0QDEuMC4xL2Rpc3QvbGVhZmxldC5qcyI+PC9zY3JpcHQ+CiAgICA8c2NyaXB0IHNyYz0iaHR0cHM6Ly9hamF4Lmdvb2dsZWFwaXMuY29tL2FqYXgvbGlicy9qcXVlcnkvMS4xMS4xL2pxdWVyeS5taW4uanMiPjwvc2NyaXB0PgogICAgPHNjcmlwdCBzcmM9Imh0dHBzOi8vbWF4Y2RuLmJvb3RzdHJhcGNkbi5jb20vYm9vdHN0cmFwLzMuMi4wL2pzL2Jvb3RzdHJhcC5taW4uanMiPjwvc2NyaXB0PgogICAgPHNjcmlwdCBzcmM9Imh0dHBzOi8vY2RuanMuY2xvdWRmbGFyZS5jb20vYWpheC9saWJzL0xlYWZsZXQuYXdlc29tZS1tYXJrZXJzLzIuMC4yL2xlYWZsZXQuYXdlc29tZS1tYXJrZXJzLmpzIj48L3NjcmlwdD4KICAgIDxzY3JpcHQgc3JjPSJodHRwczovL2NkbmpzLmNsb3VkZmxhcmUuY29tL2FqYXgvbGlicy9sZWFmbGV0Lm1hcmtlcmNsdXN0ZXIvMS4wLjAvbGVhZmxldC5tYXJrZXJjbHVzdGVyLXNyYy5qcyI+PC9zY3JpcHQ+CiAgICA8c2NyaXB0IHNyYz0iaHR0cHM6Ly9jZG5qcy5jbG91ZGZsYXJlLmNvbS9hamF4L2xpYnMvbGVhZmxldC5tYXJrZXJjbHVzdGVyLzEuMC4wL2xlYWZsZXQubWFya2VyY2x1c3Rlci5qcyI+PC9zY3JpcHQ+CiAgICA8bGluayByZWw9InN0eWxlc2hlZXQiIGhyZWY9Imh0dHBzOi8vdW5wa2cuY29tL2xlYWZsZXRAMS4wLjEvZGlzdC9sZWFmbGV0LmNzcyIgLz4KICAgIDxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iaHR0cHM6Ly9tYXhjZG4uYm9vdHN0cmFwY2RuLmNvbS9ib290c3RyYXAvMy4yLjAvY3NzL2Jvb3RzdHJhcC5taW4uY3NzIiAvPgogICAgPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBocmVmPSJodHRwczovL21heGNkbi5ib290c3RyYXBjZG4uY29tL2Jvb3RzdHJhcC8zLjIuMC9jc3MvYm9vdHN0cmFwLXRoZW1lLm1pbi5jc3MiIC8+CiAgICA8bGluayByZWw9InN0eWxlc2hlZXQiIGhyZWY9Imh0dHBzOi8vbWF4Y2RuLmJvb3RzdHJhcGNkbi5jb20vZm9udC1hd2Vzb21lLzQuNi4zL2Nzcy9mb250LWF3ZXNvbWUubWluLmNzcyIgLz4KICAgIDxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iaHR0cHM6Ly9jZG5qcy5jbG91ZGZsYXJlLmNvbS9hamF4L2xpYnMvTGVhZmxldC5hd2Vzb21lLW1hcmtlcnMvMi4wLjIvbGVhZmxldC5hd2Vzb21lLW1hcmtlcnMuY3NzIiAvPgogICAgPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBocmVmPSJodHRwczovL2NkbmpzLmNsb3VkZmxhcmUuY29tL2FqYXgvbGlicy9sZWFmbGV0Lm1hcmtlcmNsdXN0ZXIvMS4wLjAvTWFya2VyQ2x1c3Rlci5EZWZhdWx0LmNzcyIgLz4KICAgIDxsaW5rIHJlbD0ic3R5bGVzaGVldCIgaHJlZj0iaHR0cHM6Ly9jZG5qcy5jbG91ZGZsYXJlLmNvbS9hamF4L2xpYnMvbGVhZmxldC5tYXJrZXJjbHVzdGVyLzEuMC4wL01hcmtlckNsdXN0ZXIuY3NzIiAvPgogICAgPGxpbmsgcmVsPSJzdHlsZXNoZWV0IiBocmVmPSJodHRwczovL3Jhd2dpdC5jb20vcHl0aG9uLXZpc3VhbGl6YXRpb24vZm9saXVtL21hc3Rlci9mb2xpdW0vdGVtcGxhdGVzL2xlYWZsZXQuYXdlc29tZS5yb3RhdGUuY3NzIiAvPgogICAgPHN0eWxlPmh0bWwsIGJvZHkge3dpZHRoOiAxMDAlO2hlaWdodDogMTAwJTttYXJnaW46IDA7cGFkZGluZzogMDt9PC9zdHlsZT4KICAgIDxzdHlsZT4jbWFwIHtwb3NpdGlvbjphYnNvbHV0ZTt0b3A6MDtib3R0b206MDtyaWdodDowO2xlZnQ6MDt9PC9zdHlsZT4KICAgIAogICAgICAgICAgICA8c3R5bGU+ICNtYXBfYjYwYzgwNjFiNzM2NDc2Nzk5MzY5YWYwNjQ2ZWRlNjMgewogICAgICAgICAgICAgICAgcG9zaXRpb24gOiByZWxhdGl2ZTsKICAgICAgICAgICAgICAgIHdpZHRoIDogMTAwLjAlOwogICAgICAgICAgICAgICAgaGVpZ2h0OiAxMDAuMCU7CiAgICAgICAgICAgICAgICBsZWZ0OiAwLjAlOwogICAgICAgICAgICAgICAgdG9wOiAwLjAlOwogICAgICAgICAgICAgICAgfQogICAgICAgICAgICA8L3N0eWxlPgogICAgICAgIAo8L2hlYWQ+Cjxib2R5PiAgICAKICAgIAogICAgICAgICAgICA8ZGl2IGNsYXNzPSJmb2xpdW0tbWFwIiBpZD0ibWFwX2I2MGM4MDYxYjczNjQ3Njc5OTM2OWFmMDY0NmVkZTYzIiA+PC9kaXY+CiAgICAgICAgCjwvYm9keT4KPHNjcmlwdD4gICAgCiAgICAKCiAgICAgICAgICAgIAogICAgICAgICAgICAgICAgdmFyIHNvdXRoV2VzdCA9IEwubGF0TG5nKC05MCwgLTE4MCk7CiAgICAgICAgICAgICAgICB2YXIgbm9ydGhFYXN0ID0gTC5sYXRMbmcoOTAsIDE4MCk7CiAgICAgICAgICAgICAgICB2YXIgYm91bmRzID0gTC5sYXRMbmdCb3VuZHMoc291dGhXZXN0LCBub3J0aEVhc3QpOwogICAgICAgICAgICAKCiAgICAgICAgICAgIHZhciBtYXBfYjYwYzgwNjFiNzM2NDc2Nzk5MzY5YWYwNjQ2ZWRlNjMgPSBMLm1hcCgKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdtYXBfYjYwYzgwNjFiNzM2NDc2Nzk5MzY5YWYwNjQ2ZWRlNjMnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAge2NlbnRlcjogWzQwLjc2NTkzNywtNzMuOTc3MzA0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHpvb206IDExLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4Qm91bmRzOiBib3VuZHMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXllcnM6IFtdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd29ybGRDb3B5SnVtcDogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjcnM6IEwuQ1JTLkVQU0czODU3CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0pOwogICAgICAgICAgICAKICAgICAgICAKICAgIAogICAgICAgICAgICB2YXIgdGlsZV9sYXllcl9jZjU2YWYyMDFkMWU0ZmQ5YWIwNjUzNzgwYjliMmM4OCA9IEwudGlsZUxheWVyKAogICAgICAgICAgICAgICAgJ2h0dHBzOi8ve3N9LnRpbGUub3BlbnN0cmVldG1hcC5vcmcve3p9L3t4fS97eX0ucG5nJywKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICBtYXhab29tOiAxOCwKICAgICAgICAgICAgICAgICAgICBtaW5ab29tOiAxLAogICAgICAgICAgICAgICAgICAgIGNvbnRpbnVvdXNXb3JsZDogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgbm9XcmFwOiBmYWxzZSwKICAgICAgICAgICAgICAgICAgICBhdHRyaWJ1dGlvbjogJ0RhdGEgYnkgPGEgaHJlZj0iaHR0cDovL29wZW5zdHJlZXRtYXAub3JnIj5PcGVuU3RyZWV0TWFwPC9hPiwgdW5kZXIgPGEgaHJlZj0iaHR0cDovL3d3dy5vcGVuc3RyZWV0bWFwLm9yZy9jb3B5cmlnaHQiPk9EYkw8L2E+LicsCiAgICAgICAgICAgICAgICAgICAgZGV0ZWN0UmV0aW5hOiBmYWxzZQogICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICkuYWRkVG8obWFwX2I2MGM4MDYxYjczNjQ3Njc5OTM2OWFmMDY0NmVkZTYzKTsKCiAgICAgICAgCiAgICAKCiAgICAgICAgICAgIHZhciBjaXJjbGVfbWFya2VyXzdiNzQ1MTYwNzE1MzRmZjNiZWI2OWYzZThlZWEwYWU3ID0gTC5jaXJjbGVNYXJrZXIoCiAgICAgICAgICAgICAgICBbNDAuNjU1NTA2NSwtNzQuMDAzNTc1MV0sCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgY29sb3I6ICdibHVlJywKICAgICAgICAgICAgICAgICAgICB3ZWlnaHQ6IDIsCiAgICAgICAgICAgICAgICAgICAgZmlsbENvbG9yOiAnYmxhY2snLAogICAgICAgICAgICAgICAgICAgIGZpbGxPcGFjaXR5OiAwLjYKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAuc2V0UmFkaXVzKDcpCiAgICAgICAgICAgICAgICAuYWRkVG8obWFwX2I2MGM4MDYxYjczNjQ3Njc5OTM2OWFmMDY0NmVkZTYzKTsKICAgICAgICAgICAgCiAgICAKCiAgICAgICAgICAgIHZhciBjaXJjbGVfbWFya2VyXzdhOTM4Njg2YWFiMTRkZGQ4MzY5Y2IwZTk2ZGNhMWEyID0gTC5jaXJjbGVNYXJrZXIoCiAgICAgICAgICAgICAgICBbNDAuNzk2NTg5OCwtNzMuOTQ3NjA5OTA4NDE0OF0sCiAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgY29sb3I6ICdibHVlJywKICAgICAgICAgICAgICAgICAgICB3ZWlnaHQ6IDIsCiAgICAgICAgICAgICAgICAgICAgZmlsbENvbG9yOiAnYmxhY2snLAogICAgICAgICAgICAgICAgICAgIGZpbGxPcGFjaXR5OiAwLjYKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAuc2V0UmFkaXVzKDcpCiAgICAgICAgICAgICAgICAuYWRkVG8obWFwX2I2MGM4MDYxYjczNjQ3Njc5OTM2OWFmMDY0NmVkZTYzKTsKICAgICAgICAgICAgCiAgICAKCiAgICAgICAgICAgIHZhciBjaXJjbGVfbWFya2VyXzZhOTlhOWYyNWJkNzQ2OWRhOWE3MTM5MGRjNTM5YWI0ID0gTC5jaXJjbGVNYXJrZXIoCiAgICAgICAgICAgICAgICBbNDAuNzI2MDQ4MTUsLTczLjk3NTU5MjUwNDIwNzQxXSwKICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICBjb2xvcjogJ2dyZWVuJywKICAgICAgICAgICAgICAgICAgICB3ZWlnaHQ6IDIsCiAgICAgICAgICAgICAgICAgICAgZmlsbENvbG9yOiAnYmxhY2snLAogICAgICAgICAgICAgICAgICAgIGZpbGxPcGFjaXR5OiAwLjYKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAuc2V0UmFkaXVzKDcpCiAgICAgICAgICAgICAgICAuYWRkVG8obWFwX2I2MGM4MDYxYjczNjQ3Njc5OTM2OWFmMDY0NmVkZTYzKTsKICAgICAgICAgICAgCjwvc2NyaXB0Pg==\" style=\"position:absolute;width:100%;height:100%;left:0;top:0;border:none !important;\" allowfullscreen webkitallowfullscreen mozallowfullscreen></iframe></div></div>"
      ],
      "text/plain": [
       "<folium.folium.Map at 0x7f4517cfc550>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "map_nyc = folium.Map(location=[40.765937,-73.977304], zoom_start=11)\n",
    "folium.CircleMarker([latitude_U1, longitude_U1], color = 'blue',\n",
    "                   radius = 7).add_to(map_nyc)\n",
    "folium.CircleMarker([latitude_U2, longitude_U2], color = 'blue',\n",
    "                   radius = 7).add_to(map_nyc)\n",
    "folium.CircleMarker([latitude, longitude], color = 'green',\n",
    "                   radius = 7).add_to(map_nyc)\n",
    "map_nyc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "foursquare_client_id=os.environ.get('FOURSQUARE_CLIENT_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "foursquare_client_secret=os.environ.get('FOURSQUARE_CLIENT_SECRET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Foursquare credentials\n",
    "CLIENT_ID = foursquare_client_id;\n",
    "CLIENT_SECRET = foursquare_client_secret;\n",
    "VERSION = \"20170511\"\n",
    "LIMIT = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# these ids help us to find the class of places we want to\n",
    "food_id = \"4d4b7105d754a06374d81259\"\n",
    "bars_id = \"4d4b7105d754a06376d81259\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "radius = 500 # range in meters\n",
    "category_id = food_id\n",
    "price = \"2,3\" # price range. 1 very cheap, 4 very expensivecategory_id = food_id\n",
    "url=\"https://api.foursquare.com/v2/venues/explore?client_id={}&client_secret={}&ll={},{}&v={}&radius={}&categoryId={}&limit={}&price={}\".format(CLIENT_ID, CLIENT_SECRET, latitude, longitude, VERSION, radius, category_id, LIMIT, price)\n",
    "results = requests.get(url).json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['id', 'name', 'contact', 'location', 'categories', 'verified', 'stats', 'url', 'price', 'hasMenu', 'rating', 'ratingColor', 'ratingSignals', 'menu', 'allowMenuUrlEdit', 'beenHere', 'hours', 'photos', 'venuePage', 'storeId', 'hereNow'])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items = results[\"response\"][\"groups\"][0][\"items\"]\n",
    "items[0][\"venue\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function that extracts the category of the venue\n",
    "def get_category_type(row):\n",
    "   try:\n",
    "       categories_list = row[\"categories\"]\n",
    "   except:\n",
    "       categories_list = row[\"venue.categories\"]\n",
    "       \n",
    "   if len(categories_list) == 0:\n",
    "       return None\n",
    "   else:\n",
    "       return categories_list[0][\"name\"].encode('ascii',errors='ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracted places inside 500m radius "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kt12/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate_ix\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>url</th>\n",
       "      <th>categories</th>\n",
       "      <th>rating</th>\n",
       "      <th>id</th>\n",
       "      <th>isOpen</th>\n",
       "      <th>tier</th>\n",
       "      <th>address</th>\n",
       "      <th>cc</th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>crossStreet</th>\n",
       "      <th>distance</th>\n",
       "      <th>formattedAddress</th>\n",
       "      <th>labeledLatLngs</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "      <th>postalCode</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Esperanto</td>\n",
       "      <td>http://esperantony.com</td>\n",
       "      <td>b'Brazilian Restaurant'</td>\n",
       "      <td>8.9</td>\n",
       "      <td>458d06def964a520ff3f1fe3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>145 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>at E 9th St.</td>\n",
       "      <td>217</td>\n",
       "      <td>[145 Avenue C (at E 9th St.), New York, NY 100...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72545004678564...</td>\n",
       "      <td>40.725450</td>\n",
       "      <td>-73.978051</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kafana</td>\n",
       "      <td>http://www.kafananyc.com</td>\n",
       "      <td>b'Eastern European Restaurant'</td>\n",
       "      <td>8.9</td>\n",
       "      <td>49e8896cf964a5204e651fe3</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>116 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>at E 8th St</td>\n",
       "      <td>312</td>\n",
       "      <td>[116 Avenue C (at E 8th St), New York, NY 1000...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72445009594483...</td>\n",
       "      <td>40.724450</td>\n",
       "      <td>-73.978638</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Zum Schneider</td>\n",
       "      <td>http://www.zumschneider.com</td>\n",
       "      <td>b'German Restaurant'</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3fd66200f964a52022e51ee3</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>107 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>at E 7th St</td>\n",
       "      <td>346</td>\n",
       "      <td>[107 Avenue C (at E 7th St), New York, NY 1000...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72417, 'lng': ...</td>\n",
       "      <td>40.724170</td>\n",
       "      <td>-73.978871</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Virginia's</td>\n",
       "      <td>http://www.virginiasnyc.com</td>\n",
       "      <td>b'American Restaurant'</td>\n",
       "      <td>7.8</td>\n",
       "      <td>5549770d498ebdb938cbbfdb</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>647 E 11th St</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn Avenue B &amp; Avenue C</td>\n",
       "      <td>184</td>\n",
       "      <td>[647 E 11th St (btwn Avenue B &amp; Avenue C), New...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72663915585264...</td>\n",
       "      <td>40.726639</td>\n",
       "      <td>-73.977640</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bobwhite Counter</td>\n",
       "      <td>http://www.bobwhitecounter.com/</td>\n",
       "      <td>b'Fried Chicken Joint'</td>\n",
       "      <td>9.4</td>\n",
       "      <td>4f00dea9f9abd5b3917d422c</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>94 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn E 6th &amp; E 7th St</td>\n",
       "      <td>395</td>\n",
       "      <td>[94 Avenue C (btwn E 6th &amp; E 7th St), New York...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72376488808975...</td>\n",
       "      <td>40.723765</td>\n",
       "      <td>-73.979189</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Edi &amp; The Wolf</td>\n",
       "      <td>http://www.ediandthewolf.com</td>\n",
       "      <td>b'German Restaurant'</td>\n",
       "      <td>8.7</td>\n",
       "      <td>4ca0d958e9a7ef3be3085416</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>102 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn 6th &amp; 7th</td>\n",
       "      <td>385</td>\n",
       "      <td>[102 Avenue C (btwn 6th &amp; 7th), New York, NY 1...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72371374468727...</td>\n",
       "      <td>40.723714</td>\n",
       "      <td>-73.978972</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Donostia</td>\n",
       "      <td>http://www.donostianyc.com</td>\n",
       "      <td>b'Spanish Restaurant'</td>\n",
       "      <td>8.5</td>\n",
       "      <td>528bf7bb498eb53e3fc85459</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>155 Avenue B</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>at 10th St</td>\n",
       "      <td>382</td>\n",
       "      <td>[155 Avenue B (at 10th St), New York, NY 10009...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72678946581906...</td>\n",
       "      <td>40.726789</td>\n",
       "      <td>-73.980021</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Joselito</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'Spanish Restaurant'</td>\n",
       "      <td>7.5</td>\n",
       "      <td>4b81d8ccf964a520efc030e3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>125 Avenue D</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn 8th &amp; 9th</td>\n",
       "      <td>237</td>\n",
       "      <td>[125 Avenue D (btwn 8th &amp; 9th), New York, NY 1...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72392069000751...</td>\n",
       "      <td>40.723921</td>\n",
       "      <td>-73.975751</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Gnocco</td>\n",
       "      <td>http://gnocco.com</td>\n",
       "      <td>b'Italian Restaurant'</td>\n",
       "      <td>8.3</td>\n",
       "      <td>49c6a988f964a5206e571fe3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>337 E 10th St</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>Avenue B</td>\n",
       "      <td>430</td>\n",
       "      <td>[337 E 10th St (Avenue B), New York, NY 10009,...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.7272018229237,...</td>\n",
       "      <td>40.727202</td>\n",
       "      <td>-73.980457</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Gruppo</td>\n",
       "      <td>http://www.gruppothincrust.com/</td>\n",
       "      <td>b'Pizza Place'</td>\n",
       "      <td>8.8</td>\n",
       "      <td>3fd66200f964a5203ce51ee3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>98 Avenue B</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn 6th &amp; 7th Sts</td>\n",
       "      <td>499</td>\n",
       "      <td>[98 Avenue B (btwn 6th &amp; 7th Sts), New York, N...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72494964561905...</td>\n",
       "      <td>40.724950</td>\n",
       "      <td>-73.981333</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               name                              url  \\\n",
       "0         Esperanto           http://esperantony.com   \n",
       "1            Kafana         http://www.kafananyc.com   \n",
       "2     Zum Schneider      http://www.zumschneider.com   \n",
       "3        Virginia's      http://www.virginiasnyc.com   \n",
       "4  Bobwhite Counter  http://www.bobwhitecounter.com/   \n",
       "5    Edi & The Wolf     http://www.ediandthewolf.com   \n",
       "6          Donostia       http://www.donostianyc.com   \n",
       "7          Joselito                              NaN   \n",
       "8            Gnocco                http://gnocco.com   \n",
       "9            Gruppo  http://www.gruppothincrust.com/   \n",
       "\n",
       "                       categories  rating                        id isOpen  \\\n",
       "0         b'Brazilian Restaurant'     8.9  458d06def964a520ff3f1fe3   True   \n",
       "1  b'Eastern European Restaurant'     8.9  49e8896cf964a5204e651fe3   True   \n",
       "2            b'German Restaurant'     9.0  3fd66200f964a52022e51ee3  False   \n",
       "3          b'American Restaurant'     7.8  5549770d498ebdb938cbbfdb  False   \n",
       "4          b'Fried Chicken Joint'     9.4  4f00dea9f9abd5b3917d422c   True   \n",
       "5            b'German Restaurant'     8.7  4ca0d958e9a7ef3be3085416   True   \n",
       "6           b'Spanish Restaurant'     8.5  528bf7bb498eb53e3fc85459  False   \n",
       "7           b'Spanish Restaurant'     7.5  4b81d8ccf964a520efc030e3   True   \n",
       "8           b'Italian Restaurant'     8.3  49c6a988f964a5206e571fe3   True   \n",
       "9                  b'Pizza Place'     8.8  3fd66200f964a5203ce51ee3   True   \n",
       "\n",
       "   tier        address  cc      city        country               crossStreet  \\\n",
       "0     2   145 Avenue C  US  New York  United States              at E 9th St.   \n",
       "1     3   116 Avenue C  US  New York  United States               at E 8th St   \n",
       "2     2   107 Avenue C  US  New York  United States               at E 7th St   \n",
       "3     3  647 E 11th St  US  New York  United States  btwn Avenue B & Avenue C   \n",
       "4     2    94 Avenue C  US  New York  United States     btwn E 6th & E 7th St   \n",
       "5     3   102 Avenue C  US  New York  United States            btwn 6th & 7th   \n",
       "6     2   155 Avenue B  US  New York  United States                at 10th St   \n",
       "7     2   125 Avenue D  US  New York  United States            btwn 8th & 9th   \n",
       "8     2  337 E 10th St  US  New York  United States                  Avenue B   \n",
       "9     2    98 Avenue B  US  New York  United States        btwn 6th & 7th Sts   \n",
       "\n",
       "   distance                                   formattedAddress  \\\n",
       "0       217  [145 Avenue C (at E 9th St.), New York, NY 100...   \n",
       "1       312  [116 Avenue C (at E 8th St), New York, NY 1000...   \n",
       "2       346  [107 Avenue C (at E 7th St), New York, NY 1000...   \n",
       "3       184  [647 E 11th St (btwn Avenue B & Avenue C), New...   \n",
       "4       395  [94 Avenue C (btwn E 6th & E 7th St), New York...   \n",
       "5       385  [102 Avenue C (btwn 6th & 7th), New York, NY 1...   \n",
       "6       382  [155 Avenue B (at 10th St), New York, NY 10009...   \n",
       "7       237  [125 Avenue D (btwn 8th & 9th), New York, NY 1...   \n",
       "8       430  [337 E 10th St (Avenue B), New York, NY 10009,...   \n",
       "9       499  [98 Avenue B (btwn 6th & 7th Sts), New York, N...   \n",
       "\n",
       "                                      labeledLatLngs        lat        lng  \\\n",
       "0  [{'label': 'display', 'lat': 40.72545004678564...  40.725450 -73.978051   \n",
       "1  [{'label': 'display', 'lat': 40.72445009594483...  40.724450 -73.978638   \n",
       "2  [{'label': 'display', 'lat': 40.72417, 'lng': ...  40.724170 -73.978871   \n",
       "3  [{'label': 'display', 'lat': 40.72663915585264...  40.726639 -73.977640   \n",
       "4  [{'label': 'display', 'lat': 40.72376488808975...  40.723765 -73.979189   \n",
       "5  [{'label': 'display', 'lat': 40.72371374468727...  40.723714 -73.978972   \n",
       "6  [{'label': 'display', 'lat': 40.72678946581906...  40.726789 -73.980021   \n",
       "7  [{'label': 'display', 'lat': 40.72392069000751...  40.723921 -73.975751   \n",
       "8  [{'label': 'display', 'lat': 40.7272018229237,...  40.727202 -73.980457   \n",
       "9  [{'label': 'display', 'lat': 40.72494964561905...  40.724950 -73.981333   \n",
       "\n",
       "  postalCode state  \n",
       "0      10009    NY  \n",
       "1      10009    NY  \n",
       "2      10009    NY  \n",
       "3      10009    NY  \n",
       "4      10009    NY  \n",
       "5      10009    NY  \n",
       "6      10009    NY  \n",
       "7      10009    NY  \n",
       "8      10009    NY  \n",
       "9      10009    NY  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe = json_normalize(items) # flatten JSON# filter columns\n",
    "filtered_columns = ['venue.name', 'venue.url', 'venue.categories'] + [\"venue.rating\"] + \\\n",
    "                  [\"venue.id\"] + ['venue.hours.isOpen'] + ['venue.price.tier'] + \\\n",
    "                  [col for col in dataframe.columns if col.startswith('venue.location.')]\n",
    "                 \n",
    "dataframe_filtered = dataframe.ix[:, filtered_columns]# filter the category for each row\n",
    "dataframe_filtered['venue.categories'] = dataframe_filtered.apply(get_category_type, axis=1)# clean columns\n",
    "dataframe_filtered.columns = [col.split(\".\")[-1] for col in dataframe_filtered.columns]# filter just open places\n",
    "open_places = dataframe_filtered[dataframe_filtered['isOpen'] == True]\n",
    "dataframe_filtered.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting the venues ids to input into recommendation step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of places found: 30\n",
      "radius: 500 m\n",
      "price class: 2,3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['458d06def964a520ff3f1fe3',\n",
       " '49e8896cf964a5204e651fe3',\n",
       " '3fd66200f964a52022e51ee3',\n",
       " '5549770d498ebdb938cbbfdb',\n",
       " '4f00dea9f9abd5b3917d422c',\n",
       " '4ca0d958e9a7ef3be3085416',\n",
       " '528bf7bb498eb53e3fc85459',\n",
       " '4b81d8ccf964a520efc030e3',\n",
       " '49c6a988f964a5206e571fe3',\n",
       " '3fd66200f964a5203ce51ee3']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_id = list(dataframe_filtered['id'].astype(str))\n",
    "print ('number of places found:', len(list_id))\n",
    "print ('radius: %d m' %radius)\n",
    "print ('price class:', price)\n",
    "list_id[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "venues_id = list_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recommendation\n",
    "\n",
    "Our application pulls in the venues near the midpoint of the two users and their preferences which have been revealed by 'concepts' in their history. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to pull a venue's JSON using venue id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pull_foursquare_json(venue_id):\n",
    "    VERSION = \"20170511\"\n",
    "    foursquare_client_id=os.environ.get('FOURSQUARE_CLIENT_ID')\n",
    "    foursquare_client_secret=os.environ.get('FOURSQUARE_CLIENT_SECRET')\n",
    "    url=\"https://api.foursquare.com/v2/venues/{}/tips?client_id={}&client_secret={}&v={}&limit=150\".format(venue_id, foursquare_client_id, foursquare_client_secret, VERSION)\n",
    "    json_f = requests.get(url).json()\n",
    "    return json_f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper function to return list of tips from restaurant's JSON file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tips_list(json_file):\n",
    "    try:\n",
    "        num_tips = json_file['response']['tips']['count']\n",
    "        return [json_file['response']['tips']['items'][k]['text'] for k in range(num_tips)]\n",
    "    except:\n",
    "        return [\"\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine two user preference vectors and return one normalized vector and index of k important topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def combine_u_prefs(u1_dict, u2_dict, k):\n",
    "    u1_counter = Counter(u1_dict)\n",
    "    u2_counter = Counter(u2_dict)\n",
    "    u_prefs = u1_counter + u2_counter\n",
    "    # Pull k highest values\n",
    "    u_prefs_top = dict(u_prefs.most_common(k))\n",
    "    vallist = [val for val in u_prefs_top.values()]\n",
    "    factor = np.median(vallist)\n",
    "    normed_val = [val/factor for val in vallist]\n",
    "    topic_idx = [key for key in u_prefs_top.keys()]\n",
    "    pref_vec = pd.Series(data=normed_val, index=topic_idx)\n",
    "    return pref_vec, topic_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "pref_vec, topic_idx = combine_u_prefs(ma, mb, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct an empty matrix with venue id's as rows, topics as columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def construct_matrix(venue_ids, topic_idx):\n",
    "    empty_matrix = pd.DataFrame(index=venue_ids, columns=topic_idx)\n",
    "    return empty_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "empty_mat = construct_matrix(venues_id, topic_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to call Watson API to get sentiment score of a Foursquare tip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sentiment(tips):\n",
    "    # Helper function to return text sentiment analysis\n",
    "    # Load Watson credentials\n",
    "    username=os.environ.get('NLU_USERNAME')\n",
    "    password = os.environ.get('NLU_PASSWORD')\n",
    "    nlu = watson_developer_cloud.NaturalLanguageUnderstandingV1(version='2017-02-27',\n",
    "        username=username, password=password)\n",
    "    output = nlu.analyze(text=tips, features=[features.Sentiment()])\n",
    "    return output['sentiment']['document']['score']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each cell in the dataframe is now filled with the median sentiment score of the venue for only tips which include the topic\n",
    "\n",
    "Foursquare API is called to pull tips for each venue\n",
    "Watson API is called to evaluate the tip and yields a sentiment score between -1 and 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fill_sentiment_matrix(mat):\n",
    "    for j in range(mat.shape[0]):\n",
    "        venue_id = mat.index[j]\n",
    "        json_f = pull_foursquare_json(venue_id)\n",
    "        tips = tips_list(json_f)\n",
    "        for k in range(mat.shape[1]):\n",
    "            topic = mat.columns[k]\n",
    "            score = np.median([sentiment(tip) for tip in tips if topic in tip])\n",
    "            mat.loc[venue_id, topic] = score\n",
    "    return mat.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kt12/anaconda3/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2889: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/home/kt12/anaconda3/lib/python3.6/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "sent_mat = fill_sentiment_matrix(empty_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sentiment matrix is then multiplied with the user prefence matrix, yielding a weighted average for each venue.  This average is based upon the sentiment scores from Watson."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def recommend(score_mat, user_vec, venues_ids, top_n):\n",
    "    score_vec = pd.Series(np.dot(score_mat, user_vec), index=venues_ids)\n",
    "    return score_vec.sort_values(ascending=False)[:top_n].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3fd66200f964a5203ce51ee3', '3fd66200f964a5203fe51ee3',\n",
       "       '515392c4e4b0f03a43c69fce', '4ae1b55df964a520df8621e3',\n",
       "       '49e8896cf964a5204e651fe3'], dtype=object)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend(sent_mat, pref_vec, venues_id, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "recom = ['3fd66200f964a5203ce51ee3', '3fd66200f964a5203fe51ee3',\n",
    "       '515392c4e4b0f03a43c69fce', '4ae1b55df964a520df8621e3',\n",
    "       '49e8896cf964a5204e651fe3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recommendation list\n",
    "recommended = dataframe_filtered[dataframe_filtered['id'].isin(recom)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>url</th>\n",
       "      <th>categories</th>\n",
       "      <th>rating</th>\n",
       "      <th>id</th>\n",
       "      <th>isOpen</th>\n",
       "      <th>tier</th>\n",
       "      <th>address</th>\n",
       "      <th>cc</th>\n",
       "      <th>city</th>\n",
       "      <th>country</th>\n",
       "      <th>crossStreet</th>\n",
       "      <th>distance</th>\n",
       "      <th>formattedAddress</th>\n",
       "      <th>labeledLatLngs</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "      <th>postalCode</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kafana</td>\n",
       "      <td>http://www.kafananyc.com</td>\n",
       "      <td>b'Eastern European Restaurant'</td>\n",
       "      <td>8.9</td>\n",
       "      <td>49e8896cf964a5204e651fe3</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>116 Avenue C</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>at E 8th St</td>\n",
       "      <td>312</td>\n",
       "      <td>[116 Avenue C (at E 8th St), New York, NY 1000...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72445009594483...</td>\n",
       "      <td>40.724450</td>\n",
       "      <td>-73.978638</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Gruppo</td>\n",
       "      <td>http://www.gruppothincrust.com/</td>\n",
       "      <td>b'Pizza Place'</td>\n",
       "      <td>8.8</td>\n",
       "      <td>3fd66200f964a5203ce51ee3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>98 Avenue B</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn 6th &amp; 7th Sts</td>\n",
       "      <td>499</td>\n",
       "      <td>[98 Avenue B (btwn 6th &amp; 7th Sts), New York, N...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72494964561905...</td>\n",
       "      <td>40.724950</td>\n",
       "      <td>-73.981333</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Ciao For Now</td>\n",
       "      <td>http://ciaofornow.net/?page=home</td>\n",
       "      <td>b'Caf'</td>\n",
       "      <td>8.5</td>\n",
       "      <td>3fd66200f964a5203fe51ee3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>523 E 12th St</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>btwn Avenue B &amp; Avenue A</td>\n",
       "      <td>491</td>\n",
       "      <td>[523 E 12th St (btwn Avenue B &amp; Avenue A), New...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72862569265450...</td>\n",
       "      <td>40.728626</td>\n",
       "      <td>-73.980320</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Eleven B</td>\n",
       "      <td>http://www.elevenbnyc.com</td>\n",
       "      <td>b'Italian Restaurant'</td>\n",
       "      <td>6.5</td>\n",
       "      <td>4ae1b55df964a520df8621e3</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>174 Avenue B</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>11th Street</td>\n",
       "      <td>366</td>\n",
       "      <td>[174 Avenue B (11th Street), New York, NY 1000...</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.727518, 'lng':...</td>\n",
       "      <td>40.727518</td>\n",
       "      <td>-73.979478</td>\n",
       "      <td>10009</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Housewarmings!</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'Gastropub'</td>\n",
       "      <td>NaN</td>\n",
       "      <td>515392c4e4b0f03a43c69fce</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>US</td>\n",
       "      <td>New York</td>\n",
       "      <td>United States</td>\n",
       "      <td>NaN</td>\n",
       "      <td>493</td>\n",
       "      <td>[New York, NY, United States]</td>\n",
       "      <td>[{'label': 'display', 'lat': 40.72523709670373...</td>\n",
       "      <td>40.725237</td>\n",
       "      <td>-73.981347</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              name                               url  \\\n",
       "1           Kafana          http://www.kafananyc.com   \n",
       "9           Gruppo   http://www.gruppothincrust.com/   \n",
       "11    Ciao For Now  http://ciaofornow.net/?page=home   \n",
       "15        Eleven B         http://www.elevenbnyc.com   \n",
       "29  Housewarmings!                               NaN   \n",
       "\n",
       "                        categories  rating                        id isOpen  \\\n",
       "1   b'Eastern European Restaurant'     8.9  49e8896cf964a5204e651fe3   True   \n",
       "9                   b'Pizza Place'     8.8  3fd66200f964a5203ce51ee3   True   \n",
       "11                          b'Caf'     8.5  3fd66200f964a5203fe51ee3   True   \n",
       "15           b'Italian Restaurant'     6.5  4ae1b55df964a520df8621e3   True   \n",
       "29                    b'Gastropub'     NaN  515392c4e4b0f03a43c69fce    NaN   \n",
       "\n",
       "    tier        address  cc      city        country  \\\n",
       "1      3   116 Avenue C  US  New York  United States   \n",
       "9      2    98 Avenue B  US  New York  United States   \n",
       "11     2  523 E 12th St  US  New York  United States   \n",
       "15     2   174 Avenue B  US  New York  United States   \n",
       "29     3            NaN  US  New York  United States   \n",
       "\n",
       "                 crossStreet  distance  \\\n",
       "1                at E 8th St       312   \n",
       "9         btwn 6th & 7th Sts       499   \n",
       "11  btwn Avenue B & Avenue A       491   \n",
       "15               11th Street       366   \n",
       "29                       NaN       493   \n",
       "\n",
       "                                     formattedAddress  \\\n",
       "1   [116 Avenue C (at E 8th St), New York, NY 1000...   \n",
       "9   [98 Avenue B (btwn 6th & 7th Sts), New York, N...   \n",
       "11  [523 E 12th St (btwn Avenue B & Avenue A), New...   \n",
       "15  [174 Avenue B (11th Street), New York, NY 1000...   \n",
       "29                      [New York, NY, United States]   \n",
       "\n",
       "                                       labeledLatLngs        lat        lng  \\\n",
       "1   [{'label': 'display', 'lat': 40.72445009594483...  40.724450 -73.978638   \n",
       "9   [{'label': 'display', 'lat': 40.72494964561905...  40.724950 -73.981333   \n",
       "11  [{'label': 'display', 'lat': 40.72862569265450...  40.728626 -73.980320   \n",
       "15  [{'label': 'display', 'lat': 40.727518, 'lng':...  40.727518 -73.979478   \n",
       "29  [{'label': 'display', 'lat': 40.72523709670373...  40.725237 -73.981347   \n",
       "\n",
       "   postalCode state  \n",
       "1       10009    NY  \n",
       "9       10009    NY  \n",
       "11      10009    NY  \n",
       "15      10009    NY  \n",
       "29        NaN    NY  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommended"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}