main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import valentine\n",
    "from valentine import valentine_match, valentine_metrics\n",
    "from valentine.algorithms import Coma, Cupid\n",
    "import pprint\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data from dataset/monitor_specs folder, there's a folder for each source and each file json file contains the specs for a monitor\n",
    "sources2monitors = {}\n",
    "\n",
    "for folder in os.listdir('dataset/monitor_specs'):\n",
    "    monitors = []\n",
    "    for file in os.listdir('dataset/monitor_specs/' + folder):\n",
    "        with open('dataset/monitor_specs/' + folder + '/' + file) as f:\n",
    "            monitor = json.load(f)\n",
    "            monitors.append(monitor)\n",
    "            if folder in sources2monitors:\n",
    "                sources2monitors[folder].append(monitor)\n",
    "            else:\n",
    "                sources2monitors[folder] = [monitor]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4281"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources2monitors.get('www.ebay.com').__len__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from the dict source2monitors, we get all the unique attributes for each source and count how many times they appear\n",
    "sources2attributes = {}\n",
    "\n",
    "for source in sources2monitors:\n",
    "    attributes = {}\n",
    "    for monitor in sources2monitors.get(source):\n",
    "        for attribute in monitor.keys():\n",
    "            if attribute in attributes:\n",
    "                attributes[attribute] += 1\n",
    "            else:\n",
    "                attributes[attribute] = 1\n",
    "    sources2attributes[source] = attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<page title>': 4281,\n",
       " 'brand': 2148,\n",
       " 'condition': 2405,\n",
       " 'display technology': 3247,\n",
       " 'screen size': 3495,\n",
       " 'aspect ratio': 2806,\n",
       " 'brightness': 2173,\n",
       " 'contrast ratio': 2733,\n",
       " 'max resolution': 1205,\n",
       " 'model': 3055,\n",
       " 'mpn': 2400,\n",
       " 'response time': 2715,\n",
       " 'video inputs': 1654,\n",
       " 'countryregion of manufacture': 125,\n",
       " 'max. resolution': 1206,\n",
       " 'system': 1442,\n",
       " 'upc': 1762,\n",
       " 'country/region of manufacture': 90,\n",
       " 'features': 909,\n",
       " 'adjustable display angle': 269,\n",
       " 'adjustable display height': 101,\n",
       " 'audio': 97,\n",
       " 'backlight technology': 390,\n",
       " 'brand name': 381,\n",
       " 'builtin devices': 121,\n",
       " 'color': 227,\n",
       " 'color supported': 289,\n",
       " 'depth': 310,\n",
       " 'dvi': 149,\n",
       " 'general information': 159,\n",
       " 'green compliance certificateauthority': 278,\n",
       " 'hdcp supported': 220,\n",
       " 'hdmi': 117,\n",
       " 'headphone': 40,\n",
       " 'horizontal viewing angle': 312,\n",
       " 'input voltage': 148,\n",
       " 'interfacesports': 171,\n",
       " 'manufacturer': 580,\n",
       " 'manufacturer part number': 292,\n",
       " 'maximum resolution': 508,\n",
       " 'miscellaneous': 403,\n",
       " 'number of screens': 329,\n",
       " 'physical characteristics': 171,\n",
       " 'platform supported': 110,\n",
       " 'power description': 153,\n",
       " 'product model': 299,\n",
       " 'product name': 289,\n",
       " 'product series': 131,\n",
       " 'product type': 397,\n",
       " 'screen mode': 378,\n",
       " 'speakers': 198,\n",
       " 'standard refresh rate': 189,\n",
       " 'standby power consumption': 185,\n",
       " 'technical information': 170,\n",
       " 'vertical viewing angle': 354,\n",
       " 'vesa mount compatible': 161,\n",
       " 'vesa mount standard': 116,\n",
       " 'video': 174,\n",
       " 'weight approximate': 159,\n",
       " 'width': 296,\n",
       " 'manufacturer website address': 114,\n",
       " 'manufacturersupplier': 52,\n",
       " 'panel technology': 139,\n",
       " 'product line': 337,\n",
       " 'audio output': 56,\n",
       " 'cable': 42,\n",
       " 'cable details': 31,\n",
       " 'color support': 214,\n",
       " 'comments': 25,\n",
       " 'compatibility': 74,\n",
       " 'details': 25,\n",
       " 'diagonal size': 42,\n",
       " 'diagonal size metric': 43,\n",
       " 'dimensions  weight': 55,\n",
       " 'dimensions  weight details': 35,\n",
       " 'display': 52,\n",
       " 'display projector': 48,\n",
       " 'environmental standards': 51,\n",
       " 'form factor': 240,\n",
       " 'header': 58,\n",
       " 'height': 222,\n",
       " 'image aspect ratio': 46,\n",
       " 'image contrast ratio': 75,\n",
       " 'input device': 54,\n",
       " 'input signal': 48,\n",
       " 'interface': 48,\n",
       " 'interfaces': 58,\n",
       " 'lcd backlight technology': 45,\n",
       " 'nominal voltage': 39,\n",
       " 'packaged quantity': 54,\n",
       " 'power consumption operational': 57,\n",
       " 'power consumption stand by': 30,\n",
       " 'power device': 55,\n",
       " 'projector image brightness': 41,\n",
       " 'projector monitor features': 33,\n",
       " 'technology': 49,\n",
       " 'type': 173,\n",
       " 'typical response time': 38,\n",
       " 'video input': 50,\n",
       " 'weight': 164,\n",
       " 'additional mpn': 179,\n",
       " 'analog video input connector': 168,\n",
       " 'max viewing angle horizontal': 182,\n",
       " 'max viewing angle vertical': 182,\n",
       " 'monitor color': 188,\n",
       " 'monitor platform source': 113,\n",
       " 'refresh rate': 149,\n",
       " 'regulatory approval': 160,\n",
       " 'synchronization range  horizontal': 156,\n",
       " 'synchronization range  vertical': 159,\n",
       " 'backlight type': 18,\n",
       " 'brightness typical': 42,\n",
       " 'color gamut': 15,\n",
       " 'contrast ratio typical': 37,\n",
       " 'dimensions w x h x d': 16,\n",
       " 'displayable colors': 17,\n",
       " 'horizontal analogdigital': 6,\n",
       " 'input connectors': 21,\n",
       " 'lookup table': 9,\n",
       " 'native resolution': 79,\n",
       " 'operating altitude': 15,\n",
       " 'operating humidity': 19,\n",
       " 'operating temperature': 38,\n",
       " 'pixel pitch': 141,\n",
       " 'pixels per inch': 17,\n",
       " 'power consumption': 63,\n",
       " 'pwm frequency': 3,\n",
       " 'response time typical': 37,\n",
       " 'rohs': 9,\n",
       " 'storage altitude': 15,\n",
       " 'storage humidity': 16,\n",
       " 'storage temperature': 17,\n",
       " 'tco': 6,\n",
       " 'vertical': 10,\n",
       " 'vesa hole configuration': 13,\n",
       " 'viewable image size': 28,\n",
       " 'viewing angle typical': 20,\n",
       " 'green compliant': 166,\n",
       " 'mfg model': 77,\n",
       " 'product family': 59,\n",
       " 'unit of measure': 77,\n",
       " '100000001 asus smart contrast ratio': 21,\n",
       " 'advanced video technologies for exceptional visuals': 8,\n",
       " 'full hd with hdmi': 17,\n",
       " 'mount type': 175,\n",
       " 'splendid video intelligence technology': 24,\n",
       " 'color reproduction range': 13,\n",
       " 'dynamic contrast ratio': 61,\n",
       " 'virtual 4k 30hz': 4,\n",
       " 'power consumption typical': 7,\n",
       " 'antiglare coating': 17,\n",
       " 'dvid input': 17,\n",
       " 'dvid with hdcp input': 16,\n",
       " 'energy star certified': 19,\n",
       " 'hdmi input': 18,\n",
       " 'onscreen image controls': 13,\n",
       " 'screen refresh rate': 20,\n",
       " 'screen size measured diagonally': 17,\n",
       " 'speakers included': 17,\n",
       " 'specs': 20,\n",
       " 'tilt': 50,\n",
       " 'touchscreen': 63,\n",
       " 'vga input': 17,\n",
       " 'displayport': 88,\n",
       " 'adjustable display pivot': 100,\n",
       " 'offmode power consumption': 88,\n",
       " 'package contents': 182,\n",
       " 'screen surface': 111,\n",
       " 'connections': 22,\n",
       " 'dimensions': 39,\n",
       " 'input video signal': 12,\n",
       " 'maximum colors': 13,\n",
       " 'multimedia speakers': 13,\n",
       " 'on mode': 12,\n",
       " 'other features': 13,\n",
       " 'plug  play': 18,\n",
       " 'stand': 32,\n",
       " 'sync type': 12,\n",
       " 'viewing angle horizontalvertical': 15,\n",
       " 'wall mountable': 13,\n",
       " 'whats in the box': 13,\n",
       " 'additional notes': 6,\n",
       " 'category': 6,\n",
       " 'shipping dimensions': 8,\n",
       " 'shipping weight': 13,\n",
       " 'sku': 12,\n",
       " '3d technology': 5,\n",
       " 'about the brand': 5,\n",
       " 'additional features': 9,\n",
       " 'audio type': 3,\n",
       " 'audiovideo inputs': 8,\n",
       " 'builtin tuner': 4,\n",
       " 'definition': 4,\n",
       " 'description': 229,\n",
       " 'display resolution': 17,\n",
       " 'energy star': 8,\n",
       " 'family line': 3,\n",
       " 'number of speakers': 22,\n",
       " 'on mode power': 4,\n",
       " 'parental control': 7,\n",
       " 'rear input connectors': 3,\n",
       " 'remote control type originally supplied by manufacturer': 4,\n",
       " 'resolution': 49,\n",
       " 'room type': 3,\n",
       " 'sleep mode power': 4,\n",
       " 'smart tv features': 7,\n",
       " 'speaker power': 4,\n",
       " 'supported resolutions': 4,\n",
       " 'viewing angle': 64,\n",
       " 'builtin speakers': 11,\n",
       " 'builtin tv tuner': 3,\n",
       " 'cabinet color': 25,\n",
       " 'complies with': 8,\n",
       " 'connectors': 15,\n",
       " 'display colors': 41,\n",
       " 'dsub': 13,\n",
       " 'hdcp support': 5,\n",
       " 'horizontal refresh rate': 45,\n",
       " 'input video compatibility': 9,\n",
       " 'led backlight': 12,\n",
       " 'panel': 8,\n",
       " 'recommended resolution': 11,\n",
       " 'regulatory approvals': 6,\n",
       " 'stand adjustments': 22,\n",
       " 'usb 20': 4,\n",
       " 'vertical refresh rate': 50,\n",
       " 'vesa compatible': 5,\n",
       " 'widescreen': 69,\n",
       " 'normal signs of use are present on this unit': 17,\n",
       " 'height with stand': 54,\n",
       " 'marketing information': 102,\n",
       " 'screensize': 69,\n",
       " 'â': 44,\n",
       " 'digital video input standard': 111,\n",
       " 'release date': 36,\n",
       " 'vga': 110,\n",
       " 'certifications  standards': 124,\n",
       " 'depth with stand': 115,\n",
       " 'display orientation': 16,\n",
       " 'environmentconscious and reliable': 6,\n",
       " 'flexible viewing and connectivity options': 6,\n",
       " 'impressive screen clarity': 5,\n",
       " 'postconsumerwaste': 10,\n",
       " 'recycled content': 12,\n",
       " 'usb': 89,\n",
       " 'usb standard': 43,\n",
       " 'width with stand': 113,\n",
       " 'coo': 3,\n",
       " 'display diagonal size': 9,\n",
       " 'display response time': 9,\n",
       " 'display type': 60,\n",
       " 'unspsc code': 10,\n",
       " 'main features': 35,\n",
       " 'mfr part number': 223,\n",
       " 'specifications': 193,\n",
       " 'weight with stand approximate': 50,\n",
       " 'audio line out': 4,\n",
       " 'bluetooth': 4,\n",
       " 'cache': 7,\n",
       " 'display  graphics': 13,\n",
       " 'display screen type': 25,\n",
       " 'ethernet technology': 14,\n",
       " 'graphics controller manufacturer': 4,\n",
       " 'graphics controller model': 4,\n",
       " 'graphics memory accessibility': 3,\n",
       " 'graphics memory technology': 4,\n",
       " 'input devices': 12,\n",
       " 'keyboard': 7,\n",
       " 'memory': 57,\n",
       " 'memory card supported': 3,\n",
       " 'memory technology': 5,\n",
       " 'microphone': 5,\n",
       " 'multitouch screen': 9,\n",
       " 'network  communication': 4,\n",
       " 'network rj45': 7,\n",
       " 'operating system': 67,\n",
       " 'optical drive type': 5,\n",
       " 'processor  chipset': 4,\n",
       " 'processor core': 5,\n",
       " 'processor model': 5,\n",
       " 'processor speed': 61,\n",
       " 'processor type': 64,\n",
       " 'screen resolution': 18,\n",
       " 'software': 18,\n",
       " 'standard memory': 14,\n",
       " 'storage': 8,\n",
       " 'total number of usb ports': 5,\n",
       " 'wireless lan': 3,\n",
       " 'ean': 34,\n",
       " 'power supply': 38,\n",
       " 'cables included': 10,\n",
       " 'compliant standards': 60,\n",
       " 'device type': 8,\n",
       " 'dimensions wxdxh': 35,\n",
       " 'display position adjustments': 46,\n",
       " 'display type ledbacklit lcd monitor  tft active matrix': 5,\n",
       " 'energy star qualified': 25,\n",
       " 'epeat compliant': 23,\n",
       " 'flat panel mount interface': 46,\n",
       " 'humidity range operating': 19,\n",
       " 'included accessories': 10,\n",
       " 'included software': 3,\n",
       " 'manufacturer warranty': 27,\n",
       " 'max operating temperature': 41,\n",
       " 'microsoft certification': 11,\n",
       " 'min operating temperature': 40,\n",
       " 'panel type': 54,\n",
       " 'power consumption stand by  sleep': 10,\n",
       " 'product description': 7,\n",
       " 'screen coating': 8,\n",
       " 'service  support': 53,\n",
       " 'service  support 3 years warranty': 4,\n",
       " 'service  support details': 24,\n",
       " 'viewable size': 53,\n",
       " 'voltage required': 10,\n",
       " 'contemporary styling': 7,\n",
       " 'customizable color': 5,\n",
       " 'full hd resolution': 7,\n",
       " 'stunning clarity': 6,\n",
       " 'superb performance': 9,\n",
       " 'versatile connections': 9,\n",
       " 'dot acceptable': 4,\n",
       " 'plug and play capability': 4,\n",
       " 'powersaver': 4,\n",
       " 'not included': 6,\n",
       " 'ra return authroization': 5,\n",
       " 'your paypal payment email address if you pay with paypal': 5,\n",
       " 'immediate payment': 19,\n",
       " 'nonpaying bidder': 19,\n",
       " 'restocking fees': 19,\n",
       " 'tax': 19,\n",
       " 'we guarantee against doa': 19,\n",
       " 'we prefer paypal': 19,\n",
       " 'whats not included': 19,\n",
       " 'sdcstringscreen size mined': 7,\n",
       " 'bundled items': 24,\n",
       " 'warranty  labor': 8,\n",
       " 'warranty  parts': 8,\n",
       " 'country of manufacture': 78,\n",
       " 'interface  connectivity': 4,\n",
       " '3d type': 7,\n",
       " 'maximum adjustable height': 10,\n",
       " '14day moneyback guarantee': 6,\n",
       " '1501 n michael drive wood dale illinois 60191': 3,\n",
       " '57 business days': 6,\n",
       " 'monfri between 1000am and 400pm': 6,\n",
       " 'must': 6,\n",
       " 'not': 7,\n",
       " 'paypal': 6,\n",
       " 'please include your phone number': 6,\n",
       " 'limited warranty': 32,\n",
       " 'green compliance certificate/authority': 15,\n",
       " 'standby': 8,\n",
       " 'cabinet': 18,\n",
       " 'chassis': 18,\n",
       " 'country kits': 20,\n",
       " 'environmental parameters': 40,\n",
       " 'height adjustment': 23,\n",
       " 'localization': 25,\n",
       " 'power consumption sleep': 7,\n",
       " 'product color': 38,\n",
       " 'qty': 31,\n",
       " 'rotation angle': 14,\n",
       " 'service': 47,\n",
       " 'support details type': 36,\n",
       " 'tilt angle': 73,\n",
       " 'video format': 6,\n",
       " 'viewable size metric': 28,\n",
       " 'vsync rate at max res': 30,\n",
       " 'built-in devices': 69,\n",
       " 'game mode': 5,\n",
       " 'osd languages': 43,\n",
       " 'battery': 3,\n",
       " 'installed qty': 4,\n",
       " 'exterior color': 3,\n",
       " 'tv type': 4,\n",
       " 'security': 11,\n",
       " 'item condition': 7,\n",
       " 'ports': 19,\n",
       " 'size': 23,\n",
       " 'display  color support': 4,\n",
       " 'dot pitch  pixel pitch': 3,\n",
       " 'header  model': 3,\n",
       " 'service  support type': 6,\n",
       " 'product information': 71,\n",
       " 'weight (approximate)': 44,\n",
       " '162 million colors': 3,\n",
       " '19inch screen': 3,\n",
       " 'magicbright2': 3,\n",
       " 'magiccolor': 5,\n",
       " '2ms response time': 9,\n",
       " 'aspect control function': 11,\n",
       " 'assembly required': 5,\n",
       " 'country of origin': 19,\n",
       " 'digital input': 10,\n",
       " 'recycled': 16,\n",
       " 'humidity': 6,\n",
       " 'os compatibility': 7,\n",
       " 'temperature': 10,\n",
       " 'usb hub': 5,\n",
       " 'wall mount': 3,\n",
       " 'contents': 3,\n",
       " 'model number': 9,\n",
       " 'series': 12,\n",
       " 'warranty': 70,\n",
       " 'enhanced screen performance': 6,\n",
       " 'its easy to be green': 5,\n",
       " 'taking care of business never looked so stylish': 3,\n",
       " 'wide compatibility for today and tomorrow': 4,\n",
       " 'input power': 3,\n",
       " 'pc': 8,\n",
       " 'screen brightness': 12,\n",
       " 'sdc.string.response time (num)': 8,\n",
       " 'vesa mount': 11,\n",
       " 'color effect': 3,\n",
       " 'magicbrightâ³': 3,\n",
       " 'magicspeed 5ms': 3,\n",
       " 'energy star compliant': 14,\n",
       " 'maximum power consumption': 17,\n",
       " 'osd language': 3,\n",
       " 'power cable': 4,\n",
       " 'processor': 6,\n",
       " 'speaker type': 10,\n",
       " 'typical power consumption': 17,\n",
       " 'includes': 3,\n",
       " 'dimensions wxhxd': 16,\n",
       " 'international buyers': 10,\n",
       " 'resolution description': 11,\n",
       " 'room': 1,\n",
       " 'material': 4,\n",
       " 'operating power consumption': 55,\n",
       " 'touchscreen type': 43,\n",
       " 'ecomode': 8,\n",
       " 'multifunctional base design': 8,\n",
       " 'multimedia enhanced': 9,\n",
       " 'optisync digitalanalog inputs expand your connectivity': 9,\n",
       " 'stylish slimbezel design': 8,\n",
       " 'widescreen aspect ratio with 1920 x 1080 resolution': 7,\n",
       " 'conserve energy': 3,\n",
       " 'enjoy the view': 3,\n",
       " 'give your productivity a boost with the comfortable design and': 1,\n",
       " 'work comfortably': 3,\n",
       " 'swivel angle': 30,\n",
       " 'a full 14day warranty from the time the item is received': 5,\n",
       " 'ebay': 6,\n",
       " 'for more information on global shipping click': 5,\n",
       " 'auto switch to rear view': 7,\n",
       " 'brightness and contrast': 8,\n",
       " 'camera tripod screw hole': 11,\n",
       " 'image flip l  r and u  d': 10,\n",
       " 'connectivity': 6,\n",
       " 'dimensions hxwxd': 3,\n",
       " 'damaged items': 8,\n",
       " 'defective item': 8,\n",
       " 'exchanges': 8,\n",
       " 'international customers': 8,\n",
       " 'international returns': 8,\n",
       " 'refunds': 8,\n",
       " 'replacement': 8,\n",
       " 'resellers  bulk orders policy': 8,\n",
       " 'return policy': 22,\n",
       " 'return shipping': 8,\n",
       " 'returning items': 8,\n",
       " 'television returns': 8,\n",
       " 'analog tuner': 3,\n",
       " 'battery included': 5,\n",
       " 'component video': 7,\n",
       " 'digital audio output': 5,\n",
       " 'digital tuner': 3,\n",
       " 'native contrast ratio': 7,\n",
       " 'number of hdmi ports': 5,\n",
       " 'remote control': 11,\n",
       " 'scan format': 4,\n",
       " 'video signal standard': 5,\n",
       " 'created by': 8,\n",
       " 'economy': 20,\n",
       " 'expedited': 20,\n",
       " 'oneday': 20,\n",
       " 'standard': 33,\n",
       " 'magic picture controls': 3,\n",
       " 'environmental compliance': 5,\n",
       " 'input signal sync': 6,\n",
       " 'input signal video': 6,\n",
       " 'synchronization range horizontal': 6,\n",
       " 'synchronization range vertical': 6,\n",
       " 'assured compatibility': 9,\n",
       " 'designed with the environment in mind': 9,\n",
       " 'easy to set up and use': 9,\n",
       " 'improve how you view your work': 4,\n",
       " 'position the monitor for the way you work best': 9,\n",
       " 'display size': 7,\n",
       " 'graphics processing type': 39,\n",
       " 'condition details': 11,\n",
       " 'item description': 11,\n",
       " 'item id': 11,\n",
       " 'our testing': 11,\n",
       " 'sale includes': 11,\n",
       " 'video inputs1': 15,\n",
       " 'video inputs2': 15,\n",
       " 'input video': 5,\n",
       " 'assembled depth in': 4,\n",
       " 'assembled width in': 5,\n",
       " 'certifications and listings': 4,\n",
       " 'product height in': 4,\n",
       " 'product weight lb': 3,\n",
       " 'product width in': 4,\n",
       " 'connectivity details type': 28,\n",
       " 'frequency required': 28,\n",
       " 'multitouch display': 4,\n",
       " 'output power channel': 7,\n",
       " 'speakers configuration': 8,\n",
       " 'support details service included': 25,\n",
       " 'system requirements': 11,\n",
       " 'audio input': 13,\n",
       " 'chipset manufacturer': 3,\n",
       " 'chipset model': 3,\n",
       " 'hyperthreading': 3,\n",
       " 'maximum memory': 3,\n",
       " 'memory standard': 3,\n",
       " 'number of usb 30 ports': 4,\n",
       " 'operating system architecture': 3,\n",
       " 'pointing device type': 4,\n",
       " 'processor manufacturer': 4,\n",
       " 'contrast ratio to 1': 8,\n",
       " 'number of dvi': 7,\n",
       " 'screen aspect ratio': 7,\n",
       " 'viewing angle horizontal': 5,\n",
       " 'viewing angle vertical': 12,\n",
       " 'display screen coating': 23,\n",
       " 'image color temperature': 14,\n",
       " 'manufacturer selling program': 8,\n",
       " 'tft technology': 22,\n",
       " 'video bandwidth': 16,\n",
       " 'additional video inputs': 1,\n",
       " 'epeat level': 18,\n",
       " 'microsoft certifications': 8,\n",
       " 'interface type': 5,\n",
       " 'vertical frequency': 3,\n",
       " 'overall value delivered': 9,\n",
       " 'hard drive capacity': 50,\n",
       " 'operating system edition': 51,\n",
       " 'release year': 10,\n",
       " 'support details location': 9,\n",
       " 'controls  adjustments': 22,\n",
       " 'power consumption stand by sleep': 9,\n",
       " 'contact us': 6,\n",
       " 'cpu': 4,\n",
       " 'hard drive': 9,\n",
       " 'optical drive': 6,\n",
       " 'sales tax': 3,\n",
       " 'screen': 3,\n",
       " 'shipping': 4,\n",
       " 'sound': 5,\n",
       " 'enhanced compatibility': 5,\n",
       " 'see more do more': 12,\n",
       " 'smart design': 6,\n",
       " 'technology with the environment in mind': 5,\n",
       " 'total hp computing experience': 7,\n",
       " 'responsible': 3,\n",
       " 'eizo lcd utility disk': 5,\n",
       " 'signal cables': 5,\n",
       " 'chipset': 4,\n",
       " 'ram': 6,\n",
       " 'speed': 3,\n",
       " 'wireless': 3,\n",
       " 'touchscreen information': 15,\n",
       " 'ease of ownership': 4,\n",
       " 'precise vivid industrystandard color': 4,\n",
       " 'superb user experience': 5,\n",
       " 'uncompromising screen performance': 4,\n",
       " 'a/c adapter': 14,\n",
       " 'included cables': 14,\n",
       " 'webcam': 6,\n",
       " 'kvm switch': 10,\n",
       " 'kvm technology': 1,\n",
       " 'interface provided': 10,\n",
       " 'networking': 9,\n",
       " 'video output': 7,\n",
       " 'media drive': 5,\n",
       " 'included monitor accessories': 5,\n",
       " 'product brand': 2,\n",
       " 'product condition': 3,\n",
       " 'voltage': 10,\n",
       " 'media': 7,\n",
       " 'combined shipping take 3 off shipping': 9,\n",
       " 'fast response time 5ms': 3,\n",
       " 'slim and emotional design': 3,\n",
       " 'supports fullhd and movie wide': 3,\n",
       " 'vivid and clear image for multimedia contents': 3,\n",
       " 'engaging touch experience': 3,\n",
       " 'outstanding connectivity': 3,\n",
       " 'peace of mind': 4,\n",
       " 'sdc.string.screen size (mined)': 5,\n",
       " 'sdcstringresponse time num': 3,\n",
       " 'wireless technology': 1,\n",
       " 'colors': 5,\n",
       " 'power': 14,\n",
       " 'viewing angles': 13,\n",
       " 'ascr asus smart contrast ratio technology': 3,\n",
       " 'care for the earth': 5,\n",
       " 'audio system': 8,\n",
       " 'color depth ja': 5,\n",
       " 'comments ja': 4,\n",
       " 'connectivity details length': 8,\n",
       " 'connector type': 12,\n",
       " 'details ja': 4,\n",
       " 'diagonal size cm': 8,\n",
       " 'dimensions  weight shipping': 8,\n",
       " 'display format': 8,\n",
       " 'hdcp compatible': 6,\n",
       " 'hdmi ports qty': 8,\n",
       " 'lcd display technology': 3,\n",
       " 'led backlight type': 7,\n",
       " 'output power total': 4,\n",
       " 'pc interface': 9,\n",
       " 'shipping depth': 5,\n",
       " 'shipping height': 7,\n",
       " 'speaker form factor': 8,\n",
       " 'speaker qty': 7,\n",
       " 'speaker system details': 8,\n",
       " 'stands  mounts': 6,\n",
       " 'support details full contract period': 13,\n",
       " 'targeting': 8,\n",
       " 'television': 8,\n",
       " 'tv tuner': 8,\n",
       " 'tv tuner presence': 9,\n",
       " 'video interface': 9,\n",
       " 'video wall components qty': 3,\n",
       " 'audio outputs': 12,\n",
       " 'brightness cdmâ²': 12,\n",
       " 'input power source': 12,\n",
       " 'pixel pitch mm': 6,\n",
       " 'resolution pixels': 12,\n",
       " 'screen size diagonal': 12,\n",
       " 'viewing angle degrees': 12,\n",
       " 'panel backlight': 3,\n",
       " 'number of computers supported': 9,\n",
       " 'dvd media': 1,\n",
       " 'operating system ****': 2,\n",
       " 'stand included:': 1,\n",
       " 'color depth': 3,\n",
       " 'user controls': 4,\n",
       " 'sync': 10,\n",
       " 'rms output power': 4,\n",
       " 'assembled height in': 4,\n",
       " 'display area': 9,\n",
       " 'vesa wall mounting': 3,\n",
       " 'viewing angle hv': 4,\n",
       " 'frequency': 9,\n",
       " 'power management': 23,\n",
       " 'sale does not include': 7,\n",
       " 'daisy chain': 5,\n",
       " 'ps2 port': 4,\n",
       " 'backlight': 23,\n",
       " 'backlight life': 6,\n",
       " 'consumption': 8,\n",
       " 'digital': 5,\n",
       " 'mac': 5,\n",
       " 'optimum resolution': 12,\n",
       " 'panel surface': 8,\n",
       " 'physical with stand': 7,\n",
       " 'physical without stand': 7,\n",
       " 'regulations': 9,\n",
       " 'doa dead on arrival': 4,\n",
       " 'â condition': 4,\n",
       " 'â manufacturer': 4,\n",
       " 'â model number': 4,\n",
       " 'â quantity included': 4,\n",
       " 'â sales includes': 4,\n",
       " 'â warranty': 4,\n",
       " 'brightness stabilization': 8,\n",
       " 'digital uniformity equalizer': 8,\n",
       " 'grayscale tones': 10,\n",
       " 'height adjustment range': 22,\n",
       " 'hole spacing vesa standard': 15,\n",
       " 'humidity rh non condensing': 4,\n",
       " 'input terminals': 21,\n",
       " 'monitor type': 8,\n",
       " 'pivot': 21,\n",
       " 'power requirements': 26,\n",
       " 'power save mode': 26,\n",
       " 'preset modes': 18,\n",
       " 'recommended brightness for calibration': 6,\n",
       " 'selfcorrection': 4,\n",
       " 'standby mode': 6,\n",
       " 'swivel': 23,\n",
       " 'usb function': 3,\n",
       " 'viewing angles h  v typical': 15,\n",
       " 'wide gamut coverage typical': 5,\n",
       " 'a clear view to productivity': 4,\n",
       " 'stylish sturdy design': 4,\n",
       " 'monitor size': 3,\n",
       " 'virtual 4k': 10,\n",
       " 'input port': 3,\n",
       " 'dell warranty expiration date': 1,\n",
       " 'memory (ram)': 1,\n",
       " 'primary drive': 1,\n",
       " 'analog scanning frequency h  v': 11,\n",
       " 'auto ecoview  brightregulator': 12,\n",
       " 'digital scanning frequency h  v': 14,\n",
       " 'display size h x v': 14,\n",
       " 'panel size': 8,\n",
       " 'number of hdmi inputs': 6,\n",
       " 'hard drive interface': 4,\n",
       " 'ac power adapter': 3,\n",
       " 'stand/base': 3,\n",
       " 'additional information': 2,\n",
       " 'color depth number of colors': 5,\n",
       " 'color gamut cie1976': 3,\n",
       " 'contrast ratio typ': 3,\n",
       " 'ddcci': 5,\n",
       " 'dvid': 4,\n",
       " 'dynamic cr dcr': 3,\n",
       " 'epeat': 5,\n",
       " 'flicker safe': 3,\n",
       " 'hdcp': 6,\n",
       " 'intelligent auto auto resolution': 3,\n",
       " 'key lock': 3,\n",
       " 'original ratio': 3,\n",
       " 'picture mode': 3,\n",
       " 'reader mode': 3,\n",
       " 'super energy saving': 3,\n",
       " 'surface treatment': 5,\n",
       " 'contrast ratio dfc': 3,\n",
       " 'input': 12,\n",
       " 'viewing angle cr10': 5,\n",
       " 'interface output': 6,\n",
       " 'max vsync rate': 3,\n",
       " 'projector': 6,\n",
       " 'helping you go green': 3,\n",
       " 'sound system': 3,\n",
       " 'total hard drive capacity': 3,\n",
       " 'dimensions landscape w x h x d': 11,\n",
       " 'dimensions without stand w x h x d': 11,\n",
       " 'function': 11,\n",
       " 'net weight': 13,\n",
       " 'net weight without stand': 10,\n",
       " 'operating humidity rh non condensing': 11,\n",
       " 'others': 12,\n",
       " 'portsinterfaces': 4,\n",
       " 'supplied accessories': 12,\n",
       " 'sync formats': 8,\n",
       " 'ecoview sense': 5,\n",
       " 'output terminals': 7,\n",
       " 'sound adjustment': 7,\n",
       " 'active display size h v': 3,\n",
       " 'certifications and standards': 3,\n",
       " 'other settings': 4,\n",
       " 'tilt  swivel  pivot': 3,\n",
       " 'usb ports  standard': 4,\n",
       " 'viewing angles h v': 4,\n",
       " 'diagonal screen size': 14,\n",
       " 'image brightness': 14,\n",
       " 'inputs': 16,\n",
       " 'vesa compatible wall mount': 14,\n",
       " 'additional accesories': 11,\n",
       " 'battery plates': 11,\n",
       " 'connection cables': 11,\n",
       " 'depth mm': 11,\n",
       " 'height mm': 11,\n",
       " 'power adaptors': 11,\n",
       " 'product': 12,\n",
       " 'sunshade': 11,\n",
       " 'video outputs': 11,\n",
       " 'weight g': 11,\n",
       " 'width mm': 11,\n",
       " 'package type': 4,\n",
       " '60 day nondoa warranty': 12,\n",
       " 'internet connection type': 1,\n",
       " \"what's included\": 1,\n",
       " 'product depth with stand': 5,\n",
       " 'advanced features': 8,\n",
       " 'analog input': 5,\n",
       " 'colors supported': 8,\n",
       " 'epeat rated': 4,\n",
       " 'product depth without stand': 4,\n",
       " 'tm': 3,\n",
       " 'dot pitch': 5,\n",
       " 'input type': 5,\n",
       " 'number of ps2 ports': 3,\n",
       " 'number of usb ports': 5,\n",
       " 'dimensions whd': 3,\n",
       " 'viewing angle (horizontal/vertical)': 3,\n",
       " 'weight lbs': 3,\n",
       " 'part number': 1,\n",
       " 'builtin stereo speakers': 5,\n",
       " 'composite video': 6,\n",
       " 'kvm': 7,\n",
       " 'miscellaneous cdw': 3,\n",
       " 'output': 8,\n",
       " 'warranty - 3': 1,\n",
       " 'power savings mode': 3,\n",
       " 'color adjustment': 4,\n",
       " 'ecoview settings': 3,\n",
       " 'screen adjustment': 4,\n",
       " 'serial no.': 4,\n",
       " 'commercial display': 3,\n",
       " 'media player': 5,\n",
       " 'speaker system': 5,\n",
       " 'excellent visual performance': 4,\n",
       " 'stylish and userfriendly design for modern sophistication': 4,\n",
       " 'height rack units': 8,\n",
       " '3d glasses': 5,\n",
       " '3d requirements': 4,\n",
       " 'analogdigital': 4,\n",
       " 'basic': 4,\n",
       " 'eco mode': 6,\n",
       " 'light source': 5,\n",
       " 'osd': 5,\n",
       " 'choose your connection': 4,\n",
       " 'environmental footprint': 4,\n",
       " 'home is where the action is': 4,\n",
       " 'master performer': 4,\n",
       " 'slim and elegant design': 4,\n",
       " 'environmental design': 5,\n",
       " 'work smart not hard': 5,\n",
       " 'built in speakers': 3,\n",
       " 'general condition': 2,\n",
       " 'general description': 2,\n",
       " 'serial number': 5,\n",
       " 'quantity included': 4,\n",
       " 'reference': 4,\n",
       " 'sales includes': 4,\n",
       " 'colour': 5,\n",
       " 'colour support': 4,\n",
       " 'audio line in': 3,\n",
       " 'hard drive rpm': 3,\n",
       " 'number of usb 20 ports': 3,\n",
       " 'optical media supported': 3,\n",
       " 'power input': 3,\n",
       " 'no stand is included': 1,\n",
       " '1000001 dynamic contrast ratio': 3,\n",
       " 'certified green': 3,\n",
       " 'size (gauge)': 2,\n",
       " 'process notes': 2,\n",
       " 'epeat qualified': 3,\n",
       " 'gaming series': 3,\n",
       " 'ips': 3,\n",
       " 'hp': 1,\n",
       " 'upc code': 3,\n",
       " 'analog': 4,\n",
       " 'integrated speakers': 5,\n",
       " 'power consumption max': 3,\n",
       " 'advanced technology for amazing visuals': 3,\n",
       " 'contemporary style unbeatable value': 3,\n",
       " 'hd cinematic widescreen': 3,\n",
       " 'model variations': 3,\n",
       " 'network  internet multimedia': 3,\n",
       " 'shipping width': 3,\n",
       " 'usb ports qty': 3,\n",
       " 'horizontal': 4,\n",
       " 'color temperature': 3,\n",
       " 'speaker': 3,\n",
       " 'description1': 1,\n",
       " 'approved agency': 5,\n",
       " 'peripherals': 5,\n",
       " 'power connection': 5,\n",
       " 'rs232': 5,\n",
       " 'dimensions w x d x h': 4,\n",
       " 'screen size (in.)': 1,\n",
       " 'compatible brand': 3,\n",
       " 'unit damage': 2,\n",
       " 'normal signs of use are present on these units': 4,\n",
       " 'usb ports': 4,\n",
       " \"what's in the box\": 1,\n",
       " '100 lakeview pkwy vernon hills il 60061': 3,\n",
       " 'brightness cdm2': 3,\n",
       " 'number of ps/2 ports': 2,\n",
       " 'signal input': 4,\n",
       " 'screen type': 1,\n",
       " 'screen format': 1,\n",
       " 'guides included': 1,\n",
       " 'original box': 1,\n",
       " 'manufacturer/supplier: dell, inc': 1,\n",
       " 'manufacture date': 1,\n",
       " 'item #': 1,\n",
       " 'functionality': 1,\n",
       " 'item appearance': 1,\n",
       " 'missing accessories': 1,\n",
       " 'compatiblity': 1,\n",
       " 'horizontal freq.': 1,\n",
       " 'vertical freq.': 1,\n",
       " 'adjustment': 1}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources2attributes.get('www.ebay.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove attributes that appear less than 10% of the times in the products of a source\n",
    "for source in sources2attributes:\n",
    "    for attribute in list(sources2attributes.get(source).keys()):\n",
    "        if sources2attributes.get(source).get(attribute) < 0.1 * len(sources2monitors.get(source)):\n",
    "            sources2attributes.get(source).pop(attribute)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<page title>': 4281,\n",
       " 'brand': 2148,\n",
       " 'condition': 2405,\n",
       " 'display technology': 3247,\n",
       " 'screen size': 3495,\n",
       " 'aspect ratio': 2806,\n",
       " 'brightness': 2173,\n",
       " 'contrast ratio': 2733,\n",
       " 'max resolution': 1205,\n",
       " 'model': 3055,\n",
       " 'mpn': 2400,\n",
       " 'response time': 2715,\n",
       " 'video inputs': 1654,\n",
       " 'max. resolution': 1206,\n",
       " 'system': 1442,\n",
       " 'upc': 1762,\n",
       " 'features': 909,\n",
       " 'manufacturer': 580,\n",
       " 'maximum resolution': 508}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources2attributes.get('www.ebay.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a dict source2dataframe where each dataframe contains the attributes of the products of a source\n",
    "sources2dataframe = {}\n",
    "\n",
    "for source in sources2attributes:\n",
    "    attributes = list(sources2attributes.get(source).keys())\n",
    "    df = pd.DataFrame(columns=attributes)\n",
    "    for monitor in sources2monitors.get(source):\n",
    "        row = []\n",
    "        for attribute in attributes:\n",
    "            if attribute in monitor:\n",
    "                row.append(monitor.get(attribute))\n",
    "            else:\n",
    "                row.append(None)\n",
    "        df.loc[len(df)] = row\n",
    "    sources2dataframe[source] = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>&lt;page title&gt;</th>\n",
       "      <th>brand</th>\n",
       "      <th>condition</th>\n",
       "      <th>display technology</th>\n",
       "      <th>screen size</th>\n",
       "      <th>aspect ratio</th>\n",
       "      <th>brightness</th>\n",
       "      <th>contrast ratio</th>\n",
       "      <th>max resolution</th>\n",
       "      <th>model</th>\n",
       "      <th>mpn</th>\n",
       "      <th>response time</th>\n",
       "      <th>video inputs</th>\n",
       "      <th>max. resolution</th>\n",
       "      <th>system</th>\n",
       "      <th>upc</th>\n",
       "      <th>features</th>\n",
       "      <th>manufacturer</th>\n",
       "      <th>maximum resolution</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CTL Black 17\" LCD Monitor 171LX w Power and VG...</td>\n",
       "      <td>CTL</td>\n",
       "      <td>Used: An item that has been used previously. T...</td>\n",
       "      <td>LCD</td>\n",
       "      <td>17\"</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HP Compaq LE1711 17 inch LCD Monitor New Item ...</td>\n",
       "      <td>HP</td>\n",
       "      <td>None</td>\n",
       "      <td>LCD</td>\n",
       "      <td>17\"</td>\n",
       "      <td>5:4</td>\n",
       "      <td>250 cd/mÂ²</td>\n",
       "      <td>1000</td>\n",
       "      <td>1280 x 1024</td>\n",
       "      <td>LE1711</td>\n",
       "      <td>EM886A8, EM886A8#ABA</td>\n",
       "      <td>5 ms</td>\n",
       "      <td>VGA D-Sub</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dell 24\" Widescreen LED Backlight 1080p Full H...</td>\n",
       "      <td>None</td>\n",
       "      <td>New: A brand-new, unused, unopened, undamaged ...</td>\n",
       "      <td>LED LCD</td>\n",
       "      <td>24\"</td>\n",
       "      <td>16:9</td>\n",
       "      <td>250 cd/mÂ²</td>\n",
       "      <td>1000</td>\n",
       "      <td>None</td>\n",
       "      <td>E2414H</td>\n",
       "      <td>320-9776</td>\n",
       "      <td>5 ms</td>\n",
       "      <td>DVI-D</td>\n",
       "      <td>1920 x 1080</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>884116123446</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>New Samsung T27D390 Series 3 LED HDTV Monitor ...</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>New: A brand-new, unused, unopened, undamaged ...</td>\n",
       "      <td>LED LCD</td>\n",
       "      <td>27\"</td>\n",
       "      <td>16:9</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Acer 17\" Computer Monitor | eBay</td>\n",
       "      <td>Acer</td>\n",
       "      <td>Used: An item that has been used previously. T...</td>\n",
       "      <td>None</td>\n",
       "      <td>17\"</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Desktop</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        <page title>    brand  \\\n",
       "0  CTL Black 17\" LCD Monitor 171LX w Power and VG...      CTL   \n",
       "1  HP Compaq LE1711 17 inch LCD Monitor New Item ...       HP   \n",
       "2  Dell 24\" Widescreen LED Backlight 1080p Full H...     None   \n",
       "3  New Samsung T27D390 Series 3 LED HDTV Monitor ...  Samsung   \n",
       "4                   Acer 17\" Computer Monitor | eBay     Acer   \n",
       "\n",
       "                                           condition display technology  \\\n",
       "0  Used: An item that has been used previously. T...                LCD   \n",
       "1                                               None                LCD   \n",
       "2  New: A brand-new, unused, unopened, undamaged ...            LED LCD   \n",
       "3  New: A brand-new, unused, unopened, undamaged ...            LED LCD   \n",
       "4  Used: An item that has been used previously. T...               None   \n",
       "\n",
       "  screen size aspect ratio  brightness contrast ratio max resolution   model  \\\n",
       "0         17\"         None        None           None           None    None   \n",
       "1         17\"          5:4  250 cd/mÂ²           1000    1280 x 1024  LE1711   \n",
       "2         24\"         16:9  250 cd/mÂ²           1000           None  E2414H   \n",
       "3         27\"         16:9        None           None           None    None   \n",
       "4         17\"         None        None           None           None    None   \n",
       "\n",
       "                    mpn response time video inputs max. resolution   system  \\\n",
       "0                  None          None         None            None     None   \n",
       "1  EM886A8, EM886A8#ABA          5 ms    VGA D-Sub            None     None   \n",
       "2              320-9776          5 ms        DVI-D     1920 x 1080  Desktop   \n",
       "3                  None          None         None            None     None   \n",
       "4                  None          None         None            None  Desktop   \n",
       "\n",
       "            upc features manufacturer maximum resolution  \n",
       "0          None     None         None               None  \n",
       "1          None     None         None               None  \n",
       "2  884116123446     None         None               None  \n",
       "3          None     None         None               None  \n",
       "4          None     None         None               None  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources2dataframe.get('www.ebay.com').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_attribute_id</th>\n",
       "      <th>target_attribute_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>www.planet-computer.it//dimensione visibile or...</td>\n",
       "      <td>screen_size_horizontal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>www.pc-canada.com//product name</td>\n",
       "      <td>screen_size_diagonal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 source_attribute_id   target_attribute_name\n",
       "0  www.planet-computer.it//dimensione visibile or...  screen_size_horizontal\n",
       "1                    www.pc-canada.com//product name    screen_size_diagonal"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load the mediated schema from monitor_mediated_schema.txt\n",
    "with open('dataset/monitor_mediated_schema.txt', 'r') as f:\n",
    "    attributes = [line.strip() for line in f]\n",
    "mediated_schema = pd.DataFrame(columns=attributes)\n",
    "\n",
    "# load the ground truth from monitor_schema_matching_labelled_data.csv\n",
    "ground_truth = pd.read_csv('dataset/monitor_schema_matching_labelled_data.csv')\n",
    "ground_truth.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a dict source2ground_truth using a list of pairs (source_attribute_id, target_attribute_name)\n",
    "sources2ground_truth = {}\n",
    "\n",
    "for source, target in ground_truth.values:\n",
    "    # split the string to get the source and the attribute\n",
    "    source_attribute = source.split('//')\n",
    "    if source_attribute[0] in sources2ground_truth:\n",
    "        sources2ground_truth.get(source_attribute[0]).append((source_attribute[1], target))\n",
    "    else:\n",
    "        sources2ground_truth[source_attribute[0]] = [(source_attribute[1], target)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('display format', 'supported_resolution'),\n",
       " ('scan format', 'supported_resolution'),\n",
       " ('display response time', 'response_time'),\n",
       " ('image aspect ratio', 'supported_aspect_ratio'),\n",
       " ('dvid', 'dvid_port_quantity'),\n",
       " ('dvid', 'has_dvid_port'),\n",
       " ('vga input', 'has_vga_port'),\n",
       " ('microphone', 'has_microphone'),\n",
       " ('support details full contract period', 'warranty_duration'),\n",
       " ('number of dvi', 'dvi_port_quantity'),\n",
       " ('bluetooth', 'bluetooth'),\n",
       " ('screen refresh rate', 'vertical_refresh_rate_max'),\n",
       " ('screen refresh rate', 'vertical_refresh_rate_range')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources2ground_truth.get('www.ebay.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 / 26 source: ca.pcpartpicker.com\n",
      "2 / 26 source: catalog.com\n",
      "3 / 26 source: ce.yikus.com\n",
      "4 / 26 source: www.best-deal-items.com\n",
      "5 / 26 source: www.cleverboxes.com\n",
      "6 / 26 source: www.ebay.com\n",
      "7 / 26 source: www.getprice.com.au\n",
      "8 / 26 source: www.hardware-planet.it\n",
      "9 / 26 source: www.imldirect.it\n",
      "10 / 26 source: www.itenergy.co.uk\n",
      "11 / 26 source: www.jrlinton.co.uk\n",
      "12 / 26 source: www.kingsfieldcomputers.co.uk\n",
      "13 / 26 source: www.makingbuyingeasy.co.uk\n",
      "14 / 26 source: www.mediashopuk.com\n",
      "15 / 26 source: www.mrhightech.com\n",
      "16 / 26 source: www.nexus-t.co.uk\n",
      "17 / 26 source: www.odsi.co.uk\n",
      "18 / 26 source: www.officedepot.com\n",
      "19 / 26 source: www.ohc24.ch\n",
      "20 / 26 source: www.pc-canada.com\n",
      "21 / 26 source: www.pcconnection.com\n",
      "22 / 26 source: www.planet-computer.it\n",
      "23 / 26 source: www.shopmania.com\n",
      "24 / 26 source: www.softwarecity.ca\n",
      "25 / 26 source: www.vology.com\n",
      "26 / 26 source: www.xpcpro.com\n"
     ]
    }
   ],
   "source": [
    "# Instantiate matcher and run\n",
    "# Coma requires java to be installed on your machine\n",
    "matcher = Coma(max_n=10, strategy=\"COMA_OPT_INST\")\n",
    "\n",
    "output = pd.DataFrame(columns=['source_attribute_id', 'target_attribute_name'])\n",
    "i = 1\n",
    "\n",
    "for source in sources2dataframe:\n",
    "    print(i,\"/\",len(sources2dataframe), \"source:\", source)\n",
    "    df = sources2dataframe.get(source)\n",
    "    matches = valentine_match(df, mediated_schema, matcher)\n",
    "    for (_,a1), (_,a2) in matches:\n",
    "        output.loc[len(output)] = [source+'//'+a1, a2]\n",
    "    i += 1\n",
    "        \n",
    "pd.DataFrame.to_csv(output, 'coma_output_predictions.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 / 26  source: ca.pcpartpicker.com\n",
      "2 / 26  source: catalog.com\n",
      "3 / 26  source: ce.yikus.com\n",
      "4 / 26  source: www.best-deal-items.com\n",
      "5 / 26  source: www.cleverboxes.com\n",
      "6 / 26  source: www.ebay.com\n",
      "7 / 26  source: www.getprice.com.au\n",
      "8 / 26  source: www.hardware-planet.it\n",
      "9 / 26  source: www.imldirect.it\n",
      "10 / 26  source: www.itenergy.co.uk\n",
      "11 / 26  source: www.jrlinton.co.uk\n",
      "12 / 26  source: www.kingsfieldcomputers.co.uk\n",
      "13 / 26  source: www.makingbuyingeasy.co.uk\n",
      "14 / 26  source: www.mediashopuk.com\n",
      "15 / 26  source: www.mrhightech.com\n",
      "16 / 26  source: www.nexus-t.co.uk\n",
      "17 / 26  source: www.odsi.co.uk\n",
      "18 / 26  source: www.officedepot.com\n",
      "19 / 26  source: www.ohc24.ch\n",
      "20 / 26  source: www.pc-canada.com\n",
      "21 / 26  source: www.pcconnection.com\n",
      "22 / 26  source: www.planet-computer.it\n",
      "23 / 26  source: www.shopmania.com\n",
      "24 / 26  source: www.softwarecity.ca\n",
      "25 / 26  source: www.vology.com\n",
      "26 / 26  source: www.xpcpro.com\n"
     ]
    }
   ],
   "source": [
    "# Instantiate matcher and run\n",
    "\n",
    "output = pd.DataFrame(columns=['source_attribute_id', 'target_attribute_name'])\n",
    "i = 1\n",
    "\n",
    "for source in sources2dataframe:\n",
    "    matcher = Cupid(w_struct=0.4, leaf_w_struct=0.4, th_accept=0.9, parallelism=16)\n",
    "    print(i,\"/\",len(sources2dataframe), \" source:\", source)\n",
    "    df = sources2dataframe.get(source)\n",
    "    matches = valentine_match(df, mediated_schema, matcher)\n",
    "    for (_,a1), (_,a2) in matches:\n",
    "        output.loc[len(output)] = [source+'//'+a1, a2]\n",
    "    i += 1\n",
    "    \n",
    "pd.DataFrame.to_csv(output, 'cupid_output_predictions.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "taxi",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}