mining_foreign_domain_ood.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /home/ishalyminov/nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "\n",
    "import os\n",
    "import json\n",
    "import re\n",
    "import logging\n",
    "import zipfile\n",
    "\n",
    "import requests\n",
    "\n",
    "from utils.nlp_utils import profanity_check\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DSTC1_FOLDER = 'data/dstc1'\n",
    "DSTC1_URL = 'https://www.dl.dropboxusercontent.com/s/mlqqtdxunk2rlwm/dstc1.zip?dl=0'\n",
    "if not os.path.exists(DSTC1_FOLDER):\n",
    "    r = requests.get(DSTC1_URL)\n",
    "    with open('dstc1.zip', 'wb') as dstc1_out:\n",
    "        dstc1_out.write(r.content)\n",
    "    with zipfile.ZipFile('dstc1.zip') as zip_ref:\n",
    "        zip_ref.extractall('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'dstc_data_test3',\n",
       " 'dstc_data_train3_v00',\n",
       " 'test1',\n",
       " 'test2',\n",
       " 'test4',\n",
       " 'train1a',\n",
       " 'train1b',\n",
       " 'train1c',\n",
       " 'train2']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(DSTC1_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dstc1_utterances = set([])\n",
    "dstc1_start_utterances = set([])\n",
    "for root, dirs, files in os.walk(DSTC1_FOLDER):\n",
    "    for filename in files:\n",
    "        if re.match('.+\\.labels\\.json$', filename):\n",
    "            with open(os.path.join(root, filename)) as session_in:\n",
    "                session_json = json.load(session_in)\n",
    "                turns = session_json.get('turns', [])\n",
    "                if len(turns) and turns[0]['transcription-status'] == 'transcribed':\n",
    "                    dstc1_start_utterances.add(turns[0]['transcription'])\n",
    "                dstc1_utterances.update([turn['transcription']\n",
    "                                         for turn in turns\n",
    "                                         if turn.get('transcription-status') == 'transcribed'])\n",
    "                \n",
    "dstc1_utterances = [utt for utt in dstc1_utterances if utt]\n",
    "dstc1_start_utterances = [utt for utt in dstc1_start_utterances if utt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "968\n",
      "sixty one c leaving second and grand going downtown at ten oclock\n",
      "ah i would like to know how to make paytence\n",
      "can you tell me what time the next fifty one c is leaving downtown pittsburgh\n",
      "i need the bus schedule for the sixty one the ebs\n",
      "im trying to find out a bus i can take from downtown to go to kennywood park\n",
      "i want to know what time the sixty one c leaves from downtown pittsburg going toward\n",
      "wood penn hills from forbes and murray to carnegie mellon\n",
      "goodbye schedule for 59a leaving swissvale forest hills north versailles\n",
      "yes i need to know when the next sixty one a is leaving dynamo way going to swissvale station\n",
      "can you tell me what time the last fifty five m comes\n",
      "where is the next bus from squirrel hell to carnegie mellon university\n",
      "know what time the sixty one c come to duquesne and go to mckeesport\n",
      "id like to know about the sixty one for the morning hours %\n",
      "i wanted information on bus scheduling for sixty one c please\n",
      "when is there a twenty eight s at eleven oclock pm from the airport\n",
      "i need to know what bus to get to thirty_nine_hundred block of Beechwood_boulevard:p\n",
      "id like to know the scheduling of fifty four c\n",
      "i want to know the last time the sixty one c runs through homestead\n",
      "i need to know what time the sixty one c leaves pittsburgh going to mckeesport\n",
      "is the 61a:b running on it's regular route on monday:t\n",
      "id like to know when the schedule is for the sixty f\n",
      "i need to know when the next two sixty one a busses leave downtown going to wilkinsburg\n",
      "/um/ yes /um/ i wanna <whe=> when <the a=> the next 81a:b /mumble/ shows up\n",
      "can you tell me what time the fifty four c is due in oakland going to south hills village\n",
      "ah id like to know the fifty four c bus schedule leaving\n",
      "homewood know when the sixty one mckee comes to fifth at smithfield\n",
      "3l what bus that take to webster avenue to north side from carrick\n",
      "bellway penn at sixteenth north fayette av the south hills junction going to oakland\n",
      "i want to know time the fifty four come to oakland going to south hills junction\n",
      "64a eighth going to east liberty from fourth to murray\n",
      "id like the next bus from squirrel hill to downtown\n",
      "can you tell me what time the next sixty one c leaves\n",
      "id like to know the fifty four c schedule going to\n",
      "i was wondering what bus takes me on to verona from\n",
      "hello do you have the information for the seventy one d %\n",
      "u thirty the know when the next 51a leaving downtown going to dynamo way\n",
      "i need to know what time the next fifty four c leaves oakland\n",
      "what time is the next bus from cmu to squirrel hill\n",
      "can you tell me what time the last sixty one c leaves homestead at eighth avenue leaving for duquesne\n",
      "wilkins forbes and mcdevitt north neville in mckeesport going to center to wall\n",
      "can you tell me what time the next sixty one c coming at kennywood park going towards downtown\n",
      "i need the sixty one c inbound from mckeesport transportation center\n",
      "hi i would like to know what time the sixty four c stops running from seventeenth street back to oakland\n",
      "tell me what time the sixty one c comes to second and grant going to downtown\n",
      "i need the fifty four c to get % from bosmond street and mount oliver\n",
      "i need to know when the last sixty one c leaves downtown pittsburgh going to mckeesport\n",
      "when is the next eighty six b from garfield to downtown\n",
      "hello route hello toward 11e from wall the hill to\n",
      "ah i would like scheduled times for the sixty seven a\n",
      "when is the next bus from cmu to forbes and murray\n"
     ]
    }
   ],
   "source": [
    "dstc1_longer_start_utterances = [utt for utt in dstc1_start_utterances if 9 < len(utt.split())]\n",
    "print(len(dstc1_longer_start_utterances))\n",
    "print('\\n'.join(dstc1_longer_start_utterances[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/dstc1_ood.txt', 'w') as ood_out:\n",
    "    for utterance in dstc1_longer_start_utterances:\n",
    "        print(utterance, file=ood_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, get the Maluuba Frames dataset: https://www.microsoft.com/en-us/research/project/frames-dataset/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/Frames-dataset/frames.json') as frames_in:\n",
    "    frames_data = json.load(frames_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames_start_utterances = set([dialogue['turns'][0]['text'] for dialogue in frames_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1198\n",
      "We have decided to go visit my sister in Sao Paulo. What do you have out of Santa Cruz?\n",
      "I need to go to Buenos Aires.\n",
      "I'm looking to book trip for spring break for 4 people. I have a tight budget of 4600\n",
      "I need 2 get 2 Madrid from Beijing! Family emergency. Need to leave on the 6th\n",
      "PUebla to Manas for 2 adults please!\n",
      "Say, With just 1900 dollars and little more than a dream, could I make it from Seattle to North Vancouver?\n",
      "Hello you may know me as the best Pokemon Hunter in the world\n",
      "back for another crazy trip here\n",
      "I need to find a vacation between today and September 14th from Punta Cana to San Francisco\n",
      "Hello. I'd like to leave from Tel Aviv And go to Puebla\n",
      "We have 6 adults looking to book a trip\n",
      "ok the dreamer is back at it. this time i'd like to hear about trips leaving denver such as mexico city?\n",
      "Hi I need a break from Gotham City, the crime rate is just too high. I'd like to go to Hogsmead with 3 kids and 9 adults.\n",
      "Hi there. I have been tasked with bringing my big beautiful family on a trip! We need to go between August 30 and September 20. We've got 13100 dollars to spend and are leaving from Dublin. Lets hear whats available in St. Louis!\n",
      "Hi! Can you help me book a family vacation? Great! I'm looking to book a trip for 7 adults and 6 kids. We live in Campinas and we are going to Berlin. What options can you offer me?\n",
      "Hi Do you fly from Ulsan to London??\n",
      "I am SO done with my job right now. I wanna take a nice two week vacation and getting out of Rome what could you get me in Curitiba?\n",
      "Good day, please book me a trip from Vancouver, Jamaica to Recife. I would like to leave for 17 days on August 24.\n",
      "Hi I've got investors to meet with in Nagoya sometime in the near future. I can leave from anywhere\n",
      "I'd like to book a trip to Atlantis from Caprica on Saturday, August 13, 2016 for 8 adults. I have a tight budget of 1700.\n",
      "Hello there. I’m trying to get to to Sapporo from Paris\n",
      "5 adults to Cordoba from Goiania, no budget, just luxury\n",
      "I'm trying to debate taking the Amtrak, bus, or flying from Birmingham to Denver\n",
      "Hello. I need Paris to Barcelonna Sept 18 to 22\n",
      "ulsan to vitoria, 1 business class ticket\n",
      "Hello there i am looking to go on a vacation with my family to Gotham City, can you help me?\n",
      "From between Sept 6 and Sept 11, I’m on break. Leaving from Columbus\n",
      "Looking for a trip out of SL for 3 adults 7 kids\n",
      "Direct flight from san antonio to cancun, does that even exist?\n",
      "I want to leave from Tel Aviv and go to Fortaleza for 2300 with 7 adults\n",
      "I need to get away from work, how about Paris?\n",
      "Hey Im looking for a nice get away can you help me?\n",
      "I just got a tip that Madonna might be going to Burlington\n",
      "Good Morning I Am Writing To Inquire About Possible Booking Availabilities\n",
      "Just looking into vacations departing Phili. Where could I go for under 1200?\n",
      "Hey there I'm looking to go San Juan soon!\n",
      "Hello, I would like to book a 2-week trip leaving from Melbourne on August 27. I would like to go to Mannheim.\n",
      "Hi there, can you help me out with a trip? I need to get to Kingston.\n",
      "I want to throw my parents on a plane and get them out of my hair\n",
      "Hey! So me and my girlfriends are in the search for a perfect spring break trip!\n",
      "Hi, I want to take my kid to Maceio. Will you be able to help me with this trip?\n",
      "I would like to go to Hiroshima from Seoul for 3500\n",
      "I want to go to Guadalajara with my home girlz.\n",
      "Tel Aviv - San Francisco please\n",
      "I want to take my five friends with me from San Diego to Frankfurt for my birthday bash\n",
      "GET ME A FLIGHT FROM DENVER TO MILAN. LEAVE THIS FRIDAY GET ME THE BEST HOTEL MONEY CAN BUY\n",
      "I’m trying to get from Milan to a bunch of different possible internship spots. It needs to be both between Aug 31 and Sept 3\n",
      "please get me from Jerusalem to Sydney\n",
      "Hi, I'm looking for a vacation package to Chicago from Brasilia.\n",
      "Do you do flights out of San Antonio?\n"
     ]
    }
   ],
   "source": [
    "frames_longer_start_utterances = [utt.replace('\\n', ' ') for utt in frames_start_utterances if 5 < len(utt.split())]\n",
    "print(len(frames_longer_start_utterances))\n",
    "print('\\n'.join(frames_longer_start_utterances[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/frames_ood.txt', 'w') as frames_out:\n",
    "    for utterance in frames_longer_start_utterances:\n",
    "        print(utterance, file=frames_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMD_FOLDER = 'data/smd'\n",
    "SMD_URL = 'http://nlp.stanford.edu/projects/kvret/kvret_dataset_public.zip'\n",
    "\n",
    "if not os.path.exists(SMD_FOLDER):\n",
    "    r = requests.get(SMD_URL)\n",
    "    with open('smd.zip', 'wb') as smd_out:\n",
    "        smd_out.write(r.content)\n",
    "    with zipfile.ZipFile('smd.zip') as zip_ref:\n",
    "        zip_ref.extractall(SMD_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kvret_dev_public.json',\n",
       " 'kvret_entities.json',\n",
       " 'kvret_test_public.json',\n",
       " 'kvret_train_public.json']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(SMD_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "smd_utterances = set([])\n",
    "smd_start_utterances = set([])\n",
    "for filename in os.listdir(SMD_FOLDER):\n",
    "    if not '_public' in filename:\n",
    "        continue\n",
    "    with open(os.path.join(SMD_FOLDER, filename)) as dataset_in:\n",
    "        dataset_json = json.load(dataset_in)\n",
    "        for dialogue in dataset_json:\n",
    "            for idx, turn in enumerate(dialogue['dialogue']):\n",
    "                if turn['turn'] != 'driver':\n",
    "                    continue\n",
    "                smd_utterances.add(turn['data']['utterance'].lower())\n",
    "                if idx == 0:\n",
    "                    smd_start_utterances.add(turn['data']['utterance'].lower())\n",
    "                \n",
    "smd_utterances = [utt for utt in smd_utterances if utt]\n",
    "smd_start_utterances = [utt for utt in smd_start_utterances if utt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "996\n",
      "i need the quickest route to a gas station please. \n",
      "please schedule me a yoga activity with tom on the 4th at 5pm.\n",
      "is it gonna snow during the next week in atherton?\n",
      "remind me about my optometrist appointment on thursday at 11 am, my aunt is coming\n",
      "will the weather be warm in menlo park on sunday?\n",
      "look at today and tomorrow and tell me if there will be any snow in san francisco\n",
      "schedule a conference with the boss for this month on the 5th at 1pm.\n",
      "what is the date and time of my next football activity.\n",
      "car i need to reach my friends house in this local area: pick the route with the shortest distance too please\n",
      "please remind me to take my medicine at 7 pm today with my mother.\n",
      "please tell me the location, time and parties that are attending my meeting\n",
      "schedule a swimming activity with marie on sunday at 6pm.\n",
      "is there going to be frost in los angeles on tuesday? \n",
      "schedule a dentist appointment on the 8th at 1 pm. ana will be joining you.\n",
      "car i'm desiring to eat at some chinese restaurant: is there anything within 2 miles?\n",
      "will there be clear skies in compton on the weekend?\n",
      "over the next seven days will it be humid at all in redwood city?\n",
      "how is gonna be the weather forecast in san mateo for the next weekend?\n",
      "what will be the highest temperature on friday in mountain view?\n",
      "schedule a swimming activity with jon for december 10th at 3pm.\n",
      "schedule a reminder to take medicine with martha for saturday at 2pm.\n",
      "please schedule a reminder for my doctor appointment at 11am on august 3rd with alex.\n",
      "what is the weather forecast for the next 7 days\n",
      "what is the weather going to be like today and tomorrow\n",
      "please find me directions to a nearby place i can get some coffee\n",
      "what is the date of my yoga activity and who will be attending it?\n",
      "schedule a calendar reminder for yoga with jeff at 6pm on the 5th.\n",
      "please set a reminder that i have a conference call with the vp on august 14th at 6pm.\n",
      "please give me an address and directions via the fastest route to a local grocery store\n",
      "please schedule a swimming activity on sunday at 4pm with my brother.\n",
      "i need you to set a reminder for a doctor appointment at 11 am, the 7th, with alex.\n",
      "car find me a local chinese restaurant with the fastest route.\n",
      "find out if it will be cloudy on monday in cleveland.\n",
      "what time is my next swimming activity and who will be attending?\n",
      "please tell me about the time and date my next meeting is scheduled for and what i hope to accomplish it\n",
      "in mountain view, will there be frost today or tomorrow?\n",
      "can you find out for me the date and parties attending my dinner?\n",
      "i really need to have an appointment scheduled for tennis on the 7th at 11 am.\n",
      "will it be misty in redwood city during the upcoming seven day period?\n",
      "schedule a dentist appointment with jeff for next wednesday at 1pm.\n",
      "remind me about the date and time that he was going to be there and what the agenda is for his business meaning\n",
      "schedule a swimming activity with marie for friday at 1pm.\n",
      "next week in san mateo will there be any frost?\n",
      "i need a reminder, it's for a swimming activity at 10am, 14th of august, with mother.\n",
      "will the weather be hail in alhambra during the next seven days?\n",
      "please give me the address and directions via a quick route to my friend's house. \n",
      "set calendar reminder for 1pm this coming tuesday with my aunt. \n",
      "please confirm the address and give me directions via a route with no traffic to home. \n",
      "give me directions to the nearest place to get to eat\n",
      "please set a conference reminder. i need it at 10 am, the tenth, with my vice president. \n"
     ]
    }
   ],
   "source": [
    "smd_longer_start_utterances = [utt for utt in smd_start_utterances if 9 < len(utt.split())]\n",
    "print(len(smd_longer_start_utterances))\n",
    "print('\\n'.join(smd_longer_start_utterances[:50]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/stanford_ood.txt', 'w') as ood_out:\n",
    "    for utterance in smd_longer_start_utterances:\n",
    "        print(utterance, file=ood_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}