load_sungjin_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = 'data/babi_task6'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(DATA_FOLDER, 'train.json')) as data_in:\n",
    "    data = json.load(data_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "actions_0 = data['dialogs'][0]['actions_tokenized']\n",
    "for dialog in data['dialogs']:\n",
    "    assert dialog['actions_tokenized'] == actions_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['dialogs'][0]['actions_tokenized'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dataset(in_json):\n",
    "    for dialog in in_json['dialogs']:\n",
    "        db_results = None\n",
    "        for turn, action in zip(dialog['turns'], dialog['actions_tokenized']):\n",
    "            if 'db_results' in turn:\n",
    "                db_results = turn['db_results'].keys()\n",
    "                continue\n",
    "            usr = turn['input']['text_tokenized']\n",
    "            if db_results:\n",
    "                usr = ' '.join(db_results) + ' ' + usr\n",
    "                db_results = None\n",
    "            sys = turn['label']\n",
    "            print('{}|{}'.format(usr, sys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_dilaogs(in_json):\n",
    "    turns, indices = [], []\n",
    "    for dialog in in_json['dialogs']:\n",
    "        db_results = None\n",
    "        dialog_turns = []\n",
    "        for turn, action in zip(dialog['turns'], dialog['actions_tokenized']):\n",
    "            if 'db_results' in turn:\n",
    "                db_results = turn['db_results'].keys()\n",
    "                continue\n",
    "            usr = turn['input']['text_tokenized']\n",
    "            if db_results:\n",
    "                usr = ' '.join(db_results) + ' ' + usr\n",
    "                db_results = None\n",
    "            sys = turn['label']\n",
    "            dialog_turns.append((usr, sys))\n",
    "        indices.append({'start': len(turns), 'end': len(turns) + len(dialog_turns)})\n",
    "        turns += dialog_turns\n",
    "    return turns, indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_set = set([])\n",
    "for dialog in data['dialogs']:\n",
    "    for turn in dialog['turns']:\n",
    "        if 'label' in turn:\n",
    "            label_set.add(turn['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 9,\n",
       " 10,\n",
       " 11,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 15,\n",
       " 16,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 21,\n",
       " 22,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 30,\n",
       " 31,\n",
       " 32,\n",
       " 33,\n",
       " 34,\n",
       " 35,\n",
       " 36,\n",
       " 37,\n",
       " 38,\n",
       " 39,\n",
       " 40,\n",
       " 41,\n",
       " 42,\n",
       " 43,\n",
       " 44,\n",
       " 45,\n",
       " 46,\n",
       " 47,\n",
       " 48,\n",
       " 49,\n",
       " 50,\n",
       " 51,\n",
       " 52,\n",
       " 53,\n",
       " 54,\n",
       " 55,\n",
       " 56,\n",
       " 58}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['dialogs'][0]['actions'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}