look_at_babi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from operator import itemgetter\n",
    "from os import path\n",
    "\n",
    "from lib.babi import read_task\n",
    "\n",
    "\n",
    "def print_dataset_stats(in_dataset):\n",
    "    print '#Dialogs:\\t{}'.format(\n",
    "        len(in_dataset)\n",
    "    )\n",
    "    dialogues_only = map(itemgetter(1), in_dataset)\n",
    "    print 'Mean #turns:\\t{0:.3f}'.format(\n",
    "        sum(map(len, dialogues_only)) / float(len(in_dataset))\n",
    "    )\n",
    "    system_turns = reduce(\n",
    "        lambda x, y: x + filter(lambda turn: turn['agent'] == 'system', y),\n",
    "        dialogues_only,\n",
    "        []\n",
    "    )\n",
    "    user_turns = reduce(\n",
    "        lambda x, y: x + filter(lambda turn: turn['agent'] == 'user', y),\n",
    "        dialogues_only,\n",
    "        []\n",
    "    )\n",
    "    print 'Mean system turn length (words):    {0:.3f}'.format(\n",
    "        sum(map(lambda turn: len(turn['text'].split()), system_turns)) / float(len(system_turns))\n",
    "    )\n",
    "    print 'Mean user turn length (words):    {0:.3f}'.format(\n",
    "        sum(map(lambda turn: len(turn['text'].split()), user_turns)) / float(len(user_turns))\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_FOLDER = 'dialog-bAbI-tasks'\n",
    "task1_train = read_task(path.join(DATA_FOLDER, 'dialog-babi-task1-API-calls-trn.txt'))\n",
    "task1_dev = read_task(path.join(DATA_FOLDER, 'dialog-babi-task1-API-calls-dev.txt'))\n",
    "task1_test = read_task(path.join(DATA_FOLDER, 'dialog-babi-task1-API-calls-tst.txt'))\n",
    "task1_oov_test = read_task(path.join(DATA_FOLDER, 'dialog-babi-task1-API-calls-tst-OOV.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "General stats\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task 1 stats:\n",
      "train\n",
      "#Dialogs:\t1000\n",
      "Mean #turns:\t12.048\n",
      "Mean system turn length (words):    6.256\n",
      "Mean user turn length (words):    3.798\n",
      "dev\n",
      "#Dialogs:\t1000\n",
      "Mean #turns:\t12.030\n",
      "Mean system turn length (words):    6.245\n",
      "Mean user turn length (words):    3.796\n",
      "test\n",
      "#Dialogs:\t1000\n",
      "Mean #turns:\t11.872\n",
      "Mean system turn length (words):    6.242\n",
      "Mean user turn length (words):    3.835\n",
      "test_oov\n",
      "#Dialogs:\t1000\n",
      "Mean #turns:\t12.040\n",
      "Mean system turn length (words):    6.258\n",
      "Mean user turn length (words):    3.787\n"
     ]
    }
   ],
   "source": [
    "print 'Task 1 stats:'\n",
    "print 'train'\n",
    "print_dataset_stats(task1_train)\n",
    "print 'dev'\n",
    "print_dataset_stats(task1_dev)\n",
    "print 'test'\n",
    "print_dataset_stats(task1_test)\n",
    "print 'test_oov'\n",
    "print_dataset_stats(task1_oov_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "API call stats\n",
    "=="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_answers(in_dataset):\n",
    "    return reduce(\n",
    "        lambda x, y: x + y,\n",
    "        [\n",
    "            map(itemgetter('text'), filter(lambda turn: turn['agent'] == 'system', dialogue))\n",
    "            for dialogue_name, dialogue in in_dataset\n",
    "        ],\n",
    "        []\n",
    "    )\n",
    "\n",
    "def get_candidates_list(data_dir):\n",
    "    candidates_file = path.join(data_dir, 'dialog-babi-candidates.txt')\n",
    "    with open(candidates_file) as candidates_in:\n",
    "        return [line.strip().split(' ', 1)[1] for line in candidates_in]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train\n",
      "#unique answers (intelocutory): 7\n",
      "#unique answers (api call): 159\n",
      "test\n",
      "#unique answers (intelocutory): 7\n",
      "#unique answers (api call): 141\n",
      "#test api calls not present in the trainset: 141\n"
     ]
    }
   ],
   "source": [
    "train_answers = get_answers(task1_train)\n",
    "print 'train'\n",
    "print '#unique answers (intelocutory): {}'.format(\n",
    "    len(\n",
    "        set(\n",
    "            filter(lambda x: not x.startswith('api_call'), train_answers)\n",
    "        )\n",
    "    )\n",
    ")\n",
    "unique_api_calls_train = set(filter(lambda x: x.startswith('api_call'), train_answers))\n",
    "print '#unique answers (api call): {}'.format(len(unique_api_calls_train))\n",
    "\n",
    "test_answers = get_answers(task1_test)\n",
    "print 'test'\n",
    "print '#unique answers (intelocutory): {}'.format(\n",
    "    len(\n",
    "        set(\n",
    "            filter(lambda x: not x.startswith('api_call'), test_answers)\n",
    "        )\n",
    "    )\n",
    ")\n",
    "unique_api_calls_test = set(filter(lambda x: x.startswith('api_call'), test_answers))\n",
    "print '#unique answers (api call): {}'.format(len(unique_api_calls_test))\n",
    "\n",
    "print '#test api calls not present in the trainset: {}'.format(len(unique_api_calls_test.difference(unique_api_calls_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer_candidates = get_candidates_list(DATA_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_answers = reduce(\n",
    "    lambda x, y: x + y,\n",
    "    map(get_answers, [task1_train, task1_dev, task1_test, task1_oov_test])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#unique answers (api call): 460\n",
      "#all api calls not present in the candidates_list: 0\n"
     ]
    }
   ],
   "source": [
    "unique_api_calls_all = set(filter(lambda x: x.startswith('api_call'), all_answers))\n",
    "print '#unique answers (api call): {}'.format(len(unique_api_calls_all))\n",
    "\n",
    "print '#all api calls not present in the candidates_list: {}'.format(len(unique_api_calls_all.difference(answer_candidates)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_api_calls_all.difference(answer_candidates)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}