ablation_text_to_landmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "021f56cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "import numpy as np\n",
    "from lm_nav import landmark_extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c745bca",
   "metadata": {},
   "source": [
    "## Building dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2da0b4c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_routes = [\n",
    "    (5, \"Go straight toward the white building. Continue straight passing by a white truck until you reach a stop sign.\"),\n",
    "    (5, \"After passing a white building, take right next to a white truck. Then take left and go towards a square with a large tree. Go further, until you find a stop sign.\"),\n",
    "    (173, \"Start going around a building with a red-black wall and pass by a fire hydrant. Take a right and enter a grove. Continue straight and take a right, when you see a manhole cover. Go forward and left, and look for a trailer.\"),\n",
    "    (108, \"Take a right next to a stop sign. Look for a glass building, after passing by a white car.\"),\n",
    "    (247, \"Follow the road and take the right, you should see a blue semi-truck. Behind the truck, take a right next to an orange traffic cone. Go towards a blue dumpster and take left. Look for a picnic bench.\"),\n",
    "    (70, \"Go towards a white trailer. Then take left and go to the traffic lights. Take left again, and look for a traffic cone.\"),\n",
    "    (215, \"Go straight, passing by a stop sign and a  manhole cover. Next, you will see a disabled Parking spot and a red building.\"),\n",
    "    (103, \"First, you need to find a stop sign. Then take left and right and continue until you reach a square with a tree. Continue first straight, then right, until you find a white truck. The final destination is a white building.\"),\n",
    "    (103, \"Go to a stop sign. Continue straight, passing by a white truck. The final destination is a white building.\"),\n",
    "    (211, \"Go straight, until you find a glass building. Drive to a white car nearby. Drive to a stop sign.\")\n",
    "]\n",
    "landmarks_cache = eval(\"[['a white building', 'a white truck', 'a stop sign'], ['a white building', 'a white truck', 'a square with a large tree', 'a stop sign'], ['a building with a red-black wall', 'a fire hydrant', 'a grove', 'a manhole cover', 'a trailer'], ['a stop sign', 'a white car', 'a glass building'], ['a blue semi-truck', 'an orange traffic cone', 'a blue dumpster', 'a picnic bench'], ['a white trailer', 'traffic lights', 'a traffic cone'], ['a stop sign', 'a manhole cover', 'a disabled Parking spot', 'a red building'], ['a stop sign', 'a square with a tree', 'a white truck', 'a white building'], ['a stop sign', 'a white truck', 'a white building'], ['a glass building', 'a white car', 'a stop sign']]\")\n",
    "\n",
    "dataset_large = [(a[1],b) for a, b in zip(all_routes, landmarks_cache)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8d76dd32",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_routes = [\n",
    "    (180, \"Go straight towards a stop sign, take left and go until you reach a traffic cone. Take another left and then right going towards a blue box. From there take left and look for a baby stroller.\"),\n",
    "    (215, \"Go towards the blue box, take right and left until you reach a traffic cone. Take left and pass by a semi-truck until you find a big log.\"),\n",
    "    (63, \"Start at a traffic cone. Go towards a cardboard box and a parking lot. Continue driving, take a right, and pass by a picnic table. Take left and look for a stop sign.\"),\n",
    "    (160, \"Take first right towards a picnic table. Next, go to a square with a large tree, and take the left to another picnic table. Keep going until you reach a parking lot.\"),\n",
    "    (61, \"Go straight and take right next to a traffic cone. Go straight until you reach a parking lot. Take left, go through a lawn and look for a blue box.\"),\n",
    "    (219, \"Pass by a blue box and look for a big log. Take right and keep going straight, passing by a traffic cone. Take a right and finish at the parking lot.\"),\n",
    "    (186, \"Look for a traffic cone, take left and go straight until you find a square with a tree. Turn right, pass by a cardboard box and go to a picnic table.\"),\n",
    "    (75, \"Go straight pass a picnic table until you reach a street. Take right, pass by an orange trailer and take next right at an intersection. Next, take a right next to a traffic cone, take the next left, and pass by a baby stroller. Go straight and you will reach a parking lot.\"),\n",
    "    (194, \"Take a left when you see a traffic cone. Go straight passing by a semi-track and take left after you see a big log. Drive to a blue box and continue straight until you find a cardboard box next to a parking lot.\"),\n",
    "    (133, \"Take right at a traffic cone, and go straight until you reach a square with a big tree. Take right next and go straight until you find a baby stroller. Take left and right and look for an intersection.\"),\n",
    "]\n",
    "landmarks_cache = eval(\"[['a stop sign', 'a traffic cone', 'a blue box', 'a baby stroller'], ['a blue box', 'a traffic cone', 'a semi-truck', 'a big log'], ['a traffic cone', 'a cardboard box', 'a parking lot', 'a picnic table', 'a stop sign'], ['a picnic table', 'a square with a large tree', 'another picnic table', 'a parking lot'], ['a traffic cone', 'a parking lot', 'a lawn', 'a blue box'], ['a blue box', 'a big log', 'a traffic cone', 'a parking lot'], ['a traffic cone', 'a square with a tree', 'a cardboard box', 'a picnic table'], ['a picnic table', 'a street', 'an orange trailer', 'an intersection', 'a traffic cone', 'a baby stroller', 'a parking lot'], ['a traffic cone', 'a semi-track', 'a big log', 'a blue box', 'a cardboard box', 'a parking lot'], ['a traffic cone', 'a square with a big tree', 'a baby stroller', 'an intersection']]\")\n",
    "\n",
    "dataset_small = [(a[1],b) for a, b in zip(all_routes, landmarks_cache)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f3c221dd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dataset = dataset_large + dataset_small"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1a8a94d",
   "metadata": {},
   "source": [
    "## Metric: longest common subsequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a3508213",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_article(string):\n",
    "    article = [\"a\", \"an\", \"the\"]\n",
    "    words = string.split()\n",
    "    none_articles = [w for w in words if w not in article]\n",
    "    return \" \".join(none_articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d9d704e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def longest_common_subsequence(sequence, gt_sequence):\n",
    "    sequence = [remove_article(s) for s in sequence]\n",
    "    gt_sequence = [remove_article(s) for s in gt_sequence]\n",
    "    gt_lookup = {s: i for i, s in enumerate(gt_sequence)}\n",
    "    gt_index = [gt_lookup.get(s, -1) for s in sequence]\n",
    "    longest_dp = [0] * len(sequence)\n",
    "    for i in range(len(sequence)):\n",
    "        if gt_index[i] >= 0:\n",
    "            best = 1\n",
    "            for j in range(i):\n",
    "                if gt_index[j] < gt_index[i] and best < longest_dp[j]+1:\n",
    "                    best = longest_dp[j]+1\n",
    "            longest_dp[i] = best\n",
    "    return max(longest_dp) / len(gt_sequence)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6305cf94",
   "metadata": {},
   "source": [
    "## Evaluation of different models\n",
    "In order to run the experiment you need to setup keys to OpenAI and Goose AI API, either by specifying them on the command line or by uncommenting the next cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68843ab6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.environ[\"OPENAI_API_KEY\"] = \"sk-[real api key here]\"\n",
    "# os.environ[\"GOOSE_API_KEY\"] = \"sk-[real api key here]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2cf2916d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8775000000000001"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spacy_answers = [landmark_extraction.text_to_landmarks_spacy(text) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(spacy_answers, dataset)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccb2ec7f",
   "metadata": {},
   "source": [
    "## Simple prompt\n",
    "\n",
    "> Look for a library, after taking a right turn next to a statue.\n",
    ">\n",
    "> Landmarks:\n",
    "> 1. a statue\n",
    "> 2. a library\n",
    ">\n",
    "> Look for a statue. Then look for a library. Then go towards a pink house.\n",
    ">\n",
    "> Landmarks:\n",
    "> 1. a statue\n",
    "> 2. a library\n",
    "> 3. a pink house\n",
    "> \n",
    "> {Description}\n",
    ">\n",
    "> Landmarks:\n",
    "> 1. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fcb2dc7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt3_answers_simple = [landmark_extraction.text_to_landmarks_gpt3(text, simple_prompt=True) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(gpt3_answer, gt_answer) for gpt3_answer, (_, gt_answer) in zip(gpt3_answers_simple, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "520338b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.804404761904762"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_j_6b_answers_simple = [landmark_extraction.text_to_landmarks_goose_ai(text, \"gpt-j-6b\", simple_prompt=True) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(gpt6_answer, gt_answer) for gpt6_answer, (_, gt_answer) in zip(gpt_j_6b_answers_simple, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "35654881",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7615476190476189"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fairseq_13_answers_simple = [landmark_extraction.text_to_landmarks_goose_ai(text, \"fairseq-13b\", simple_prompt=True) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(fairseq_13_answers_simple, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "32a42860",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.723452380952381"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_neo_20b_answers_simple = [landmark_extraction.text_to_landmarks_goose_ai(text, \"gpt-neo-20b\", simple_prompt=True) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(gpt_neo_20b_answers_simple, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8eb451ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5242857142857144"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fairseq_1_3_answers_simple = [landmark_extraction.text_to_landmarks_goose_ai(text, \"fairseq-1-3b\", simple_prompt=True) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(fairseq_1_3_answers_simple, dataset)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afe04706",
   "metadata": {},
   "source": [
    "## More complicated prompt\n",
    "\n",
    "> Take right next to an old white building. Look for a fire station, which you will see after passing by a school.\n",
    "> Ordered landmarks:\n",
    "> 1. an old white building\n",
    "> 2. a school\n",
    "> 3. a fire station\n",
    ">\n",
    "> Go straight for two blocks. Take right at a roundabout, before it you will pass a big, blue tree.\n",
    "> Ordered landmarks:\n",
    "> 1. a big, blue tree\n",
    "> 2. a roundabout\n",
    ">\n",
    "> Look for a library, after taking a right turn next to a statue.\n",
    "> Ordered landmarks:\n",
    "> 1. a statue\n",
    "> 2. a library\n",
    ">\n",
    "> {Description}\n",
    ">\n",
    "> Ordered landmarks:\n",
    "> 1. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "594f28c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9857142857142858"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# verify how good is GPT3\n",
    "gpt3_answers = [landmark_extraction.text_to_landmarks_gpt3(text) for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(gpt3_answer, gt_answer) for gpt3_answer, (_, gt_answer) in zip(gpt3_answers, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "06866328",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5313095238095238"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_j_6b_answers = [landmark_extraction.text_to_landmarks_goose_ai(text, \"gpt-j-6b\") for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(gpt6_answer, gt_answer) for gpt6_answer, (_, gt_answer) in zip(gpt_j_6b_answers, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "60ec4906",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7998809523809524"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fairseq_13_answers = [landmark_extraction.text_to_landmarks_goose_ai(text, \"fairseq-13b\") for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(fairseq_13_answers, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "cec2ad22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.573095238095238"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_neo_20b_answers = [landmark_extraction.text_to_landmarks_goose_ai(text, \"gpt-neo-20b\") for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(gpt_neo_20b_answers, dataset)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6cb80436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48261904761904767"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fairseq_1_3_answers = [landmark_extraction.text_to_landmarks_goose_ai(text, \"fairseq-1-3b\") for text, _ in dataset]\n",
    "np.mean([longest_common_subsequence(answer, gt_answer) for answer, (_, gt_answer) in zip(fairseq_1_3_answers, dataset)])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}