simulation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10-final"
    },
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.7.10 64-bit ('durp': conda)",
      "metadata": {
        "interpreter": {
          "hash": "ad3b2c1c9ba1d7362819af9951b90963271aba6bda6a6815a50c7fa2bbb724c2"
        }
      }
    },
    "colab": {
      "name": "simulation.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "source": [
        "# Distributed UAV RL Protocol\n",
        "An implementation of a distributed protocol for cooperative sensing and sending operations of Unmanned Aerial Vehicles (UAVs). It is built on top of <a href=\"https://www.tensorflow.org/agents\">TensorFlow Agents</a> and uses reinforcement learning (e.g. Deep Q-Learning, Actor-Critic) techniques to compute ideal trajectories."
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "source": [
        "## Setup the environment and import dependencies"
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DsqsLLlabJnZ"
      },
      "source": [
        "try:\n",
        "  # Install colab required dependencies\n",
        "  import google.colab\n",
        "  !pip install -q tf-agents\n",
        "except:\n",
        "  pass"
      ],
      "execution_count": 849,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sA1ej8o2776K"
      },
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import random\n",
        "import math\n",
        "\n",
        "import tensorflow as tf\n",
        "from tf_agents.networks import q_network\n",
        "from tf_agents.agents.dqn import dqn_agent\n",
        "from tf_agents.agents.ddpg import ddpg_agent, critic_network, actor_network, critic_rnn_network, actor_rnn_network\n",
        "from tf_agents.specs import tensor_spec, array_spec\n",
        "from tf_agents.environments import py_environment\n",
        "from tf_agents.environments import tf_py_environment\n",
        "from tf_agents.replay_buffers import tf_uniform_replay_buffer\n",
        "from tf_agents.trajectories import trajectory\n",
        "from tf_agents.policies import random_tf_policy\n",
        "from tf_agents import utils\n",
        "from tf_agents.trajectories import time_step as ts"
      ],
      "execution_count": 850,
      "outputs": []
    },
    {
      "source": [
        "## Data visualization"
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "fIFTvGAg776L",
        "outputId": "be39e32a-6b9e-40d0-d355-e635c0b16f88"
      },
      "source": [
        "# Number of sensing locations\n",
        "locations_num = 10\n",
        "\n",
        "# Size of the map (the map is squared)\n",
        "map_size = 100.0\n",
        "\n",
        "# Coordinates of the sensing locations\n",
        "sensing_locations = (np.random.rand(locations_num, 2) * map_size) - (map_size / 2)\n",
        "\n",
        "_ = plt.scatter(sensing_locations[:,0], sensing_locations[:,1])"
      ],
      "execution_count": 851,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<Figure size 432x288 with 1 Axes>",
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"248.518125pt\" version=\"1.1\" viewBox=\"0 0 377.304687 248.518125\" width=\"377.304687pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <metadata>\n  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n   <cc:Work>\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n    <dc:date>2021-03-03T18:53:22.137341</dc:date>\n    <dc:format>image/svg+xml</dc:format>\n    <dc:creator>\n     <cc:Agent>\n      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\n     </cc:Agent>\n    </dc:creator>\n   </cc:Work>\n  </rdf:RDF>\n </metadata>\n <defs>\n  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 248.518125 \nL 377.304687 248.518125 \nL 377.304687 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 35.304688 224.64 \nL 370.104688 224.64 \nL 370.104688 7.2 \nL 35.304688 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"PathCollection_1\">\n    <defs>\n     <path d=\"M 0 3 \nC 0.795609 3 1.55874 2.683901 2.12132 2.12132 \nC 2.683901 1.55874 3 0.795609 3 0 \nC 3 -0.795609 2.683901 -1.55874 2.12132 -2.12132 \nC 1.55874 -2.683901 0.795609 -3 0 -3 \nC -0.795609 -3 -1.55874 -2.683901 -2.12132 -2.12132 \nC -2.683901 -1.55874 -3 -0.795609 -3 0 \nC -3 0.795609 -2.683901 1.55874 -2.12132 2.12132 \nC -1.55874 2.683901 -0.795609 3 0 3 \nz\n\" id=\"m5a396422fc\" style=\"stroke:#1f77b4;\"/>\n    </defs>\n    <g clip-path=\"url(#p6413f6c84f)\">\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"293.866442\" xlink:href=\"#m5a396422fc\" y=\"203.838945\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"50.522869\" xlink:href=\"#m5a396422fc\" y=\"214.756364\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"260.656212\" xlink:href=\"#m5a396422fc\" y=\"61.748056\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"164.301758\" xlink:href=\"#m5a396422fc\" y=\"17.083636\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"298.021116\" xlink:href=\"#m5a396422fc\" y=\"173.915064\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"255.262867\" xlink:href=\"#m5a396422fc\" y=\"183.300596\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"188.683329\" xlink:href=\"#m5a396422fc\" y=\"163.3063\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"145.764399\" xlink:href=\"#m5a396422fc\" y=\"91.770055\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"354.886506\" xlink:href=\"#m5a396422fc\" y=\"213.388478\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"78.455197\" xlink:href=\"#m5a396422fc\" y=\"90.099322\"/>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m585b0b775a\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"69.410136\" xlink:href=\"#m585b0b775a\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- −40 -->\n      <g transform=\"translate(58.857792 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 10.59375 35.5 \nL 73.1875 35.5 \nL 73.1875 27.203125 \nL 10.59375 27.203125 \nz\n\" id=\"DejaVuSans-8722\"/>\n        <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n        <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"133.213145\" xlink:href=\"#m585b0b775a\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- −20 -->\n      <g transform=\"translate(122.660802 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"197.016155\" xlink:href=\"#m585b0b775a\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- 0 -->\n      <g transform=\"translate(193.834905 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"260.819165\" xlink:href=\"#m585b0b775a\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- 20 -->\n      <g transform=\"translate(254.456665 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"324.622174\" xlink:href=\"#m585b0b775a\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 40 -->\n      <g transform=\"translate(318.259674 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_6\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"mb5cc2e04e2\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#mb5cc2e04e2\" y=\"193.846314\"/>\n      </g>\n     </g>\n     <g id=\"text_6\">\n      <!-- −40 -->\n      <g transform=\"translate(7.2 197.645532)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#mb5cc2e04e2\" y=\"150.335609\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- −20 -->\n      <g transform=\"translate(7.2 154.134827)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#mb5cc2e04e2\" y=\"106.824904\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 0 -->\n      <g transform=\"translate(21.942188 110.624122)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_9\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#mb5cc2e04e2\" y=\"63.314199\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 20 -->\n      <g transform=\"translate(15.579688 67.113417)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#mb5cc2e04e2\" y=\"19.803494\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- 40 -->\n      <g transform=\"translate(15.579688 23.602712)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 35.304688 224.64 \nL 35.304688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 370.104688 224.64 \nL 370.104688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 35.304688 224.64 \nL 370.104688 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 35.304688 7.2 \nL 370.104688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p6413f6c84f\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"35.304688\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAO6ElEQVR4nO3dYYhcZ73H8e/vrrEueCVKU2028SZwYzBaIbIEoW+EWrdqaUNBiKAEFIpQoYJGu+3rUiGgvlBfBBUKFkrBmAZRYlr1xX1RNW2sIca1QdF2E228EBRcNMn93xc7aSftJu1kZjKzz3w/b3bOc87M+T9L8tszzznPOakqJElt+o9RFyBJGh5DXpIaZshLUsMMeUlqmCEvSQ17w6gL6Hb99dfXpk2bRl2GJK0qTz/99N+qat1K68Yq5Ddt2sSRI0dGXYYkrSpJ/nS5dQ7XSFLDDHlJapghL0kNM+QlqWGGvCQ1bKyurtHkOnB0kb2HFjh1don1a6fZM7eVndtnRl2WtOoZ8hq5A0cXmd9/jKVzFwBYPLvE/P5jAAa91CeHazRyew8tvBTwFy2du8DeQwsjqkhqx8BCPslUkqNJfthZfluSw0me6/x866D2pbacOrvUU7uk12+QR/L3Aie6lu8DnqyqLcCTnWXpVdavne6pXdLrN5CQT7IB+Bjw7a7mO4GHO68fBnYOYl9qz565rUyvmbqkbXrNFHvmto6oIqkdgzrx+nXgS8B/drW9vapOA1TV6SQ3rPTGJHcDdwO8853vHFA5Wk0unlz16hpp8PoO+SS3Ay9W1dNJPtjr+6tqH7APYHZ21gfOTqid22cMdWkIBnEkfzNwR5KPAm8C3pLke8Bfk9zYOYq/EXhxAPuSJPWg7zH5qpqvqg1VtQnYBfy0qj4JHAR2dzbbDTze774kSb0Z5nXyXwFuTfIccGtnWZJ0DQ10xmtV/Rz4eef1/wK3DPLzJUm9ccarJDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ17A2jLkDSeDhwdJG9hxY4dXaJ9Wun2TO3lZ3bZ0ZdlvrU95F8ko1JfpbkRJLjSe7ttL8tyeEkz3V+vrX/ciUNw4Gji8zvP8bi2SUKWDy7xPz+Yxw4ujjq0tSnQQzXnAe+UFXvBj4A3JNkG3Af8GRVbQGe7CxLGkN7Dy2wdO7CJW1L5y6w99DCiCrSoPQd8lV1uqqe6bz+B3ACmAHuBB7ubPYwsLPffUkajlNnl3pq1+ox0BOvSTYB24FfAG+vqtOw/IcAuOEy77k7yZEkR86cOTPIciS9TuvXTvfUrtVjYCGf5M3A94HPV9XfX+/7qmpfVc1W1ey6desGVY6kHuyZ28r0mqlL2qbXTLFnbuuIKtKgDOTqmiRrWA74R6pqf6f5r0lurKrTSW4EXhzEviQN3sWraLy6pj19h3ySAN8BTlTVV7tWHQR2A1/p/Hy8331JGp6d22cM9QYN4kj+ZuBTwLEkv+603c9yuD+W5DPAn4GPD2BfkqQe9B3yVfU/QC6z+pZ+P1+SdPW8rYEkNcyQl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYN4hmvE+XA0UWfaC9p1TDke3Dg6CLz+4+xdO4CAItnl5jffwzAoJc0lgz5Huw9tPBSwF+0dO4Cew8tTGzI+81GGm+GfA9OnV3qqb11frORxp8nXnuwfu10T+2tu9I3G0njwZDvwZ65rUyvmbqkbXrNFHvmto6ootHym400/gz5HuzcPsNDd93EzNppAsysneahu26a2KEJv9lI488x+R7t3D4zsaH+Snvmtl4yJg+T/c1GGkeGvK7axT92Xl0jjS9DXn3xm4003hyTl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYUMP+SS3JVlIcjLJfcPenyTpZUMN+SRTwDeBjwDbgE8k2TbMfUqSXjbsI/kdwMmq+kNV/Rt4FLhzyPuUJHUMO+RngOe7ll/otL0kyd1JjiQ5cubMmSGXI0mTZdghnxXa6pKFqn1VNVtVs+vWrRtyOZI0WYYd8i8AG7uWNwCnhrxPSVLHsEP+V8CWJJuTvBHYBRwc8j4lSR1DvdVwVZ1P8jngEDAFfLeqjg9zn5Kklw39fvJV9SPgR8PejyTp1ZzxKkkNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhhrwkNcyQl6SGGfKS1DBDXpIaZshLUsMMeUlqmCEvSQ0z5CWpYYa8JDXMkJekhhnyktQwQ16SGmbIS1LDDHlJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhpmyEtSwwx5SWqYIS9JDTPkJalhfYV8kr1JfpfkN0l+kGRt17r5JCeTLCSZ67tSSVLP+j2SPwy8t6reB/wemAdIsg3YBbwHuA34VpKpPvclSepRXyFfVT+pqvOdxaeADZ3XdwKPVtW/quqPwElgRz/7kiT1bpBj8p8Gftx5PQM837XuhU7bqyS5O8mRJEfOnDkzwHIkSW94rQ2SPAG8Y4VVD1TV451tHgDOA49cfNsK29dKn19V+4B9ALOzsytuI0m6Oq8Z8lX1oSutT7IbuB24paouhvQLwMauzTYAp662SEnS1en36prbgC8Dd1TVP7tWHQR2JbkuyWZgC/DLfvYlSerdax7Jv4ZvANcBh5MAPFVVn62q40keA37L8jDOPVV1oc99SZJ61FfIV9V/X2Hdg8CD/Xy+JKk/zniVpIYZ8pLUMENekhpmyEtSw/q9ukZaNQ4cXWTvoQVOnV1i/dpp9sxtZef2FSdiS80w5DURDhxdZH7/MZbOLV/Ju3h2ifn9xwAMejXN4RpNhL2HFl4K+IuWzl1g76GFEVUkXRuGvCbCqbNLPbVLrTDkNRHWr53uqV1qhSGvibBnbivTay59bs30min2zG0dUUXSteGJV02EiydXvbpGk8aQ18TYuX3GUNfEcbhGkhpmyEtSwwx5SWqYY/KSxpa3ouifIS9pLHkrisFwuEbSWPJWFINhyEsaS96KYjAMeUljyVtRDIYhL2kseSuKwfDEq6Sx5K0oBsOQlzS2vBVF/xyukaSGeSQvTRgnGE0WQ16aIE4wmjwO10gTxAlGk8eQlyaIE4wmjyEvTRAnGE0eQ16aIE4wmjyeeJUmiBOMJo8hL00YJxhNFodrJKlhhrwkNcyQl6SGDSTkk3wxSSW5vqttPsnJJAtJ5gaxH0lSb/o+8ZpkI3Ar8Oeutm3ALuA9wHrgiSTvqqoLK3+KJGkYBnEk/zXgS0B1td0JPFpV/6qqPwIngR0D2JckqQd9hXySO4DFqnr2FatmgOe7ll/otK30GXcnOZLkyJkzZ/opR5L0Cq85XJPkCeAdK6x6ALgf+PBKb1uhrVZoo6r2AfsAZmdnV9xGknR1XjPkq+pDK7UnuQnYDDybBGAD8EySHSwfuW/s2nwDcKrvaiVJPbnq4ZqqOlZVN1TVpqraxHKwv7+q/gIcBHYluS7JZmAL8MuBVCxJet2GcluDqjqe5DHgt8B54B6vrJGka29gId85mu9efhB4cFCfL0nqnTNeJalh3oVS0tjwIeODZ8hLGgs+ZHw4HK6RNBZ8yPhwGPKSxoIPGR8OQ17SWPAh48NhyEsaCz5kfDg88SppLPiQ8eEw5CWNDR8yPngO10hSwwx5SWqYwzWSNELDnuXbRMg7FVrSanQtZvmu+uGai7+kxbNLFC//kg4cXRx1aZJ0Rddilu+qD3mnQktara7FLN9VH/JOhZa0Wl2LWb6rPuSdCi1ptboWs3xXfcg7FVrSarVz+wwP3XUTM2unCTCzdpqH7rrJq2u6ORVa0mo27Fm+qz7kwanQknQ5q364RpJ0eYa8JDXMkJekhhnyktQwQ16SGpaqGnUNL0lyBvjTqOsArgf+NuoiRsj+23/7v7r8V1WtW2nFWIX8uEhypKpmR13HqNh/+2//2+m/wzWS1DBDXpIaZsivbN+oCxgx+z/Z7H9DHJOXpIZ5JC9JDTPkJalhhvwKknwxSSW5vqttPsnJJAtJ5kZZ37Ak2Zvkd0l+k+QHSdZ2rZuE/t/W6d/JJPeNup5hS7Ixyc+SnEhyPMm9nfa3JTmc5LnOz7eOutZhSjKV5GiSH3aWm+q/If8KSTYCtwJ/7mrbBuwC3gPcBnwrydTKn7CqHQbeW1XvA34PzMNk9L/Tn28CHwG2AZ/o9Ltl54EvVNW7gQ8A93T6fB/wZFVtAZ7sLLfsXuBE13JT/TfkX+1rwJeA7jPSdwKPVtW/quqPwElgxyiKG6aq+klVne8sPgVs6LyehP7vAE5W1R+q6t/Aoyz3u1lVdbqqnum8/gfLQTfDcr8f7mz2MLBzJAVeA0k2AB8Dvt3V3FT/DfkuSe4AFqvq2VesmgGe71p+odPWsk8DP+68noT+T0IfLyvJJmA78Avg7VV1Gpb/EAA3jLC0Yfs6ywd1/9fV1lT/m3gyVC+SPAG8Y4VVDwD3Ax9e6W0rtK3Ka0+v1P+qeryzzQMsf5V/5OLbVth+Vfb/CiahjytK8mbg+8Dnq+rvyUq/ivYkuR14saqeTvLBEZczNBMX8lX1oZXak9wEbAae7fwj3wA8k2QHy0d1G7s23wCcGnKpQ3G5/l+UZDdwO3BLvTyJopn+X8Ek9PFVkqxhOeAfqar9nea/Jrmxqk4nuRF4cXQVDtXNwB1JPgq8CXhLku/RWP8drumoqmNVdUNVbaqqTSz/p39/Vf0FOAjsSnJdks3AFuCXIyx3KJLcBnwZuKOq/tm1ahL6/ytgS5LNSd7I8onmgyOuaaiyfDTzHeBEVX21a9VBYHfn9W7g8Wtd27VQVfNVtaHz/30X8NOq+iSN9X/ijuSvRlUdT/IY8FuWhzHuqaoLIy5rGL4BXAcc7nybeaqqPjsJ/a+q80k+BxwCpoDvVtXxEZc1bDcDnwKOJfl1p+1+4CvAY0k+w/JVZh8fTXkj01T/va2BJDXM4RpJapghL0kNM+QlqWGGvCQ1zJCXpIYZ8pLUMENekhr2/1GL1Ak4aYU0AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjicDDkW776M"
      },
      "source": [
        "# Number of drones\n",
        "drones_num = 5\n",
        "\n",
        "# Length of a single cycle in seconds\n",
        "cycle_length = 1.0\n",
        "\n",
        "# Maximum speed of drones in points / seconds\n",
        "max_speed = 5.0\n",
        "\n",
        "# Definition of the global state\n",
        "\n",
        "# Coordinates of the drones locations\n",
        "drones_locations = np.zeros((drones_num, 2), dtype=float)\n",
        "\n",
        "# Amounts of sensing data to send per drone\n",
        "sensing_data_amounts = np.zeros(drones_num, dtype=float)\n",
        "\n",
        "# AoIs (age of information) per sensing location. Denotes the last cycle in which\n",
        "# any drone performed sensing at a specific sensing location\n",
        "aois = np.zeros(locations_num, dtype=int)\n",
        "\n",
        "# Indices of the chosen locations per drone. They are chosen in the decision cycles.\n",
        "# The initial values are not significant.\n",
        "chosen_locations = np.zeros(drones_num, dtype=int)\n",
        "\n",
        "# Stage of the cycle for every drone\n",
        "# 0 -> decision, 1 -> empty, 2 -> sensing, 3 -> transmission\n",
        "cycle_stages = np.zeros(drones_num, dtype=int)\n",
        "\n",
        "# Current cycle\n",
        "current_cycle = [0]\n",
        "\n",
        "\n",
        "# Returns the AoI of the given location\n",
        "def get_location_aoi(cycle, location_index):\n",
        "    return (cycle * cycle_length) - ((aois[location_index] // cycle_length) * cycle_length)\n",
        "\n",
        "# Returns the accumulated AoI (of all the sensing locations)\n",
        "def get_accumulated_aoi(cycle):\n",
        "    return max([sum([get_location_aoi(cycle, location) for location in range(locations_num)]), 0.0001])\n",
        "\n",
        "# Returns the trajectory for the current cycle \n",
        "def get_trajectory(drone_index):\n",
        "    chosen_location_index = chosen_locations[drone_index]\n",
        "    chosen_location = sensing_locations[chosen_location_index]\n",
        "    drone_location = drones_locations[drone_index]\n",
        "    distance = np.linalg.norm(chosen_location - drone_location)\n",
        "    if distance <= max_speed * cycle_length:\n",
        "        return chosen_location - drone_location\n",
        "    else:\n",
        "        return ((chosen_location - drone_location) / distance) * max_speed * cycle_length\n",
        "\n",
        "# Resets all the AoIs to 0\n",
        "def reset_aois():\n",
        "    for i in range(locations_num): aois[i] = 0\n",
        "\n",
        "# Resets all the drones locations to 0, 0\n",
        "def reset_drones_locations():\n",
        "    for i in range(drones_num): drones_locations[i] = np.array([0.0,0.0])"
      ],
      "execution_count": 852,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5yp0-M0776M"
      },
      "source": [
        "# Simplifying transmission step: only considering a bandwidth and a total amount of\n",
        "# data for every location. Assumption: tasks are equal, so the amount of data is the same.\n",
        "\n",
        "# Transmission bandwidth per cycle\n",
        "bandwidth = 0.5\n",
        "\n",
        "# Total amount of sensing data per location\n",
        "total_location_data = 1.5\n",
        "\n",
        "# Amount of data transmissible per cycle \n",
        "data_transmission_cycle = bandwidth * cycle_length"
      ],
      "execution_count": 853,
      "outputs": []
    },
    {
      "source": [],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "source": [
        "## Agents definition"
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x5X_eV09776N"
      },
      "source": [
        "'''\n",
        "Using Deep Q Learning. Having current drone location (x,y), \n",
        "potential sensing location (x',y') (could also directly consider distance\n",
        "between the two locations) and AoI of the sensing location\n",
        "as features, the network should output a Q value. The higher\n",
        "the value the better this sensing location choice. Executing the network\n",
        "at the decision cycle for every sensing location and choosing the one\n",
        "that outputs the highest Q value. After each decision, the Q network\n",
        "should be updated by performing backpropagation and considering the error.\n",
        "'''\n",
        "\n",
        "\n",
        "class DurpEnv(py_environment.PyEnvironment):\n",
        "    def __init__(self, drone):\n",
        "        self._drone = drone\n",
        "        self._action_spec = array_spec.BoundedArraySpec(\n",
        "        shape=(), dtype=np.int32, minimum=0, maximum=locations_num-1, name='action')\n",
        "        self._observation_spec = array_spec.BoundedArraySpec(shape=(2,),\n",
        "            minimum=(-map_size//2), maximum=(map_size//2), dtype=np.float64)\n",
        "        \n",
        "\n",
        "\n",
        "    def action_spec(self):\n",
        "        return self._action_spec\n",
        "\n",
        "    def observation_spec(self):\n",
        "        return self._observation_spec\n",
        "\n",
        "    def _reset(self):\n",
        "        return ts.restart(np.array([0.0,0.0], dtype=np.float64))\n",
        "\n",
        "    def _step(self, action):\n",
        "        ''' Performing next step '''\n",
        "        chosen_location_index = int(action)\n",
        "        accumulated_aoi = get_accumulated_aoi(current_cycle[0])\n",
        "\n",
        "        # Updating AoI of chosen destination\n",
        "        aois[chosen_location_index] = current_cycle[0]\n",
        "\n",
        "        # Calculating reward considering new accumulated AoI (sum of AoIs) \n",
        "        # and distance from chosen location\n",
        "        new_accumulated_aoi = get_accumulated_aoi(current_cycle[0])\n",
        "\n",
        "        aoi_multiplier = 0.05\n",
        "\n",
        "        # Using sigmoid function to obtain a value between 0 and 1. The \n",
        "        # higher the aoi difference the better, so the higher the reward.\n",
        "        normalized_diff_aoi_component = \\\n",
        "            ((1/(1+np.exp(- (accumulated_aoi - new_accumulated_aoi) * aoi_multiplier))) - 0.5) * 2.0\n",
        "\n",
        "\n",
        "        # Taking distance between current location and chosen location\n",
        "        chosen_location = sensing_locations[chosen_location_index]\n",
        "        drone_location = drones_locations[self._drone]\n",
        "        distance = np.linalg.norm(chosen_location - drone_location)\n",
        "\n",
        "        distance_multiplier = 0.03\n",
        "\n",
        "        # Normalizing distance between 0 and 1. The smaller the distance, the higher the reward.\n",
        "        normalized_location_distance = (1 - (1/(1+np.exp(-distance * distance_multiplier)))) * 2.0\n",
        "\n",
        "        # Computing reward using weights\n",
        "        aoi_weight = 1.0\n",
        "        distance_weight = 0.0\n",
        "        reward = normalized_diff_aoi_component * aoi_weight + normalized_location_distance * distance_weight + np.random.random() * (1.0 - aoi_weight - distance_weight)\n",
        "\n",
        "        return ts.transition(chosen_location, reward=reward)\n",
        "\n",
        "        \n",
        "learning_rate = 0.001\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "\n",
        "environments = []\n",
        "agents = []\n",
        "for drone in range(drones_num):\n",
        "    durp_env = DurpEnv(drone)\n",
        "    train_env = tf_py_environment.TFPyEnvironment(durp_env)\n",
        "\n",
        "    '''\n",
        "    # Using Actor-Critic methods\n",
        "    actor_net = actor_network.ActorNetwork(\n",
        "        train_env.observation_spec(), \n",
        "        train_env.action_spec()\n",
        "    )\n",
        "\n",
        "    critic_net = critic_network.CriticNetwork(\n",
        "        (train_env.observation_spec(), train_env.action_spec()), \n",
        "        #output_activation_fn=tf.keras.activations.sigmoid,\n",
        "        #activation_fn=tf.nn.relu\n",
        "    )\n",
        "\n",
        "    agent = ddpg_agent.DdpgAgent(\n",
        "    train_env.time_step_spec(),\n",
        "    train_env.action_spec(),\n",
        "    critic_network=critic_net,\n",
        "    actor_network=actor_net,\n",
        "    critic_optimizer=optimizer,\n",
        "    actor_optimizer=optimizer,\n",
        "    #td_errors_loss_fn=utils.common.element_wise_squared_loss,\n",
        "    #train_step_counter=tf.Variable(0))\n",
        "    '''\n",
        "    \n",
        "    # Using Deep-Q learning\n",
        "    q_net = q_network.QNetwork(\n",
        "    train_env.observation_spec(),\n",
        "    train_env.action_spec())\n",
        "\n",
        "    train_step_counter = tf.Variable(0)\n",
        "\n",
        "    agent = dqn_agent.DqnAgent(\n",
        "    train_env.time_step_spec(),\n",
        "    train_env.action_spec(),\n",
        "    q_network=q_net,\n",
        "    optimizer=optimizer,\n",
        "    train_step_counter=train_step_counter)\n",
        "\n",
        "    agent.initialize()\n",
        "    agents.append(agent)\n",
        "    environments.append(train_env)\n",
        "\n",
        "\n",
        "def get_best_action(drone):\n",
        "    highest_reward = 0\n",
        "    best_action = 0\n",
        "\n",
        "    for action in range(locations_num):\n",
        "        chosen_location_index = action\n",
        "        aoi = get_location_aoi(current_cycle[0], chosen_location_index)\n",
        "        aoi_multiplier = 0.05\n",
        "\n",
        "        # Using sigmoid function to obtain a value between 0 and 1. The \n",
        "        # higher the aoi difference the better, so the higher the reward.\n",
        "        normalized_diff_aoi_component = \\\n",
        "            ((1/(1+np.exp(- aoi * aoi_multiplier))) - 0.5) * 2.0\n",
        "\n",
        "        # Taking distance between current location and chosen location\n",
        "        chosen_location = sensing_locations[chosen_location_index]\n",
        "        drone_location = drones_locations[drone]\n",
        "        distance = np.linalg.norm(chosen_location - drone_location)\n",
        "\n",
        "        # Normalizing distance between 0 and 1. The smaller the distance, the higher the reward.\n",
        "        distance_multiplier = 0.03\n",
        "        normalized_location_distance = (1 - (1/(1+np.exp(-distance * distance_multiplier)))) * 2.0\n",
        "\n",
        "        # Define weights\n",
        "        aoi_weight = 0.7\n",
        "        distance_weight = 0.3\n",
        "        reward = normalized_diff_aoi_component * aoi_weight + normalized_location_distance * distance_weight + np.random.random() * (1.0 - aoi_weight - distance_weight)\n",
        "\n",
        "        # Choose the action that maximizes the reward\n",
        "        if reward >= highest_reward:\n",
        "            best_action = chosen_location_index\n",
        "            highest_reward = reward\n",
        "\n",
        "    return best_action\n"
      ],
      "execution_count": 854,
      "outputs": []
    },
    {
      "source": [
        "## Training phase"
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "cell_type": "code",
      "execution_count": 855,
      "metadata": {
        "tags": []
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:AutoGraph could not transform <function TFUniformReplayBuffer._as_dataset.<locals>.get_next at 0x7fef8c1a0b90> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "WARNING: AutoGraph could not transform <function TFUniformReplayBuffer._as_dataset.<locals>.get_next at 0x7fef8c1a0b90> and will run it as-is.\n",
            "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
            "Cause: 'arguments' object has no attribute 'posonlyargs'\n",
            "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
            "---------------- 0\n",
            "Loss: tf.Tensor(3.163754, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(5.722932, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.730577, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.757555, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(7.2097893, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.2469454, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(4.8487415, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(4.402645, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.799264, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(6.134297, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.305144, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.6240702, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.0088875, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.231114, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(6.1737137, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.8165405, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.118977, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.4701276, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.7805703, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(5.3589, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.5914962, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.0993454, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.3443305, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.8826976, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(4.358063, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.304094, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.0886424, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.2321074, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.8285276, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.7535775, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.6846552, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.8201432, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.8810005, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.5110269, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.7049086, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.1090739, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.1037955, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.2023289, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.8562783, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(2.1480346, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.6396972, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.7560372, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.0021197, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.3762894, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.1367512, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.3615632, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(3.028058, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(0.7557937, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(1.3337517, shape=(), dtype=float32)\n",
            "Loss: tf.Tensor(0.9656687, shape=(), dtype=float32)\n"
          ]
        }
      ],
      "source": [
        "# Define model hyperparameters\n",
        "num_iterations = 10 # @param {type:\"integer\"}\n",
        "intermediate_iterations = 300 # @param {type:\"integer\"}\n",
        "eval_interval = 10 # @param {type:\"integer\"}\n",
        "initial_collect_steps = 1 # @param {type:\"integer\"} \n",
        "collect_steps_per_iteration = 1  # @param {type:\"integer\"}\n",
        "batch_size = 64  # @param {type:\"integer\"}\n",
        "\n",
        "def compute_avg_return(environment, policy, num_episodes=10):\n",
        "  total_return = 0.0\n",
        "\n",
        "  for _ in range(num_episodes):\n",
        "    time_step = environment.reset()\n",
        "    episode_return = 0.0\n",
        "\n",
        "    for m in range(10):\n",
        "      action_step = policy.action(time_step)\n",
        "      time_step = environment.step(action_step.action)\n",
        "      episode_return += time_step.reward\n",
        "    total_return += episode_return\n",
        "\n",
        "  avg_return = total_return / num_episodes\n",
        "  return avg_return.numpy()[0]\n",
        "\n",
        "def collect_step(environment, policy, buffer, drone, iteration):\n",
        "  time_step = environment.current_time_step()\n",
        "  best_action = get_best_action(drone)\n",
        "  action_step = policy.action(time_step)\n",
        "\n",
        "  if np.random.random() < (iteration /num_iterations):\n",
        "      action_step = action_step.replace(action=tf.constant([best_action], dtype=np.int32))\n",
        "  else:\n",
        "      action_step = action_step.replace(action=tf.constant([random.randrange(locations_num)], dtype=np.int32))\n",
        "\n",
        "  next_time_step = environment.step(action_step.action)\n",
        "  drones_locations[drone] = next_time_step.observation\n",
        "  traj = trajectory.from_transition(time_step, action_step, next_time_step)\n",
        "\n",
        "  # Add trajectory to the replay buffer\n",
        "  buffer.add_batch(traj)\n",
        "\n",
        "def collect_data(env, policy, buffer, steps, drone, iteration):\n",
        "  for step in range(1, steps + 1):\n",
        "    current_cycle[0] = step\n",
        "    collect_step(env, policy, buffer, drone, iteration)\n",
        "\n",
        "replay_buffer = tf_uniform_replay_buffer.TFUniformReplayBuffer(\n",
        "        data_spec=agents[0].collect_data_spec,\n",
        "        batch_size=environments[0].batch_size)\n",
        "\n",
        "random_policy = random_tf_policy.RandomTFPolicy(environments[0].time_step_spec(),\n",
        "                                            environments[0].action_spec())\n",
        "                                            \n",
        "reset_aois()\n",
        "reset_drones_locations()\n",
        "\n",
        "collect_data(environments[0], random_policy, replay_buffer, initial_collect_steps, 0, 0)\n",
        "\n",
        "reset_aois()\n",
        "reset_drones_locations()\n",
        "\n",
        "dataset = replay_buffer.as_dataset(\n",
        "num_parallel_calls=3, \n",
        "sample_batch_size=batch_size, \n",
        "num_steps=2).prefetch(3)\n",
        "iterator = iter(dataset)\n",
        "\n",
        "returns = np.zeros((drones_num, (num_iterations // eval_interval) + 1), dtype=np.float64)\n",
        "for k in range(len(agents)):\n",
        "\n",
        "  avg_return = compute_avg_return(environments[k], agents[k].policy)\n",
        "  returns[k][0] = avg_return\n",
        "\n",
        "for i in range(num_iterations):\n",
        "  \n",
        "    reset_aois()\n",
        "    reset_drones_locations()\n",
        "\n",
        "    if i % 100 == 0:        \n",
        "        print(\"----------------\", i)\n",
        "\n",
        "    for j in range(intermediate_iterations):\n",
        "      for k in range(len(agents)):\n",
        "        agent = agents[k]\n",
        "        env = environments[k]\n",
        "        \n",
        "        # Collect a few steps using collect_policy and save to the replay buffer.\n",
        "        collect_data(env, agent.collect_policy, replay_buffer, collect_steps_per_iteration, k, i)\n",
        "        \n",
        "      \n",
        "    for k in range(len(agents)):\n",
        "        agent = agents[k]\n",
        "\n",
        "        # Sample a batch of data from the buffer and update the agent's network.\n",
        "        experience, unused_info = next(iterator)\n",
        "        train_loss = agent.train(experience).loss\n",
        "\n",
        "        print(\"Loss:\", train_loss)\n",
        "    \n",
        "    if i % eval_interval == 0:\n",
        "      for k in range(len(agents)):\n",
        "        agent = agents[k]\n",
        "\n",
        "        avg_return = compute_avg_return(environments[k], agent.policy)\n",
        "        returns[k][(num_iterations//eval_interval)] = avg_return\n",
        "\n"
      ]
    },
    {
      "source": [
        "## Test phase"
      ],
      "cell_type": "markdown",
      "metadata": {}
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KS0CyA4G776N",
        "tags": []
      },
      "source": [
        "time_steps = []\n",
        "for drone in range(drones_num):\n",
        "    time_steps.append(environments[drone].reset())\n",
        "\n",
        "# Number of total cycles for the simulation\n",
        "cycles_num = 200\n",
        "\n",
        "reset_aois()\n",
        "reset_drones_locations()\n",
        "\n",
        "# Running simulation (one cycle at a time for every drone)\n",
        "for cycle in range(1, cycles_num + 1):\n",
        "    if cycle % 10 == 1:\n",
        "        print(get_accumulated_aoi(cycle))\n",
        "        print(aois)\n",
        "        print(\"----------------\")\n",
        "\n",
        "    for drone in range(drones_num):\n",
        "        if cycle_stages[drone] == 0:\n",
        "            # Decision cycle\n",
        "            agent = agents[drone]\n",
        "            env = environments[drone]\n",
        "            current_cycle[0] = cycle\n",
        "            policy_step = agent.policy.action(time_steps[drone])\n",
        "            new_step = env.step(policy_step.action)\n",
        "            time_steps[drone] = new_step\n",
        "            chosen_locations[drone] = int(policy_step.action)\n",
        "            cycle_stages[drone] =  1\n",
        "        elif (drones_locations[drone] != sensing_locations[chosen_locations[drone]]).all():\n",
        "            # Empty cycle\n",
        "            traj = get_trajectory(drone)\n",
        "            new_location = drones_locations[drone] + traj\n",
        "            drones_locations[drone] = new_location\n",
        "        elif sensing_data_amounts[drone] == 0.0:\n",
        "            # Sensing cycle\n",
        "            cycle_stages[drone] =  2\n",
        "            sensing_data_amounts[drone] = total_location_data\n",
        "            cycle_stages[drone] =  3\n",
        "        else:\n",
        "            # Transmission cycle\n",
        "            sensing_data_amounts[drone] = \\\n",
        "                np.max([sensing_data_amounts[drone] - data_transmission_cycle, 0.0])\n",
        "            if sensing_data_amounts[drone] == 0.0:\n",
        "                cycle_stages[drone] =  0"
      ],
      "execution_count": 856,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "10.0\n",
            "[0 0 0 0 0 0 0 0 0 0]\n",
            "----------------\n",
            "107.0\n",
            "[0 1 0 0 0 0 0 0 1 1]\n",
            "----------------\n",
            "135.0\n",
            "[20  1  0 20  0  0 14  0  1 19]\n",
            "----------------\n",
            "216.0\n",
            "[29  1  0 20  0  0 14  0  1 29]\n",
            "----------------\n",
            "296.0\n",
            "[39  1  0 20  0  0 14  0  1 39]\n",
            "----------------\n",
            "345.0\n",
            "[49  1  0 20  0  0 45  0  1 49]\n",
            "----------------\n",
            "425.0\n",
            "[59  1  0 20  0  0 45  0  1 59]\n",
            "----------------\n",
            "461.0\n",
            "[69  1  0 64  0  0 45  0  1 69]\n",
            "----------------\n",
            "512.0\n",
            "[79  1  0 64  0  0 74  0  1 79]\n",
            "----------------\n",
            "583.0\n",
            "[89  1  0 64  0  0 83  0  1 89]\n",
            "----------------\n",
            "663.0\n",
            "[99  1  0 64  0  0 83  0  1 99]\n",
            "----------------\n",
            "684.0\n",
            "[109   1   0 102   0   0 104   0   1 109]\n",
            "----------------\n",
            "764.0\n",
            "[119   1   0 102   0   0 104   0   1 119]\n",
            "----------------\n",
            "827.0\n",
            "[129   1   0 102   0   0 121   0   1 129]\n",
            "----------------\n",
            "856.0\n",
            "[139   1   0 140   0   0 134   0   1 139]\n",
            "----------------\n",
            "936.0\n",
            "[149   1   0 140   0   0 134   0   1 149]\n",
            "----------------\n",
            "991.0\n",
            "[159   1   0 140   0   0 159   0   1 159]\n",
            "----------------\n",
            "1066.0\n",
            "[169   1   0 140   0   0 164   0   1 169]\n",
            "----------------\n",
            "1108.0\n",
            "[179   1   0 178   0   0 164   0   1 179]\n",
            "----------------\n",
            "1188.0\n",
            "[189   1   0 178   0   0 164   0   1 189]\n",
            "----------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 857,
      "metadata": {},
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<Figure size 432x288 with 1 Axes>",
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"248.518125pt\" version=\"1.1\" viewBox=\"0 0 377.304687 248.518125\" width=\"377.304687pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <metadata>\n  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n   <cc:Work>\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n    <dc:date>2021-03-03T18:55:40.547115</dc:date>\n    <dc:format>image/svg+xml</dc:format>\n    <dc:creator>\n     <cc:Agent>\n      <dc:title>Matplotlib v3.3.4, https://matplotlib.org/</dc:title>\n     </cc:Agent>\n    </dc:creator>\n   </cc:Work>\n  </rdf:RDF>\n </metadata>\n <defs>\n  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 248.518125 \nL 377.304687 248.518125 \nL 377.304687 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 35.304688 224.64 \nL 370.104688 224.64 \nL 370.104688 7.2 \nL 35.304688 7.2 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"PathCollection_1\">\n    <defs>\n     <path d=\"M 0 3 \nC 0.795609 3 1.55874 2.683901 2.12132 2.12132 \nC 2.683901 1.55874 3 0.795609 3 0 \nC 3 -0.795609 2.683901 -1.55874 2.12132 -2.12132 \nC 1.55874 -2.683901 0.795609 -3 0 -3 \nC -0.795609 -3 -1.55874 -2.683901 -2.12132 -2.12132 \nC -2.683901 -1.55874 -3 -0.795609 -3 0 \nC -3 0.795609 -2.683901 1.55874 -2.12132 2.12132 \nC -1.55874 2.683901 -0.795609 3 0 3 \nz\n\" id=\"md65b3f7983\" style=\"stroke:#1f77b4;\"/>\n    </defs>\n    <g clip-path=\"url(#p0d9b7ddfb0)\">\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"179.458142\" xlink:href=\"#md65b3f7983\" y=\"17.083636\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"147.398207\" xlink:href=\"#md65b3f7983\" y=\"127.39802\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"50.522869\" xlink:href=\"#md65b3f7983\" y=\"69.074759\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"354.886506\" xlink:href=\"#md65b3f7983\" y=\"214.756364\"/>\n     <use style=\"fill:#1f77b4;stroke:#1f77b4;\" x=\"50.522869\" xlink:href=\"#md65b3f7983\" y=\"69.074759\"/>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m2d21b52cc5\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"37.742721\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- −40 -->\n      <g transform=\"translate(27.190377 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 10.59375 35.5 \nL 73.1875 35.5 \nL 73.1875 27.203125 \nL 10.59375 27.203125 \nz\n\" id=\"DejaVuSans-8722\"/>\n        <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n        <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_2\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"82.817703\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- −30 -->\n      <g transform=\"translate(72.265359 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-51\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_3\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"127.892685\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- −20 -->\n      <g transform=\"translate(117.340341 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"172.967667\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- −10 -->\n      <g transform=\"translate(162.415324 239.238437)scale(0.1 -0.1)\">\n       <defs>\n        <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n       </defs>\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-49\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_5\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"218.042649\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 0 -->\n      <g transform=\"translate(214.861399 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_6\">\n     <g id=\"line2d_6\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"263.117631\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_6\">\n      <!-- 10 -->\n      <g transform=\"translate(256.755131 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-49\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_7\">\n     <g id=\"line2d_7\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"308.192614\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 20 -->\n      <g transform=\"translate(301.830114 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_8\">\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"353.267596\" xlink:href=\"#m2d21b52cc5\" y=\"224.64\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 30 -->\n      <g transform=\"translate(346.905096 239.238437)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-51\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_9\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"md00cc62222\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"201.95746\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- −40 -->\n      <g transform=\"translate(7.2 205.756678)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"174.092459\"/>\n      </g>\n     </g>\n     <g id=\"text_10\">\n      <!-- −30 -->\n      <g transform=\"translate(7.2 177.891678)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-51\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_11\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"146.227458\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- −20 -->\n      <g transform=\"translate(7.2 150.026677)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"118.362457\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- −10 -->\n      <g transform=\"translate(7.2 122.161676)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-49\"/>\n       <use x=\"147.412109\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_13\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"90.497456\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 0 -->\n      <g transform=\"translate(21.942188 94.296675)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_6\">\n     <g id=\"line2d_14\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"62.632456\"/>\n      </g>\n     </g>\n     <g id=\"text_14\">\n      <!-- 10 -->\n      <g transform=\"translate(15.579688 66.431674)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-49\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_7\">\n     <g id=\"line2d_15\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"35.304688\" xlink:href=\"#md00cc62222\" y=\"34.767455\"/>\n      </g>\n     </g>\n     <g id=\"text_15\">\n      <!-- 20 -->\n      <g transform=\"translate(15.579688 38.566674)scale(0.1 -0.1)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 35.304688 224.64 \nL 35.304688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 370.104688 224.64 \nL 370.104688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 35.304688 224.64 \nL 370.104688 224.64 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 35.304688 7.2 \nL 370.104688 7.2 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p0d9b7ddfb0\">\n   <rect height=\"217.44\" width=\"334.8\" x=\"35.304688\" y=\"7.2\"/>\n  </clipPath>\n </defs>\n</svg>\n",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQl0lEQVR4nO3df6jdd33H8edraeyCbtSuqW1+dMlYDKYqtlyiY2MwWk0VaeKGkDK2gkIQKlOY0caIMlxAF9B/NnEBZQWrpWCbZlOX/kCRwdp6+zP9FRvtbJPUNiJZHYbaxvf+uN+0p+lJk5tzzj3nfvJ8wOV+v5/vOd/Pq+nt6377Od+Tk6pCktSm3xl3AEnS6FjyktQwS16SGmbJS1LDLHlJathZ4w7Q67zzzqsVK1aMO4YkzSv33HPPL6pqcb9jE1XyK1asYHp6etwxJGleSfKzEx1zuUaSGmbJS1LDLHlJapglL0kNs+QlqWETdXeNdDp23neA7bv3cvDwEZacs4jN61az4ZKl444lTQRLXvPazvsOsOWmPRx54SgABw4fYctNewAsegmXazTPbd+996WCP+bIC0fZvnvvmBJJk8WS17x28PCRWY1LZxpLXvPaknMWzWpcOtNY8prXNq9bzaKFC14xtmjhAjavWz2mRNJk8YVXzWvHXlz17hqpP0te896GS5Za6tIJuFwjSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWEDl3yS5Um+n+TRJA8n+Vg3fm6S25I83n1/4+BxJUmzMYwr+ReBv6+qtwDvAq5Jsga4FrijqlYBd3T7kqQ5NHDJV9XTVXVvt/0r4FFgKbAeuK572HXAhkHnkiTNzlDX5JOsAC4B7gLeVFVPw8wvAuD8EzxnU5LpJNOHDh0aZhxJOuMNreSTvAH4NvDxqnruVJ9XVTuqaqqqphYvXjysOJIkhlTySRYyU/DXV9VN3fAzSS7sjl8IPDuMuSRJp24Yd9cE+BrwaFV9qefQLuDqbvtq4JZB55Ikzc4wPhnqT4G/AfYkub8b+zTwBeDGJB8GngQ+OIS5JEmzMHDJV9V/ATnB4csGPb8k6fT5jldJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNcySl6SGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYUMp+SRfT/Jskod6xs5NcluSx7vvbxzGXJKkUzesK/l/A644buxa4I6qWgXc0e1LkubQUEq+qn4I/PK44fXAdd32dcCGYcwlSTp1o1yTf1NVPQ3QfT+/34OSbEoynWT60KFDI4wjSWeesb/wWlU7qmqqqqYWL1487jiS1JRRlvwzSS4E6L4/O8K5JEl9nDXCc+8Crga+0H2/ZVQTfWbnHr5111McrWJBwlXvXM4/bnjbqKaTpHljWLdQfgv4b2B1kv1JPsxMub87yePAu7v9ofvMzj18484nOVoFwNEqvnHnk3xm555RTCdJ88pQruSr6qoTHLpsGOd/Ld+666kTjns1L+lMN/YXXgd17Ar+VMcl6Uwy70t+QTKrcUk6k8z7kr/qnctnNS5JZ5JR3l0zJ46tu3t3jSS9WmqC1q6npqZqenp63DEkaV5Jck9VTfU7Nu+XayRJJ2bJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhIy/5JFck2ZtkX5JrRz2fJOllIy35JAuAfwHeC6wBrkqyZpRzSpJeNuor+bXAvqr6aVX9BrgBWD/iOSVJnVGX/FLgqZ79/d2YJGkOjLrk+32a9is+iirJpiTTSaYPHTo04jiSdGYZdcnvB3o/UXsZcLD3AVW1o6qmqmpq8eLFI44jSWeWUZf8j4BVSVYmeR2wEdg14jklSZ2zRnnyqnoxyUeB3cAC4OtV9fAo55QkvWykJQ9QVd8FvjvqeSRJr+Y7XiWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNcySl6SGDVTyST6Y5OEkv00yddyxLUn2JdmbZN1gMSVJp2PQD/J+CPhL4F97B5OsATYCFwNLgNuTvLmqjg44nyRpFga6kq+qR6tqb59D64Ebqur5qnoC2AesHWQuSdLsjWpNfinwVM/+/m5MkjSHTrpck+R24II+h7ZW1S0nelqfsTrB+TcBmwAuuuiik8WRJM3CSUu+qi4/jfPuB5b37C8DDp7g/DuAHQBTU1N9fxFIkk7PqJZrdgEbk5ydZCWwCrh7RHNJkk5g0FsoP5BkP/AnwHeS7AaoqoeBG4FHgP8ErvHOGkmaewPdQllVNwM3n+DYNmDbIOeXJA3Gd7xKUsMGfTOUxM77DrB9914OHj7CknMWsXndajZc4h2z0iSw5DWQnfcdYMtNezjywsxLLgcOH2HLTXsALHppArhco4Fs3733pYI/5sgLR9m+u98boSXNNUteAzl4+MisxiXNLUteA1lyzqJZjUuaW5a8BrJ53WoWLVzwirFFCxewed3qMSWS1MsXXjWQYy+ueneNNJkseQ1swyVLLXVpQrlcI0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhg36Q9/YkjyV5MMnNSc7pObYlyb4ke5OsGzipJGnWBr2Svw14a1W9HfgxsAUgyRpgI3AxcAXwlSQLTngWSdJIDFTyVXVrVb3Y7d4JLOu21wM3VNXzVfUEsA9YO8hckqTZG+aa/IeA73XbS4Gneo7t78ZeJcmmJNNJpg8dOjTEOJKkk/5Vw0luBy7oc2hrVd3SPWYr8CJw/bGn9Xl89Tt/Ve0AdgBMTU31fYwk6fSctOSr6vLXOp7kauD9wGVVdayk9wPLex62DDh4uiElSadn0LtrrgA+BVxZVb/uObQL2Jjk7CQrgVXA3YPMJUmavUE/GeqfgbOB25IA3FlVH6mqh5PcCDzCzDLONVV1dMC5JEmzNFDJV9Ufv8axbcC2Qc4vSRqM73iVpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNcySl6SGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGjZQySf5fJIHk9yf5NYkS3qObUmyL8neJOsGjypJmq1Br+S3V9Xbq+odwH8AnwVIsgbYCFwMXAF8JcmCAeeSJM3SQCVfVc/17L4eqG57PXBDVT1fVU8A+4C1g8wlSZq9swY9QZJtwN8C/wv8RTe8FLiz52H7u7F+z98EbAK46KKLBo0jSepx0iv5JLcneajP13qAqtpaVcuB64GPHntan1NVnzGqakdVTVXV1OLFi0/3n0OS1MdJr+Sr6vJTPNc3ge8An2Pmyn15z7FlwMFZp5MkDWTQu2tW9exeCTzWbe8CNiY5O8lKYBVw9yBzSZJmb9A1+S8kWQ38FvgZ8BGAqno4yY3AI8CLwDVVdXTAuSRJszRQyVfVX73GsW3AtkHOL0kajO94laSGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekho2lJJP8okkleS8nrEtSfYl2Ztk3TDmkSTNzkAf5A2QZDnwbuDJnrE1wEbgYmAJcHuSN1fV0UHnkySdumFcyX8Z+CRQPWPrgRuq6vmqegLYB6wdwlySpFkYqOSTXAkcqKoHjju0FHiqZ39/N9bvHJuSTCeZPnTo0CBxJEnHOelyTZLbgQv6HNoKfBp4T7+n9RmrPmNU1Q5gB8DU1FTfx0iSTs9JS76qLu83nuRtwErggSQAy4B7k6xl5sp9ec/DlwEHB04rSZqV016uqao9VXV+Va2oqhXMFPulVfVzYBewMcnZSVYCq4C7h5JYknTKBr67pp+qejjJjcAjwIvANd5ZI0lzb2gl313N9+5vA7YN6/ySpNnzHa+S1DBLXpIaZslLUsMseUlq2EjurpEknZqd9x1g++69HDx8hCXnLGLzutVsuKTvXxBwWix5SRqTnfcdYMtNezjywswd5gcOH2HLTXsAhlb0LtdI0phs3733pYI/5sgLR9m+e+/Q5rDkJWlMDh4+Mqvx02HJS9KYLDln0azGT4clL0ljsnndahYtXPCKsUULF7B53eqhzeELr5I0JsdeXPXuGklq1IZLlg611I/nco0kNcySl6SGWfKS1DBLXpIaZslLUsNSVePO8JIkh4CfdbvnAb8YY5zTYea5Yea5Yea5MYzMf1hVi/sdmKiS75Vkuqqmxp1jNsw8N8w8N8w8N0ad2eUaSWqYJS9JDZvkkt8x7gCnwcxzw8xzw8xzY6SZJ3ZNXpI0uEm+kpckDciSl6SGTWTJJ/lEkkpyXs/YliT7kuxNsm6c+Xol+XySB5Pcn+TWJEt6jk1q5u1JHuty35zknJ5jE5kZIMkHkzyc5LdJpo47Nsm5r+hy7Uty7bjz9JPk60meTfJQz9i5SW5L8nj3/Y3jzHi8JMuTfD/Jo93Pxce68YnNneR3k9yd5IEu8z9046PLXFUT9QUsB3Yz86ao87qxNcADwNnASuAnwIJxZ+2y/X7P9t8BX50Hmd8DnNVtfxH44qRn7vK9BVgN/ACY6hmf2NzAgi7PHwGv63KuGXeuPjn/HLgUeKhn7J+Aa7vta4/9nEzKF3AhcGm3/XvAj7ufhYnNDQR4Q7e9ELgLeNcoM0/ilfyXgU8Cva8IrwduqKrnq+oJYB+wdhzhjldVz/Xsvp6Xc09y5lur6sVu905gWbc9sZkBqurRqur3CceTnHstsK+qflpVvwFuYCbvRKmqHwK/PG54PXBdt30dsGEuM51MVT1dVfd2278CHgWWMsG5a8b/dbsLu69ihJknquSTXAkcqKoHjju0FHiqZ39/NzYRkmxL8hTw18Bnu+GJztzjQ8D3uu35kvl4k5x7krOdzJuq6mmYKVTg/DHnOaEkK4BLmLkynujcSRYkuR94Fritqkaaec4/GSrJ7cAFfQ5tBT7NzFLCq57WZ2zO7v18rcxVdUtVbQW2JtkCfBT4HBOeuXvMVuBF4PpjT+vz+Dm9x/ZUcvd7Wp+xSbk3eJKzNSHJG4BvAx+vqueSfn/kk6OqjgLv6F4LuznJW0c535yXfFVd3m88yduYWU99oPuXtAy4N8laZq5+lvc8fBlwcMRRX3KizH18E/gOMyU/0ZmTXA28H7isuoVAxpwZZvVn3WvsuV/DJGc7mWeSXFhVTye5kJkrz4mSZCEzBX99Vd3UDU98boCqOpzkB8AVjDDzxCzXVNWeqjq/qlZU1Qpm/uO4tKp+DuwCNiY5O8lKYBVw9xjjviTJqp7dK4HHuu1JznwF8Cngyqr6dc+hic18EpOc+0fAqiQrk7wO2MhM3vlgF3B1t301cKL/kxqLzFwNfg14tKq+1HNoYnMnWXzsbrYki4DLmemM0WUe96vNr/Eq9P/Q3V3T7W9l5i6FvcB7x52vJ9e3gYeAB4F/B5bOg8z7mFknvr/7+uqkZ+6yfYCZX/7PA88Au+dJ7vcxc+fHT5hZdhp7pj4ZvwU8DbzQ/Rl/GPgD4A7g8e77uePOeVzmP2Nm6evBnp/l901ybuDtwH1d5oeAz3bjI8vsX2sgSQ2bmOUaSdLwWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYf8PUnr3wbaB/xMAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Visualize the position of the drones at the end of the simulation \n",
        "_ = plt.scatter(drones_locations[:,0], drones_locations[:,1])"
      ]
    }
  ]
}