Trial_1_Oversample.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n9MuWuhpDO8c",
        "outputId": "b030ef41-c15e-4e2a-f697-9999bbe62587"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgJerUsTCo_I",
        "outputId": "36e8b715-6366-4b4e-918b-7764ba008c54"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.3.1.tar.gz (281.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.4 MB 43 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.5\n",
            "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 65.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.3.1-py2.py3-none-any.whl size=281845513 sha256=71ba7df10961d26e826b8183aed96ceac254f715c90548369ecf7f7cda6f0869\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/59/f5/79a5bf931714dcd201b26025347785f087370a10a3329a899c\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.5 pyspark-3.3.1\n"
          ]
        }
      ],
      "source": [
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession"
      ],
      "metadata": {
        "id": "1Wcm87mzCsfd"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark = SparkSession.builder.appName('test_1').getOrCreate()"
      ],
      "metadata": {
        "id": "LNGoVrLGDCvB"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark.read.csv('/content/drive/MyDrive/BDAI Project/unbalaced_20_80_dataset.csv',inferSchema=True, header=True)"
      ],
      "metadata": {
        "id": "yKyEi_ZfDIvh"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PYUsDqU-Hf5a",
        "outputId": "e27ca18b-4153-4bd7-842b-84700cedde61"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------------------+--------------+--------+-------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+------------------+---------------+---------------+------------------+------------------+------------------+------------------+------------------+-------------------+------------+------------+-----------+------------------+------------------+-----------+-----------+-----------+--------------------+-------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+----------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+-----------+--------+-----------+-----------+-----+\n",
            "|    _c0|             Flow ID|        Src IP|Src Port|       Dst IP|Dst Port|Protocol|           Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|   Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|  Bwd Pkt Len Mean|   Bwd Pkt Len Std|       Flow Byts/s|       Flow Pkts/s|     Flow IAT Mean|       Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|      Fwd IAT Mean|       Fwd IAT Std|Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|        Bwd IAT Mean|        Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|         Fwd Pkts/s|         Bwd Pkts/s|Pkt Len Min|Pkt Len Max|      Pkt Len Mean|       Pkt Len Std|       Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|      Pkt Size Avg|Fwd Seg Size Avg|  Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|  Idle Mean|Idle Std|   Idle Max|   Idle Min|Label|\n",
            "+-------+--------------------+--------------+--------+-------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+------------------+---------------+---------------+------------------+------------------+------------------+------------------+------------------+-------------------+------------+------------+-----------+------------------+------------------+-----------+-----------+-----------+--------------------+-------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+----------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+-----------+--------+-----------+-----------+-----+\n",
            "|1739476|172.31.69.25-18.2...| 18.219.193.20|   37882| 172.31.69.25|      80|       6|16/02/2018 11:27:...|         8660|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0|230.94688221709012|            8660.0|                0.0|      8660.0|      8660.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32| 115.47344110854506| 115.47344110854506|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1822666|172.31.69.28-18.2...|  172.31.69.28|      80|   18.219.9.1|   63287|       6|22/02/2018 12:13:...|         5829|           4|           3|          935.0|          298.0|          935.0|            0.0|          233.75|467.50000000000006|          298.0|            0.0| 99.33333333333331|172.05038021850845|211528.56407617085|1200.8920912677988|             971.5| 2104.1249725242083|      5260.0|         7.0|     5822.0|1940.6666666666665| 3119.411856958509|     5541.0|       46.0|     5540.0|              2770.0| 3521.3917703090065|     5260.0|      280.0|            0|            0|            0|            0|            92|            72|  686.2240521530281|   514.668039114771|        0.0|      935.0|154.12500000000003| 332.3064151824766|110427.55357142855|           0|           1|           0|           0|           0|           0|             1|           1|          0.0|176.14285714285714|          233.75| 99.33333333333331|             0|             0|               0|             0|             0|               0|               4|             935|               3|             298|               -1|            32768|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "| 905739|172.31.69.28-52.1...| 52.14.136.135|   63095| 172.31.69.28|      80|       6|22/02/2018 12:14:...|         3396|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0| 588.9281507656066|            3396.0|                0.0|      3396.0|      3396.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            20|            20|  294.4640753828033|  294.4640753828033|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|            32738|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1143064|172.31.69.28-18.2...|18.216.200.189|   52341| 172.31.69.28|      80|       6|22/02/2018 12:28:...|         2390|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0| 836.8200836820082|            2390.0|                0.0|      2390.0|      2390.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            20|            20| 418.41004184100416| 418.41004184100416|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|            32738|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1934016|172.31.69.28-18.2...|  172.31.69.28|      80|18.218.55.126|   57459|       6|22/02/2018 12:19:...|        17362|           4|           3|          935.0|          314.0|          935.0|            0.0|          233.75|467.50000000000006|          314.0|            0.0|104.66666666666669| 181.2879845255425| 71938.71673770303|403.17935721691055| 2893.666666666667|   6770.57766122409|     16711.0|         7.0|    17355.0|            5785.0| 9779.680414001268|    17077.0|       38.0|    17074.0|              8537.0|  11559.78165883768|    16711.0|      363.0|            0|            0|            0|            0|            92|            72| 230.38820412394887| 172.79115309296162|        0.0|      935.0|           156.125|333.34256832445334|111117.26785714287|           0|           1|           0|           0|           0|           0|             1|           1|          0.0| 178.4285714285714|          233.75|104.66666666666669|             0|             0|               0|             0|             0|               0|               4|             935|               3|             314|               -1|            32768|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1916750|172.31.69.28-18.2...|  172.31.69.28|      80|   18.219.9.1|   56276|       6|22/02/2018 12:18:...|       201316|           4|           3|          935.0|          274.0|          935.0|            0.0|          233.75|467.50000000000006|          274.0|            0.0| 91.33333333333331|158.19397375795745| 6005.483915833814|34.771205468020426|33552.666666666664|  67513.71346227865|    168848.0|         6.0|   201308.0| 67102.66666666666| 89590.87364979391|   168848.0|       30.0|    32432.0|             16216.0|  22575.09109616172|    32179.0|      253.0|            0|            0|            0|            0|            92|            72| 19.869260267440247| 14.901945200580185|        0.0|      935.0|           151.125| 330.9274575233327|109512.98214285717|           0|           1|           0|           0|           0|           0|             1|           1|          0.0|172.71428571428572|          233.75| 91.33333333333331|             0|             0|               0|             0|             0|               0|               4|             935|               3|             274|               -1|            32768|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|2170484|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   40594|       6|16/02/2018 11:17:...|      4265070|           4|           4|          935.0|          351.0|          935.0|            0.0|          233.75|467.50000000000006|          351.0|            0.0|             87.75|             175.5| 301.5190840947511|1.8757019228289336| 609295.7142857143| 1557317.0768343562|   4140224.0|         5.0|   124841.0| 41613.66666666667| 41745.29595455437|    83507.0|       18.0|  4181537.0|  1393845.6666666665|  2378520.243271504|  4140224.0|      332.0|            0|            0|            0|            0|           136|           136| 0.9378509614144668| 0.9378509614144668|        0.0|      935.0|142.88888888888889|318.91826713299304|101708.86111111114|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|            160.75|          233.75|             87.75|             0|             0|               0|             0|             0|               0|               4|             935|               4|             351|               -1|              211|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1891912|172.31.69.28-18.2...|  172.31.69.28|      80|  18.219.5.43|   53734|       6|22/02/2018 12:17:...|         1461|           4|           3|          935.0|          339.0|          935.0|            0.0|          233.75|467.50000000000006|          339.0|            0.0|             113.0|195.72174125528312| 872005.4757015742| 4791.238877481177|             243.5|  348.4111077448593|       906.0|         6.0|     1455.0|             485.0| 658.1116926479881|     1239.0|       26.0|     1236.0|               618.0| 407.29350596345137|      906.0|      330.0|            0|            0|            0|            0|            92|            72| 2737.8507871321012|  2053.388090349076|        0.0|      935.0|            159.25| 335.1463433018052|112323.07142857145|           0|           1|           0|           0|           0|           0|             1|           1|          0.0|             182.0|          233.75|             113.0|             0|             0|               0|             0|             0|               0|               4|             935|               3|             339|               -1|            32768|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "| 262892|172.31.69.25-18.2...| 18.219.193.20|   41228| 172.31.69.25|      80|       6|16/02/2018 11:16:...|        12708|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0|157.38117721120554|           12708.0|                0.0|     12708.0|     12708.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32|  78.69058860560277|  78.69058860560277|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "| 170886|172.31.69.25-18.2...| 18.219.193.20|   33826| 172.31.69.25|      80|       6|16/02/2018 11:15:...|        51238|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0| 39.03352980210001|           51238.0|                0.0|     51238.0|     51238.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32| 19.516764901050006| 19.516764901050006|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "| 847148|172.31.69.25-18.2...| 18.219.193.20|   43072| 172.31.69.25|      80|       6|16/02/2018 11:20:...|        13813|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0|144.79113878230655|           13813.0|                0.0|     13813.0|     13813.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32|  72.39556939115327|  72.39556939115327|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|3142101|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   56310|       6|16/02/2018 11:24:...|      4497448|           4|           4|          935.0|          309.0|          935.0|            0.0|          233.75|467.50000000000006|          309.0|            0.0|             77.25|             154.5|276.60130811962694|1.7787865473930993| 642492.5714285714| 1642641.1620094082|   4366923.0|         6.0|   130518.0|           43506.0| 42015.30460439386|    83886.0|       27.0|  4413529.0|  1471176.3333333337|  2507895.590364426|  4366923.0|      309.0|            0|            0|            0|            0|           136|           136| 0.8893932736965496| 0.8893932736965496|        0.0|      935.0|138.22222222222223| 315.7842530026544| 99719.69444444444|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|             155.5|          233.75|             77.25|             0|             0|               0|             0|             0|               0|               4|             935|               4|             309|               -1|              211|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|2964667|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   41060|       6|16/02/2018 11:22:...|     61541276|           4|           5|          935.0|          338.0|          935.0|            0.0|          233.75|467.50000000000006|          338.0|            0.0|              67.6|151.15819527898574|20.685303957623496|0.1462433115621457|         7692659.5|2.171257927076242E7| 6.1428536E7|         7.0|   104462.0| 34820.66666666667|33696.968444258215|    67319.0|       41.0|6.1504125E7|1.5376031249999998E7|3.070167230756893E7|6.1428536E7|     8271.0|            0|            0|            0|            0|           136|           168|0.06499702736095364|0.08124628420119205|        0.0|      935.0|             127.3| 303.0247697979673| 91824.01111111107|           1|           0|           0|           0|           1|           0|             0|           0|          1.0|141.44444444444446|          233.75|              67.6|             0|             0|               0|             0|             0|               0|               4|             935|               5|             338|               -1|              211|                1|               0|     8271.0|       0.0|    8271.0|    8271.0|6.1428536E7|     0.0|6.1428536E7|6.1428536E7| ddos|\n",
            "|2845398|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   48258|       6|16/02/2018 11:22:...|      4320844|           4|           4|          935.0|          374.0|          935.0|            0.0|          233.75|467.50000000000006|          374.0|            0.0|              93.5|             187.0|302.95007179152964| 1.851490125540288| 617263.4285714285| 1597285.1585744752|   4239423.0|         6.0|    81415.0|27138.333333333336| 24077.95411436224|    46024.0|       25.0|  4285447.0|  1428482.3333333337| 2434346.1596267144|  4239423.0|    22205.0|            0|            0|            0|            0|           136|           136|  0.925745062770144|  0.925745062770144|        0.0|      935.0|145.44444444444446| 320.8804727274282|102964.27777777775|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|           163.625|          233.75|              93.5|             0|             0|               0|             0|             0|               0|               4|             935|               4|             374|               -1|              211|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1661439|172.31.69.25-18.2...| 18.219.193.20|   41464| 172.31.69.25|      80|       6|16/02/2018 11:26:...|          483|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0| 4140.786749482401|             483.0|                0.0|       483.0|       483.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32| 2070.3933747412007| 2070.3933747412007|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1346670|172.31.69.28-18.2...|  172.31.69.28|      80|18.218.55.126|   64285|       6|21/02/2018 11:52:...|         1341|           4|           3|          935.0|          272.0|          935.0|            0.0|          233.75|467.50000000000006|          272.0|            0.0| 90.66666666666669|157.03927321957818| 900074.5712155108| 5219.985085756898|             223.5| 275.96068560575804|       706.0|         6.0|     1335.0|             445.0|475.29674941030265|      962.0|       27.0|      960.0|               480.0|  319.6122650963195|      706.0|      254.0|            0|            0|            0|            0|            92|            72| 2982.8486204325127|  2237.136465324385|        0.0|      935.0|           150.875| 330.8221090815166|109443.26785714287|           0|           1|           0|           0|           0|           0|             1|           1|          0.0| 172.4285714285714|          233.75| 90.66666666666669|             0|             0|               0|             0|             0|               0|               4|             935|               3|             272|               -1|            32768|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1700729|172.31.69.25-18.2...| 18.219.193.20|   40198| 172.31.69.25|      80|       6|16/02/2018 11:27:...|          676|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0| 2958.579881656805|             676.0|                0.0|       676.0|       676.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            32|            32| 1479.2899408284025| 1479.2899408284025|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|              225|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|2585319|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   35928|       6|16/02/2018 11:20:...|      4383793|           4|           4|          935.0|          310.0|          935.0|            0.0|          233.75|467.50000000000006|          310.0|            0.0|              77.5|             155.0|284.00063597893416|   1.8249036850052| 626256.1428571428| 1600815.1614760137|   4255935.0|         8.0|   127850.0| 42616.66666666667|  44195.1537199876|    88272.0|       43.0|  4344195.0|           1448065.0| 2431953.2028137385|  4255935.0|     8131.0|            0|            0|            0|            0|           136|           136| 0.9124518425025999| 0.9124518425025999|        0.0|      935.0|138.33333333333334| 315.8520223142476|           99762.5|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|           155.625|          233.75|              77.5|             0|             0|               0|             0|             0|               0|               4|             935|               4|             310|               -1|              211|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|3151335|172.31.69.25-18.2...|  172.31.69.25|      80|18.219.193.20|   54882|       6|16/02/2018 11:24:...|      4136996|           4|           4|          935.0|          358.0|          935.0|            0.0|          233.75|467.50000000000006|          358.0|            0.0|              89.5|             179.0| 312.5456248930384|1.9337703009623408| 590999.4285714285| 1519772.2016687081|   4037103.0|         5.0|    99888.0|           33296.0|29231.011545959198|    54886.0|       32.0|  4091986.0|  1363995.3333333335| 2315138.8116275733|  4037103.0|      252.0|            0|            0|            0|            0|           136|           136| 0.9668851504811704| 0.9668851504811704|        0.0|      935.0|143.66666666666666|319.49726133411536|          102078.5|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|           161.625|          233.75|              89.5|             0|             0|               0|             0|             0|               0|               4|             935|               4|             358|               -1|              211|                1|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "|1006757|172.31.69.28-18.2...|    18.219.9.1|   57282| 172.31.69.28|      80|       6|22/02/2018 12:19:...|       172168|           1|           1|            0.0|            0.0|            0.0|            0.0|             0.0|               0.0|            0.0|            0.0|               0.0|               0.0|               0.0|11.616560568746806|          172168.0|                0.0|    172168.0|    172168.0|        0.0|               0.0|               0.0|        0.0|        0.0|        0.0|                 0.0|                0.0|        0.0|        0.0|            0|            0|            0|            0|            20|            20|  5.808280284373403|  5.808280284373403|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          1.0|               0.0|             0.0|               0.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|               0|               -1|            32738|                0|               0|        0.0|       0.0|       0.0|       0.0|        0.0|     0.0|        0.0|        0.0| ddos|\n",
            "+-------+--------------------+--------------+--------+-------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+------------------+---------------+---------------+------------------+------------------+------------------+------------------+------------------+-------------------+------------+------------+-----------+------------------+------------------+-----------+-----------+-----------+--------------------+-------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+----------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+-----------+--------+-----------+-----------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wLlM_SMlDMmT",
        "outputId": "cd394b19-5637-429a-ff1d-aa04cdd24015"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- _c0: integer (nullable = true)\n",
            " |-- Flow ID: string (nullable = true)\n",
            " |-- Src IP: string (nullable = true)\n",
            " |-- Src Port: integer (nullable = true)\n",
            " |-- Dst IP: string (nullable = true)\n",
            " |-- Dst Port: integer (nullable = true)\n",
            " |-- Protocol: integer (nullable = true)\n",
            " |-- Timestamp: string (nullable = true)\n",
            " |-- Flow Duration: integer (nullable = true)\n",
            " |-- Tot Fwd Pkts: integer (nullable = true)\n",
            " |-- Tot Bwd Pkts: integer (nullable = true)\n",
            " |-- TotLen Fwd Pkts: double (nullable = true)\n",
            " |-- TotLen Bwd Pkts: double (nullable = true)\n",
            " |-- Fwd Pkt Len Max: double (nullable = true)\n",
            " |-- Fwd Pkt Len Min: double (nullable = true)\n",
            " |-- Fwd Pkt Len Mean: double (nullable = true)\n",
            " |-- Fwd Pkt Len Std: double (nullable = true)\n",
            " |-- Bwd Pkt Len Max: double (nullable = true)\n",
            " |-- Bwd Pkt Len Min: double (nullable = true)\n",
            " |-- Bwd Pkt Len Mean: double (nullable = true)\n",
            " |-- Bwd Pkt Len Std: double (nullable = true)\n",
            " |-- Flow Byts/s: double (nullable = true)\n",
            " |-- Flow Pkts/s: double (nullable = true)\n",
            " |-- Flow IAT Mean: double (nullable = true)\n",
            " |-- Flow IAT Std: double (nullable = true)\n",
            " |-- Flow IAT Max: double (nullable = true)\n",
            " |-- Flow IAT Min: double (nullable = true)\n",
            " |-- Fwd IAT Tot: double (nullable = true)\n",
            " |-- Fwd IAT Mean: double (nullable = true)\n",
            " |-- Fwd IAT Std: double (nullable = true)\n",
            " |-- Fwd IAT Max: double (nullable = true)\n",
            " |-- Fwd IAT Min: double (nullable = true)\n",
            " |-- Bwd IAT Tot: double (nullable = true)\n",
            " |-- Bwd IAT Mean: double (nullable = true)\n",
            " |-- Bwd IAT Std: double (nullable = true)\n",
            " |-- Bwd IAT Max: double (nullable = true)\n",
            " |-- Bwd IAT Min: double (nullable = true)\n",
            " |-- Fwd PSH Flags: integer (nullable = true)\n",
            " |-- Bwd PSH Flags: integer (nullable = true)\n",
            " |-- Fwd URG Flags: integer (nullable = true)\n",
            " |-- Bwd URG Flags: integer (nullable = true)\n",
            " |-- Fwd Header Len: integer (nullable = true)\n",
            " |-- Bwd Header Len: integer (nullable = true)\n",
            " |-- Fwd Pkts/s: double (nullable = true)\n",
            " |-- Bwd Pkts/s: double (nullable = true)\n",
            " |-- Pkt Len Min: double (nullable = true)\n",
            " |-- Pkt Len Max: double (nullable = true)\n",
            " |-- Pkt Len Mean: double (nullable = true)\n",
            " |-- Pkt Len Std: double (nullable = true)\n",
            " |-- Pkt Len Var: double (nullable = true)\n",
            " |-- FIN Flag Cnt: integer (nullable = true)\n",
            " |-- SYN Flag Cnt: integer (nullable = true)\n",
            " |-- RST Flag Cnt: integer (nullable = true)\n",
            " |-- PSH Flag Cnt: integer (nullable = true)\n",
            " |-- ACK Flag Cnt: integer (nullable = true)\n",
            " |-- URG Flag Cnt: integer (nullable = true)\n",
            " |-- CWE Flag Count: integer (nullable = true)\n",
            " |-- ECE Flag Cnt: integer (nullable = true)\n",
            " |-- Down/Up Ratio: double (nullable = true)\n",
            " |-- Pkt Size Avg: double (nullable = true)\n",
            " |-- Fwd Seg Size Avg: double (nullable = true)\n",
            " |-- Bwd Seg Size Avg: double (nullable = true)\n",
            " |-- Fwd Byts/b Avg: integer (nullable = true)\n",
            " |-- Fwd Pkts/b Avg: integer (nullable = true)\n",
            " |-- Fwd Blk Rate Avg: integer (nullable = true)\n",
            " |-- Bwd Byts/b Avg: integer (nullable = true)\n",
            " |-- Bwd Pkts/b Avg: integer (nullable = true)\n",
            " |-- Bwd Blk Rate Avg: integer (nullable = true)\n",
            " |-- Subflow Fwd Pkts: integer (nullable = true)\n",
            " |-- Subflow Fwd Byts: integer (nullable = true)\n",
            " |-- Subflow Bwd Pkts: integer (nullable = true)\n",
            " |-- Subflow Bwd Byts: integer (nullable = true)\n",
            " |-- Init Fwd Win Byts: integer (nullable = true)\n",
            " |-- Init Bwd Win Byts: integer (nullable = true)\n",
            " |-- Fwd Act Data Pkts: integer (nullable = true)\n",
            " |-- Fwd Seg Size Min: integer (nullable = true)\n",
            " |-- Active Mean: double (nullable = true)\n",
            " |-- Active Std: double (nullable = true)\n",
            " |-- Active Max: double (nullable = true)\n",
            " |-- Active Min: double (nullable = true)\n",
            " |-- Idle Mean: double (nullable = true)\n",
            " |-- Idle Std: double (nullable = true)\n",
            " |-- Idle Max: double (nullable = true)\n",
            " |-- Idle Min: double (nullable = true)\n",
            " |-- Label: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.groupBy('Label').count().show()"
      ],
      "metadata": {
        "id": "a832XQqQINKC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "56af5dd2-234d-4861-97af-c6cdae63ac49"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-------+\n",
            "| Label|  count|\n",
            "+------+-------+\n",
            "|  ddos|1294529|\n",
            "|Benign|6321980|\n",
            "+------+-------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col,isnan, when, count"
      ],
      "metadata": {
        "id": "ioXMrKAaFoAR"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.select([count(when(isnan(c), c)).alias(c) for c in df.columns]).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bdjUZrjVEvsE",
        "outputId": "bffb4d91-9375-409a-bf78-ac8560a2515f"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+-------+------+--------+------+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "|_c0|Flow ID|Src IP|Src Port|Dst IP|Dst Port|Protocol|Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|Fwd Pkt Len Mean|Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|Bwd Pkt Len Mean|Bwd Pkt Len Std|Flow Byts/s|Flow Pkts/s|Flow IAT Mean|Flow IAT Std|Flow IAT Max|Flow IAT Min|Fwd IAT Tot|Fwd IAT Mean|Fwd IAT Std|Fwd IAT Max|Fwd IAT Min|Bwd IAT Tot|Bwd IAT Mean|Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|Fwd Pkts/s|Bwd Pkts/s|Pkt Len Min|Pkt Len Max|Pkt Len Mean|Pkt Len Std|Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|Pkt Size Avg|Fwd Seg Size Avg|Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|Active Mean|Active Std|Active Max|Active Min|Idle Mean|Idle Std|Idle Max|Idle Min|Label|\n",
            "+---+-------+------+--------+------+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "|  0|      0|     0|       0|     0|       0|       0|        0|            0|           0|           0|              0|              0|              0|              0|               0|              0|              0|              0|               0|              0|          0|          0|            0|           0|           0|           0|          0|           0|          0|          0|          0|          0|           0|          0|          0|          0|            0|            0|            0|            0|             0|             0|         0|         0|          0|          0|           0|          0|          0|           0|           0|           0|           0|           0|           0|             0|           0|            0|           0|               0|               0|             0|             0|               0|             0|             0|               0|               0|               0|               0|               0|                0|                0|                0|               0|          0|         0|         0|         0|        0|       0|       0|       0|    0|\n",
            "+---+-------+------+--------+------+--------+--------+---------+-------------+------------+------------+---------------+---------------+---------------+---------------+----------------+---------------+---------------+---------------+----------------+---------------+-----------+-----------+-------------+------------+------------+------------+-----------+------------+-----------+-----------+-----------+-----------+------------+-----------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+----------+----------+-----------+-----------+------------+-----------+-----------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------+----------------+----------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+-----------+----------+----------+----------+---------+--------+--------+--------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### NO NULLS"
      ],
      "metadata": {
        "id": "3Qy38ZtlH4Xx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_ddos = df.filter(df['Label']=='ddos')\n",
        "df_benign = df.filter(df['Label']=='Benign')"
      ],
      "metadata": {
        "id": "71g6O2LTFo7j"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "oversample_ratio = df_benign.count()/df_ddos.count()"
      ],
      "metadata": {
        "id": "0VT9bVI8Jb7l"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_ddos_oversampled = df_ddos.sample(withReplacement=True, fraction=oversample_ratio, seed=1)"
      ],
      "metadata": {
        "id": "iXTa1YhtJmrS"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_final = df_benign.unionAll(df_ddos_oversampled)"
      ],
      "metadata": {
        "id": "UdTlYPJyKIo7"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_final.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Hd2l0vHKRjK",
        "outputId": "b6828042-0e7b-4482-97ea-3b8960ae89b6"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------------------+---------------+--------+---------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+------------------+------------------+---------------+---------------+------------------+------------------+--------------------+-------------------+------------------+------------------+------------+------------+------------+------------------+------------------+-----------+-----------+------------+------------------+------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+------------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------------+------------------+----------+----------+-------------------+------------------+-----------+-----------+------+\n",
            "|    _c0|             Flow ID|         Src IP|Src Port|         Dst IP|Dst Port|Protocol|           Timestamp|Flow Duration|Tot Fwd Pkts|Tot Bwd Pkts|TotLen Fwd Pkts|TotLen Bwd Pkts|Fwd Pkt Len Max|Fwd Pkt Len Min|  Fwd Pkt Len Mean|   Fwd Pkt Len Std|Bwd Pkt Len Max|Bwd Pkt Len Min|  Bwd Pkt Len Mean|   Bwd Pkt Len Std|         Flow Byts/s|        Flow Pkts/s|     Flow IAT Mean|      Flow IAT Std|Flow IAT Max|Flow IAT Min| Fwd IAT Tot|      Fwd IAT Mean|       Fwd IAT Std|Fwd IAT Max|Fwd IAT Min| Bwd IAT Tot|      Bwd IAT Mean|       Bwd IAT Std|Bwd IAT Max|Bwd IAT Min|Fwd PSH Flags|Bwd PSH Flags|Fwd URG Flags|Bwd URG Flags|Fwd Header Len|Bwd Header Len|         Fwd Pkts/s|         Bwd Pkts/s|Pkt Len Min|Pkt Len Max|      Pkt Len Mean|       Pkt Len Std|       Pkt Len Var|FIN Flag Cnt|SYN Flag Cnt|RST Flag Cnt|PSH Flag Cnt|ACK Flag Cnt|URG Flag Cnt|CWE Flag Count|ECE Flag Cnt|Down/Up Ratio|      Pkt Size Avg|  Fwd Seg Size Avg|  Bwd Seg Size Avg|Fwd Byts/b Avg|Fwd Pkts/b Avg|Fwd Blk Rate Avg|Bwd Byts/b Avg|Bwd Pkts/b Avg|Bwd Blk Rate Avg|Subflow Fwd Pkts|Subflow Fwd Byts|Subflow Bwd Pkts|Subflow Bwd Byts|Init Fwd Win Byts|Init Bwd Win Byts|Fwd Act Data Pkts|Fwd Seg Size Min|       Active Mean|        Active Std|Active Max|Active Min|          Idle Mean|          Idle Std|   Idle Max|   Idle Min| Label|\n",
            "+-------+--------------------+---------------+--------+---------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+------------------+------------------+---------------+---------------+------------------+------------------+--------------------+-------------------+------------------+------------------+------------+------------+------------+------------------+------------------+-----------+-----------+------------+------------------+------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+------------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------------+------------------+----------+----------+-------------------+------------------+-----------+-----------+------+\n",
            "|1445871|172.31.0.2-172.31...|   172.31.66.76|   55330|     172.31.0.2|      53|      17| 20/02/2018 10:36:27|        22123|           2|           2|           72.0|          194.0|           36.0|           36.0|              36.0|               0.0|           97.0|           97.0|              97.0|               0.0|  12023.685756904579| 180.80730461510646| 7374.333333333334|11315.938597099817|     20432.0|       429.0|     21694.0|           21694.0|               0.0|    21694.0|    21694.0|     20861.0|           20861.0|               0.0|    20861.0|    20861.0|            0|            0|            0|            0|            16|            16|  90.40365230755323|  90.40365230755323|       36.0|       97.0|              60.4|33.411076007815126|            1116.3|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              75.5|              36.0|              97.0|             0|             0|               0|             0|             0|               0|               2|              72|               2|             194|               -1|               -1|                1|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "| 283411|172.217.9.228-192...|  192.168.10.15|   53799|  172.217.9.228|     443|       6|04/07/2017 12:15:...|      3095495|           4|           2|           77.0|           92.0|           46.0|            0.0|             19.25|23.056091024571646|           46.0|           46.0|              46.0|               0.0|   54.59546857610818| 1.9383006595067993|          619099.0|1322575.2609619235|   2984691.0|         3.0|   3001930.0|1000643.3333333331|1718257.2862573094|  2984691.0|        3.0|     37178.0|           37178.0|               0.0|    37178.0|    37178.0|            0|            1|            0|            0|            80|            40| 1.2922004396711997| 0.6461002198355997|        0.0|       46.0| 30.71428571428571|21.684973950597573| 470.2380952380952|           0|           0|           0|           1|           1|           0|             0|           0|          0.0|35.833333333333336|             19.25|              46.0|             0|             0|               0|             0|             0|               0|               4|              77|               2|              92|               -1|              360|                2|               0|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|6626404|129.158.72.21-172...|  129.158.72.21|   56889|  172.31.66.121|    3389|       6| 20/02/2018 08:55:58|      1127340|           8|           7|         1052.0|         1429.0|          581.0|            0.0|             131.5|196.37209577737872|         1149.0|            0.0|204.14285714285708| 420.4990798925441|   2200.755761349726| 13.305657565596892| 80524.28571428571| 255987.1821350643|    968741.0|         3.0|   1127340.0|161048.57142857142| 381944.9179108846|  1026973.0|        3.0|   1114653.0|          185775.5|384101.91055278544|   968741.0|    13853.0|            0|            0|            0|            0|           172|           152|  7.096350701651676| 6.2093068639452165|        0.0|     1149.0|155.06249999999997| 302.8970053224913| 91746.59583333335|           0|           0|           1|           1|           0|           0|             0|           1|          0.0|             165.4|             131.5|204.14285714285717|             0|             0|               0|             0|             0|               0|               8|            1052|               7|            1429|             8192|            62948|                5|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|6314547|172.31.65.117-23....|  172.31.65.117|   51263|   23.218.55.94|     443|       6| 20/02/2018 11:53:46|    105120546|          16|          24|         1789.0|         3492.0|          270.0|            0.0|          111.8125|116.18675699063124|         1053.0|            0.0|145.50000000000006|235.55577203526653|   50.23756250276706| 0.3805155273832006|2695398.6153846155| 5764629.090818004| 1.4914967E7|         1.0|1.05120546E8|7008036.3999999985| 7603898.590261698|1.4914967E7|   126951.0|1.05099616E8|  4569548.52173913| 6974719.669465895|1.4936234E7|        1.0|            1|            0|            0|            0|           320|           480|0.15220621095328024| 0.2283093164299204|        0.0|     1053.0|134.09756097560975|193.43084098432297| 37415.49024390244|           0|           1|           0|           0|           1|           0|             0|           0|          1.0|            137.45|          111.8125|             145.5|             0|             0|               0|             0|             0|               0|              16|            1789|              24|            3492|              254|             2105|                7|              20| 137708.7142857143|11188.587008112117|  158091.0|  126951.0|1.486127557142857E7| 31810.08468948772|1.4914967E7|1.4824695E7|Benign|\n",
            "|7887478|172.31.67.95-94.2...|  94.25.169.113|   35818|   172.31.67.95|     445|       6| 20/02/2018 10:06:32|      1088596|           6|           6|          389.0|          209.0|          103.0|            0.0| 64.83333333333333|  50.5308486636299|          131.0|            0.0| 34.83333333333333| 50.83863360345818|   549.3314324138615| 11.023373225696218| 98963.27272727274| 155094.1939397416|    495922.0|         3.0|   1088531.0|          217706.2|178285.36356554905|   496014.0|        3.0|   1088550.0|          217710.0|166943.79749933808|   495963.0|    43404.0|            0|            0|            0|            0|           132|           144|  5.511686612848108|  5.511686612848108|        0.0|      131.0|              46.0| 50.56513950671285| 2556.833333333333|           0|           0|           0|           1|           0|           0|             0|           0|          1.0| 49.83333333333334| 64.83333333333333|34.833333333333336|             0|             0|               0|             0|             0|               0|               6|             389|               6|             209|             8192|                0|                4|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|7280736|172.31.0.2-172.31...|   172.31.64.74|   62374|     172.31.0.2|      53|      17| 20/02/2018 11:28:42|         1062|           1|           1|           33.0|           49.0|           33.0|           33.0|              33.0|               0.0|           49.0|           49.0|              49.0|               0.0|   77212.80602636535| 1883.2391713747645|            1062.0|               0.0|      1062.0|      1062.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8|  941.6195856873824|  941.6195856873824|       33.0|       49.0|38.333333333333336| 9.237604307034012| 85.33333333333333|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              57.5|              33.0|              49.0|             0|             0|               0|             0|             0|               0|               1|              33|               1|              49|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|5581176|172.31.0.2-172.31...|   172.31.69.10|   51869|     172.31.0.2|      53|      17| 20/02/2018 12:13:16|          468|           1|           1|           40.0|           56.0|           40.0|           40.0|              40.0|               0.0|           56.0|           56.0|              56.0|               0.0|  205128.20512820513|  4273.504273504273|             468.0|               0.0|       468.0|       468.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8| 2136.7521367521367| 2136.7521367521367|       40.0|       56.0| 45.33333333333334| 9.237604307034012| 85.33333333333333|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              68.0|              40.0|              56.0|             0|             0|               0|             0|             0|               0|               1|              40|               1|              56|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|5402899|172.31.68.25-178....|   172.31.68.25|   50941|   178.255.83.1|      80|       6| 20/02/2018 10:16:50|         1031|           2|           0|            0.0|            0.0|            0.0|            0.0|               0.0|               0.0|            0.0|            0.0|               0.0|               0.0|                 0.0| 1939.8642095053347|            1031.0|               0.0|      1031.0|      1031.0|      1031.0|            1031.0|               0.0|     1031.0|     1031.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|            40|             0| 1939.8642095053349|                0.0|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          0.0|               0.0|               0.0|               0.0|             0|             0|               0|             0|             0|               0|               2|               0|               0|               0|              253|               -1|                0|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|3501431|172.31.0.2-172.31...|  172.31.66.124|   64680|     172.31.0.2|      53|      17| 20/02/2018 04:22:35|        20468|           1|           1|           40.0|          100.0|           40.0|           40.0|              40.0|               0.0|          100.0|          100.0|             100.0|               0.0|   6839.945280437756|  97.71350400625366|           20468.0|               0.0|     20468.0|     20468.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8|  48.85675200312683|  48.85675200312683|       40.0|      100.0|              60.0| 34.64101615137755|            1200.0|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              90.0|              40.0|             100.0|             0|             0|               0|             0|             0|               0|               1|              40|               1|             100|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "| 757749|172.31.0.2-172.31...|   172.31.64.65|   50108|     172.31.0.2|      53|      17| 20/02/2018 08:56:55|         1015|           1|           1|           58.0|          144.0|           58.0|           58.0|              58.0|               0.0|          144.0|          144.0|             144.0|               0.0|  199014.77832512313| 1970.4433497536945|            1015.0|               0.0|      1015.0|      1015.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8|  985.2216748768473|  985.2216748768473|       58.0|      144.0| 86.66666666666669|49.652123150307816| 2465.333333333333|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|             130.0|              58.0|             144.0|             0|             0|               0|             0|             0|               0|               1|              58|               1|             144|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|5047913|172.31.0.2-172.31...|    172.31.67.6|   65507|     172.31.0.2|      53|      17| 20/02/2018 10:36:01|          599|           1|           1|           30.0|           94.0|           30.0|           30.0|              30.0|               0.0|           94.0|           94.0|              94.0|               0.0|  207011.68614357262|   3338.89816360601|             599.0|               0.0|       599.0|       599.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8| 1669.4490818030054| 1669.4490818030054|       30.0|       94.0| 51.33333333333333| 36.95041722813605|1365.3333333333333|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              77.0|              30.0|              94.0|             0|             0|               0|             0|             0|               0|               1|              30|               1|              94|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|  49062|162.208.22.34-192...|   192.168.10.5|   53552|  162.208.22.34|     443|       6|03/07/2017 06:39:...|       131782|           5|           7|          334.0|         3776.0|          208.0|            0.0|              66.8| 95.95415572032302|         1460.0|            0.0| 539.4285714285714| 663.3397463198019|   31187.87087766159|  91.05947701507034|11980.181818181818|19103.766009968727|     42277.0|         1.0|     90404.0|           22601.0|24371.454176283085|    44821.0|      125.0|    131544.0|           21924.0| 22830.76527845705|    45221.0|       48.0|            0|            0|            0|            0|           100|           164| 37.941448756279314|  53.11802825879103|        0.0|     1460.0|316.15384615384613| 535.1594538318847|286395.64102564106|           0|           1|           0|           0|           0|           0|             0|           0|          1.0|             342.5|              66.8| 539.4285714285714|             0|             0|               0|             0|             0|               0|               5|             334|               7|            3776|               -1|               60|                2|               0|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|7650986|172.31.65.87-185....|   172.31.65.87|    3389|185.129.148.228|   21724|       6| 20/02/2018 05:10:09|     84568617|           2|           0|            0.0|            0.0|            0.0|            0.0|               0.0|               0.0|            0.0|            0.0|               0.0|               0.0|                 0.0|0.02364943487251305|       8.4568617E7|               0.0| 8.4568617E7| 8.4568617E7| 8.4568617E7|       8.4568617E7|               0.0|8.4568617E7|8.4568617E7|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|            40|             0|0.02364943487251305|                0.0|        0.0|        0.0|               0.0|               0.0|               0.0|           0|           0|           0|           0|           1|           0|             0|           0|          0.0|               0.0|               0.0|               0.0|             0|             0|               0|             0|             0|               0|               2|               0|               0|               0|            64000|               -1|                0|              20|               0.0|               0.0|       0.0|       0.0|        8.4568617E7|               0.0|8.4568617E7|8.4568617E7|Benign|\n",
            "|3544600|145.239.183.169-1...|145.239.183.169|   57775|   172.31.66.13|    3389|       6| 20/02/2018 02:42:50|      5872037|          13|           7|         1148.0|         1581.0|          677.0|            0.0| 88.30769230769229| 188.7817896476356|         1173.0|            0.0|225.85714285714286|430.09860441974166|  464.74502800305925|  3.405973089066026| 309054.5789473684| 749405.9411215375|   3210450.0|        39.0|   5872037.0|489336.41666666657|  926095.529385648|  3210450.0|     2867.0|   5724493.0| 954082.1666666664|1233374.5123514456|  3346760.0|    94349.0|            0|            0|            0|            0|           272|           152|  2.213882507892917|  1.192090581173109|        0.0|     1173.0|129.95238095238096|286.47765640455737| 82069.44761904767|           0|           0|           1|           1|           0|           0|             0|           1|          0.0|            136.45| 88.30769230769229|225.85714285714286|             0|             0|               0|             0|             0|               0|              13|            1148|               7|            1581|             8192|            62852|                5|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "| 124057|192.168.10.5-52.1...|  52.14.156.245|     443|   192.168.10.5|   57803|       6|03/07/2017 08:01:...|            3|           1|           1|            0.0|           31.0|            0.0|            0.0|               0.0|               0.0|           31.0|           31.0|              31.0|               0.0|1.0333333333333334E7|  666666.6666666666|               3.0|               0.0|         3.0|         3.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            1|            0|            0|            20|            20| 333333.33333333326| 333333.33333333326|        0.0|       31.0|20.666666666666664|  17.8978583448784|320.33333333333337|           0|           0|           0|           1|           1|           0|             0|           0|          1.0|              31.0|               0.0|              31.0|             0|             0|               0|             0|             0|               0|               1|               0|               1|              31|               -1|              163|                0|               0|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|2270044|172.31.0.2-172.31...|   172.31.64.44|   51894|     172.31.0.2|      53|      17| 20/02/2018 10:38:11|          297|           1|           1|           43.0|           59.0|           43.0|           43.0|              43.0|               0.0|           59.0|           59.0|              59.0|               0.0|   343434.3434343434| 6734.0067340067335|             297.0|               0.0|       297.0|       297.0|         0.0|               0.0|               0.0|        0.0|        0.0|         0.0|               0.0|               0.0|        0.0|        0.0|            0|            0|            0|            0|             8|             8| 3367.0033670033667| 3367.0033670033667|       43.0|       59.0| 48.33333333333334| 9.237604307034012| 85.33333333333333|           0|           0|           0|           0|           0|           0|             0|           0|          1.0|              72.5|              43.0|              59.0|             0|             0|               0|             0|             0|               0|               1|              43|               1|              59|               -1|               -1|                0|               8|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|7746528|169.254.169.254-1...|   172.31.67.26|   49411|169.254.169.254|      80|       6| 20/02/2018 10:01:49|          683|           3|           4|          161.0|          488.0|          161.0|            0.0| 53.66666666666666| 92.95339333952974|          488.0|            0.0|             122.0|             244.0|   950219.6193265008| 10248.901903367496|113.83333333333331| 83.19475143701473|       228.0|        17.0|       354.0|             177.0| 96.16652224137043|      245.0|      109.0|       455.0|151.66666666666669|127.70408502993679|      275.0|       20.0|            0|            0|            0|            0|            72|            92|  4392.386530014641|  5856.515373352855|        0.0|      488.0|            81.125|173.78758586274225|         30202.125|           0|           0|           1|           1|           0|           0|             0|           1|          1.0| 92.71428571428572| 53.66666666666666|             122.0|             0|             0|               0|             0|             0|               0|               3|             161|               4|             488|             8192|              149|                1|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "| 288893|192.168.10.17-8.2...|  192.168.10.17|   39733|   8.26.201.125|      80|       6|04/07/2017 12:25:...|      8864563|          13|          16|          333.0|        19111.0|          333.0|            0.0|25.615384615384606| 92.35758267150065|         2896.0|            0.0|1194.4375000000002| 962.1060903905902|  2193.4527398586933| 3.2714528623689625| 316591.5357142857| 1650738.128801611|   8739343.0|        16.0|     93377.0| 7781.416666666668|13392.643180345713|    32000.0|      153.0|   8864563.0| 590970.8666666667| 2254232.547566969|  8739392.0|      191.0|            0|            0|            0|            0|           416|           528| 1.4665133520964315| 1.8049395102725312|        0.0|     2896.0| 648.1333333333333| 913.8814807837724| 835179.3609195402|           0|           1|           0|           0|           0|           0|             0|           0|          1.0| 670.4827586206898|25.615384615384606|         1194.4375|             0|             0|               0|             0|             0|               0|              13|             333|              16|           19111|               -1|              122|                1|               0|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|3235974|172.31.65.8-31.13...|    31.13.65.36|     443|    172.31.65.8|   51748|       6| 20/02/2018 11:17:24|           86|           3|           1|           77.0|            0.0|           46.0|            0.0|25.666666666666664|23.459184413217212|            0.0|            0.0|               0.0|               0.0|   895348.8372093022|  46511.62790697674|28.666666666666664|27.024680078279058|        55.0|         1.0|        86.0|              43.0| 59.39696961966999|       85.0|        1.0|         0.0|               0.0|               0.0|        0.0|        0.0|            1|            0|            0|            0|            60|            20| 34883.720930232565| 11627.906976744185|        0.0|       46.0|              24.6| 23.27659768952499|             541.8|           0|           1|           0|           0|           1|           0|             0|           0|          0.0|             30.75| 25.66666666666667|               0.0|             0|             0|               0|             0|             0|               0|               3|              77|               1|               0|              132|                0|                1|              20|               0.0|               0.0|       0.0|       0.0|                0.0|               0.0|        0.0|        0.0|Benign|\n",
            "|2866596|172.31.65.27-52.8...|   172.31.65.27|   52095|   52.85.117.75|      80|       6| 20/02/2018 02:54:06|    115146691|          17|          15|          901.0|         1942.0|          445.0|            0.0| 53.00000000000001|147.54406121562462|          971.0|            0.0| 129.4666666666666|341.66166779153787|  24.690244898136065|0.27790637943733876|3714409.3870967724| 4817859.301081922| 1.0003673E7|        15.0|1.15146691E8|      7196668.1875| 4456274.002696262|1.0015505E7|      133.0|1.10208958E8| 7872068.428571427| 4259818.858983158|1.0018548E7|     5731.0|            0|            0|            0|            0|           352|           444| 0.1476377640760862|0.13026861536125253|        0.0|      971.0| 86.15151515151518|252.29039830274482|63650.445075757576|           0|           0|           1|           1|           0|           0|             0|           1|          0.0|          88.84375|              53.0|129.46666666666667|             0|             0|               0|             0|             0|               0|              17|             901|              15|            1942|             8192|              123|               13|              20|19976.545454545456|26900.645220379516|  101085.0|   11830.0|   9999014.09090909|15258.863958070659|1.0003673E7|  9953007.0|Benign|\n",
            "+-------+--------------------+---------------+--------+---------------+--------+--------+--------------------+-------------+------------+------------+---------------+---------------+---------------+---------------+------------------+------------------+---------------+---------------+------------------+------------------+--------------------+-------------------+------------------+------------------+------------+------------+------------+------------------+------------------+-----------+-----------+------------+------------------+------------------+-----------+-----------+-------------+-------------+-------------+-------------+--------------+--------------+-------------------+-------------------+-----------+-----------+------------------+------------------+------------------+------------+------------+------------+------------+------------+------------+--------------+------------+-------------+------------------+------------------+------------------+--------------+--------------+----------------+--------------+--------------+----------------+----------------+----------------+----------------+----------------+-----------------+-----------------+-----------------+----------------+------------------+------------------+----------+----------+-------------------+------------------+-----------+-----------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_final.groupBy('Label').count().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5RN6WCOLKTy8",
        "outputId": "ea144383-0eb8-4cbb-9050-51e1f3657d47"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-------+\n",
            "| Label|  count|\n",
            "+------+-------+\n",
            "|Benign|6321980|\n",
            "|  ddos|6326177|\n",
            "+------+-------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "f61DtUhaKeAh"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}