4_model_inference_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "947a21f1-12f5-4d2c-baf9-0ce5a9b9a84a",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Inference & Evaluation for Our Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "306b625a-daa6-4594-8f5e-47c8a7d31e46",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Importing Data & Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "80f13e99-8f71-423b-a4ee-1db8639872e8",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.8/site-packages (1.9.3)\n",
      "Collecting scipy\n",
      "  Using cached scipy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.5 MB)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.8/site-packages (2.8.6)\n",
      "Collecting networkx\n",
      "  Using cached networkx-3.0-py3-none-any.whl (2.0 MB)\n",
      "Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /opt/conda/lib/python3.8/site-packages (from scipy) (1.23.5)\n",
      "Installing collected packages: scipy, networkx\n",
      "  Attempting uninstall: scipy\n",
      "    Found existing installation: scipy 1.9.3\n",
      "    Uninstalling scipy-1.9.3:\n",
      "      Successfully uninstalled scipy-1.9.3\n",
      "  Attempting uninstall: networkx\n",
      "    Found existing installation: networkx 2.8.6\n",
      "    Uninstalling networkx-2.8.6:\n",
      "      Successfully uninstalled networkx-2.8.6\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "ydata-profiling 4.0.0 requires scipy<1.10,>=1.4.1, but you have scipy 1.10.1 which is incompatible.\u001b[0m\n",
      "Successfully installed networkx-3.0 scipy-1.10.1\n",
      "Collecting nltk\n",
      "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.5 MB 7.3 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting click\n",
      "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
      "\u001b[K     |████████████████████████████████| 96 kB 13.4 MB/s  eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from nltk) (1.2.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /opt/conda/lib/python3.8/site-packages (from nltk) (2022.10.31)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.8/site-packages (from nltk) (4.62.3)\n",
      "Installing collected packages: click, nltk\n",
      "Successfully installed click-8.1.3 nltk-3.8.1\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.8/site-packages (1.10.1)\n",
      "Requirement already satisfied: numpy<1.27.0,>=1.19.5 in /opt/conda/lib/python3.8/site-packages (from scipy) (1.23.5)\n",
      "Collecting rouge\n",
      "  Downloading rouge-1.0.1-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from rouge) (1.16.0)\n",
      "Installing collected packages: rouge\n",
      "Successfully installed rouge-1.0.1\n",
      "Collecting sentence-transformers\n",
      "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
      "\u001b[K     |████████████████████████████████| 85 kB 3.6 MB/s eta 0:00:011\n",
      "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.6.0 in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (4.26.1)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (4.62.3)\n",
      "Requirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (1.13.1)\n",
      "Collecting torchvision\n",
      "  Downloading torchvision-0.14.1-cp38-cp38-manylinux1_x86_64.whl (24.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 24.2 MB 6.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (1.23.5)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (1.2.1)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (1.10.1)\n",
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (3.8.1)\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3 MB 118.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.4.0 in /opt/conda/lib/python3.8/site-packages (from sentence-transformers) (0.12.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (6.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.4.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2.27.1)\n",
      "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.9.0)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /opt/conda/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (8.5.0.96)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /opt/conda/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (11.10.3.66)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /opt/conda/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /opt/conda/lib/python3.8/site-packages (from torch>=1.6.0->sentence-transformers) (11.7.99)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.6.0->sentence-transformers) (58.0.4)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.8/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.6.0->sentence-transformers) (0.37.1)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.8/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.13.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2022.10.31)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from nltk->sentence-transformers) (1.2.0)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from nltk->sentence-transformers) (8.1.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2022.12.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2.0.4)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from scikit-learn->sentence-transformers) (3.1.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.8/site-packages (from torchvision->sentence-transformers) (9.4.0)\n",
      "Building wheels for collected packages: sentence-transformers\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125938 sha256=9fe6d3aedb7ef7e6f14f3f5ca7dd5d566cf128e0c5ac2c528cca1f684d560294\n",
      "  Stored in directory: /home/sarchakr/.cache/pip/wheels/5e/6f/8c/d88aec621f3f542d26fac0342bef5e693335d125f4e54aeffe\n",
      "Successfully built sentence-transformers\n",
      "Installing collected packages: torchvision, sentencepiece, sentence-transformers\n",
      "Successfully installed sentence-transformers-2.2.2 sentencepiece-0.1.97 torchvision-0.14.1\n"
     ]
    }
   ],
   "source": [
    "! pip install --upgrade scipy networkx\n",
    "! pip3 install nltk\n",
    "! pip install --upgrade scipy\n",
    "! pip install rouge\n",
    "!pip install -U sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "701720be-e704-4599-b6fe-599845ac6e86",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/sarchakr/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scipy\n",
    "import networkx as nx\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pickle\n",
    "import nltk\n",
    "import json\n",
    "from queue import Queue\n",
    "import matplotlib.pyplot as mp\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from statistics import mean\n",
    "from bs4 import BeautifulSoup\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c053919-6ca6-44eb-a2e8-3f9aa34c6440",
   "metadata": {},
   "outputs": [],
   "source": [
    "g = pickle.load(open('./cg_kg.gpickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df5b2863-b79d-4768-9c5e-3082073ff5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cso_df = pd.read_csv(\"CSO_data/CSO_entities_ensembled_Sign_bart-large.csv\", index_col=0)\n",
    "cso_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9582b863-bd99-4834-a6ed-5458dd6f02b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "alert_df = pd.read_csv(\"./Alerts_data/alert_id_mapping.csv\", index_col=0)\n",
    "alert_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c800cc89-9e50-4ca0-af70-a132dc1a7c12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the important information from the alerts and the outage like started time, symptom etc.\n",
    "\n",
    "df_alerts_sorted = pd.read_csv('./Alerts_data/all_alerts_parsed.csv', index_col=0)\n",
    "df_alerts_sorted['created_at'] = pd.to_datetime(df_alerts_sorted['created_at'])\n",
    "df_alerts_sorted = df_alerts_sorted.sort_values(by = 'created_at')\n",
    "timelist_alerts = list(df_alerts_sorted['created_at'])\n",
    "numberlist_alerts = list(df_alerts_sorted['incident_number'])\n",
    "\n",
    "with open('./CSO_data/CSO.json', 'r') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "cso_data = {'cso_number':[], 'start_time':[], 'end_time':[]}\n",
    "for cso, value in data.items():\n",
    "    cso_data['cso_number'].append(cso)\n",
    "    cso_data['start_time'].append(value['primaryIncident']['u_cso_started'])\n",
    "    cso_data['end_time'].append(value['primaryIncident']['u_cso_ended'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0423cbcb-5558-4c3b-ad5e-5f2f643c4651",
   "metadata": {},
   "outputs": [],
   "source": [
    "cso_time_df = pd.DataFrame(cso_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "02c057b6-7d11-4cc6-b2d6-0c8dc06dc5ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "cg = pickle.load(open('./causal_graph_data/cg_unweighted_filtered.gpickle', 'rb'))\n",
    "cg_alert_id = nx.get_node_attributes(cg, \"alert_id\")\n",
    "alert_ids = [v for k,v in cg_alert_id.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48b7009b-e036-40c0-a872-a9492051e365",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_file = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "13da2fa9-b65f-43be-a0d6-4506919f2246",
   "metadata": {},
   "outputs": [],
   "source": [
    "alert_names = df_alerts_sorted.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "98776eab-492b-4db8-9196-c363230881b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cso_descr = pd.read_csv(\"./CSO_data/cso_alert.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cadb0b59-9d03-4dab-8c7c-6dcf37412c0e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Preprocessing some data (specific to our dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1718190b-5a78-4bac-9495-71d636c34daf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def slicer(my_str,sub):\n",
    "    index=my_str.find(sub)\n",
    "    if index !=-1 :\n",
    "        return my_str[index+8:] \n",
    "    else :\n",
    "        raise Exception(my_str,' -----Sub string not found!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2bb04ac6-6e7b-47c0-9eb9-df6afdf8a24c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def html_stripper(code):\n",
    "    return BeautifulSoup(code).get_text().replace('\\r',' ').replace('\\xa0',' ').replace('\\n',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f4f940b7-c548-48cf-9027-2fd26cd7d66c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Removing region name from outage-alert description\n",
    "cso_descr_data = []\n",
    "for index, row in cso_descr.iterrows():\n",
    "    z = row['cso_number']\n",
    "    x = row['description']\n",
    "    try:\n",
    "        y = slicer(x,\"<br><br>\")\n",
    "        cso_descr_data.append([z,y])\n",
    "    except:\n",
    "        continue\n",
    "cso_descr_df = pd.DataFrame(cso_descr_data, columns=['cso_number', 'descr'])\n",
    "len(cso_descr_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bdcc7b4-0bbe-4873-9fc2-fddaae9522b9",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Functions to Retrieve Required Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9b9f7fab-11b0-489a-a249-9a057ec1ab5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "inc_dict = {}\n",
    "for i, inc in enumerate(numberlist_alerts):\n",
    "    inc_dict[str(inc)] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16dcf4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For each outage with a start and end time, find the alerts that were fired within 1 hour ago of the start of the outage and till the outage persists\n",
    "def get_alerts_by_time(start_time, end_time, num):                                           # the input time and the number of different recorded past alert times, we want alerts from\n",
    "    lst = []\n",
    "    low = 0\n",
    "    mx = len(timelist_alerts)\n",
    "    high = mx\n",
    "    while(low<high):\n",
    "        if(timelist_alerts[int((low+high)/2)] > start_time):\n",
    "            high = int((low+high)/2)\n",
    "        else:\n",
    "            low = int((low+high)/2)+1\n",
    "    \n",
    "    ind = low-1\n",
    "    j = 0\n",
    "    k = 0\n",
    "    while(j < low-1 and (start_time - timelist_alerts[low-1-j]) < pd.Timedelta(hours=1)):\n",
    "        lst.append(numberlist_alerts[low - 1 -j])\n",
    "        while(j<low-1 and timelist_alerts[low-1-j] == timelist_alerts[low-2-j]):\n",
    "            lst.append(numberlist_alerts[low-2-j])\n",
    "            j = j + 1\n",
    "        j = j + 1\n",
    "        k = k + 1\n",
    "    \n",
    "    j = low\n",
    "    k = 0\n",
    "    while(j<mx and (end_time - timelist_alerts[j]) > pd.Timedelta(minutes=1)):\n",
    "        lst.append(numberlist_alerts[j])\n",
    "        while(j<mx-1 and timelist_alerts[j] == timelist_alerts[j+1]):\n",
    "            lst.append(numberlist_alerts[j+1])\n",
    "            j = j + 1\n",
    "        j = j + 1\n",
    "        k = k + 1\n",
    "    return list(set(lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6ba5b61e-aa21-44ac-b6f2-4126f22368f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FInd the alerts that are inked to an outage\n",
    "def get_alerts_by_cso(cso, n=3):\n",
    "    timestr = cso_time_df[cso_time_df['cso_number'] == cso]['start_time'].iloc[0]   # gets cso timestr\n",
    "    timeend = cso_time_df[cso_time_df['cso_number'] == cso]['end_time'].iloc[0]\n",
    "    \n",
    "    timestr = pd.Timestamp(timestr, tz=\"UTC\")\n",
    "    timeend = pd.Timestamp(timeend, tz=\"UTC\")\n",
    "\n",
    "    if timestr < timelist_alerts[0]:\n",
    "        timestr = timelist_alerts[0]\n",
    "    \n",
    "    if timeend < timelist_alerts[0]:\n",
    "        timeend = timelist_alerts[0]\n",
    "        \n",
    "    alerts_timeframe = get_alerts_by_time(timestr, timeend, n)\n",
    "    new_alerts_timeframe = [df_alerts_sorted['alert_id'].iloc[inc_dict[str(alert)]] for alert in alerts_timeframe]\n",
    "    \n",
    "    return new_alerts_timeframe\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3a5406ba-1d84-4f93-b257-f24d6f58748f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_top_n_rc(cso_dict, n):\n",
    "    return dict(sorted(cso_dict.items(), key=lambda x: x[1], reverse=True)[:n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2a54bfae-b51f-4071-9522-48b8a88651fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_alert_name_from_id(alert_id):\n",
    "    temp = alert_df.index[alert_df['id']==alert_id].tolist()\n",
    "    for i in temp:\n",
    "        return alert_df['alert_name'][i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d5089e12-bf75-4d3b-9776-f9c24ae39f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sym_description(cso_number):\n",
    "    temp = cso_df.index[cso_df['cso_number']==int(cso_number)].tolist()[0]\n",
    "    return cso_df['symptom'][temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "77c3ee20-7efc-471d-93f0-28b0bf1e9ccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rc_description(cso_number):\n",
    "    temp = cso_df.index[cso_df['cso_number']==int(cso_number)].tolist()[0]\n",
    "    return cso_df['root_cause'][temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d2c0ee82-751a-4a59-bb1a-340c1fc4f1c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rem_description(cso_number):\n",
    "    temp = cso_df.index[cso_df['cso_number']==int(cso_number)].tolist()[0]\n",
    "    return cso_df['remediations'][temp]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0360026-6d00-406c-a1e3-a26e2be48021",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Inference Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15f78fdb-8dcf-47fd-b43a-c9ab17eb3107",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Pred: path-based inference method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "01f93040-cd90-487d-893c-fc511013bf01",
   "metadata": {},
   "outputs": [],
   "source": [
    "## path_based_inference\n",
    "def path_based_inf(alert_id, k, g, q, visited_dict):\n",
    "    alert_id_cg = {value:key for key,value in cg_alert_id.items()}\n",
    "    \n",
    "    rc = {}\n",
    "    while(not q.empty()):\n",
    "        (node, k_) = q.get()\n",
    "        if k_ > 0:            \n",
    "            for (u, v, dic) in g.out_edges(alert_id_cg[node], data = True):\n",
    "                if (not visited_dict[v]):\n",
    "                    if (dic['label'] == 'caused by') or (dic['label'] == 'causes'):\n",
    "                        visited_dict[v] = True\n",
    "                        q.put((cg_alert_id[v], k_ - 1))\n",
    "                    elif (dic['label'] == 'caused CSO') or (dic['label'] == 'alert-symptom link'):\n",
    "                        if v[:-4] in rc.keys():\n",
    "                            rc[v[:-4]] += 1/(k-k_+1)\n",
    "                        else:\n",
    "                            rc[v[:-4]] = 1/(k-k_+1)\n",
    "                \n",
    "    return rc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e0379394-0014-4940-8798-47b20f0981ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_rc_for_set(alert_ids,k,g):\n",
    "    rc = {}\n",
    "    visited_dict = {}\n",
    "    for node in g.nodes():\n",
    "        visited_dict[node] = False\n",
    "\n",
    "    q = Queue()\n",
    "\n",
    "    if(isinstance(alert_ids,list)):\n",
    "        for alert_id in alert_ids:\n",
    "            visited_dict[alert_id] = True\n",
    "            q.put((alert_id,k))\n",
    "            temp = path_based_inf(alert_id, k, g, q, visited_dict)\n",
    "\n",
    "            for foo in temp.keys():\n",
    "                if foo in rc.keys():\n",
    "                    rc[foo] += temp[foo]\n",
    "                else:\n",
    "                    rc[foo] = temp[foo]\n",
    "                    \n",
    "        for node in g.nodes():\n",
    "            visited_dict[node] = False\n",
    "    else:\n",
    "        visited_dict[alert_ids] = True\n",
    "        q.put((alert_ids,k))\n",
    "        rc = path_based_inf(alert_ids, k, g, q, visited_dict)\n",
    "\n",
    "    foo= dict(sorted(rc.items(), key=lambda x: x[1], reverse=True)[::])\n",
    "    poo = {}\n",
    "    for i in foo.keys():\n",
    "        poo[int(i)] = foo[i]\n",
    "    return poo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0db18efa-52a2-490f-af05-af26fe7f1118",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_rc_for_set_normalised(alert_ids,k,g):\n",
    "    rc = {}\n",
    "    visited_dict = {}\n",
    "    for node in g.nodes():\n",
    "        visited_dict[node] = False\n",
    "    q = Queue()\n",
    "    \n",
    "    for alert_id in alert_ids:\n",
    "        visited_dict[alert_id] = True\n",
    "        q.put((alert_id,k))\n",
    "        temp = path_based_inf(alert_id, k, g, q, visited_dict)\n",
    "        for foo in temp.keys():\n",
    "            if foo in rc.keys():\n",
    "                rc[foo] += temp[foo]\n",
    "            else:\n",
    "                rc[foo] = temp[foo]\n",
    "    for node in g.nodes():\n",
    "        visited_dict[node] = False\n",
    "    \n",
    "    foo= dict(sorted(rc.items(), key=lambda x: x[1], reverse=True)[::])\n",
    "    norm_factor = sum(list(foo.values()))\n",
    "    poo = {}\n",
    "    for i in foo.keys():\n",
    "        poo[int(i)] = foo[i]/norm_factor\n",
    "    return poo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2694e09-2d06-4f33-bc41-16a0315b5372",
   "metadata": {},
   "source": [
    "### Sim: Similarity-based Inference Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "af2cf0fc-5263-4416-8370-d4470ed42262",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "\n",
    "model_bert = BertModel.from_pretrained('bert-base-uncased', output_hidden_states = True,)\n",
    "tokenizer_bert = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "26f2dd30-1d52-462d-9c75-3586f593abd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_text_preparation(text, tokenizer):\n",
    "    \"\"\"Preparing the input for BERT\n",
    "    \n",
    "    Takes a string argument and performs\n",
    "    pre-processing like adding special tokens,\n",
    "    tokenization, tokens to ids, and tokens to\n",
    "    segment ids. All tokens are mapped to seg-\n",
    "    ment id = 1.\n",
    "    \n",
    "    Args:\n",
    "        text (str): Text to be converted\n",
    "        tokenizer (obj): Tokenizer object\n",
    "            to convert text into BERT-re-\n",
    "            adable tokens and ids\n",
    "        \n",
    "    Returns:\n",
    "        list: List of BERT-readable tokens\n",
    "        obj: Torch tensor with token ids\n",
    "        obj: Torch tensor segment ids\n",
    "    \n",
    "    \n",
    "    \"\"\"\n",
    "    marked_text = \"[CLS] \" + text + \" [SEP]\"\n",
    "    tokenized_text = tokenizer.tokenize(marked_text)\n",
    "    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
    "    segments_ids = [1]*len(indexed_tokens)\n",
    "\n",
    "    # Convert inputs to PyTorch tensors\n",
    "    tokens_tensor = torch.tensor([indexed_tokens])\n",
    "    segments_tensors = torch.tensor([segments_ids])\n",
    "\n",
    "    return tokenized_text, tokens_tensor, segments_tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "ff35eed8-5e1b-45fa-95a8-34d1afdc227e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bert_embeddings(tokens_tensor, segments_tensors, model):\n",
    "    \"\"\"Get embeddings from an embedding model\n",
    "    \n",
    "    Args:\n",
    "        tokens_tensor (obj): Torch tensor size [n_tokens]\n",
    "            with token ids for each token in text\n",
    "        segments_tensors (obj): Torch tensor size [n_tokens]\n",
    "            with segment ids for each token in text\n",
    "        model (obj): Embedding model to generate embeddings\n",
    "            from token and segment ids\n",
    "    \n",
    "    Returns:\n",
    "        list: List of list of floats of size\n",
    "            [n_tokens, n_embedding_dimensions]\n",
    "            containing embeddings for each token\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    # Gradient calculation id disabled\n",
    "    # Model is in inference mode\n",
    "    with torch.no_grad():\n",
    "        outputs = model(tokens_tensor, segments_tensors)\n",
    "        # Removing the first hidden state\n",
    "        # The first state is the input state\n",
    "        hidden_states = outputs[2][1:]\n",
    "\n",
    "    # Getting embeddings from the final BERT layer\n",
    "    token_embeddings = hidden_states[-1]\n",
    "    # Collapsing the tensor into 1-dimension\n",
    "    token_embeddings = torch.squeeze(token_embeddings, dim=0)\n",
    "    # Converting torchtensors to lists\n",
    "    list_token_embeddings = [token_embed.tolist() for token_embed in token_embeddings]\n",
    "\n",
    "    return list_token_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "f1f49c00-cfff-45d2-b38e-0ba6cc4cc3c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_symptom_embeddings():\n",
    "    symptom_list = cso_df['symptom'].to_list()\n",
    "    \n",
    "    embeddings = []\n",
    "    for i in range(len(symptom_list)):\n",
    "        tokenized_text, tokens_tensor, segments_tensors = bert_text_preparation(symptom_list[i], tokenizer_bert)\n",
    "        list_token_embeddings = np.array(get_bert_embeddings(tokens_tensor, segments_tensors, model_bert))\n",
    "        sentence_embed = np.mean(list_token_embeddings, axis=0)\n",
    "        embeddings.append(sentence_embed)\n",
    "    \n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25686bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine(u, v):\n",
    "    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "9e44d04e-a8f6-4bd2-b56d-f8f4aa2a9051",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_cso_from_description_bert(description, embeddings):\n",
    "    tokenized_text, tokens_tensor, segments_tensors = bert_text_preparation(description, tokenizer_bert)\n",
    "    list_token_embeddings = np.array(get_bert_embeddings(tokens_tensor, segments_tensors, model_bert))\n",
    "    sentence_embed = np.mean(list_token_embeddings, axis=0)\n",
    "    \n",
    "    similarity_dict = {}\n",
    "    for i in range(len(embeddings)):\n",
    "        similarity_dict[int(cso_df.iloc[i]['cso_number'])] = cosine(sentence_embed, embeddings[i])\n",
    "    return similarity_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "ed888f05-8687-45fd-bb5e-e02fc499deab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_cso_from_description_bert_with_embed(cso_embed, embeddings):    \n",
    "    similarity_dict = {}\n",
    "    for i in range(len(embeddings)):\n",
    "        similarity_dict[int(cso_df.iloc[i]['cso_number'])] = cosine(cso_embed, embeddings[i])\n",
    "    return similarity_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "15171273-f38c-48e7-b009-15022765c220",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cso_similarity_bert(cso1, emb2):\n",
    "    tokenized_text, tokens_tensor, segments_tensors = bert_text_preparation(cso1, tokenizer_bert)\n",
    "    list_token_embeddings = np.array(get_bert_embeddings(tokens_tensor, segments_tensors, model_bert))\n",
    "    emb1 = np.mean(list_token_embeddings, axis=0)\n",
    "    \n",
    "    return cosine(emb1, emb2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3250cc0-6061-41a9-8512-65d088e0f5c9",
   "metadata": {},
   "source": [
    "### Clust: Cluster based inference method\n",
    "### Merge CG and KG graph with KG clusters into effect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "337beb53-c5d2-4559-bc71-31113b49258a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_symptom_cluster_for_set(alert_ids, k, g, cso_cluster_map):\n",
    "    cluster = {}\n",
    "    visited_dict = {}\n",
    "    for node in g.nodes():\n",
    "        visited_dict[node] = False\n",
    "        \n",
    "    q = Queue()\n",
    "    if not isinstance(alert_ids, list):\n",
    "        new_alert_ids = [alert_ids]\n",
    "    else:\n",
    "        new_alert_ids = alert_ids\n",
    "        \n",
    "    for alert_id in new_alert_ids:\n",
    "        visited_dict[alert_id] = True\n",
    "        q.put((alert_id, k))\n",
    "        temp = path_based_inf(alert_id, k, g, q, visited_dict)\n",
    "\n",
    "        for foo in temp.keys():\n",
    "            cl = cso_cluster_map[foo]\n",
    "            if cl in cluster:\n",
    "                cluster[cl] += 1\n",
    "            else:\n",
    "                cluster[cl] = 1\n",
    "\n",
    "    for node in g.nodes():\n",
    "        visited_dict[node] = False\n",
    "                    \n",
    "    foo= dict(sorted(cluster.items(), key=lambda x: x[1], reverse=True)[::])\n",
    "    return foo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5f40bfb0-16bd-4406-a0e6-10c949a351ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_rf_inference_df(alert_ids, alert_id_map):\n",
    "    reverse_map = {v:k for k,v in alert_id_map.items()}\n",
    "    all_alerts = list(reverse_map.keys())\n",
    "    \n",
    "    l = np.zeros(shape=len(all_alerts))\n",
    "    \n",
    "    for alert in alert_ids:\n",
    "        l[reverse_map[alert]] = 1\n",
    "    \n",
    "    df = pd.DataFrame([l], columns=all_alerts)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "fe47d623-d045-4458-86d5-aeadff51f7d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def rank_cluster(clusters, n_clusters, rf_model, X):\n",
    "    y_pred_proba = rf_model.predict_proba(X).reshape(-1)\n",
    "    \n",
    "    rf_clusters = {float(i):y_pred_proba[i] for i in range(len(y_pred_proba))}\n",
    "    cluster_rank = dict(sorted(rf_clusters.items(), key=lambda x: x[1], reverse=True)[::])\n",
    "    \n",
    "    return cluster_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "aecaca16-a028-4d44-bdb0-48bc14b2efbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_rc_set_bert(clusters, K, cso_cluster_map, cso_embedding):\n",
    "    cluster_cso_map = {v:k for k,v in cso_cluster_map.items()}\n",
    "    \n",
    "    combined_pred = {}\n",
    "    count = 0\n",
    "    for cl, _ in sorted(clusters.items(), reverse=True):\n",
    "        all_csos = [v for k,v in cluster_cso_map.items() if k == cl]\n",
    "        for i in all_csos:\n",
    "            combined_pred[i] = cso_similarity_bert(cso_df[cso_df['cso_number'] == int(i)]['symptom'].item(), cso_embedding)\n",
    "        count += 1\n",
    "    \n",
    "    return combined_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ccd22bf4-a9df-49e9-86d1-c63540cf8d85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Debugging code to check whether Clust inference method is working or not\n",
    "\n",
    "'''\n",
    "node_cluster_map = nx.get_node_attributes(g, \"cluster_id\")\n",
    "cso_cluster_map = {k.split('_')[0]:v for k,v in node_cluster_map.items()}\n",
    "n_clusters = len(set([v for v in cso_cluster_map.values()]))\n",
    "rf = pickle.load(open('random_forest.pickle', 'rb'))\n",
    "\n",
    "i=14757\n",
    "\n",
    "df = create_rf_inference_df(get_alerts_by_cso(str(i)), nx.get_node_attributes(g, \"alert_id\"))\n",
    "\n",
    "clust1 = find_symptom_cluster_for_set(get_alerts_by_cso(str(i)), 10, g, cso_cluster_map)\n",
    "clust2 = rank_cluster(clust1, n_clusters, rf, df)\n",
    "\n",
    "s = sum([1.0/v for k,v in clust1.items()])\n",
    "clust1_1 = {k: 1.0/(v*s) for k,v in clust1.items()}\n",
    "\n",
    "clust = {}\n",
    "for i in range(n_clusters):\n",
    "    cl = float(i)\n",
    "    r1 = clust1_1[cl] if cl in clust1_1 else 0\n",
    "    r2 = clust2[cl] if cl in clust2 else 0\n",
    "    clust[cl] = r1+r2\n",
    "# find_rc_set(clust2, 1, cso_cluster_map, csos_descr[i])\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b375c94-013f-43da-ae22-2201775a10fa",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Evaluation Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "aabd3596-9957-460e-82e0-108613a43489",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_cso_from_dict(cso_dict1, exception_cso_number):\n",
    "    cso_dict = cso_dict1.copy()\n",
    "    if int(exception_cso_number) in cso_dict.keys():\n",
    "        del cso_dict[int(exception_cso_number)]\n",
    "    if str(exception_cso_number) in cso_dict.keys():\n",
    "        del cso_dict[str(exception_cso_number)]\n",
    "    return cso_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc36c921-e72a-4a61-80a1-1b3cdd627b95",
   "metadata": {
    "tags": []
   },
   "source": [
    "### ROUGE Score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "6bbe58f2-4146-4e83-b5a2-1fd8afa967e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge import Rouge\n",
    "rouge = Rouge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "3a6168a8-1369-412c-ac78-32af40861c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take max rouge among top-n RCs\n",
    "def max_rouge(cso_dict, actual_rc, actual_rem, type_rpf, n):\n",
    "    max_rc = 0\n",
    "    max_rem = 0\n",
    "    rc_cso = 0\n",
    "    rem_cso = 0\n",
    "    top_dic = get_top_n_rc(cso_dict,n)\n",
    "    for i in top_dic.keys():\n",
    "        foo = cso_df.iloc[cso_df.index[cso_df['cso_number']==int(i)].tolist()[0]]\n",
    "        trc = rouge.get_scores(str(foo['root_cause']), str(actual_rc))[0]['rouge-1'][type_rpf]\n",
    "        if trc >= max_rc:\n",
    "            rc_cso = i\n",
    "            max_rc = trc\n",
    "        trm = rouge.get_scores(str(foo['remediations']), str(actual_rem))[0]['rouge-1'][type_rpf]\n",
    "        if trm >= max_rem:\n",
    "            rem_cso = i\n",
    "            max_rem = trm\n",
    "    return max_rc, max_rem, rc_cso, rem_cso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "71f47f2b-a6a4-4e61-b018-b2256fea71e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take max rouge among top-n RCs by removing the case where prediction == inout outage\n",
    "def max_rouge_minus1(cso_dict, actual_rc, actual_rem, type_rpf, n):\n",
    "    max_rc = 0\n",
    "    max_rem = 0\n",
    "    rc_cso = 0\n",
    "    rem_cso = 0\n",
    "    top_dic = get_top_n_rc(cso_dict,n)\n",
    "    for i in top_dic.keys():\n",
    "        foo = cso_df.iloc[cso_df.index[cso_df['cso_number']==int(i)].tolist()[0]]\n",
    "        trc = rouge.get_scores(str(foo['root_cause']), str(actual_rc))[0]['rouge-1'][type_rpf]\n",
    "        if trc < 1 and trc >= max_rc:\n",
    "            rc_cso = i\n",
    "            max_rc = trc\n",
    "        trm = rouge.get_scores(str(foo['remediations']), str(actual_rem))[0]['rouge-1'][type_rpf]\n",
    "        if trm < 1 and trm >= max_rem:\n",
    "            rem_cso = i\n",
    "            max_rem = trm\n",
    "    return max_rc, max_rem, rc_cso, rem_cso"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e5e0d4b-4860-4245-a3a3-dd50af2b992e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6b08818-032d-450c-b116-bdedbf63c733",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## generating random (and valid) CSOs\n",
    "random_cso_list = []\n",
    "n = 50 #number of random CSOs needed\n",
    "while True:\n",
    "    x = list(cso_time_df.sample()['cso_number'])[0]\n",
    "    timestr = cso_time_df[cso_time_df['cso_number'] == x]['start_time'].iloc[0]\n",
    "    timestr = pd.Timestamp(timestr, tz=\"UTC\")\n",
    "    if len(get_alerts_by_cso(x)) == 0:\n",
    "        continue\n",
    "\n",
    "    cond3 = type(list(cso_df[cso_df['cso_number'] == int(x)]['root_cause'])[0]) is str\n",
    "    cond4 = type(list(cso_df[cso_df['cso_number'] == int(x)]['remediations'])[0]) is str\n",
    "    if x not in random_cso_list:\n",
    "        if cond3 and cond4:\n",
    "            random_cso_list.append(int(x))\n",
    "\n",
    "    if len(random_cso_list)==n:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3025c108-d0a2-490b-910f-d375d60e13c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_cso_list = [14757,\n",
    " 6704,\n",
    " 15019,\n",
    " 15126,\n",
    " 8365,\n",
    " 12070,\n",
    " 14965,\n",
    " 6524,\n",
    " 14886,\n",
    " 16742,\n",
    " 9131,\n",
    " 6119,\n",
    " 9144,\n",
    " 15484,\n",
    " 16516,\n",
    " 6894,\n",
    " 13738,\n",
    " 9560,\n",
    " 10190,\n",
    " 7599,\n",
    " 9242,\n",
    " 6920,\n",
    " 17510,\n",
    " 9060,\n",
    " 9828,\n",
    " 15215,\n",
    " 15005,\n",
    " 15558,\n",
    " 12686,\n",
    " 8548,\n",
    " 9139,\n",
    " 7653,\n",
    " 8653,\n",
    " 13678,\n",
    " 15461,\n",
    " 8754,\n",
    " 14055,\n",
    " 10999,\n",
    " 15334,\n",
    " 7872,\n",
    " 9624,\n",
    " 6577,\n",
    " 14886,\n",
    " 14902,\n",
    " 14797,\n",
    " 10384,\n",
    " 10961,\n",
    " 12052,\n",
    " 9563,\n",
    " 6704]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "239f095c-8575-4d55-8970-d0ae5d21d270",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Inference Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "44c8c84d-72ec-409e-b39e-7792efe42f35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csos_descr = {}\n",
    "for cso in random_cso_list:\n",
    "    x = get_alerts_by_cso(str(cso))\n",
    "    descr = \"\"\n",
    "    for i in x:\n",
    "        j = (list(alert_names.index[alert_names['alert_id']==i])[0])\n",
    "        descr += \"\"\n",
    "        descr+= alert_names['alert_identifier'][j].replace(\"_\",\" \")\n",
    "    csos_descr[cso] = descr\n",
    "len(csos_descr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "5b9d079c-9d7a-4f76-91a4-1a12a6fc3d2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "csos_descr_embed = {}\n",
    "for cso in random_cso_list:\n",
    "    x = get_alerts_by_cso(str(cso))\n",
    "    embed = []\n",
    "    for i in x:\n",
    "        j = (list(alert_names.index[alert_names['alert_id']==i])[0])\n",
    "        descr = alert_names['alert_identifier'][j].replace(\"_\",\" \")\n",
    "        tokenized_text, tokens_tensor, segments_tensors = bert_text_preparation(descr, tokenizer_bert)\n",
    "        list_token_embeddings = np.array(get_bert_embeddings(tokens_tensor, segments_tensors, model_bert))\n",
    "        descr_embed = np.mean(list_token_embeddings, axis=0)\n",
    "        embed.append(descr_embed)\n",
    "        \n",
    "    embed = np.array(embed)\n",
    "    cso_embed = np.mean(embed, axis=0)\n",
    "    csos_descr_embed[cso] = cso_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "2153c552-64ae-4444-8497-a5416ae2d6cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#### Sim\n",
    "embeddings = get_symptom_embeddings()\n",
    "nlp_pred_random = {}\n",
    "for i in random_cso_list:\n",
    "    nlp_pred_random[i] = remove_cso_from_dict(get_similar_cso_from_description_bert_with_embed(csos_descr_embed[i], embeddings),i)\n",
    "len(nlp_pred_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "58d2f64f-b85b-4c42-a801-2bcb6de771ca",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#### Pred\n",
    "pred_g1_random = {}\n",
    "for i in random_cso_list:\n",
    "    pred_g1_random[i] = remove_cso_from_dict(find_rc_for_set(get_alerts_by_cso(str(i)), 10, g), i)\n",
    "    \n",
    "len(pred_g1_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "7d74b40a-f8ed-4c97-8656-8e38c6651e1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#### Clust #######\n",
    "node_cluster_map = nx.get_node_attributes(g, \"cluster_id\")\n",
    "cso_cluster_map = {k.split('_')[0]:v for k,v in node_cluster_map.items()}\n",
    "n_clusters = len(set([v for v in cso_cluster_map.values()]))\n",
    "rf = pickle.load(open('random_forest.pickle', 'rb'))\n",
    "\n",
    "combined_cluster_pred_random = {}\n",
    "for i in random_cso_list:\n",
    "    df = create_rf_inference_df(get_alerts_by_cso(str(i)), nx.get_node_attributes(g, \"alert_id\"))\n",
    "    \n",
    "    clust1 = find_symptom_cluster_for_set(get_alerts_by_cso(str(i)), 10, g, cso_cluster_map)\n",
    "    clust2 = rank_cluster(clust1, n_clusters, rf, df)\n",
    "    \n",
    "    s = sum([1.0/v for k,v in clust1.items()])\n",
    "    clust1_1 = {k: 1.0/(v*s) for k,v in clust1.items()}\n",
    "\n",
    "    clust = {}\n",
    "    for x in range(n_clusters):\n",
    "        cl = float(x)\n",
    "        r1 = clust1_1[cl] if cl in clust1_1 else 0\n",
    "        r2 = clust2[cl] if cl in clust2 else 0\n",
    "        clust[cl] = r1+r2\n",
    "    \n",
    "    clust = dict(sorted(clust.items(), key=lambda x: x[1], reverse=True)[::])\n",
    "    \n",
    "    combined_cluster_pred_random[i] = find_rc_set_bert(clust, 3, cso_cluster_map, csos_descr_embed[i])\n",
    "    \n",
    "len(combined_cluster_pred_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cb0bbe4-0bad-49f9-a170-436225334fdc",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Results\n",
    "- take max of ROUGE-l (mode) scores among top z predictions from each method\n",
    "- average maxes over entire test set to generate average score for each method"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05e1e19a-5300-4dcf-aa52-e117c2414727",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Extracted vs Extracted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a874dd1-a042-46db-a1d5-72be25a016b6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## exracted vs extracted\n",
    "cols = ['cso_number','comdoublepred_rc','compred_rc','doublepred_rc','pred_rc','nlp_rc','cluster_pred_rc','oracle_rc','comdoublepred_rem','compred_rem','doublepred_rem','pred_rem','nlp_rem','cluster_pred_rem','oracle_rem']\n",
    "random_cso_rouge_recall_df = pd.DataFrame(columns = cols)\n",
    "mode = 'f'\n",
    "z = 5\n",
    "for i in random_cso_list:\n",
    "    series = {}\n",
    "    series['cso_number'] = int(i)\n",
    "    index = cso_df.index[cso_df['cso_number'] == int(i)].tolist()[0]\n",
    "    actual_rc = cso_df['root_cause'][index]\n",
    "    actual_rem = cso_df['remediations'][index]\n",
    "    \n",
    "    \n",
    "    series['nlp_rc'],series['nlp_rem'],a,b  = max_rouge_minus1(nlp_pred_random[i], actual_rc, actual_rem, mode,z)\n",
    "    series['pred_rc'],series['pred_rem'],a,b = max_rouge_minus1(pred_g1_random[i], actual_rc, actual_rem,mode,z)\n",
    "    series['cluster_pred_rc'],series['cluster_pred_rem'],a,b  = max_rouge_minus1(combined_cluster_pred_random[i], actual_rc, actual_rem, mode,z)\n",
    "       \n",
    "    random_cso_rouge_recall_df = pd.concat([random_cso_rouge_recall_df, pd.DataFrame([series], columns=cols)], ignore_index=True)\n",
    "random_cso_rouge_recall_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "a1be2fb4-7b65-4b8a-9104-d3de6689fc83",
   "metadata": {},
   "outputs": [],
   "source": [
    "show = ['pred_rc','nlp_rc','cluster_pred_rc','pred_rem','nlp_rem','cluster_pred_rem']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5c36856-71eb-44de-a236-e76823e650ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_cso_rouge_recall_df[show].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a827a0d9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}