Assessing_the_Impact_of_Membership_Inference_Attacks_on_Classical_Machine_Learning_Algorithms.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7au3_uHCdO9c"
      },
      "outputs": [],
      "source": [
        "!pip install adversarial-robustness-toolbox "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gUQouQCkXVaJ"
      },
      "source": [
        "# Preparación"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "97ZZ7tCqdHzN"
      },
      "source": [
        "## Libraries. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_mQSBNCXze8G"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "import time\n",
        "\n",
        "import plotly.graph_objects as go\n",
        "from plotly.subplots import make_subplots"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_rW9sB3TCWS3"
      },
      "source": [
        "Datasets used."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AhJa5jL5CXCB"
      },
      "outputs": [],
      "source": [
        "from sklearn.datasets import load_iris\n",
        "from sklearn.datasets import load_digits\n",
        "from sklearn.datasets import load_breast_cancer\n",
        "from art.utils import load_cifar10\n",
        "from art.utils import load_mnist\n",
        "from art.utils import load_nursery\n",
        "from art.utils import load_nursery"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beOSwCXrzgk5"
      },
      "source": [
        "Scikit-learn's utils used."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ilqqOcEUEtzb"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "import sklearn.preprocessing as preprocessing\n",
        "from sklearn.preprocessing import OrdinalEncoder\n",
        "\n",
        "#Estadisticas\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7OTudvdQCmuo"
      },
      "source": [
        "Adversarial Robustness Toolbox"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GBsKUC_kC9c1"
      },
      "outputs": [],
      "source": [
        "from art.attacks.inference.membership_inference import MembershipInferenceBlackBox\n",
        "from art.attacks.inference.membership_inference import LabelOnlyDecisionBoundary\n",
        "\n",
        "from art.attacks.inference.attribute_inference import AttributeInferenceBaseline\n",
        "from art.attacks.inference.attribute_inference import AttributeInferenceBlackBox\n",
        "from art.attacks.inference.attribute_inference import AttributeInferenceMembership"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HbOmZ0pqj7yo"
      },
      "source": [
        "Models."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FO_SxnfqfWle"
      },
      "outputs": [],
      "source": [
        "#General\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnClassifier\n",
        "\n",
        "#DecisionTreeClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnDecisionTreeClassifier\n",
        "\n",
        "#ExtraTreeClassifier\n",
        "from sklearn.tree import ExtraTreeClassifier\n",
        "from  art.estimators.classification.scikitlearn import ScikitlearnExtraTreeClassifier\n",
        "\n",
        "#Adaboost\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnAdaBoostClassifier\n",
        "\n",
        "#BaggingClassifier\n",
        "from sklearn.ensemble import BaggingClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnBaggingClassifier\n",
        "\n",
        "#ExtraTreesClassifier Ensemble\n",
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnExtraTreesClassifier\n",
        "\n",
        "#GradientBoostingClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnGradientBoostingClassifier\n",
        "\n",
        "#RandomForest\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnRandomForestClassifier\n",
        "\n",
        "#Logistic Regresion\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnLogisticRegression\n",
        "\n",
        "#SVC Classifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.svm import LinearSVC\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnSVC\n",
        "\n",
        "# Gaussian\n",
        "from art.estimators.classification.scikitlearn import ScikitlearnGaussianNB"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D1FgxWwisetX"
      },
      "source": [
        "# Metodologia"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2hFWCk_fdHzL"
      },
      "source": [
        "## Load Datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MtN9zj0XeG1s"
      },
      "source": [
        "### Dataset Nursery"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zE1p-DijA9rN"
      },
      "outputs": [],
      "source": [
        "train_url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/nursery/nursery.data\"\n",
        "\n",
        "features = [\"parents\", \"has_nurs\", \"form\", \"children\", \"housing\", \"finance\", \"social\", \"health\", \"label\"] \n",
        "\n",
        "nursery_df = pd.read_csv(train_url, names=features, sep=r'\\s*,\\s*', engine='python', na_values=\"?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iH1BLme-qSmr"
      },
      "outputs": [],
      "source": [
        "nursery_df.drop(nursery_df.loc[nursery_df[\"label\"] == \"recommend\"].index, axis=0, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y7p8WG14vUm0"
      },
      "outputs": [],
      "source": [
        "categories_parents = ['usual', 'pretentious', 'great_pret']\n",
        "categories_has_nurs = ['proper', 'less_proper', 'improper', 'critical', 'very_crit']\n",
        "categories_form = ['complete', 'completed', 'incomplete', 'foster']\n",
        "categories_children = ['1','2', '3', 'more']\n",
        "categories_housing = ['convenient','less_conv', 'critical']\n",
        "categories_finance = ['convenient','inconv']\n",
        "categories_social = ['nonprob','slightly_prob', 'problematic']\n",
        "categories_health = ['recommended','priority', 'not_recom']\n",
        "categories_label = ['not_recom','very_recom', 'priority', 'spec_prior']\n",
        "\n",
        "encoderX = OrdinalEncoder(categories=[categories_parents, categories_has_nurs, categories_form, \n",
        "                                     categories_children, categories_housing, categories_finance,\n",
        "                                      categories_social, categories_health])\n",
        "\n",
        "encoderY = OrdinalEncoder(categories=[categories_label]) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ElESbK6nw1ey"
      },
      "outputs": [],
      "source": [
        "X = encoderX.fit_transform(nursery_df[['parents', 'has_nurs', 'form', 'children', 'housing', 'finance', 'social', 'health']])\n",
        "y = encoderY.fit_transform(nursery_df[['label']])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RtIzpGH0MGWC"
      },
      "outputs": [],
      "source": [
        "x_train_nursery, x_test_nursery, y_train_nursery, y_test_nursery = train_test_split(X, y, test_size=0.2, random_state=0, stratify=y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QP-2wPRKMGWD"
      },
      "outputs": [],
      "source": [
        "x_train_nursery = x_train_nursery.astype(int)\n",
        "x_test_nursery = x_test_nursery.astype(int)\n",
        "y_train_nursery = np.ravel(y_train_nursery.astype(int))\n",
        "y_test_nursery = np.ravel(y_test_nursery.astype(int))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z1VxFSNHWI9f"
      },
      "outputs": [],
      "source": [
        "X_nursery = np.concatenate((x_train_nursery, x_test_nursery))\n",
        "Y_nursery = np.concatenate((y_train_nursery, y_test_nursery))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xN-rLOL1eIGl"
      },
      "source": [
        "### Dataset Mnist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v2p-cqqIKYmK"
      },
      "outputs": [],
      "source": [
        "(x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist), min_mnist, max_mnist = load_mnist()\n",
        "\n",
        "x_train_mnist = x_train_mnist.reshape(x_train_mnist.shape[0], x_train_mnist.shape[1] * x_train_mnist.shape[2] * x_train_mnist.shape[3])[:10000]\n",
        "x_test_mnist = x_test_mnist.reshape(x_test_mnist.shape[0], x_test_mnist.shape[1] * x_test_mnist.shape[2] * x_test_mnist.shape[3])[:10000]\n",
        "\n",
        "y_train_mnist = np.argmax(y_train_mnist, axis=1)[:10000]\n",
        "y_test_mnist = np.argmax(y_test_mnist, axis=1)[:10000]\n",
        "\n",
        "######\n",
        "\n",
        "X_mnist = np.concatenate((x_train_mnist, x_test_mnist))\n",
        "Y_mnist = np.concatenate((y_train_mnist, y_test_mnist))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bAHdc3V9eKII"
      },
      "source": [
        "### Dataset Breast Cancer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3Vdpk_QYdnzV"
      },
      "outputs": [],
      "source": [
        "X_BreastCancer, Y_BreastCancer = load_breast_cancer(return_X_y=True)\n",
        "x_train_BreastCancer, x_test_BreastCancer, y_train_BreastCancer, y_test_BreastCancer = train_test_split(X_BreastCancer, Y_BreastCancer, random_state=0, stratify=Y_BreastCancer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C-XaEdRzR1Na"
      },
      "outputs": [],
      "source": [
        "X_BreastCancer = np.concatenate((x_train_BreastCancer, x_test_BreastCancer))\n",
        "Y_BreastCancer = np.concatenate((y_train_BreastCancer, y_test_BreastCancer))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4rldlQbheSI9"
      },
      "source": [
        "### Dataset Adult"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DFBB0jlpeA5w"
      },
      "outputs": [],
      "source": [
        "features = [\"Age\", \"Workclass\", \"fnlwgt\", \"Education\", \"Education-Num\", \"Martial Status\",\n",
        "        \"Occupation\", \"Relationship\", \"Race\", \"Sex\", \"Capital Gain\", \"Capital Loss\",\n",
        "        \"Hours per week\", \"Country\", \"Target\"] \n",
        "\n",
        "adult_train = pd.read_csv(\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\", \n",
        "                             names=features, sep=r'\\s*,\\s*',  engine='python', na_values=\"?\")\n",
        "adult_test = pd.read_csv(\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test\", \n",
        "                            names=features, sep=r'\\s*,\\s*',  engine='python', na_values=\"?\", skiprows=1)\n",
        "\n",
        "train_len = len(adult_train)\n",
        "\n",
        "adultdf = pd.concat([adult_train, adult_test])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w32Se_TvHthR"
      },
      "outputs": [],
      "source": [
        "labels = adultdf['Target']\n",
        "labels = labels.replace('<=50K', 0).replace('>50K', 1)\n",
        "labels = labels.replace('<=50K.', 0).replace('>50K.', 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vs2mG1HgeA5y"
      },
      "outputs": [],
      "source": [
        "del adultdf[\"Education\"]\n",
        "del adultdf[\"Target\"]\n",
        "\n",
        "binary_data = pd.get_dummies(adultdf)\n",
        "feature_cols = binary_data[binary_data.columns[:-2]]\n",
        "scaler = preprocessing.StandardScaler()\n",
        "data = pd.DataFrame(scaler.fit_transform(feature_cols), columns=feature_cols.columns)\n",
        "\n",
        "x_train_adult = data[:train_len].to_numpy()\n",
        "y_train_adult = labels[:train_len].to_numpy() \n",
        "x_test_adult = data[train_len:].to_numpy()\n",
        "y_test_adult = labels[train_len:].to_numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JH6xLqEYSEB8"
      },
      "outputs": [],
      "source": [
        "X_adult = np.concatenate((x_train_adult, x_test_adult))\n",
        "Y_adult = np.concatenate((y_train_adult, y_test_adult))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gYZyeEO9jW2b"
      },
      "source": [
        "### Car Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ApS5VGKY4C5K"
      },
      "outputs": [],
      "source": [
        "train_url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/car/car.data\"\n",
        "\n",
        "features = [\"buying\", \"maint\", \"doors\", \"persons\", \"lug_boot\", \"safety\", \"label\"] \n",
        "\n",
        "car_df = pd.read_csv(train_url, names=features, sep=r'\\s*,\\s*', \n",
        "                             engine='python', na_values=\"?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gLmCuWJ8ZYKT"
      },
      "outputs": [],
      "source": [
        "categories_buying = ['low', 'med', 'high', 'vhigh']\n",
        "categories_maint = ['low', 'med', 'high', 'vhigh']\n",
        "categories_doors = ['2', '3', '4', '5more']\n",
        "categories_persons = ['2','4', 'more']\n",
        "categories_lugboot = ['small','med', 'big']\n",
        "categories_safety = ['low','med', 'high']\n",
        "categories_label = ['unacc','acc', 'good', 'vgood']\n",
        "\n",
        "encoderX = OrdinalEncoder(categories=[categories_buying, categories_maint, categories_doors, \n",
        "                                     categories_persons, categories_lugboot, categories_safety])\n",
        "\n",
        "encoderY = OrdinalEncoder(categories=[categories_label])     "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0xlZXZ_kce9K"
      },
      "outputs": [],
      "source": [
        "X = encoderX.fit_transform(car_df[['buying', 'maint', 'doors', 'persons', 'lug_boot', 'safety']])\n",
        "y = encoderY.fit_transform(car_df[['label']])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VskQuneQCePz"
      },
      "outputs": [],
      "source": [
        "prueba = encoderX.fit_transform(car_df[['buying', 'maint', 'doors', 'persons', 'lug_boot', 'safety']])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RcuQPyifYWjF"
      },
      "outputs": [],
      "source": [
        "x_train_car, x_test_car, y_train_car, y_test_car = train_test_split(X, y, test_size=0.2, random_state=0, stratify=y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_8k41cwkyD03"
      },
      "outputs": [],
      "source": [
        "x_train_car = x_train_car.astype(int)\n",
        "x_test_car = x_test_car.astype(int)\n",
        "y_train_car = np.ravel(y_train_car.astype(int))\n",
        "y_test_car = np.ravel(y_test_car.astype(int))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xTql7Na5SZTs"
      },
      "outputs": [],
      "source": [
        "X_car = np.concatenate((x_train_car, x_test_car))\n",
        "Y_car = np.concatenate((y_train_car, y_test_car))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iyvT6i8Ix05r"
      },
      "source": [
        "### Titanic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2oqmV4-1x0Z2"
      },
      "outputs": [],
      "source": [
        "titanic_train = pd.read_csv('datasets/titanic/train.csv')\n",
        "\n",
        "titanic_test = pd.read_csv('datasets/titanic/test.csv')\n",
        "\n",
        "titanic_test_label = pd.read_csv('datasets/titanic/gender_submission.csv')\n",
        "\n",
        "titanic_test = titanic_test.merge(titanic_test_label, on=\"PassengerId\")\n",
        "titanic = pd.concat([titanic_train, titanic_test])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LL39JnlyCUAu"
      },
      "outputs": [],
      "source": [
        "del titanic['Name']\n",
        "del titanic['Ticket']\n",
        "del titanic['Cabin']\n",
        "del titanic['PassengerId']\n",
        "\n",
        "titanic['Age'] = titanic['Age'].fillna(titanic['Age'].mean())\n",
        "titanic['Fare'] = titanic['Fare'].fillna(titanic['Fare'].mean())\n",
        "titanic = titanic.dropna()\n",
        "\n",
        "titanic['Sex'] = titanic['Sex'].replace('male', 0).replace('female', 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RsdLkJOZ72fm"
      },
      "outputs": [],
      "source": [
        "titanic_label = titanic['Survived']\n",
        "del titanic['Survived']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R-6aMUHyM53G"
      },
      "outputs": [],
      "source": [
        "titanic = pd.get_dummies(titanic) \n",
        "scaler = preprocessing.StandardScaler()\n",
        "data_titanic = pd.DataFrame( titanic, columns=titanic.columns)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OIIgnwwjbmD9"
      },
      "outputs": [],
      "source": [
        "x_train_titanic, x_test_titanic, y_train_titanic, y_test_titanic = train_test_split(data_titanic, titanic_label, test_size=0.2, random_state=0,  stratify=titanic_label)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hhBvqlm7c8mr"
      },
      "outputs": [],
      "source": [
        "x_train_titanic = x_train_titanic.to_numpy()\n",
        "x_test_titanic = x_test_titanic.to_numpy()\n",
        "y_train_titanic = y_train_titanic.to_numpy()\n",
        "y_test_titanic = y_test_titanic.to_numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0AA_ZG2LSoaW"
      },
      "outputs": [],
      "source": [
        "X_titanic = np.concatenate((x_train_titanic, x_test_titanic))\n",
        "Y_titanic = np.concatenate((y_train_titanic, y_test_titanic))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z9G4dKLCu4y6"
      },
      "source": [
        "## Auxiliary functions\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XSdsMHkUHMzK"
      },
      "source": [
        "Encapsulates a SKlearn object in an ART library object."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9UzqBnED-AdH"
      },
      "outputs": [],
      "source": [
        "def objART(sklearnObject):\n",
        "  if type(sklearnObject) == DecisionTreeClassifier:\n",
        "    return ScikitlearnDecisionTreeClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == ExtraTreeClassifier:\n",
        "    return ScikitlearnExtraTreeClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == AdaBoostClassifier:\n",
        "    return ScikitlearnAdaBoostClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == BaggingClassifier:\n",
        "    return ScikitlearnBaggingClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == ExtraTreesClassifier:\n",
        "    return ScikitlearnExtraTreesClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == GradientBoostingClassifier:\n",
        "    return ScikitlearnGradientBoostingClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == RandomForestClassifier:\n",
        "    return ScikitlearnRandomForestClassifier(sklearnObject)\n",
        "  elif type(sklearnObject) == LogisticRegression:\n",
        "    return ScikitlearnLogisticRegression(sklearnObject)\n",
        "  elif type(sklearnObject) == SVC or type(sklearnObject) == LinearSVC:\n",
        "    return ScikitlearnSVC(sklearnObject)\n",
        "  else:\n",
        "    return ScikitlearnClassifier(sklearnObject)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HoPtYefAqtAY"
      },
      "source": [
        "## Calcular estadisticas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iMVMjqmFmbrP"
      },
      "outputs": [],
      "source": [
        "def calcularDatasetAtacante(train, test, ratio=0.8):\n",
        "  train_size = int(len(train) * ratio)\n",
        "  test_size = int(len(test) * ratio)\n",
        "\n",
        "  return train_size, test_size"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zPlyZsiXZJBV"
      },
      "outputs": [],
      "source": [
        "def obtenerEstadisticas(inferred_train_bb, inferred_test_bb):\n",
        "\n",
        "  real_train_bb = np.ones(len(inferred_train_bb))\n",
        "  real_test_bb = np.zeros(len(inferred_test_bb))\n",
        "\n",
        "  inferred = np.concatenate((inferred_train_bb, inferred_test_bb))\n",
        "  real = np.concatenate((real_train_bb, real_test_bb))\n",
        "\n",
        "  train_acc = accuracy_score(real_train_bb, inferred_train_bb)\n",
        "  test_acc = accuracy_score(real_test_bb, inferred_test_bb)\n",
        "\n",
        "  acc = accuracy_score(real, inferred)\n",
        "\n",
        "  return acc\n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EALWTpi6wApq"
      },
      "outputs": [],
      "source": [
        "def entrenarModeloAtacante(modeloAtaque, x_train, y_train, x_test, y_test, ratio=0.5):\n",
        "  attack_train_size, attack_test_size = calcularDatasetAtacante( x_train, x_test, ratio=ratio)\n",
        "\n",
        "  modeloAtaque.fit(x_train[:attack_train_size], y_train[:attack_train_size],  x_test[:attack_test_size], y_test[:attack_test_size])\n",
        "\n",
        "  inferred_train_bb = modeloAtaque.infer(x_train[-attack_test_size:], y_train[-attack_test_size:])\n",
        "  inferred_test_bb = modeloAtaque.infer(x_test[-attack_test_size:], y_test[-attack_test_size:])\n",
        "\n",
        "  acc = obtenerEstadisticas(inferred_train_bb, inferred_test_bb)\n",
        "  \n",
        "  return acc\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TrovwUzxsCVT"
      },
      "source": [
        "# Experiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TY-vYBgmflrb"
      },
      "source": [
        "## Membership Inference: Shadow Training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UdiD7mKUF3U8"
      },
      "outputs": [],
      "source": [
        "def cargarModelos():\n",
        "  modelos = []\n",
        "\n",
        "  modelos.append(ExtraTreesClassifier())\n",
        "  modelos.append(RandomForestClassifier())\n",
        "  modelos.append(BaggingClassifier())\n",
        "  modelos.append(DecisionTreeClassifier())\n",
        "  modelos.append(AdaBoostClassifier())\n",
        "  modelos.append(GradientBoostingClassifier())\n",
        "  modelos.append(LogisticRegression(max_iter=2000))\n",
        "  modelos.append(SVC())\n",
        "  modelos.append(LinearSVC(max_iter=2000))\n",
        "\n",
        "  return modelos\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JfEdKNBUmz-h"
      },
      "outputs": [],
      "source": [
        "def experimento1(modelos, X, Y, Dataset, tipoModeloAtacante= \"rf\", repeticiones = 1 ):\n",
        "\n",
        "  kf = KFold(n_splits=5)\n",
        "  kf.get_n_splits(X)\n",
        "\n",
        "  experimentdf = pd.DataFrame(columns=[\"Dataset\", \"Modelo\", 'Hiperparametro','Metric', 'Score'])\n",
        "  for modelo in modelos:\n",
        "    tmp_score_train = list()\n",
        "    tmp_score_test = list()\n",
        "    tmp_accuracy = list()\n",
        "    tmp_precision = list()\n",
        "    tmp_recall = list()\n",
        "    for i in range(repeticiones):\n",
        "      for train_index, test_index in kf.split(X):\n",
        "        x_train, x_test = X[train_index], X[test_index]\n",
        "        y_train, y_test = Y[train_index], Y[test_index]\n",
        "        #\n",
        "        modelo.fit(x_train,y_train)\n",
        "        modeloAtaque = MembershipInferenceBlackBox(objART(modelo), attack_model_type = tipoModeloAtacante)\n",
        "        acc = entrenarModeloAtacante(modeloAtaque, x_train, y_train, x_test, y_test)\n",
        "\n",
        "        tmp_score_train.append( modelo.score(x_train, y_train))\n",
        "        tmp_score_test.append( modelo.score(x_test, y_test))\n",
        "        tmp_accuracy.append(acc)\n",
        "\n",
        "    sco_train_avg = np.average(tmp_score_train)\n",
        "    sco_test_avg = np.average(tmp_score_test)\n",
        "    acc_avg = np.average(tmp_accuracy)\n",
        "\n",
        "    temporaldf = pd.DataFrame({\n",
        "      'Dataset': [Dataset]*3,\n",
        "      'Modelo': [type(modelo).__name__]*3,\n",
        "      'Metric': [\"Train Acc\", \"Test Acc\",\"MI\"],\n",
        "      'Score': [sco_train_avg, sco_test_avg, acc_avg ]\n",
        "      })\n",
        "    experimentdf = pd.concat([experimentdf, temporaldf])\n",
        "\n",
        "  return experimentdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oJ-hUNi8mz-i"
      },
      "outputs": [],
      "source": [
        "def experimentoCompararModelos(modelos):\n",
        "\n",
        "  Car_df = experimento1(modelos, X_car, Y_car, \"Car\")\n",
        "  Nursery_df = experimento1(modelos, X_nursery, Y_nursery, \"Nursery\")\n",
        "  Titanic_df = experimento1(modelos, X_titanic, Y_titanic, \"Titanic\")\n",
        "  BreastCancer_df = experimento1(modelos, X_BreastCancer, Y_BreastCancer, \"Breast Cancer\")\n",
        "  Adult_df = experimento1(modelos, X_adult, Y_adult, \"Adult\")\n",
        "  MNIST_df = experimento1(modelos, X_mnist, Y_mnist, \"MNIST\")\n",
        "\n",
        "  df = pd.concat([Car_df, Nursery_df, Titanic_df, BreastCancer_df, Adult_df, MNIST_df])\n",
        "\n",
        "  return df\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FW6jTFzuzrDt"
      },
      "outputs": [],
      "source": [
        "def visualiceResults(df):\n",
        "  gfg = pd.pivot_table(\n",
        "    df,\n",
        "\t  index=['Dataset',\"Modelo\"],\n",
        "    columns='Metric',\n",
        "    values='Score',\n",
        "    aggfunc='first'\n",
        "  )\n",
        "  df = gfg\n",
        "  df = df.reindex(df.sort_values(by='MI', ascending=True).index)\n",
        "\n",
        "  df.MI = df.MI * -1\n",
        "  Diverging = make_subplots( rows=2, cols=3, shared_yaxes='all', shared_xaxes='all', vertical_spacing=0.1,\n",
        "                            subplot_titles=(\"Adult\", \"Breast Cancer\", \"Car\", \"MNIST\", \"Nursery\", \"Titanic\"))\n",
        "\t\n",
        "  Diverging.update_xaxes(tickvals=[-1, -0.5, 0, 0.5, 1], ticktext=[1, 0.5, 0, 0.5, 1])\n",
        "  n = {\"Adult\": [1,1], \"Breast Cancer\": [1,2], \"Car\": [1,3], \"MNIST\": [2,1], \"Nursery\": [2,2], \"Titanic\": [2,3]}\n",
        "  for k in n:\n",
        "    leyend =  k =='Adult'\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['MI'], y=df.loc[k].index, orientation='h', name='MI',marker_color='rgb(188,128,189)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['Train Acc'], y=df.loc[k].index, orientation='h', name='Train Acc',marker_color='rgba(251,128,114,0.9)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['Test Acc'], y=df.loc[k].index, orientation='h', name='Test Acc',marker_color='rgb(128,177,211)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "\n",
        "  Diverging.update_layout(barmode='overlay', height=700, width=1300,  \n",
        "              legend = dict(font = dict(size = 15)),\n",
        "\t\t\t\t\t\t\tbargap=0.3, legend_orientation='h', legend_x=0.36, legend_y=1.13 )\n",
        "  return Diverging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMP_1B-4myjM"
      },
      "source": [
        "---"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N7mCu_3y_yXu"
      },
      "outputs": [],
      "source": [
        "modelos = cargarModelos()\n",
        "df_results = experimentoCompararModelos(modelos)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "izjsxiNX_y1E"
      },
      "outputs": [],
      "source": [
        "visualiceResults(df_results)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTZllSzxBJz9"
      },
      "source": [
        "## Hyperparameter Selection\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_ghqrze8M7pR"
      },
      "outputs": [],
      "source": [
        "def experimentoHiperparametrosEjecucion(modelos, hiperparametros, X, Y, Dataset, tipoModeloAtacante= \"rf\", repeticiones = 1 ):\n",
        "  kf = KFold(n_splits=5)\n",
        "  kf.get_n_splits(X)\n",
        "\n",
        "  experimentdf = pd.DataFrame(columns=[\"Dataset\", \"Modelo\", 'Hiperparametro','Metric', 'Score'])\n",
        "  for modelo,hiperparametro in zip(modelos,hiperparametros):\n",
        "    tmp_score_train = list()\n",
        "    tmp_score_test = list()\n",
        "    tmp_accuracy = list()\n",
        "    for i in range(repeticiones):\n",
        "      for train_index, test_index in kf.split(X):\n",
        "        x_train, x_test = X[train_index], X[test_index]\n",
        "        y_train, y_test = Y[train_index], Y[test_index]\n",
        "        #\n",
        "        modelo.fit(x_train,y_train)\n",
        "        modeloAtaque = MembershipInferenceBlackBox(objART(modelo), attack_model_type = tipoModeloAtacante)\n",
        "        acc = entrenarModeloAtacante(modeloAtaque, x_train, y_train, x_test, y_test)\n",
        "\n",
        "        tmp_score_train.append( modelo.score(x_train, y_train))\n",
        "        tmp_score_test.append( modelo.score(x_test, y_test))\n",
        "        tmp_accuracy.append(acc)\n",
        "\n",
        "    sco_train_avg = np.average(tmp_score_train)\n",
        "    sco_test_avg = np.average(tmp_score_test)\n",
        "    acc_avg = np.average(tmp_accuracy)\n",
        "\n",
        "    temporaldf = pd.DataFrame({\n",
        "      'Dataset': [Dataset]*3,\n",
        "      'Modelo': [type(modelo).__name__]*3,\n",
        "      'Hiperparametro': [hiperparametro]*3,\n",
        "      'Metric': [\"Train Acc\", \"Test Acc\",\"MI\"],\n",
        "      'Score': [sco_train_avg, sco_test_avg, acc_avg]\n",
        "      })\n",
        "    experimentdf = pd.concat([experimentdf, temporaldf])\n",
        "\n",
        "  return experimentdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "locjwZ-rBJz_"
      },
      "outputs": [],
      "source": [
        "def experimentoHiperparametros(modelos, hiperparametros):\n",
        "\n",
        "  Adult_df = experimentoHiperparametrosEjecucion(modelos, hiperparametros, X_adult, Y_adult, \"Adult\")\n",
        "\n",
        "  return Adult_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jPWjr3baTnYp"
      },
      "source": [
        "### Decision Tree"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bR9SfNpEFimp"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"Gini criterion\", \"Best splitter\",\"Low max_features\", \"Low max_depth\",\"min_samples_split\", \"min_samples_leaf\", \"Pruning\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(DecisionTreeClassifier())\n",
        "modelos.append(DecisionTreeClassifier(criterion='gini'))\n",
        "modelos.append(DecisionTreeClassifier(splitter='best'))\n",
        "modelos.append(DecisionTreeClassifier(max_features=1))\n",
        "modelos.append(DecisionTreeClassifier(max_depth=10))\n",
        "modelos.append(DecisionTreeClassifier(min_samples_split=25))\n",
        "modelos.append(DecisionTreeClassifier(min_samples_leaf=25))\n",
        "modelos.append(DecisionTreeClassifier(ccp_alpha=2.9346628038314405e-05))\n",
        "\n",
        "\n",
        "decisiontreedf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZbdSSEJVBJ0A"
      },
      "source": [
        "### Bagging"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C8UNZmpLKjW_"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"High n_estimators\", \"No bootstrap\",\"bootstrap_features\", \"Low max_features\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(BaggingClassifier())\n",
        "modelos.append(BaggingClassifier(n_estimators=300))\n",
        "modelos.append(BaggingClassifier(bootstrap=False))\n",
        "modelos.append(BaggingClassifier(bootstrap_features=True))\n",
        "modelos.append(BaggingClassifier(max_features=2))\n",
        "\n",
        "baggingdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMuRN4GqBJ0D"
      },
      "source": [
        "### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "REkohqnPUIFD"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"High n_estimators\", \"Entropy criterion\",\"Low max_features\", \"No bootstrap\", \"Low max_depth\", \"min_samples_split\", \"min_samples_leaf\", \"Pruning\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(RandomForestClassifier())\n",
        "modelos.append(RandomForestClassifier(n_estimators=300))\n",
        "modelos.append(RandomForestClassifier(criterion='entropy'))\n",
        "modelos.append(RandomForestClassifier(max_features=1))\n",
        "modelos.append(RandomForestClassifier(bootstrap=False))\n",
        "modelos.append(RandomForestClassifier(max_depth=10))\n",
        "modelos.append(RandomForestClassifier(min_samples_split=25))\n",
        "modelos.append(RandomForestClassifier(min_samples_leaf=25))\n",
        "modelos.append(RandomForestClassifier(ccp_alpha=2.9346628038314405e-05))\n",
        "\n",
        "randomforestdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "606X4ofJBJ0B"
      },
      "source": [
        "### ExtraTreesClassifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jopukqBLOz9y"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"High n_estimators\", \"Entropy criterion\", \"bootstrap\", \"Low max_features\", \"Low max_depth\", \"min_samples_split\", \"min_samples_leaf\", \"Pruning\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(ExtraTreesClassifier())\n",
        "modelos.append(ExtraTreesClassifier(n_estimators=300))\n",
        "modelos.append(ExtraTreesClassifier(criterion='entropy'))\n",
        "modelos.append(ExtraTreesClassifier(bootstrap=True))\n",
        "modelos.append(ExtraTreesClassifier(max_features=1))\n",
        "modelos.append(ExtraTreesClassifier(max_depth=10))\n",
        "modelos.append(ExtraTreesClassifier(min_samples_split=25))\n",
        "modelos.append(ExtraTreesClassifier(min_samples_leaf=25))\n",
        "modelos.append(ExtraTreesClassifier(ccp_alpha=2.9346628038314405e-05))\n",
        "\n",
        "extratreesdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GKyv__c9BJ0A"
      },
      "source": [
        "### AdaBoost"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LUvY8SpqI1K6"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"High n_estimators\", \"Low learning_rate\",\"SAMME.R algorithm\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(AdaBoostClassifier())\n",
        "modelos.append(AdaBoostClassifier(n_estimators=300))\n",
        "modelos.append(AdaBoostClassifier(learning_rate=0.5))\n",
        "modelos.append(AdaBoostClassifier(algorithm='SAMME'))\n",
        "\n",
        "\n",
        "adaboostdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b04_QFoXBJ0C"
      },
      "source": [
        "### Gradient Boosting"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HFLfiH75R-kq"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"High n_estimators\", \"Low learning_rate\", \"Low max_features\", \"Low max_depth\", \"min_samples_split\", \"min_samples_leaf\", \"Pruning\"]\n",
        "\n",
        "modelos = list()\n",
        "modelos.append(GradientBoostingClassifier())\n",
        "modelos.append(GradientBoostingClassifier(n_estimators=300))\n",
        "modelos.append(GradientBoostingClassifier(learning_rate=0.5))\n",
        "modelos.append(GradientBoostingClassifier(max_features=1))\n",
        "modelos.append(GradientBoostingClassifier(max_depth=10))\n",
        "modelos.append(GradientBoostingClassifier(min_samples_split=25))\n",
        "modelos.append(GradientBoostingClassifier(min_samples_leaf=25))\n",
        "modelos.append(GradientBoostingClassifier(ccp_alpha=2.9346628038314405e-05))\n",
        "\n",
        "gradientboostingdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJj0grSMBJ0D"
      },
      "source": [
        "### Logistic Regresion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XDeScRPR1sR6"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"l2 with newton-cg\", \"l2 with lbfgs\", \"l1 with liblinear\", \"l2 with liblinear\", \"l2 with sag\", \"elasticnet with saga\", \"l1 with saga\", \"l2 with saga\", \"C\"]\n",
        "\n",
        "modelos = list()\n",
        "\n",
        "modelos.append(LogisticRegression())\n",
        "modelos.append(LogisticRegression(penalty=\"l2\", solver=\"newton-cg\"))\n",
        "modelos.append(LogisticRegression(penalty=\"l2\", solver=\"lbfgs\"))\n",
        "modelos.append(LogisticRegression(penalty=\"l1\", solver=\"liblinear\"))\n",
        "modelos.append(LogisticRegression(penalty=\"l2\", solver=\"liblinear\"))\n",
        "modelos.append(LogisticRegression(penalty=\"l2\", solver=\"sag\"))\n",
        "modelos.append(LogisticRegression(penalty=\"elasticnet\", solver=\"saga\", l1_ratio=0.5))\n",
        "modelos.append(LogisticRegression(penalty=\"l1\", solver=\"saga\"))\n",
        "modelos.append(LogisticRegression(penalty=\"l2\", solver=\"saga\"))\n",
        "modelos.append(LogisticRegression(C=1e2))\n",
        "\n",
        "\n",
        "LogisticRegressiondf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mp8LqgofBJ0E"
      },
      "source": [
        "### SVC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LHRRD0sZ9Hd3"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"poly kernel\", \"auto gamma\", \"C\"]\n",
        "\n",
        "modelos = list()\n",
        "\n",
        "modelos.append(SVC())\n",
        "modelos.append(SVC(kernel=\"poly\"))\n",
        "modelos.append(SVC(gamma=\"auto\"))\n",
        "modelos.append(SVC(C=1e-5))\n",
        "\n",
        "\n",
        "SVCdf = experimentoHiperparametros(modelos, hiperparametros)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqbjYoG4Rp1r"
      },
      "source": [
        "### Linear SVC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BIEerXgXGxiZ"
      },
      "outputs": [],
      "source": [
        "hiperparametros = [\"default\", \"l1 penalty\", \"hinge loss\", \"C\"]\n",
        "\n",
        "modelos = list()\n",
        "\n",
        "modelos.append(LinearSVC())\n",
        "modelos.append(LinearSVC(penalty=\"l1\", dual=False))\n",
        "modelos.append(LinearSVC(loss=\"hinge\"))\n",
        "modelos.append(LinearSVC(C=1e-5))\n",
        "\n",
        "LinearSVCdf = experimentoHiperparametros(modelos, hiperparametros)\n",
        "LinearSVCdf.to_csv('LinearSVCdf.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k8zC9fqz56vE"
      },
      "source": [
        "## Results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jsk7QReg588l"
      },
      "outputs": [],
      "source": [
        "todosdf = pd.concat([adaboostdf, decisiontreedf, extratreesdf, randomforestdf, gradientboostingdf, baggingdf, LogisticRegressiondf, SVCdf, LinearSVCdf], ignore_index=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6buziKMnbxGB"
      },
      "outputs": [],
      "source": [
        "def visualiceResultsHyper(df):\n",
        "  df = todosdf\n",
        "  gfg = pd.pivot_table(\n",
        "    df,\n",
        "    index=[\"Modelo\",'Hiperparametro'],\n",
        "    columns='Metric',\n",
        "    values='Score',\n",
        "    aggfunc='first'\n",
        "  )\n",
        "  df = gfg\n",
        "\n",
        "  df = df.reindex(df.sort_values(by='Train Acc', ascending=True).sort_values(by='MI', ascending=True).index)\n",
        "\n",
        "  df.MI = df.MI * -1\n",
        "  Diverging = make_subplots( rows=3, cols=3,\n",
        "                            shared_yaxes=False, shared_xaxes='all', \n",
        "                            vertical_spacing=0.04,\n",
        "                            horizontal_spacing = 0.1,\n",
        "                            subplot_titles=(\"DecisionTreeClassifier\", \"BaggingClassifier\", \"RandomForestClassifier\", \"ExtraTreesClassifier\", \"AdaBoostClassifier\", \"GradientBoostingClassifier\", \"LogisticRegression\", \"SVC\", \"LinearSVC\"))\n",
        "\n",
        "  Diverging.update_xaxes(tickvals=[-1, -0.5, 0, 0.5, 1], ticktext=[1, 0.5, 0, 0.5, 1])\n",
        "  n = {\"DecisionTreeClassifier\": [1,1], \"BaggingClassifier\": [1,2], \"RandomForestClassifier\": [1,3], \"ExtraTreesClassifier\": [2,1], \"AdaBoostClassifier\": [2,2], \"GradientBoostingClassifier\": [2,3], \"LogisticRegression\": [3,1], \"SVC\": [3,2], \"LinearSVC\": [3,3]}\n",
        "  for k in n:\n",
        "    leyend =  k =='DecisionTreeClassifier'\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['MI'], y=df.loc[k].index, orientation='h', name='MI',marker_color='rgb(188,128,189)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['Train Acc'], y=df.loc[k].index, orientation='h', name='Train Acc',marker_color='rgb(251,128,114)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "    Diverging.add_trace(go.Bar(x=df.loc[k]['Test Acc'], y=df.loc[k].index, orientation='h', name='Test Acc',marker_color='rgb(128,177,211)',showlegend=leyend), row=n[k][0], col=n[k][1])\n",
        "\n",
        "  Diverging.update_layout(barmode='overlay', xaxis_range=[-1.1,1.1],\n",
        "              height=1100, width=1600,\n",
        "              legend = dict(font = dict(size = 15)),\n",
        "              bargap=0.3, legend_orientation='h', legend_x=0.39, legend_y=1.07\n",
        "              )\n",
        "\n",
        "  return Diverging\n",
        "\n",
        "visualiceResultsHyper(todosdf)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "97ZZ7tCqdHzN",
        "MtN9zj0XeG1s",
        "xN-rLOL1eIGl",
        "bAHdc3V9eKII",
        "4rldlQbheSI9",
        "gYZyeEO9jW2b",
        "iyvT6i8Ix05r",
        "z9G4dKLCu4y6",
        "HoPtYefAqtAY",
        "TY-vYBgmflrb",
        "fTZllSzxBJz9",
        "jPWjr3baTnYp",
        "ZbdSSEJVBJ0A",
        "QMuRN4GqBJ0D",
        "606X4ofJBJ0B",
        "GKyv__c9BJ0A",
        "b04_QFoXBJ0C",
        "UJj0grSMBJ0D",
        "aqbjYoG4Rp1r",
        "k8zC9fqz56vE"
      ],
      "name": "Assessing the Impact of Membership Inference Attacks on Classical Machine Learning Algorithms.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}