Reinforcement_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ad6V490MxnYC"
   },
   "source": [
    "## Tensorflow 2.x Setup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pcauSkcpxnYE",
    "outputId": "ae8a05ee-2fad-4f31-cac8-43879d477901"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "1.9.0\n"
     ]
    }
   ],
   "source": [
    "# Uncomment the following code to enable GPU\n",
    "# import tensorflow as tf\n",
    "# device_name = tf.test.gpu_device_name()\n",
    "# if device_name != '/device:GPU:0':\n",
    "#   raise SystemError('GPU device not found')\n",
    "# print('Found GPU at: {}'.format(device_name))\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# tensorflow\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "\n",
    "# pytorch\n",
    "import torch\n",
    "from torch import nn\n",
    "print(torch.__version__)\n",
    "\n",
    "# scikit-learn\n",
    "import sklearn\n",
    "from sklearn import cluster, decomposition, manifold, metrics\n",
    "import pandas as pd\n",
    "\n",
    "import scipy\n",
    "from scipy.stats import entropy\n",
    "from scipy.stats import dirichlet\n",
    "\n",
    "# access google file system\n",
    "# from google.colab import drive\n",
    "# drive.mount('/content/drive')\n",
    "cwd = 'rl-starter-files/storage/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EL_sc3uaOnLL"
   },
   "source": [
    "### Get action probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "t-gM0q2pNKkH"
   },
   "outputs": [],
   "source": [
    "# Example:\n",
    "# shadow (seed 0~4)\n",
    "# IN: label_0 ... label_4\n",
    "# OUT: label_5 ... label_9\n",
    "\n",
    "# attack\n",
    "# concat(shadow_seed_0, label_0) = IN\n",
    "# concat(shadow_seed_5, label_5) = OUT\n",
    "\n",
    "# attack on concat(shadow_seed_1, label_1)\n",
    "\n",
    "# victim, whether seed 3 is used in training\n",
    "# attack on concat(victim_seed_3, label_3) ==> IN/OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "-oNNIhQMnz0c"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(cwd+'mr_label_1/probabilities.csv')\n",
    "df0 = pd.read_csv(cwd+'mr_label_5/probabilities.csv')\n",
    "df1 = pd.read_csv(cwd+'mr_shadow_1/probabilities.csv')\n",
    "df2 = pd.read_csv(cwd+'mr_shadow_5/probabilities.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YIGA5uDupMQg",
    "outputId": "b308d583-d91c-4695-cec9-0b7a03b561d4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1600, 7, 128)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def reshape_data(dataframe):\n",
    "    # reshape the data into the shape that fit the attack model\n",
    "    data = dataframe.to_numpy()\n",
    "    data = data.reshape((len(data)//128, 128, 7))\n",
    "    data = data.swapaxes(1,2)\n",
    "    return data\n",
    "\n",
    "data0 = reshape_data(df)\n",
    "data00 = reshape_data(df0)\n",
    "data1 = reshape_data(df1)\n",
    "data2 = reshape_data(df2)\n",
    "data0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sCz8z80Oo-SM",
    "outputId": "6dc14072-0c3f-4519-d66e-d3018da0a008"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3200, 14, 128), (3200, 14, 128))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_data = np.concatenate([np.concatenate([data0[:1600], data1[:1600]], axis=1),\n",
    "                          np.concatenate([data00[:1600], data2[:1600]], axis=1)])\n",
    "\n",
    "out_data = np.concatenate([np.concatenate([data0[:1600], data2[:1600]], axis=1),\n",
    "                           np.concatenate([data00[:1600], data1[:1600]], axis=1)])\n",
    "in_data.shape, out_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7dO4kbeNptK2"
   },
   "outputs": [],
   "source": [
    "def get_att_data(in_data, out_data):\n",
    "    # Generate IN/OUT labels for the data\n",
    "    in_label = [1.0]*len(in_data)\n",
    "    out_label = [0.0]*len(out_data)\n",
    "    labels = in_label + out_label\n",
    "    in_data = [d for d in in_data]\n",
    "    out_data = [d for d in out_data]\n",
    "    data = in_data + out_data\n",
    "\n",
    "    c = list(zip(data, labels))\n",
    "    random.shuffle(c)\n",
    "    data, labels = zip(*c)\n",
    "    return np.array(data), np.array(labels)\n",
    "\n",
    "def get_label_vector(labels):\n",
    "    # Convert labels into vector form (one-hot embedding)\n",
    "    label_vectors = np.zeros((len(labels), np.max(labels)+1))\n",
    "    for i in range(len(labels)):\n",
    "        label_vectors[i, labels[i]] = 1\n",
    "    return label_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6en96QQeH1EV",
    "outputId": "d7ade9eb-de9a-412f-d79e-16726e9905d9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4400, 14, 128), (4400, 2))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data, labels = get_att_data(in_data, out_data)\n",
    "labels = get_label_vector(np.array(labels, dtype=np.int))\n",
    "\n",
    "test_size = 2000\n",
    "data, v_data = data[:-test_size], data[-test_size:]\n",
    "labels, v_labels = labels[:-test_size], labels[-test_size:]\n",
    "\n",
    "data.shape, labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-_IYphsf9nUm"
   },
   "source": [
    "### Attack Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mSDk4ZNSqON1"
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Softmax, LSTM\n",
    "def build_att():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(LSTM(48, return_sequences=True))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(32, activation='relu'))\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dense(2))\n",
    "    model.add(Softmax())\n",
    "    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
    "    precision = keras.metrics.Precision(class_id=0)\n",
    "    recall = keras.metrics.Recall(class_id=0)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy', precision, recall])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gSUapK_bqmot",
    "outputId": "505ba4a3-e7ca-4719-d999-dde18294fa54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "69/69 [==============================] - 4s 11ms/step - loss: 0.6952 - accuracy: 0.5064 - precision_2: 0.5074 - recall_2: 0.5102\n",
      "Epoch 2/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.6963 - accuracy: 0.4973 - precision_2: 0.4988 - recall_2: 0.6385\n",
      "Epoch 3/15\n",
      "69/69 [==============================] - 1s 11ms/step - loss: 0.6938 - accuracy: 0.5109 - precision_2: 0.5090 - recall_2: 0.6812\n",
      "Epoch 4/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.6931 - accuracy: 0.5005 - precision_2: 0.5020 - recall_2: 0.4009\n",
      "Epoch 5/15\n",
      "69/69 [==============================] - 1s 11ms/step - loss: 0.6873 - accuracy: 0.5611 - precision_2: 0.5492 - recall_2: 0.6934\n",
      "Epoch 6/15\n",
      "69/69 [==============================] - 1s 11ms/step - loss: 0.6571 - accuracy: 0.6464 - precision_2: 0.6591 - recall_2: 0.6095\n",
      "Epoch 7/15\n",
      "69/69 [==============================] - 1s 11ms/step - loss: 0.5159 - accuracy: 0.8093 - precision_2: 0.8153 - recall_2: 0.8009\n",
      "Epoch 8/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.2233 - accuracy: 0.9568 - precision_2: 0.9561 - recall_2: 0.9578\n",
      "Epoch 9/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.1099 - accuracy: 0.9800 - precision_2: 0.9818 - recall_2: 0.9782\n",
      "Epoch 10/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0406 - accuracy: 0.9950 - precision_2: 0.9955 - recall_2: 0.9946\n",
      "Epoch 11/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0124 - accuracy: 1.0000 - precision_2: 1.0000 - recall_2: 1.0000\n",
      "Epoch 12/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0076 - accuracy: 1.0000 - precision_2: 1.0000 - recall_2: 1.0000\n",
      "Epoch 13/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0041 - accuracy: 1.0000 - precision_2: 1.0000 - recall_2: 1.0000\n",
      "Epoch 14/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0026 - accuracy: 1.0000 - precision_2: 1.0000 - recall_2: 1.0000\n",
      "Epoch 15/15\n",
      "69/69 [==============================] - 1s 12ms/step - loss: 0.0019 - accuracy: 1.0000 - precision_2: 1.0000 - recall_2: 1.0000\n",
      "32/32 [==============================] - 1s 8ms/step - loss: 0.0157 - accuracy: 0.9945 - precision_2: 0.9950 - recall_2: 0.9940\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.015675978735089302,\n",
       " 0.9944999814033508,\n",
       " 0.9949698448181152,\n",
       " 0.9939698576927185]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attack_model = build_att()\n",
    "attack_model.fit(x=data, y=labels, batch_size=64, epochs=15, verbose=True)\n",
    "attack_model.evaluate(x=v_data, y=v_labels, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6GFTXWY_PzVy"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "RL_Men_Att_multiroom",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}