PGD-TRADES-Defenses-on-Neural-Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(28 * 28, 200)\n",
    "        self.fc2 = nn.Linear(200, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view((-1, 28 * 28))\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Normalize(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return (x - 0.1307) / 0.3081\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fc7fdc7a9b0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "batch_size = 512\n",
    "seed = 42\n",
    "learning_rate = 0.01\n",
    "num_epochs = 10\n",
    "eps = 0.1\n",
    "k = 7\n",
    "trades_lambda = 1.0\n",
    "\n",
    "# Setting the random number generator\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Datasets\n",
    "train_dataset = datasets.MNIST('mnist_data/', train=True, download=True, transform=transforms.Compose([transforms.ToTensor()]))\n",
    "test_dataset = datasets.MNIST('mnist_data/', train=False, download=True, transform=transforms.Compose([transforms.ToTensor()]))\n",
    "\n",
    "# Data loaders\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add data normalization as a first \"layer\" to the network\n",
    "# This allows us to search for adversarial examples to the real image,\n",
    "# rather than to the normalized image\n",
    "model = nn.Sequential(Normalize(), Net())\n",
    "model = model.to(device)\n",
    "\n",
    "opt = optim.Adam(params=model.parameters(), lr=learning_rate, weight_decay=1e-5)\n",
    "scheduler = optim.lr_scheduler.StepLR(opt, 15)\n",
    "ce_loss = torch.nn.CrossEntropyLoss()\n",
    "kl_loss = torch.nn.KLDivLoss(reduction='batchmean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pgd(model, x_batch, target, k, eps, eps_step, kl_loss: bool = False):\n",
    "    if kl_loss:\n",
    "        # Loss function for the case that target is a distribution rather than a label (used for TRADES)\n",
    "        loss_fn = torch.nn.KLDivLoss(reduction='sum')\n",
    "    else:\n",
    "        # Standard PGD\n",
    "        loss_fn = torch.nn.CrossEntropyLoss(reduction='sum')\n",
    "    \n",
    "    # Disable gradients here\n",
    "    with torch.no_grad():\n",
    "        # Initialize with a random point inside the considered perturbation region\n",
    "        x_adv = x_batch.detach() + eps * (2 * torch.rand_like(x_batch) - 1)\n",
    "        \n",
    "       # Project back to the image domain\n",
    "        x_adv.clamp(min=0.0, max=1.0)\n",
    "\n",
    "        for step in range(k):\n",
    "            # Make sure we don't have a previous compute graph and enable gradient computation\n",
    "            x_adv.detach_().requires_grad_()\n",
    "\n",
    "            # Re-enable gradients\n",
    "            with torch.enable_grad():\n",
    "                # Run the model and obtain the loss\n",
    "                out = F.log_softmax(model(x_adv), dim=1)\n",
    "                model.zero_grad()\n",
    "\n",
    "                # Compute gradient\n",
    "                loss_fn(out, target).backward()\n",
    "            \n",
    "            # Compute step\n",
    "            step = eps_step * x_adv.grad.sign()\n",
    "\n",
    "            # Project to eps ball\n",
    "            x_adv = x_batch + (x_adv + step - x_batch).clamp(min=-eps, max=eps)\n",
    "\n",
    "            # Clamp back to image domain: we clamp at each step\n",
    "            x_adv.clamp_(min=0.0, max=1.0)\n",
    "    \n",
    "    return x_adv.detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test_accuracies_using_defense(defense, num_epochs, train_loader, test_loader, k, eps):\n",
    "    for epoch in range(1, num_epochs + 1):\n",
    "        # Training\n",
    "        for _, (x_batch, y_batch) in enumerate(tqdm(train_loader)):\n",
    "\n",
    "            x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n",
    "            \n",
    "            if defense == 'PGD':\n",
    "                # PGD attack to generate adversarial examples\n",
    "                \n",
    "                # Switch model to eval mode, to ensure it is deterministic\n",
    "                model.eval()\n",
    "\n",
    "                x_adv = pgd(\n",
    "                    model, \n",
    "                    x_batch=x_batch, \n",
    "                    target=y_batch,\n",
    "                    eps=eps, \n",
    "                    k=k, \n",
    "                    eps_step = 2.5 * eps / k\n",
    "                )\n",
    "\n",
    "                # Switch back to training mode\n",
    "                model.train()\n",
    "                out_pgd = model(x_adv)\n",
    "\n",
    "                # Compute loss\n",
    "                loss = ce_loss(out_pgd, y_batch)\n",
    "\n",
    "            elif defense == 'TRADES':\n",
    "                # Switch to training mode\n",
    "                model.train()\n",
    "                out_nat = model(x_batch)\n",
    "                target = F.softmax(out_nat.detach(), dim=1)\n",
    "\n",
    "                # Do PGD attack to generate adversarial examples\n",
    "                \n",
    "                # Switch network to eval mode, to ensure it is deterministic\n",
    "                model.eval()\n",
    "\n",
    "                x_adv = pgd(\n",
    "                    model, \n",
    "                    x_batch=x_batch, \n",
    "                    target=target, \n",
    "                    k=k, \n",
    "                    eps=eps,\n",
    "                    eps_step=2.5 * eps / k,\n",
    "                    kl_loss=True\n",
    "                )\n",
    "\n",
    "                # Calculate loss\n",
    "                \n",
    "                # Switch to training mode\n",
    "                model.train()\n",
    "                out_adv = F.log_softmax(model(x_adv), dim=1)\n",
    "                \n",
    "                loss_nat = ce_loss(out_nat, y_batch)\n",
    "                loss_adv = kl_loss(out_adv, target)\n",
    "                loss = loss_nat + trades_lambda * loss_adv\n",
    "                \n",
    "            elif defense == 'none':                \n",
    "                model.train()\n",
    "                out_nat = model(x_batch)\n",
    "                loss = ce_loss(out_nat, y_batch)\n",
    "\n",
    "            opt.zero_grad()\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "\n",
    "        # Testing\n",
    "        model.eval()\n",
    "\n",
    "        tot_test, tot_acc, tot_adv_acc = 0.0, 0.0, 0.0\n",
    "\n",
    "        for _, (x_batch, y_batch) in enumerate(tqdm(test_loader)):\n",
    "            x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n",
    "\n",
    "            # Prediction by the model on each batch\n",
    "            out = model(x_batch)\n",
    "            pred = torch.max(out, dim=1)[1]\n",
    "            acc = pred.eq(y_batch).sum().item()\n",
    "\n",
    "            x_adv = pgd(\n",
    "                model,\n",
    "                x_batch=x_batch,\n",
    "                target=y_batch,\n",
    "                k=k,\n",
    "                eps=eps,\n",
    "                eps_step=2.5 * eps / k\n",
    "            )\n",
    "\n",
    "            # Prediction of the model on the adversarial batch\n",
    "            out_adv = model(x_adv)\n",
    "            pred_adv = torch.max(out_adv, dim=1)[1]\n",
    "            acc_adv = pred_adv.eq(y_batch).sum().item()\n",
    "\n",
    "            # Add to total accuracies for both regular and adversarial accuracies\n",
    "            tot_acc += acc\n",
    "            tot_adv_acc += acc_adv\n",
    "            tot_test += x_batch.size()[0]\n",
    "\n",
    "        scheduler.step()\n",
    "\n",
    "        print('Epoch %d: Accuracy %.5lf, Adv Accuracy %.5lf' %\n",
    "            (epoch, tot_acc / tot_test, tot_adv_acc / tot_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:07<00:00, 16.62it/s]\n",
      "100%|██████████| 20/20 [00:06<00:00,  3.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Accuracy 0.98690, Adv Accuracy 0.54680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 18.04it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Accuracy 0.98690, Adv Accuracy 0.54730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 18.93it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Accuracy 0.98680, Adv Accuracy 0.53310\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 19.01it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: Accuracy 0.98670, Adv Accuracy 0.53690\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 18.68it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5: Accuracy 0.98720, Adv Accuracy 0.52480\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 18.92it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: Accuracy 0.98690, Adv Accuracy 0.52470\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 18.30it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7: Accuracy 0.98620, Adv Accuracy 0.52730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 19.65it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8: Accuracy 0.98660, Adv Accuracy 0.53170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:06<00:00, 19.44it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: Accuracy 0.98690, Adv Accuracy 0.52450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:05<00:00, 20.00it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: Accuracy 0.98690, Adv Accuracy 0.52230\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate model using standard training, no defense\n",
    "train_and_test_accuracies_using_defense(\n",
    "    defense='none', num_epochs=num_epochs, train_loader=train_loader, test_loader=test_loader, k=k, eps=eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.56it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Accuracy 0.98460, Adv Accuracy 0.80610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:32<00:00,  3.61it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Accuracy 0.98520, Adv Accuracy 0.82600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.56it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Accuracy 0.98520, Adv Accuracy 0.83650\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:34<00:00,  3.45it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: Accuracy 0.98540, Adv Accuracy 0.84520\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.48it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5: Accuracy 0.98550, Adv Accuracy 0.85170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.51it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: Accuracy 0.98530, Adv Accuracy 0.85380\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.55it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7: Accuracy 0.98520, Adv Accuracy 0.85780\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:32<00:00,  3.60it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8: Accuracy 0.98500, Adv Accuracy 0.85950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:35<00:00,  3.34it/s]\n",
      "100%|██████████| 20/20 [00:06<00:00,  3.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: Accuracy 0.98540, Adv Accuracy 0.86080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.52it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: Accuracy 0.98530, Adv Accuracy 0.86350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Evaluate model using PGD defense\n",
    "train_and_test_accuracies_using_defense(\n",
    "    defense='PGD', num_epochs=num_epochs, train_loader=train_loader, test_loader=test_loader, k=k, eps=eps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.50it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Accuracy 0.98520, Adv Accuracy 0.86570\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.56it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Accuracy 0.98520, Adv Accuracy 0.86630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:35<00:00,  3.34it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Accuracy 0.98540, Adv Accuracy 0.86600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:34<00:00,  3.46it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4: Accuracy 0.98530, Adv Accuracy 0.86730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:33<00:00,  3.51it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5: Accuracy 0.98570, Adv Accuracy 0.86640\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:35<00:00,  3.35it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6: Accuracy 0.98550, Adv Accuracy 0.86770\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:42<00:00,  2.77it/s]\n",
      "100%|██████████| 20/20 [00:07<00:00,  2.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7: Accuracy 0.98540, Adv Accuracy 0.86610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:39<00:00,  2.97it/s]\n",
      "100%|██████████| 20/20 [00:06<00:00,  3.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8: Accuracy 0.98540, Adv Accuracy 0.86780\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:35<00:00,  3.29it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9: Accuracy 0.98530, Adv Accuracy 0.86740\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [00:35<00:00,  3.37it/s]\n",
      "100%|██████████| 20/20 [00:05<00:00,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10: Accuracy 0.98530, Adv Accuracy 0.86790\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Evaluate model using PGD defense\n",
    "train_and_test_accuracies_using_defense(\n",
    "    defense='TRADES', num_epochs=num_epochs, train_loader=train_loader, test_loader=test_loader, k=k, eps=eps)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "39d7dbb437d063c754c82cf8e6b839566e777d8b07cf6ccd9b0451f2a2b8e0f1"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 64-bit ('deeppoly': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}