examples/maug_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5cc633a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## DATA\n",
    "import torch\n",
    "\n",
    "N = 100\n",
    "T = 12\n",
    "\n",
    "D1 = 5\n",
    "D2 = 3\n",
    "D3 = 8\n",
    "\n",
    "V1 = torch.rand(N, T, D1) #view 1\n",
    "V2 = torch.rand(N, T, D2) #view 2\n",
    "V3 = torch.rand(N, D3) #view 3 with different shape\n",
    "\n",
    "n_labels = 5\n",
    "labels = torch.randint(n_labels, size=(N,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6b5630b",
   "metadata": {},
   "source": [
    "## Data Loader for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd90aa20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'index': 2,\n",
       " 'views': {'view 1': tensor([[0.6037, 0.9473, 0.6308, 0.3833, 0.7511],\n",
       "          [0.5716, 0.5351, 0.4556, 0.5908, 0.2210],\n",
       "          [0.7186, 0.5884, 0.0608, 0.8996, 0.9142],\n",
       "          [0.3415, 0.4466, 0.3904, 0.7918, 0.4546],\n",
       "          [0.4569, 0.2663, 0.4702, 0.4472, 0.3186],\n",
       "          [0.0086, 0.8844, 0.8416, 0.5689, 0.2238],\n",
       "          [0.3750, 0.0367, 0.6954, 0.1767, 0.8864],\n",
       "          [0.8374, 0.7627, 0.4890, 0.6574, 0.5600],\n",
       "          [0.9713, 0.1575, 0.9749, 0.4535, 0.3132],\n",
       "          [0.2204, 0.8840, 0.4557, 0.9976, 0.6552],\n",
       "          [0.7092, 0.4781, 0.3984, 0.4223, 0.6815],\n",
       "          [0.5515, 0.0493, 0.9711, 0.7360, 0.5470]]),\n",
       "  'view 2': tensor([[0.9491, 0.4606, 0.1015],\n",
       "          [0.3673, 0.7667, 0.8475],\n",
       "          [0.2531, 0.9373, 0.0419],\n",
       "          [0.2623, 0.9134, 0.1321],\n",
       "          [0.8031, 0.9152, 0.3755],\n",
       "          [0.8543, 0.4214, 0.6850],\n",
       "          [0.0835, 0.1835, 0.2483],\n",
       "          [0.1269, 0.1443, 0.8095],\n",
       "          [0.4972, 0.2590, 0.1855],\n",
       "          [0.9246, 0.6335, 0.9758],\n",
       "          [0.0978, 0.0618, 0.8760],\n",
       "          [0.1873, 0.3366, 0.7742]]),\n",
       "  'view 3': tensor([0.3416, 0.4824, 0.0357, 0.0283, 0.0995, 0.4031, 0.8098, 0.4982])},\n",
       " 'target': tensor([2])}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class DataViews_torch(torch.utils.data.Dataset):\n",
    "    def __init__(self, input_views: dict, target):\n",
    "        super(DataViews_torch,self).__init__()\n",
    "        self.target = target\n",
    "        self.view_names = list(input_views.keys())\n",
    "        self.views = [input_views[v_name] for v_name in self.view_names]\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.views[0])\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        target = self.target[index]\n",
    "        views = {view_n: view[index] for view, view_n in zip(self.views,self.view_names)}\n",
    "        \n",
    "        return {\"index\": index, \"views\": views, \"target\": target}\n",
    "\n",
    "train_dataset = DataViews_torch(input_views={\"view 1\": V1, \"view 2\": V2, \"view 3\": V3}, target=labels)\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(train_dataset,batch_size=32,drop_last=False,shuffle=True)\n",
    "\n",
    "train_dataset[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b904b811",
   "metadata": {},
   "source": [
    "## Model encoder, prediction head and merge function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68ac1c14",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### MODEL\n",
    "from mvlearning.single.models import create_model\n",
    "from mvlearning.merge_module import MergeModule\n",
    "from mvlearning.utils import get_dic_emb_dims\n",
    "from mvlearning.fusion import FeatureFusion\n",
    "\n",
    "DIM_EMBEDDING = 32\n",
    "\n",
    "encoder_models = {}\n",
    "for name, inp_dim in {\"view 1\": D1, \"view 2\":D2}.items():\n",
    "    encoder_models[name] = create_model(inp_dim, DIM_EMBEDDING, model_type=\"lstm\")\n",
    "encoder_models[\"view 3\"] = create_model(D3, DIM_EMBEDDING, model_type=\"mlp\") #different model architecture\n",
    "\n",
    "EMBEDDING_DIC= get_dic_emb_dims(encoder_models)\n",
    "merge_function = MergeModule(EMBEDDING_DIC, mode=\"concat\")\n",
    "\n",
    "prediction_head = create_model(sum(EMBEDDING_DIC.values()), n_labels, model_type=\"mlp\", encoder=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14c226c7",
   "metadata": {},
   "source": [
    "## Define MAUG (Sensor Dropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6b423409",
   "metadata": {},
   "outputs": [],
   "source": [
    "maug_arguments = {\"maug\": \"sensd\", \"maug_args\": {\"drop_ratio\":0.3}}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25285f13",
   "metadata": {},
   "source": [
    "### Illustration "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "082f3d2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random augmented views from list =  ['S1', 'S2']\n",
      ">> more random augmented views from list =  ['weather']\n",
      ">> more random augmented views from list =  ['S2']\n",
      ">> more random augmented views from list =  ['S2', 'weather']\n",
      ">> more random augmented views from list =  ['S2']\n",
      ">> more random augmented views from list =  ['S1']\n",
      "Random augmented list with 10perc of dropout ratio =  ['S1', 'S2', 'weather']\n",
      ">> more random augmented list with 10perc of dropout ratio =  ['S1', 'S2', 'weather']\n",
      ">> more random augmented list with 10perc of dropout ratio =  ['S2', 'weather']\n",
      ">> more random augmented list with 10perc of dropout ratio =  ['S2', 'weather']\n",
      ">> more random augmented list with 10perc of dropout ratio =  ['weather']\n",
      ">> more random augmented list with 10perc of dropout ratio =  ['S2', 'weather']\n",
      "Random augmented list with 10perc of dropout ratio =  ['S1']\n",
      ">> mode random augmented list with 10perc of dropout ratio =  ['S2']\n",
      ">> mode random augmented list with 10perc of dropout ratio =  ['weather']\n",
      ">> mode random augmented list with 10perc of dropout ratio =  ['S1', 'S2']\n",
      ">> mode random augmented list with 10perc of dropout ratio =  ['weather']\n",
      ">> mode random augmented list with 10perc of dropout ratio =  ['S2', 'weather']\n"
     ]
    }
   ],
   "source": [
    "from mvlearning.missing.simulate import augment_random_missing, augment_randomlist_missing\n",
    "\n",
    "view_names = [ \"S1\", \"S2\",\"weather\"]\n",
    "\n",
    "print(\"Random augmented views from list = \",augment_randomlist_missing(view_names))\n",
    "for _ in range (5):\n",
    "    print(\">> more random augmented views from list = \",augment_randomlist_missing(view_names))\n",
    "\n",
    "print(\"Random augmented list with 10perc of dropout ratio = \",augment_random_missing(view_names, drop_ratio=0.10))\n",
    "for _ in range(5):\n",
    "    print(\">> more random augmented list with 10perc of dropout ratio = \",augment_random_missing(view_names, drop_ratio=0.10))\n",
    "\n",
    "print(\"Random augmented list with 10perc of dropout ratio = \",augment_random_missing(view_names, drop_ratio=0.90))\n",
    "for _ in range(5):\n",
    "    print(\">> mode random augmented list with 10perc of dropout ratio = \",augment_random_missing(view_names, drop_ratio=0.90))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dceddcf6",
   "metadata": {},
   "source": [
    "# Training based on Pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af2a7e7d",
   "metadata": {},
   "source": [
    "### Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2eb94e47",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy tensor(0.1760)\n"
     ]
    }
   ],
   "source": [
    "## MODEL DEFINITION\n",
    "mv_model = FeatureFusion(encoder_models, merge_function, prediction_head, **maug_arguments)\n",
    "\n",
    "output_ = mv_model( {\"view 1\":V1, \"view 2\": V2, \"view 3\": V3})\n",
    "\n",
    "print(\"Accuracy\", torch.mean(((output_[\"prediction\"] > 0.5) == labels).to(torch.float32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1b6be54c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.5194, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## LOSS FUNCTION AND OPTIMIZER\n",
    "optimizer = torch.optim.Adam(mv_model.parameters())\n",
    "loss_function=torch.nn.CrossEntropyLoss()\n",
    "\n",
    "loss_function(output_[\"prediction\"], torch.squeeze(labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31150e6a",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "33630cf3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH 1: with LOSS 0.7880977392196655:\n",
      "EPOCH 2: with LOSS 0.5458723306655884:\n",
      "EPOCH 3: with LOSS 0.4586859345436096:\n",
      "EPOCH 4: with LOSS 0.49492549896240234:\n",
      "EPOCH 5: with LOSS 0.40600499510765076:\n",
      "EPOCH 6: with LOSS 0.39157378673553467:\n",
      "EPOCH 7: with LOSS 0.3639375567436218:\n",
      "EPOCH 8: with LOSS 0.4806094765663147:\n",
      "EPOCH 9: with LOSS 0.33303287625312805:\n",
      "EPOCH 10: with LOSS 0.3622744083404541:\n",
      "EPOCH 11: with LOSS 0.4265765845775604:\n",
      "EPOCH 12: with LOSS 0.3206545114517212:\n",
      "EPOCH 13: with LOSS 0.38373202085494995:\n",
      "EPOCH 14: with LOSS 0.24350401759147644:\n",
      "EPOCH 15: with LOSS 0.2805960774421692:\n",
      "EPOCH 16: with LOSS 0.35794103145599365:\n",
      "EPOCH 17: with LOSS 0.24888888001441956:\n",
      "EPOCH 18: with LOSS 0.266275018453598:\n",
      "EPOCH 19: with LOSS 0.20832347869873047:\n",
      "EPOCH 20: with LOSS 0.22012090682983398:\n",
      "EPOCH 21: with LOSS 0.3731854557991028:\n",
      "EPOCH 22: with LOSS 0.19205619394779205:\n",
      "EPOCH 23: with LOSS 0.18372194468975067:\n",
      "EPOCH 24: with LOSS 0.17249898612499237:\n",
      "EPOCH 25: with LOSS 0.14190427958965302:\n",
      "EPOCH 26: with LOSS 0.15757852792739868:\n",
      "EPOCH 27: with LOSS 0.15805453062057495:\n",
      "EPOCH 28: with LOSS 0.11455365270376205:\n",
      "EPOCH 29: with LOSS 0.31297171115875244:\n",
      "EPOCH 30: with LOSS 0.11068545281887054:\n",
      "EPOCH 31: with LOSS 0.26010629534721375:\n",
      "EPOCH 32: with LOSS 0.0986231118440628:\n",
      "EPOCH 33: with LOSS 0.15146031975746155:\n",
      "EPOCH 34: with LOSS 0.14548151195049286:\n",
      "EPOCH 35: with LOSS 0.11484862864017487:\n",
      "EPOCH 36: with LOSS 0.1578913778066635:\n",
      "EPOCH 37: with LOSS 0.09443806111812592:\n",
      "EPOCH 38: with LOSS 0.06798696517944336:\n",
      "EPOCH 39: with LOSS 0.05422143638134003:\n",
      "EPOCH 40: with LOSS 0.05098383501172066:\n",
      "EPOCH 41: with LOSS 0.041963376104831696:\n",
      "EPOCH 42: with LOSS 0.036839261651039124:\n",
      "EPOCH 43: with LOSS 0.0375889353454113:\n",
      "EPOCH 44: with LOSS 0.030392080545425415:\n",
      "EPOCH 45: with LOSS 0.048548467457294464:\n",
      "EPOCH 46: with LOSS 0.020615650340914726:\n",
      "EPOCH 47: with LOSS 0.020147476345300674:\n",
      "EPOCH 48: with LOSS 0.019329633563756943:\n",
      "EPOCH 49: with LOSS 0.01517552137374878:\n",
      "EPOCH 50: with LOSS 0.013516226783394814:\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 50\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    mv_model.train(True)\n",
    "    running_loss = 0.\n",
    "    for i, data in enumerate(train_dataloader):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = mv_model(data[\"views\"])\n",
    "        loss = loss_function(outputs[\"prediction\"], torch.squeeze(data[\"target\"]))\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "        running_loss += loss\n",
    "    print('EPOCH {}: with LOSS {}:'.format(epoch + 1, running_loss/len(train_dataloader)))\n",
    "\n",
    "    ## VALIDATION/EARLY STOPPING IF REQUIRED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "02da705b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy tensor(0.1760)\n"
     ]
    }
   ],
   "source": [
    "output_ = mv_model( {\"view 1\":V1, \"view 2\": V2, \"view 3\": V3})\n",
    "\n",
    "print(\"Accuracy\", torch.mean(((output_[\"prediction\"] > 0.5) == labels).to(torch.float32)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93713659",
   "metadata": {},
   "source": [
    "# Training based on Pytorch Lightning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2422e63",
   "metadata": {},
   "source": [
    "### Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9cff5e13",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adam (\n",
       "Parameter Group 0\n",
       "    amsgrad: False\n",
       "    betas: (0.9, 0.999)\n",
       "    capturable: False\n",
       "    differentiable: False\n",
       "    eps: 1e-08\n",
       "    foreach: None\n",
       "    fused: None\n",
       "    lr: 0.001\n",
       "    maximize: False\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pytorch_lightning as pl\n",
    "\n",
    "## MODEL DEFINITION with LOSS FUNCTION and OPTIMIZER INSIDE\n",
    "mv_model = FeatureFusion(encoder_models, merge_function, prediction_head, \n",
    "                         loss_function=torch.nn.CrossEntropyLoss(), optimizer=\"adam\", **maug_arguments)\n",
    "\n",
    "mv_model.configure_optimizers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "527d849f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'objective': tensor(3.5678, grad_fn=<NllLossBackward0>)}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mv_model.loss_batch({\n",
    "    \"views\": {\"view 1\":V1, \"view 2\": V2, \"view 3\": V3},\n",
    "    \"target\": labels\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "fcf47046",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy tensor(0.1760)\n"
     ]
    }
   ],
   "source": [
    "output_ = mv_model( {\"view 1\":V1, \"view 2\": V2, \"view 3\": V3})\n",
    "\n",
    "print(\"Accuracy\", torch.mean(((output_[\"prediction\"] > 0.5) == labels).to(torch.float32)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e887a91",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f860da19",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3080 Laptop GPU') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name            | Type             | Params\n",
      "-----------------------------------------------------\n",
      "0 | views_encoder   | ModuleDict       | 431 K \n",
      "1 | merge_module    | MergeModule      | 0     \n",
      "2 | prediction_head | Generic_Decoder  | 29.6 K\n",
      "3 | loss_function   | CrossEntropyLoss | 0     \n",
      "-----------------------------------------------------\n",
      "461 K     Trainable params\n",
      "0         Non-trainable params\n",
      "461 K     Total params\n",
      "1.844     Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49: 100%|██████████| 4/4 [00:00<00:00, 114.53it/s, v_num=1, train_objective=0.0972] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=50` reached.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49: 100%|██████████| 4/4 [00:00<00:00, 74.28it/s, v_num=1, train_objective=0.0972] \n"
     ]
    }
   ],
   "source": [
    "trainer = pl.Trainer(max_epochs=50, accelerator=\"gpu\", devices = 1)\n",
    "trainer.fit(mv_model, train_dataloader, val_dataloaders=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "76810f3b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy tensor(0.1780)\n"
     ]
    }
   ],
   "source": [
    "output_ = mv_model( {\"view 1\":V1, \"view 2\": V2, \"view 3\": V3})\n",
    "\n",
    "print(\"Accuracy\", torch.mean(((output_[\"prediction\"] > 0.5) == labels).to(torch.float32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a127339",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mvlearn",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}