Retrieving_human_classification_without_supervision.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPE4LyhZhCT9GbbBrxuW/aU"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# The Pursuit of Human Labeling: A New Perspective on Unsupervised Learning"
      ],
      "metadata": {
        "id": "yr0rDrIBi4I5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "A new approach to unsupervised classification based on existing vision models"
      ],
      "metadata": {
        "id": "ESojbm-PifZo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Implementation of the research paper published by Artyom Gadetsky and Maria Brbic from the EPFL AI Reasearch Center. The abstract can be found [here](https://openreview.net/pdf?id=3GpIeVYw8X)."
      ],
      "metadata": {
        "id": "7deGeYjuiMhX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Approach** : Human labeled points are linearly separable in a sufficiently strong\n",
        "representation space, and are invariant to the underlying model and resulting representation space."
      ],
      "metadata": {
        "id": "JmFiA1x6jNmi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torchvision.models as models\n",
        "import numpy as np\n",
        "\n",
        "phi2model=torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14')\n",
        "\n",
        "phi2model.eval()\n",
        "statedict=phi2model.state_dict()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M7KMbEYAIUI3",
        "outputId": "730ac2ab-7678-4de1-a42e-f8df8bd4a5a2"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading: \"https://github.com/facebookresearch/dinov2/zipball/main\" to /root/.cache/torch/hub/main.zip\n",
            "/root/.cache/torch/hub/facebookresearch_dinov2_main/dinov2/layers/swiglu_ffn.py:51: UserWarning: xFormers is not available (SwiGLU)\n",
            "  warnings.warn(\"xFormers is not available (SwiGLU)\")\n",
            "/root/.cache/torch/hub/facebookresearch_dinov2_main/dinov2/layers/attention.py:33: UserWarning: xFormers is not available (Attention)\n",
            "  warnings.warn(\"xFormers is not available (Attention)\")\n",
            "/root/.cache/torch/hub/facebookresearch_dinov2_main/dinov2/layers/block.py:40: UserWarning: xFormers is not available (Block)\n",
            "  warnings.warn(\"xFormers is not available (Block)\")\n",
            "Downloading: \"https://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pth\" to /root/.cache/torch/hub/checkpoints/dinov2_vits14_pretrain.pth\n",
            "100%|██████████| 84.2M/84.2M [00:01<00:00, 77.5MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# convert to numpy\n",
        "numpy_params = {key: value.numpy() for key, value in statedict.items()}\n",
        "#save npy file\n",
        "np.save('dinov2basic.npy', numpy_params)"
      ],
      "metadata": {
        "id": "1XdydVGBJJcn"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Getting the First representation phi1"
      ],
      "metadata": {
        "id": "EniyMDhPLLcF"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finetuned on CIFAR10"
      ],
      "metadata": {
        "id": "9PYYFzWQLPx1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision.datasets import CIFAR10\n",
        "from torchvision import models\n",
        "\n",
        "\n",
        "# create a transform class for applying the normalization\n",
        "transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])  # normalize the representations to have unit norm\n",
        "])"
      ],
      "metadata": {
        "id": "6ndJJn6PJPC-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load the data\n",
        "train_dataset = CIFAR10(root='./data', train=True, transform=transform, download=True)\n",
        "test_dataset = CIFAR10(root='./data', train=False, transform=transform, download=True)"
      ],
      "metadata": {
        "id": "JuzGgwlLLZgP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc4503ef-4d93-40c6-b1a2-45e415f0fafd"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 170498071/170498071 [00:03<00:00, 43219421.81it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define DataLoader\n",
        "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=2)\n",
        "test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=2)"
      ],
      "metadata": {
        "id": "o2vsxF_LLrCe"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device=torch.device('cuda')"
      ],
      "metadata": {
        "id": "JHhW_JyRRxqB"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wFb0UXd1R7Kx",
        "outputId": "cbecd79b-5731-43f3-8092-05b32a066ed2"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We need to check the architecture of the chosen model and modify the last layer according to the original layer in the head"
      ],
      "metadata": {
        "id": "_sb9IrpVPeJ1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torchvision import models\n",
        "\n",
        "resnet = models.resnet50(True)\n",
        "vgg16=models.vgg16(True)\n",
        "num_classes = 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7mTL9YmtLukF",
        "outputId": "c3ac0c12-d6f8-4f30-f7a3-160d138ac89d"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:135: UserWarning: Using 'weights' as positional parameter(s) is deprecated since 0.13 and may be removed in the future. Please use keyword parameter(s) instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet50_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet50_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/resnet50-0676ba61.pth\" to /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth\n",
            "100%|██████████| 97.8M/97.8M [00:00<00:00, 170MB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/vgg16-397923af.pth\" to /root/.cache/torch/hub/checkpoints/vgg16-397923af.pth\n",
            "100%|██████████| 528M/528M [00:03<00:00, 154MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for name, param in resnet.named_parameters():\n",
        "  param.requires_grad = False"
      ],
      "metadata": {
        "id": "JXSklTRjOt1_"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for name, param in vgg16.named_parameters():\n",
        "  param.requires_grad = False"
      ],
      "metadata": {
        "id": "YcmUPP1NPec4"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resnet.fc = nn.Linear(resnet.fc.in_features, num_classes)"
      ],
      "metadata": {
        "id": "c3pOlgvfbIIK"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vgg16.classifier[6] = nn.Linear(vgg16.classifier[6].in_features,10)"
      ],
      "metadata": {
        "id": "FaPvJOP7PldP"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for name, param in vgg16.named_parameters():\n",
        "  print(name, param.requires_grad)"
      ],
      "metadata": {
        "id": "U5FMPKGBlp5R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resnet=resnet.to(device)"
      ],
      "metadata": {
        "id": "wNFVliPCR_nK"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vgg16=vgg16.to(device)"
      ],
      "metadata": {
        "id": "auP_BoxQPwA_"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(vgg16.parameters(), lr = 0.001, momentum=0.9, weight_decay=0.0001)\n",
        "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)\n",
        "\n",
        "num_epochs = 10"
      ],
      "metadata": {
        "id": "2fgv0AdfP3Gz"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(num_epochs):\n",
        "\n",
        "  vgg16.train()\n",
        "  for images, labels in train_loader:\n",
        "    images, labels = images.to(device), labels.to(device)\n",
        "\n",
        "    outputs = vgg16(images)\n",
        "    loss = criterion(outputs, labels)\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "  print(f'Epoch [{epoch+1}/{num_epochs}], Training Loss: {loss.item()}')\n",
        "\n",
        "  scheduler.step()\n",
        "\n",
        "  vgg16.eval()\n",
        "  total_loss = 0\n",
        "  correct = 0\n",
        "  total = 0\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for images, labels in test_loader:\n",
        "      images, labels = images.to(device), labels.to(device)\n",
        "      outputs = vgg16(images)\n",
        "      loss = criterion(outputs, labels)\n",
        "      total_loss += loss.item() * labels.size(0)\n",
        "      total += labels.size(0)\n",
        "\n",
        "      _, predicted = torch.max(outputs.data, 1)\n",
        "      correct += (predicted == labels).sum().item()\n",
        "\n",
        "\n",
        "  average = total_loss / total\n",
        "  print(f'Epoch [{epoch+1}/{num_epochs}], Validation Loss: {average:.4f}')\n",
        "  accuracy = correct / total\n",
        "  print(f'Epoch [{epoch+1}/{num_epochs}], Test Accuracy: {100 * accuracy:.2f}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5jBq-7w-PEHS",
        "outputId": "a7323a27-de19-42af-8f63-5d21ce0712c0"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [1/10], Training Loss: 0.9663029909133911\n",
            "Epoch [1/10], Validation Loss: 1.1510\n",
            "Epoch [1/10], Test Accuracy: 59.65%\n",
            "Epoch [2/10], Training Loss: 1.3450578451156616\n",
            "Epoch [2/10], Validation Loss: 1.1284\n",
            "Epoch [2/10], Test Accuracy: 60.44%\n",
            "Epoch [3/10], Training Loss: 0.8741094470024109\n",
            "Epoch [3/10], Validation Loss: 1.1287\n",
            "Epoch [3/10], Test Accuracy: 60.30%\n",
            "Epoch [4/10], Training Loss: 1.268105149269104\n",
            "Epoch [4/10], Validation Loss: 1.1059\n",
            "Epoch [4/10], Test Accuracy: 60.79%\n",
            "Epoch [5/10], Training Loss: 1.1579726934432983\n",
            "Epoch [5/10], Validation Loss: 1.1050\n",
            "Epoch [5/10], Test Accuracy: 61.25%\n",
            "Epoch [6/10], Training Loss: 1.1116459369659424\n",
            "Epoch [6/10], Validation Loss: 1.1022\n",
            "Epoch [6/10], Test Accuracy: 60.62%\n",
            "Epoch [7/10], Training Loss: 1.4910484552383423\n",
            "Epoch [7/10], Validation Loss: 1.1068\n",
            "Epoch [7/10], Test Accuracy: 61.39%\n",
            "Epoch [8/10], Training Loss: 1.1516261100769043\n",
            "Epoch [8/10], Validation Loss: 1.1071\n",
            "Epoch [8/10], Test Accuracy: 61.20%\n",
            "Epoch [9/10], Training Loss: 1.5885508060455322\n",
            "Epoch [9/10], Validation Loss: 1.0986\n",
            "Epoch [9/10], Test Accuracy: 60.94%\n",
            "Epoch [10/10], Training Loss: 0.992675244808197\n",
            "Epoch [10/10], Validation Loss: 1.1225\n",
            "Epoch [10/10], Test Accuracy: 60.83%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vgg16.to(\"cpu\")\n",
        "vgg16.eval()\n",
        "# sae the model state\n",
        "state_dictphi1 = vgg16.state_dict()\n",
        "\n",
        "tonumpystate= {key: value.numpy() for key, value in state_dictphi1.items()}\n",
        "\n",
        "# Save the NumPy parameters to a .npy file\n",
        "np.save('vgg16cifar10.npy', tonumpystate)"
      ],
      "metadata": {
        "id": "DTVH9KWYRyzw"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating a task with those two representations"
      ],
      "metadata": {
        "id": "bmoW7feIXV8w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip3 install learn2learn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "eudgdF41n7xp",
        "outputId": "f8a1f457-05a7-43b3-f345-c85b195e6004"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting learn2learn\n",
            "  Downloading learn2learn-0.2.0.tar.gz (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.10/dist-packages (from learn2learn) (1.25.2)\n",
            "Requirement already satisfied: gym>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from learn2learn) (0.25.2)\n",
            "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from learn2learn) (2.1.0+cu121)\n",
            "Requirement already satisfied: torchvision>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from learn2learn) (0.16.0+cu121)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from learn2learn) (1.11.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from learn2learn) (2.31.0)\n",
            "Collecting gsutil (from learn2learn)\n",
            "  Downloading gsutil-5.27.tar.gz (3.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m51.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from learn2learn) (4.66.1)\n",
            "Collecting qpth>=0.0.15 (from learn2learn)\n",
            "  Downloading qpth-0.0.16.tar.gz (16 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from gym>=0.14.0->learn2learn) (2.2.1)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.10/dist-packages (from gym>=0.14.0->learn2learn) (0.0.8)\n",
            "Requirement already satisfied: cvxpy>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from qpth>=0.0.15->learn2learn) (1.3.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (4.9.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.1.0->learn2learn) (2.1.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision>=0.3.0->learn2learn) (9.4.0)\n",
            "Collecting argcomplete>=1.9.4 (from gsutil->learn2learn)\n",
            "  Downloading argcomplete-3.2.2-py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.3/42.3 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting crcmod>=1.7 (from gsutil->learn2learn)\n",
            "  Downloading crcmod-1.7.tar.gz (89 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m89.7/89.7 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting fasteners>=0.14.1 (from gsutil->learn2learn)\n",
            "  Downloading fasteners-0.19-py3-none-any.whl (18 kB)\n",
            "Collecting gcs-oauth2-boto-plugin>=3.0 (from gsutil->learn2learn)\n",
            "  Downloading gcs-oauth2-boto-plugin-3.0.tar.gz (20 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting google-apitools>=0.5.32 (from gsutil->learn2learn)\n",
            "  Downloading google_apitools-0.5.32-py3-none-any.whl (135 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m135.7/135.7 kB\u001b[0m \u001b[31m20.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httplib2==0.20.4 (from gsutil->learn2learn)\n",
            "  Downloading httplib2-0.20.4-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.6/96.6 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting google-reauth>=0.1.0 (from gsutil->learn2learn)\n",
            "  Downloading google_reauth-0.1.1-py2.py3-none-any.whl (17 kB)\n",
            "Collecting monotonic>=1.4 (from gsutil->learn2learn)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: pyOpenSSL>=0.13 in /usr/local/lib/python3.10/dist-packages (from gsutil->learn2learn) (24.0.0)\n",
            "Collecting retry_decorator>=1.0.0 (from gsutil->learn2learn)\n",
            "  Downloading retry_decorator-1.1.1.tar.gz (3.9 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: six>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from gsutil->learn2learn) (1.16.0)\n",
            "Requirement already satisfied: google-auth[aiohttp]>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from gsutil->learn2learn) (2.27.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2==0.20.4->gsutil->learn2learn) (3.1.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->learn2learn) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->learn2learn) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->learn2learn) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->learn2learn) (2024.2.2)\n",
            "Requirement already satisfied: osqp>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.1.0->qpth>=0.0.15->learn2learn) (0.6.2.post8)\n",
            "Requirement already satisfied: ecos>=2 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.1.0->qpth>=0.0.15->learn2learn) (2.0.13)\n",
            "Requirement already satisfied: scs>=1.1.6 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.1.0->qpth>=0.0.15->learn2learn) (3.2.4.post1)\n",
            "Requirement already satisfied: setuptools>65.5.1 in /usr/local/lib/python3.10/dist-packages (from cvxpy>=1.1.0->qpth>=0.0.15->learn2learn) (67.7.2)\n",
            "Collecting rsa==4.7.2 (from gcs-oauth2-boto-plugin>=3.0->gsutil->learn2learn)\n",
            "  Downloading rsa-4.7.2-py3-none-any.whl (34 kB)\n",
            "Collecting boto>=2.29.1 (from gcs-oauth2-boto-plugin>=3.0->gsutil->learn2learn)\n",
            "  Downloading boto-2.49.0-py2.py3-none-any.whl (1.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m55.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: oauth2client>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from gcs-oauth2-boto-plugin>=3.0->gsutil->learn2learn) (4.1.3)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.10/dist-packages (from rsa==4.7.2->gcs-oauth2-boto-plugin>=3.0->gsutil->learn2learn) (0.5.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (0.3.0)\n",
            "Requirement already satisfied: aiohttp<4.0.0.dev0,>=3.6.2 in /usr/local/lib/python3.10/dist-packages (from google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (3.9.3)\n",
            "Collecting pyu2f (from google-reauth>=0.1.0->gsutil->learn2learn)\n",
            "  Downloading pyu2f-0.1.5.tar.gz (27 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: cryptography<43,>=41.0.5 in /usr/local/lib/python3.10/dist-packages (from pyOpenSSL>=0.13->gsutil->learn2learn) (42.0.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.1.0->learn2learn) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.1.0->learn2learn) (1.3.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0.dev0,>=3.6.2->google-auth[aiohttp]>=2.5.0->gsutil->learn2learn) (4.0.3)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.10/dist-packages (from cryptography<43,>=41.0.5->pyOpenSSL>=0.13->gsutil->learn2learn) (1.16.0)\n",
            "Requirement already satisfied: qdldl in /usr/local/lib/python3.10/dist-packages (from osqp>=0.4.1->cvxpy>=1.1.0->qpth>=0.0.15->learn2learn) (0.1.7.post0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.12->cryptography<43,>=41.0.5->pyOpenSSL>=0.13->gsutil->learn2learn) (2.21)\n",
            "Building wheels for collected packages: learn2learn, qpth, gsutil, crcmod, gcs-oauth2-boto-plugin, retry_decorator, pyu2f\n",
            "  Building wheel for learn2learn (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for learn2learn: filename=learn2learn-0.2.0-cp310-cp310-linux_x86_64.whl size=1200683 sha256=4cb321c42897f90507adeb7d82dcc7a840ae023619cf0c6f75d20f48ff53c857\n",
            "  Stored in directory: /root/.cache/pip/wheels/89/2c/13/c538cd229cdfc6c15a9d3cf64d2bb8220e205ea0f63ecb5fbe\n",
            "  Building wheel for qpth (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for qpth: filename=qpth-0.0.16-py3-none-any.whl size=19522 sha256=e366742c078ef28a0d072aa8ea79ff83f076df96defc9c4903d63a6e06612a9c\n",
            "  Stored in directory: /root/.cache/pip/wheels/9e/f2/44/9121d0b84e8b35b1de55a8744c6452201ed697a5f89dae4f1d\n",
            "  Building wheel for gsutil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gsutil: filename=gsutil-5.27-py3-none-any.whl size=3785291 sha256=259d6dcc78c380656a34bc0506db6dee59414067f83510d4410998cac2afce7f\n",
            "  Stored in directory: /root/.cache/pip/wheels/03/0c/cd/24471307835689188d49827232a1430bdd36587f913d8f707f\n",
            "  Building wheel for crcmod (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for crcmod: filename=crcmod-1.7-cp310-cp310-linux_x86_64.whl size=31405 sha256=588f0136dc12d58ebb2c09f3070bd3f103bfc6e2e224beddf9d97ec95af59079\n",
            "  Stored in directory: /root/.cache/pip/wheels/85/4c/07/72215c529bd59d67e3dac29711d7aba1b692f543c808ba9e86\n",
            "  Building wheel for gcs-oauth2-boto-plugin (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gcs-oauth2-boto-plugin: filename=gcs_oauth2_boto_plugin-3.0-py3-none-any.whl size=23213 sha256=2a9be7eed6d0ac5b4b97e26075f451da14dbc9f9ff09eff9052c599c9f6f4611\n",
            "  Stored in directory: /root/.cache/pip/wheels/77/2a/cd/c7b51b51563ebefe01488e3ecb19df401ea6725771937d3bc6\n",
            "  Building wheel for retry_decorator (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for retry_decorator: filename=retry_decorator-1.1.1-py2.py3-none-any.whl size=3636 sha256=358f33636fbaf24f990c9dad43d7736d1b6f0c04b607ea8fb058e7ce6255e853\n",
            "  Stored in directory: /root/.cache/pip/wheels/dd/ac/77/8c54eac0d373d9eacfbe42599710c9bf91b4c5985297f6922a\n",
            "  Building wheel for pyu2f (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyu2f: filename=pyu2f-0.1.5-py3-none-any.whl size=39404 sha256=ef744eb6fce95461de5ea25f8bec704f906368593ea0a845c344b76feaa8fc3f\n",
            "  Stored in directory: /root/.cache/pip/wheels/ab/7b/27/66d7389916ad9e6caabb23cb55a4cf483dbbe5e6fc2a5cb428\n",
            "Successfully built learn2learn qpth gsutil crcmod gcs-oauth2-boto-plugin retry_decorator pyu2f\n",
            "Installing collected packages: retry_decorator, monotonic, crcmod, boto, rsa, pyu2f, httplib2, fasteners, argcomplete, google-reauth, google-apitools, gcs-oauth2-boto-plugin, qpth, gsutil, learn2learn\n",
            "  Attempting uninstall: rsa\n",
            "    Found existing installation: rsa 4.9\n",
            "    Uninstalling rsa-4.9:\n",
            "      Successfully uninstalled rsa-4.9\n",
            "  Attempting uninstall: httplib2\n",
            "    Found existing installation: httplib2 0.22.0\n",
            "    Uninstalling httplib2-0.22.0:\n",
            "      Successfully uninstalled httplib2-0.22.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires openai, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed argcomplete-3.2.2 boto-2.49.0 crcmod-1.7 fasteners-0.19 gcs-oauth2-boto-plugin-3.0 google-apitools-0.5.32 google-reauth-0.1.1 gsutil-5.27 httplib2-0.20.4 learn2learn-0.2.0 monotonic-1.6 pyu2f-0.1.5 qpth-0.0.16 retry_decorator-1.1.1 rsa-4.7.2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "httplib2"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import pickle\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import learn2learn as l2l\n",
        "import numpy as np\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "9bn-IdIQXVWJ"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "exp_path=\"/content/linear/\"\n",
        "phi1_path=\"/content/vgg16cifar10.npy\"\n",
        "phi2_path=\"/content/dinov2basic.npy\"\n",
        "device = torch.device(\"cuda\")\n",
        "\n",
        "labels_path='/content/cifar10train_targets.npy'\n",
        "\n",
        "classes=10\n",
        "seed=98"
      ],
      "metadata": {
        "id": "Xe-ypQwcXq-R"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phi1 = np.load(phi1_path, allow_pickle=True)\n",
        "val=1\n",
        "top = np.array(phi1.item()[key] for key in phi1.item())\n",
        "top.astype(np.float32)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "DZAYRC0CsPIx",
        "outputId": "e161e4e0-dbb7-425d-d1b4-decb4274bfac"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "float() argument must be a string or a real number, not 'generator'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-a1f0c984ccf5>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphi1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mphi1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mtop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m: float() argument must be a string or a real number, not 'generator'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.path.exists(exp_path):\n",
        "        os.makedirs(exp_path)\n",
        "\n",
        "phi1 = np.load(phi1_path, allow_pickle=True).astype(np.float32)\n",
        "phi2 = np.load(phi2_path, allow_pickle=True).astype(np.float32)\n",
        "\n",
        "phi1_val = np.copy(phi1)\n",
        "phi2_val = np.copy(phi2)\n",
        "\n",
        "ylabels_val=np.load(labels_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "eyUU82FHXhNw",
        "outputId": "ccf45744-3386-4bce-aed5-767d02179eaf"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "float() argument must be a string or a real number, not 'dict'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-24-5c15f48a6f0a>\u001b[0m in \u001b[0;36m<cell line: 4>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m         \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mphi1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphi1_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_pickle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mphi2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mphi2_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_pickle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: float() argument must be a string or a real number, not 'dict'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "assert phi1.shape[0] == phi2.shape[0]\n",
        "assert phi1_val.shape[0] == phi2_val.shape[0]\n",
        "assert phi1_val.shape[0] == y_true_val.shape[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 176
        },
        "id": "6B1soGWlpbak",
        "outputId": "5c6a75d6-9c38-4350-84e0-3db4cb3f424a"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'phi1' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-e5e126284004>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32massert\u001b[0m \u001b[0mphi1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mphi2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mphi1_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mphi2_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mphi1_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0my_true_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'phi1' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n_train = phi1.shape[0]\n",
        "#dimensions of representations\n",
        "d1, d2 = phi2.shape[1], phi1.shape[1]\n",
        "\n",
        "inner_linear = nn.Linear(d1, classes, bias=True).to(device)\n",
        "inner_lr=0.001\n",
        "\n",
        "# MAML Algorithm  - Meta optimization algorithm to perform the optimization on the cross task distribution =\n",
        "inner_linear = l2l.algorithms.MAML(inner_linear, lr=inner_lr)"
      ],
      "metadata": {
        "id": "A7875JsFX1fy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "optimize the cross task distribution - > solve a multi class logistic regression"
      ],
      "metadata": {
        "id": "3hUO6NEybokv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate task encoder with orthogonal weights parametrization (Equation 3)\n",
        "task_encoder = nn.Linear(d2, classes, bias=False).to(device)\n",
        "task_encoder = nn.utils.parametrizations.orthogonal(task_encoder)\n",
        "\n",
        "outer_lr=0.001\n",
        "temperature=0.1\n",
        "\n",
        "optimizer = torch.optim.Adam(task_encoder.parameters(), lr=outer_lr)\n",
        "scheduler = torch.optim.lr_scheduler.MultiStepLR(\n",
        "        optimizer, milestones=[100, 200],\n",
        "        gamma=0.1)"
      ],
      "metadata": {
        "id": "BU8pSCcdbx_P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Implementing the SparseMax activation function"
      ],
      "metadata": {
        "id": "-bJXPZobfEIt"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Sparsemax function operates element-wise on a real-valued vector and outputs a probability distribution that is sparse. Given an input vector\n",
        "z, the Sparsemax function is defined as:\n",
        "\n",
        "\n",
        "\n",
        "> Sparsemax(z)i=max(0,zi−τ(z))\n",
        "\n",
        "\n",
        "\n",
        "where **τ(z)** is a threshold computed based on the input vector\n",
        "\n",
        "\n",
        "The threshold is determined in such a way that the output is a probability distribution with a specified number of non-zero values.\n",
        "\n",
        "\n",
        "\n",
        "The Sparsemax transforms logits into probabilities and when encouraging sparsity, it creates more distinctive representations for each class in the self-supervised learning task."
      ],
      "metadata": {
        "id": "8pJFeVfQfKmk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Author: Mathieu Blondel\n",
        "# License: Simplified BSD\n",
        "\n",
        "\"\"\"\n",
        "PyTorch implementation of\n",
        "\n",
        "Learning Classifiers with Fenchel-Young Losses:\n",
        "    Generalized Entropies, Margins, and Algorithms.\n",
        "Mathieu Blondel, André F. T. Martins, Vlad Niculae.\n",
        "https://arxiv.org/abs/1805.09717\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "import torch\n",
        "\n",
        "# begin: From OpenNMT-py\n",
        "def threshold_and_support(z, dim=0):\n",
        "    \"\"\"\n",
        "    z: any dimension\n",
        "    dim: dimension along which to apply the sparsemax\n",
        "    \"\"\"\n",
        "    sorted_z, _ = torch.sort(z, descending=True, dim=dim)\n",
        "    z_sum = sorted_z.cumsum(dim) - 1  # sort of a misnomer\n",
        "    k = torch.arange(1, sorted_z.size(dim) + 1, device=z.device).type(z.dtype).view(\n",
        "        torch.Size([-1] + [1] * (z.dim() - 1))\n",
        "    ).transpose(0, dim)\n",
        "    support = k * sorted_z > z_sum\n",
        "\n",
        "    k_z_indices = support.sum(dim=dim).unsqueeze(dim)\n",
        "    k_z = k_z_indices.type(z.dtype)\n",
        "    tau_z = z_sum.gather(dim, k_z_indices - 1) / k_z\n",
        "    return tau_z, k_z\n",
        "\n",
        "\n",
        "class SparsemaxFunction(torch.autograd.Function):\n",
        "\n",
        "    @staticmethod\n",
        "    def forward(ctx, input, dim=0):\n",
        "        \"\"\"\n",
        "        input (FloatTensor): any shape\n",
        "        returns (FloatTensor): same shape with sparsemax computed on given dim\n",
        "        \"\"\"\n",
        "        ctx.dim = dim\n",
        "        tau_z, k_z = threshold_and_support(input, dim=dim)\n",
        "        output = torch.clamp(input - tau_z, min=0)\n",
        "        ctx.save_for_backward(k_z, output)\n",
        "        return output\n",
        "\n",
        "    @staticmethod\n",
        "    def backward(ctx, grad_output):\n",
        "        k_z, output = ctx.saved_tensors\n",
        "        dim = ctx.dim\n",
        "        grad_input = grad_output.clone()\n",
        "        grad_input[output == 0] = 0\n",
        "\n",
        "        v_hat = (grad_input.sum(dim=dim) / k_z.squeeze()).unsqueeze(dim)\n",
        "        grad_input = torch.where(output != 0, grad_input - v_hat, grad_input)\n",
        "        return grad_input, None\n",
        "\n",
        "\n",
        "sparsemax = SparsemaxFunction.apply\n",
        "\n",
        "\n",
        "class Sparsemax(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, dim=0):\n",
        "        self.dim = dim\n",
        "        super(Sparsemax, self).__init__()\n",
        "\n",
        "    def forward(self, input):\n",
        "        return sparsemax(input, self.dim)\n",
        "# end: From OpenNMT-py"
      ],
      "metadata": {
        "id": "Hcq5VbNFct1i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Cross validation function"
      ],
      "metadata": {
        "id": "LiNI431ClfhT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import KFold, cross_val_score\n",
        "\n",
        "def cv_score(X, y):\n",
        "    cv = KFold(n_splits=10, random_state=1, shuffle=True)\n",
        "    clf = LogisticRegression(penalty=None)\n",
        "    scores = cross_val_score(clf, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
        "    return np.mean(scores)"
      ],
      "metadata": {
        "id": "21WbPNwjllID"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Main training loop"
      ],
      "metadata": {
        "id": "2asQqxKYgeTY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        " sparsemax= Sparsemax(dim=1)"
      ],
      "metadata": {
        "id": "TbcZYSMdgakH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "iterations=1000\n",
        "\n",
        "linear_steps=300\n",
        "#Number of inner iterations to fit linear model"
      ],
      "metadata": {
        "id": "CLq_MrDVgvV6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "number_subsets=20\n",
        "#Number of (Xtr, Xte) subsets for averaging HUME's loss\n",
        "size_subet=10000\n",
        "#Size of union of each (Xtr, Xte) subset"
      ],
      "metadata": {
        "id": "6iQR-2wlg4t1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in tqdm(range(iterations)):\n",
        "        optimizer.zero_grad()\n",
        "        mean_train_error = 0.0\n",
        "        mean_valid_error = 0.0\n",
        "        mean_valid_acc = 0.0\n",
        "        mean_train_acc = 0.0\n",
        "        mean_label_dist = 0.0\n",
        "        mean_sparsity = 0.0\n",
        "\n",
        "        for j in range(number_subsets):\n",
        "            # Sample X_tr and X_te\n",
        "            subset = np.random.choice(n_train, size=size_subset, replace=False)\n",
        "            subset_tr = subset[:int(subset_size * 0.9)]\n",
        "            subset_te = subset[int(subset_size * 0.9):]\n",
        "\n",
        "            phi1_tr = torch.from_numpy(phi1[subset_tr]).to(device)\n",
        "            phi1_te = torch.from_numpy(phi1[subset_te]).to(device)\n",
        "            phi2_tr = torch.from_numpy(phi2[subset_tr]).to(device)\n",
        "            phi2_te = torch.from_numpy(phi2[subset_te]).to(device)\n",
        "\n",
        "            # Get labels using current task encoder\n",
        "            task_labels_tr = sparsemax_act(task_encoder(phi1_tr) / temperature)\n",
        "            task_labels_te = sparsemax_act(task_encoder(phi1_te) / temperature)\n",
        "            task_labels_all = torch.cat((task_labels_tr, task_labels_te))\n",
        "\n",
        "            \"\"\"\n",
        "            Perform inner optimization from the random initialization or\n",
        "            from fixed w0 (corresponds to Cold Start BLO for Equation 5)\n",
        "            \"\"\"\n",
        "\n",
        "            learner = inner_linear.clone()\n",
        "\n",
        "            for step in range(linear_steps):\n",
        "                train_error = F.cross_entropy(learner(phi2_tr), task_labels_tr)\n",
        "                learner.adapt(train_error)\n",
        "\n",
        "            # Compute HUME's objective (Equation 7)\n",
        "            label_dist = task_labels_all.mean(0)\n",
        "            entr = torch.special.entr(label_dist)\n",
        "            valid_error = F.cross_entropy(learner(phi2_te), task_labels_te)\n",
        "\n",
        "            # Accumulate gradients across args.num_subsets\n",
        "            (valid_error - float(10)* entr.sum()).backward()\n",
        "\n",
        "            # Compute training stats\n",
        "            mean_train_error += train_error.item()\n",
        "            mean_train_acc += torch.eq(learner(phi2_tr).argmax(1),task_labels_tr.argmax(1)).float().mean().item()\n",
        "            mean_valid_error += valid_error.item()\n",
        "            mean_valid_acc += torch.eq(learner(phi2_te).argmax(1),task_labels_te.argmax(1)).float().mean().item()\n",
        "            mean_label_dist += label_dist.detach().cpu().numpy()\n",
        "            mean_sparsity += task_labels_all[torch.arange(task_labels_all.shape[0]),task_labels_all.argmax(1)].mean().item()\n",
        "\n",
        "        # Average gradients over subsets and update the task encoder parameters\n",
        "        for p in task_encoder.parameters():\n",
        "            p.grad.data.mul_(1.0 / number_subsets)\n",
        "            print(f\"Grad norm: {torch.norm(p.grad.data).item()}\")\n",
        "        nn.utils.clip_grad_norm_(task_encoder.parameters(), 1.0)\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "\n",
        "        # Anneal step size and temperature\n",
        "        if scheduler.get_last_lr()[0] != outer_lr:\n",
        "            print(\"Annealed Learning rate\")\n",
        "            outer_lr = scheduler.get_last_lr()[0]\n",
        "            print(\"Annealed Temperature\")\n",
        "            temperature = temperature / 10\n",
        "            print()\n",
        "\n",
        "        # Print train stats\n",
        "        print(\"Train stats:\")\n",
        "        print(f\"Mean TrainError {mean_train_error / number_subsets}\")\n",
        "        print(f\"Mean ValidError {mean_valid_error /number_subsets}\")\n",
        "        print(f\"Mean TrainAcc {mean_train_acc / number_subsets}\")\n",
        "        print(f\"Mean ValidAcc {mean_valid_acc / number_subsets}\")\n",
        "        print(f\"Mean Sparsity {mean_sparsity / number_subsets}\")\n",
        "        print(\"Mean Label Dist:\", mean_label_dist / number_subsets)\n",
        "        print()\n",
        "\n",
        "\n",
        "        # Compute cross-validation accuracy w.r.t. found task and save the results\n",
        "        out_all_val = task_encoder(torch.from_numpy(phi1_val).to(device))\n",
        "        task_val = torch.argmax(out_all_val, dim=1).detach().cpu().numpy()\n",
        "        crossvalscore= cv_score(phi2_val, task_val)\n",
        "        with open(exp_path + f\"results_{seed}.pickle\", \"wb\") as handle:\n",
        "            pickle.dump({\"CV_Score\": crossvalscore}, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "        torch.save(task_encoder.state_dict(), exp_path + f\"linear_task_{seed}.pt\")"
      ],
      "metadata": {
        "id": "2G8i1dUigrRn"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}