shallow_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from icl.util_classes.arg_classes import ShallowArgs, DeepArgs, CompressArgs\n",
    "from sklearn.metrics import accuracy_score\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "change ShallowArgs to ShallowNonLabelArgs if you want to get the results of that"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_label(y):\n",
    "    return y.predictions[0].argmax(-1)\n",
    "\n",
    "def get_logits(y):\n",
    "    if y.predictions[2].shape[-1] > 30000:\n",
    "        return y.predictions[2]\n",
    "    else:\n",
    "        return y.predictions[3]\n",
    "\n",
    "def get_topk(y, k):\n",
    "    logits = get_logits(y)\n",
    "    indices = np.argpartition(logits, -k,axis=1)[:,-k:]\n",
    "    return indices\n",
    "\n",
    "def jaccard(a,b):\n",
    "    scores = []\n",
    "    for single_a, single_b in zip(a,b):\n",
    "        set_a = set(single_a)\n",
    "        set_b = set(single_b)\n",
    "        score = len(set_a.intersection(set_b))/len(set_a.union(set_b))\n",
    "        scores.append(score)\n",
    "    return np.array(scores).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from icl.utils.load_huggingface_dataset import load_huggingface_dataset_train_and_test\n",
    "import warnings\n",
    "\n",
    "def calculate_average_scores(seeds, task_name,sample_size=1000,model_name='gpt2-xl',mask_layer_num=5,demonstration_shot=1):\n",
    "    scores = []  \n",
    "    jaccards = []\n",
    "    dataset = load_huggingface_dataset_train_and_test(task_name)\n",
    "    if len(dataset['test']) < sample_size:\n",
    "        warnings.warn(\n",
    "            f\"sample_size: {sample_size} is larger than test set size: {len(dataset['test'])},\"\n",
    "            f\"actual_sample_size is {len(dataset['test'])}\")\n",
    "        actual_sample_size = len(dataset['test'])\n",
    "    else:\n",
    "        actual_sample_size = sample_size\n",
    "    \n",
    "    for seed in tqdm(seeds):\n",
    "        args = ShallowArgs(task_name=task_name, seeds=[seed],sample_size=sample_size,model_name=model_name,\n",
    "            mask_layer_pos='first',mask_layer_num=mask_layer_num,demonstration_shot=demonstration_shot)\n",
    "        y_first, = args.load_result()[0]\n",
    "        args = ShallowArgs(task_name=task_name, seeds=[seed],sample_size=sample_size,model_name=model_name,\n",
    "            mask_layer_pos='last',mask_layer_num=mask_layer_num,demonstration_shot=demonstration_shot)\n",
    "        y_last, = args.load_result()[0]\n",
    "        try:\n",
    "            args = CompressArgs(task_name=task_name, seeds=[seed],sample_size=sample_size,\n",
    "            model_name=model_name,demonstration_shot=demonstration_shot)\n",
    "            _, y_true, _, _ = args.load_result()[0]\n",
    "        except KeyboardInterrupt:\n",
    "            break\n",
    "        except:\n",
    "            args = DeepArgs(task_name=task_name, seeds=[seed],sample_size=sample_size,\n",
    "            model_name=model_name,demonstration_shot=demonstration_shot)\n",
    "            y_true, = args.load_result()[0]\n",
    "            \n",
    "        label_first, label_last, label_true = [get_label(_) for _ in [y_first,y_last,y_true]]\n",
    "\n",
    "        score_first = accuracy_score(label_true, label_first)\n",
    "        score_last = accuracy_score(label_true, label_last)\n",
    "        score_true = accuracy_score(label_true, label_true)\n",
    "        \n",
    "        scores.append((score_first, score_last, score_true))\n",
    "\n",
    "        jaccard_first = jaccard(get_topk(y_true, 10), get_topk(y_first, 10))\n",
    "        jaccard_last = jaccard(get_topk(y_true, 10), get_topk(y_last, 10))\n",
    "        jaccard_true = jaccard(get_topk(y_true, 10), get_topk(y_true, 10))\n",
    "        jaccards.append((jaccard_first, jaccard_last, jaccard_true))\n",
    "\n",
    "\n",
    "    average_scores = np.mean(scores, axis=0) \n",
    "    accuracy_jaccards = np.mean(jaccards, axis=0)\n",
    "    return average_scores, accuracy_jaccards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model_name = 'gpt2-xl'\n",
    "layers = [1,3,5,7]\n",
    "demonstration_shot = 1\n",
    "\n",
    "tables = []\n",
    "for layer in layers:\n",
    "    seeds = [42,43,44,45 ,46]\n",
    "    tasks = ['sst2', 'agnews', 'trec', 'emo']\n",
    "    single_table = []\n",
    "\n",
    "    for task in tqdm(tasks):\n",
    "        average_scores = calculate_average_scores(seeds, task,sample_size=1000,model_name=model_name,mask_layer_num=layer,demonstration_shot=demonstration_shot)\n",
    "        row = [task] + list(average_scores)\n",
    "        single_table.append(row)\n",
    "\n",
    "    header = ['Task', 'label loyalty','word loyalty']\n",
    "    tables.append(single_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data = np.array(np.array([[_[1:] for _ in table] for table in tables])) # omit task name"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.lines import Line2D\n",
    "\n",
    "mean_data = data.mean(1)\n",
    "layer_num, metric_num, _ = mean_data.shape\n",
    "colors = ['b', 'r', 'g']\n",
    "linestyles = ['-', '--']\n",
    "Metric_name = ['Label Loyalty', 'Word Loyalty']\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "for i in range(metric_num):\n",
    "    for j in range(2):\n",
    "        ax.plot(range(layer_num), mean_data[:, i, j], color=colors[i], linestyle=linestyles[j])\n",
    "\n",
    "handles = []\n",
    "labels = []\n",
    "for i in range(metric_num):\n",
    "    handles.append(plt.Line2D([], [], color=colors[i], linestyle='-'))\n",
    "    labels.append(f'%s (First)' % Metric_name[i])\n",
    "    handles.append(plt.Line2D([], [], color=colors[i], linestyle='--'))\n",
    "    labels.append(f'%s (Last)' % Metric_name[i])\n",
    "\n",
    "ax.legend(handles, labels)\n",
    "\n",
    "ax.set_xlabel('Isolation Layer Num')\n",
    "ax.set_ylabel('Loyalty')\n",
    "\n",
    "xticks = list(map(str,layers))\n",
    "ax.set_xticks(range(layer_num))\n",
    "ax.set_xticklabels(xticks)\n",
    "\n",
    "plt.show()\n",
    "fig.savefig(f'aggregation_loyalty_{model_name}_{demonstration_shot}.pdf', dpi=300, bbox_inches='tight')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.16 ('bbtv2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "fa84ff37f7354d5baacf3f95c54ec9bb9436f05eafb6bc27ab368dac8f7f3b02"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}