duplicates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a81f5ab7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import torch\n",
    "import torchvision.transforms as transforms\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "from torchvision.datasets import CelebA, CIFAR10, CIFAR100, MNIST\n",
    "from IPython.display import display\n",
    "from PIL import Image\n",
    "from main import DatasetLoader\n",
    "import util\n",
    "\n",
    "# Parameters\n",
    "params = [#'runtime.dataset_dir=./', # Where to load the datasets\n",
    "         ]\n",
    "args = util.get_config(params)\n",
    "\n",
    "\n",
    "DATASETS = (CIFAR100, CelebA,)\n",
    "label_names = {}\n",
    "label_names[CIFAR100] = ['apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock', 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur', 'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster', 'house', 'kangaroo', 'keyboard', 'lamp', 'lawn_mower', 'leopard', 'lion', 'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain', 'mouse', 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear', 'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy', 'porcupine', 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket', 'rose', 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake', 'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table', 'tank', 'telephone', 'television', 'tiger', 'tractor', 'train', 'trout', 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 'worm']\n",
    "\n",
    "def lname(dataset, l):\n",
    "    if dataset in label_names:\n",
    "        return label_names[dataset][l]\n",
    "    else:\n",
    "        return str(l)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "041c6bc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Duplicate Detection\n",
    "#\n",
    "# You may skip this cell. Results can be found in duplicates.json and are loaded subsequently.\n",
    "\n",
    "duplicates = defaultdict(dict)\n",
    "#indices = np.random.choice(116412, size=5, replace=False)\n",
    "\n",
    "\n",
    "for dataset in DATASETS:\n",
    "    test_in_train = defaultdict(list)\n",
    "    print(f\"Doing {dataset.__name__} ...\")\n",
    "    \n",
    "    if dataset in (CelebA,):\n",
    "        ds_train = dataset(args.runtime.dataset_dir, split='train', target_type='identity')\n",
    "        ds_test = dataset(args.runtime.dataset_dir, split='test', target_type='identity')\n",
    "    else:\n",
    "        ds_train = dataset(args.runtime.dataset_dir, train=True)\n",
    "        ds_test = dataset(args.runtime.dataset_dir, train=False)\n",
    "        \n",
    "\n",
    "    ids_train = defaultdict(list)\n",
    "    print(f\"  training:\")\n",
    "    for i, sample in enumerate(tqdm(ds_train)):\n",
    "        x = np.array(sample[0]).flatten()\n",
    "        x.flags.writeable = False\n",
    "        ids_train[hash(x.data.tobytes())].append(i)\n",
    "        \n",
    "    ids_test = defaultdict(list)\n",
    "    dups_test_in_train = dict()\n",
    "    print(f\"  testing:\")\n",
    "    for i, sample in enumerate(tqdm(ds_test)):\n",
    "        x = np.array(sample[0]).flatten()\n",
    "        x.flags.writeable = False\n",
    "        ids_test[hash(x.data.tobytes())].append(i)\n",
    "        if hash(x.data.tobytes()) in ids_train:\n",
    "            dups_test_in_train[i] = ids_train[hash(x.data.tobytes())]\n",
    "        \n",
    "    \n",
    "    for ids, part in ((ids_train, 'training'), (ids_test, 'testing')):\n",
    "        dups = []\n",
    "        for v in ids.values():\n",
    "            if len(v) > 1:\n",
    "                dups.append(v)\n",
    "        print(f\"Found {len(dups)} {part} duplicates\")\n",
    "        duplicates[dataset.__name__][f\"{part}_duplicates\"] = dups\n",
    "    \n",
    "    print(f\"Found {len(dups_test_in_train)} duplicates in training and testing\")    \n",
    "    duplicates[dataset.__name__][\"test_in_train\"] = dups_test_in_train\n",
    "\n",
    "    with open('duplicates.json', 'w') as fp:\n",
    "        json.dump(duplicates, fp, indent=1)\n",
    "\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04c98839",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print LaTeX Table with duplicates and save PNG files to ./imgs\n",
    "\n",
    "\n",
    "# Detection\n",
    "with open('duplicates.json', 'r') as fp:\n",
    "    duplicates = json.load(fp)\n",
    "    \n",
    "include_cmd = (r\"\\includegraphics[align=c,width=.8cm]{\", r\"}\\vspace{1px}\")\n",
    "    \n",
    "for dataset in DATASETS:\n",
    "    print(f\"% {dataset.__name__}:\")\n",
    "    print(r\"\"\"\\begin{tabular}{cll}\n",
    "  \\toprule\n",
    "  Image & Training (ID:Label) & Testing (ID:Label) \\\\\n",
    "  \\midrule\"\"\")\n",
    "    train_dups = duplicates[dataset.__name__]['training_duplicates']\n",
    "    test_dups = duplicates[dataset.__name__]['testing_duplicates']\n",
    "    test_in_train = duplicates[dataset.__name__]['test_in_train']\n",
    "    if dataset in (CelebA,):\n",
    "        ds_train = dataset(args.runtime.dataset_dir, split='train', target_type='identity')\n",
    "        ds_test = dataset(args.runtime.dataset_dir, split='test', target_type='identity')\n",
    "    else:\n",
    "        ds_train = dataset(args.runtime.dataset_dir, train=True)\n",
    "        ds_test = dataset(args.runtime.dataset_dir, train=False)\n",
    "    \n",
    "    for dups, ds, part in zip ( (train_dups, test_dups), (ds_train, ds_test), ('TRAIN', 'TEST')):\n",
    "        print(f\"  % {part}\")\n",
    "        i_dups = 0\n",
    "        i_dups_wrong_label = 0\n",
    "        for v in dups:\n",
    "            x = np.array([np.array(ds[i][0]) for i in v])\n",
    "            y = np.array([ds[i][1] for i in v])\n",
    "            D = np.abs(x - x[0:1,:]).sum()\n",
    "            Dy = y - y[0]\n",
    "            \n",
    "            assert (D == 0).all()\n",
    "            i_dups += 1\n",
    "            if Dy.sum() != 0:\n",
    "                i_dups_wrong_label += 1\n",
    "            \n",
    "            names = [lname(dataset, i).replace(\"_\", \"\\\\_\") for i in y]\n",
    "\n",
    "            img_name = f'./imgs/{dataset.__name__!s}_{part}_{v[0]}.png'\n",
    "            img_txt = \", \".join([f\"{i}:\\\\texttt{{{name}}}\" for name, i in zip(names, v)])\n",
    "            ds[v[0]][0].save(img_name)\n",
    "            if part == \"TRAIN\":\n",
    "                print(f'  {include_cmd[0]}{img_name}{include_cmd[1]} & {img_txt} & -- \\\\\\\\')\n",
    "            if part == \"TEST\":\n",
    "                print(f'  {include_cmd[0]}{img_name}{include_cmd[1]} & -- & {img_txt} \\\\\\\\')\n",
    "        \n",
    "        print(f\"  % {part}: {i_dups} duplicates with {i_dups_wrong_label} label mismatch\")\n",
    "                                                  \n",
    "    print(f\"  % TEST and TRAIN\")\n",
    "    i_dups = 0\n",
    "    i_dups_wrong_label = 0\n",
    "    for k, v in test_in_train.items():\n",
    "        k = int(k)\n",
    "        for v0 in test_dups:\n",
    "            assert k not in v0\n",
    "        y0 = int(ds_test[k][1])\n",
    "        y1 = np.array([ds_train[i][1] for i in v])\n",
    "        Dy = y1 - y0\n",
    "        i_dups += 1\n",
    "        if Dy.sum() != 0:\n",
    "            i_dups_wrong_label += 1\n",
    "        name0 = lname(dataset,y0).replace(\"_\", \"\\\\_\")\n",
    "        names1 = [lname(dataset,i).replace(\"_\", \"\\\\_\") for i in y1]\n",
    "        img_name = f'./imgs/{dataset.__name__!s}_TEST_{k}.png'\n",
    "        img_txt1 = \", \".join([f\"{i}:\\\\texttt{{{name}}}\" for name, i in zip(names1, v)])\n",
    "        img_txt0 = f\"{k}:\\\\texttt{{{name0}}}\"\n",
    "        ds_test[k][0].save(img_name)\n",
    "        print(f'  {include_cmd[0]}{img_name}{include_cmd[1]} & {img_txt1} & {img_txt0} \\\\\\\\')\n",
    "    print(f\"  % TRAIN in TEST: {i_dups} duplicates with {i_dups_wrong_label} label mismatch\")\n",
    "    print(r\"\\bottomrule\")    \n",
    "    print(r\"\\end{tabular}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c02be16e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display Duplicates\n",
    "for dataset in DATASETS:\n",
    "    print(f\"\\n\\n{dataset.__name__}:\")\n",
    "    if dataset in (CelebA,):\n",
    "        ds_train = dataset(args.runtime.dataset_dir, split='train', target_type='identity')\n",
    "        ds_test = dataset(args.runtime.dataset_dir, split='test', target_type='identity')\n",
    "    else:\n",
    "        ds_train = dataset(args.runtime.dataset_dir, train=True)\n",
    "        ds_test = dataset(args.runtime.dataset_dir, train=False)\n",
    "    \n",
    "    for ds, part in ((ds_train, 'training'), (ds_test, 'testing')):\n",
    "\n",
    "        dups=duplicates[dataset.__name__][f\"{part}_duplicates\"]\n",
    "\n",
    "        print(f\"\\n  {part}\")\n",
    "\n",
    "        i_dups = 0\n",
    "        i_dups_wrong_label = 0\n",
    "        for v in dups:\n",
    "            x = np.array([np.array(ds[i][0]) for i in v])\n",
    "            y = np.array([ds[i][1] for i in v])\n",
    "            D = np.abs(x - x[0:1,:]).sum()\n",
    "            Dy = y - y[0]\n",
    "\n",
    "            assert (D == 0).all()\n",
    " \n",
    "            i_dups += 1\n",
    "            if Dy.sum() != 0:\n",
    "                i_dups_wrong_label += 1\n",
    "\n",
    "            names = [lname(dataset,i) for i in y]\n",
    "\n",
    "            img_txt = \", \".join([f\"{i}:{name}\" for name, i in zip(names, v)])\n",
    "            img1 = ds[v[0]][0]\n",
    "            img2 = ds[v[1]][0]\n",
    "            display(img1)\n",
    "            print(f'    {img_txt}')\n",
    "            \n",
    "        print(f\"{part} had {i_dups} duplicates with {i_dups_wrong_label} label mismatches\")\n",
    "        \n",
    "    i_dups = 0\n",
    "    i_dups_wrong_label = 0\n",
    "    for i, v in duplicates[dataset.__name__][\"test_in_train\"].items():\n",
    "        i = int(i)\n",
    "        img1 = ds_test[i][0]\n",
    "        y0 = int(ds_test[i][1])\n",
    "        y1 = np.array([ds_train[j][1] for j in v])\n",
    "        Dy = y1 - y0\n",
    "        i_dups += 1\n",
    "        if Dy.sum() != 0:\n",
    "            i_dups_wrong_label += 1\n",
    "        name0 = lname(dataset,y0)\n",
    "        names1 = [lname(dataset,i) for i in y1]\n",
    "        display(img1)\n",
    "        img_txt = f\"testing:: {i}:{name0} -- training:: \" + \",\".join([f\"{j}:{name}\" for name, j in zip(names1, v)])\n",
    "        print(f'    {img_txt}')\n",
    "    print(f\"{i_dups} duplicates in training and testing with {i_dups_wrong_label} label mismatches\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf78906",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8db45950",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9",
   "language": "python",
   "name": "python39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}