Day #20 DeepFake Detection Kaggle Challenge (1_2).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/skyprince999/100-Days-Of-ML/blob/master/Day_20_DeepFake_Detection_Kaggle_Challenge_(1_2).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Mpg7qdKPrn-K"
   },
   "source": [
    "This is one of the two notebooks that were submitted for the challenge:\n",
    "\n",
    "\n",
    "Name of notebook: ensembling-learning-learning-rates.ipynb\n",
    "\n",
    "Public Score: 0.43236\n",
    "Private Score: 0.56004\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1PJAXYUxrmR8"
   },
   "source": [
    "## Resnext Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1jWllKS0rmR_"
   },
   "outputs": [],
   "source": [
    "import os, sys, time\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oymbTB8ermSN",
    "outputId": "24291f15-a8b7-450e-aae3-99338e599c3d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n",
    "\n",
    "test_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\n",
    "frame_h = 5\n",
    "frame_l = 5\n",
    "len(test_videos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aLdiadssrmSc",
    "outputId": "a4a90bb1-cd9d-41dc-d151-4fe9d236a9bc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version: 1.3.0\n",
      "CUDA version: 10.0.130\n",
      "cuDNN version: 7603\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version:\", torch.__version__)\n",
    "print(\"CUDA version:\", torch.version.cuda)\n",
    "print(\"cuDNN version:\", torch.backends.cudnn.version())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ns3QtsK-rmSn",
    "outputId": "d5cbfdad-07c1-4ac4-b890-39f817b3ae6a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jMHZ4bwlrmSx"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\n",
    "sys.path.insert(0, \"/kaggle/input/deepfakes-inference-demo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vFc60lDarmS6"
   },
   "outputs": [],
   "source": [
    "from blazeface import BlazeFace\n",
    "facedet = BlazeFace().to(gpu)\n",
    "facedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\n",
    "facedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n",
    "_ = facedet.train(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D-sis1FhrmTC"
   },
   "outputs": [],
   "source": [
    "from helpers.read_video_1 import VideoReader\n",
    "from helpers.face_extract_1 import FaceExtractor\n",
    "\n",
    "frames_per_video = 64 #frame_h * frame_l\n",
    "video_reader = VideoReader()\n",
    "video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)\n",
    "face_extractor = FaceExtractor(video_read_fn, facedet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p4rlWlzGrmTP"
   },
   "outputs": [],
   "source": [
    "input_size = 224"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z5DwPBVormTY"
   },
   "outputs": [],
   "source": [
    "from torchvision.transforms import Normalize\n",
    "\n",
    "mean = [0.485, 0.456, 0.406]\n",
    "std = [0.229, 0.224, 0.225]\n",
    "normalize_transform = Normalize(mean, std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MJBYTmL8rmTg"
   },
   "outputs": [],
   "source": [
    "def isotropically_resize_image(img, size, resample=cv2.INTER_AREA):\n",
    "    h, w = img.shape[:2]\n",
    "    if w > h:\n",
    "        h = h * size // w\n",
    "        w = size\n",
    "    else:\n",
    "        w = w * size // h\n",
    "        h = size\n",
    "\n",
    "    resized = cv2.resize(img, (w, h), interpolation=resample)\n",
    "    return resized\n",
    "\n",
    "\n",
    "def make_square_image(img):\n",
    "    h, w = img.shape[:2]\n",
    "    size = max(h, w)\n",
    "    t = 0\n",
    "    b = size - h\n",
    "    l = 0\n",
    "    r = size - w\n",
    "    return cv2.copyMakeBorder(img, t, b, l, r, cv2.BORDER_CONSTANT, value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CS6dtL0srmTo"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torchvision.models as models\n",
    "\n",
    "class MyResNeXt(models.resnet.ResNet):\n",
    "    def __init__(self, training=True):\n",
    "        super(MyResNeXt, self).__init__(block=models.resnet.Bottleneck,\n",
    "                                        layers=[3, 4, 6, 3], \n",
    "                                        groups=32, \n",
    "                                        width_per_group=4)\n",
    "        self.fc = nn.Linear(2048, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2dQ--UEurmTy"
   },
   "outputs": [],
   "source": [
    "checkpoint = torch.load(\"/kaggle/input/deepfakes-inference-demo/resnext.pth\", map_location=gpu)\n",
    "\n",
    "model = MyResNeXt().to(gpu)\n",
    "model.load_state_dict(checkpoint)\n",
    "_ = model.eval()\n",
    "\n",
    "del checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WyYP0_VOrmT7"
   },
   "outputs": [],
   "source": [
    "def predict_on_video(video_path, batch_size):\n",
    "    try:\n",
    "        # Find the faces for N frames in the video.\n",
    "        faces = face_extractor.process_video(video_path)\n",
    "\n",
    "        # Only look at one face per frame.\n",
    "        face_extractor.keep_only_best_face(faces)\n",
    "        \n",
    "        if len(faces) > 0:\n",
    "            # NOTE: When running on the CPU, the batch size must be fixed\n",
    "            # or else memory usage will blow up. (Bug in PyTorch?)\n",
    "            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n",
    "\n",
    "            # If we found any faces, prepare them for the model.\n",
    "            n = 0\n",
    "            for frame_data in faces:\n",
    "                for face in frame_data[\"faces\"]:\n",
    "                    # Resize to the model's required input size.\n",
    "                    # We keep the aspect ratio intact and add zero\n",
    "                    # padding if necessary.                    \n",
    "                    resized_face = isotropically_resize_image(face, input_size)\n",
    "                    resized_face = make_square_image(resized_face)\n",
    "\n",
    "                    if n < batch_size:\n",
    "                        x[n] = resized_face\n",
    "                        n += 1\n",
    "                    else:\n",
    "                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n",
    "                    \n",
    "                    # Test time augmentation: horizontal flips.\n",
    "                    # TODO: not sure yet if this helps or not\n",
    "                    #x[n] = cv2.flip(resized_face, 1)\n",
    "                    #n += 1\n",
    "\n",
    "            if n > 0:\n",
    "                x = torch.tensor(x, device=gpu).float()\n",
    "\n",
    "                # Preprocess the images.\n",
    "                x = x.permute((0, 3, 1, 2))\n",
    "\n",
    "                for i in range(len(x)):\n",
    "                    x[i] = normalize_transform(x[i] / 255.)\n",
    "\n",
    "                # Make a prediction, then take the average.\n",
    "                with torch.no_grad():\n",
    "                    y_pred = model(x)\n",
    "                    y_pred = torch.sigmoid(y_pred.squeeze())\n",
    "                    return y_pred[:n].mean().item()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n",
    "\n",
    "    return 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JqroBl2ZrmUD"
   },
   "outputs": [],
   "source": [
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "def predict_on_video_set(videos, num_workers):\n",
    "    def process_file(i):\n",
    "        filename = videos[i]\n",
    "        y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)\n",
    "        return y_pred\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n",
    "        predictions = ex.map(process_file, range(len(videos)))\n",
    "\n",
    "    return list(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U6DVwelTrmUL"
   },
   "outputs": [],
   "source": [
    "speed_test = True  # you have to enable this manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RAtrEXyermUT",
    "outputId": "89476932-43ff-48d4-ddaf-c46c504aea11"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed 24.536189 sec. Average per video: 4.907238 sec.\n"
     ]
    }
   ],
   "source": [
    "if speed_test:\n",
    "    start_time = time.time()\n",
    "    speedtest_videos = test_videos[:5]\n",
    "    predictions = predict_on_video_set(speedtest_videos, num_workers=4)\n",
    "    elapsed = time.time() - start_time\n",
    "    print(\"Elapsed %f sec. Average per video: %f sec.\" % (elapsed, elapsed / len(speedtest_videos)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JtimNGyjrmUb"
   },
   "outputs": [],
   "source": [
    "predictions = predict_on_video_set(test_videos, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Gf4NsClQrmUm"
   },
   "outputs": [],
   "source": [
    "submission_df_resnext = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\n",
    "submission_df_resnext.to_csv(\"submission_resnext.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pD8McQuprmUu"
   },
   "source": [
    "## Xception Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G4MKcyyGrmUw"
   },
   "outputs": [],
   "source": [
    "!pip install ../input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Hz9wb4qkrmU3",
    "outputId": "92cb1e73-0cc1-4273-f075-6da0035cb3ad"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n",
    "\n",
    "test_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\n",
    "len(test_videos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XeuU4uxxrmVC",
    "outputId": "39d1e50b-b9f4-4c5a-946b-8606ad165dda"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6oth-8vdrmVL"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/kaggle/input/blazeface-pytorch\")\n",
    "sys.path.insert(0, \"/kaggle/input/deepfakes-inference-demo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DJKEGJTlrmVS"
   },
   "outputs": [],
   "source": [
    "from blazeface import BlazeFace\n",
    "facedet = BlazeFace().to(gpu)\n",
    "facedet.load_weights(\"/kaggle/input/blazeface-pytorch/blazeface.pth\")\n",
    "facedet.load_anchors(\"/kaggle/input/blazeface-pytorch/anchors.npy\")\n",
    "_ = facedet.train(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tUE0ek97rmVa"
   },
   "outputs": [],
   "source": [
    "from helpers.read_video_1 import VideoReader\n",
    "from helpers.face_extract_1 import FaceExtractor\n",
    "\n",
    "frames_per_video = 64 # originally 4\n",
    "\n",
    "video_reader = VideoReader()\n",
    "video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)\n",
    "face_extractor = FaceExtractor(video_read_fn, facedet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "62gSaVOfrmVi"
   },
   "outputs": [],
   "source": [
    "input_size = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o1grk9G8rmVt"
   },
   "outputs": [],
   "source": [
    "from torchvision.transforms import Normalize\n",
    "\n",
    "mean = [0.485, 0.456, 0.406]\n",
    "std = [0.229, 0.224, 0.225]\n",
    "normalize_transform = Normalize(mean, std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lS7o6twbrmV1"
   },
   "outputs": [],
   "source": [
    "def isotropically_resize_image(img, size, resample=cv2.INTER_AREA):\n",
    "    h, w = img.shape[:2]\n",
    "    if w > h:\n",
    "        h = h * size // w\n",
    "        w = size\n",
    "    else:\n",
    "        w = w * size // h\n",
    "        h = size\n",
    "\n",
    "    resized = cv2.resize(img, (w, h), interpolation=resample)\n",
    "    return resized\n",
    "\n",
    "\n",
    "def make_square_image(img):\n",
    "    h, w = img.shape[:2]\n",
    "    size = max(h, w)\n",
    "    t = 0\n",
    "    b = size - h\n",
    "    l = 0\n",
    "    r = size - w\n",
    "    return cv2.copyMakeBorder(img, t, b, l, r, cv2.BORDER_CONSTANT, value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MOAep5DmrmV8",
    "outputId": "f18213b4-7bd4-4458-a652-d8903e5b94fe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.pth     model_v2.1.pth  pytorchcv-0.0.55-py2.py3-none-any.whl\r\n",
      "model_v0.pth  model_v2.pth\r\n",
      "model_v1.pth  model_v3.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/deepfake-xception-trained-model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TNpItvzVrmWE",
    "outputId": "2ba3ccc2-dd99-4988-f07a-03b170cfbc6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "google_cloud_compute_engine_launch_vm.png\r\n",
      "google_cloud_vm.png\r\n",
      "google_cloud_vm_deepfake_training_screenshot.png\r\n",
      "lr_15e-2_epochs_42_patience_5.png\r\n",
      "lr_2e-3_epochs_10_patience_5.png\r\n",
      "lr_2e-3_epochs_20_patience_5.png\r\n",
      "lr_4e-3_epochs_12_patience_2.png\r\n",
      "lr_4e-3_epochs_30_patience_2.png\r\n",
      "model_50epochs_lr0001_patience5_factor01_batchsize32.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/deepfake-kernel-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AfgM10sarmWN"
   },
   "outputs": [],
   "source": [
    "from pytorchcv.model_provider import get_model\n",
    "model = get_model(\"xception\", pretrained=False)\n",
    "model = nn.Sequential(*list(model.children())[:-1]) # Remove original output layer\n",
    "\n",
    "class Pooling(nn.Module):\n",
    "  def __init__(self):\n",
    "    super(Pooling, self).__init__()\n",
    "    \n",
    "    self.p1 = nn.AdaptiveAvgPool2d((1,1))\n",
    "    self.p2 = nn.AdaptiveMaxPool2d((1,1))\n",
    "\n",
    "  def forward(self, x):\n",
    "    x1 = self.p1(x)\n",
    "    x2 = self.p2(x)\n",
    "    return (x1+x2) * 0.5\n",
    "\n",
    "model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d((1,1)))\n",
    "\n",
    "class Head(torch.nn.Module):\n",
    "  def __init__(self, in_f, out_f):\n",
    "    super(Head, self).__init__()\n",
    "    \n",
    "    self.f = nn.Flatten()\n",
    "    self.l = nn.Linear(in_f, 512)\n",
    "    self.d = nn.Dropout(0.5)\n",
    "    self.o = nn.Linear(512, out_f)\n",
    "    self.b1 = nn.BatchNorm1d(in_f)\n",
    "    self.b2 = nn.BatchNorm1d(512)\n",
    "    self.r = nn.ReLU()\n",
    "\n",
    "  def forward(self, x):\n",
    "    x = self.f(x)\n",
    "    x = self.b1(x)\n",
    "    x = self.d(x)\n",
    "\n",
    "    x = self.l(x)\n",
    "    x = self.r(x)\n",
    "    x = self.b2(x)\n",
    "    x = self.d(x)\n",
    "\n",
    "    out = self.o(x)\n",
    "    return out\n",
    "\n",
    "class FCN(torch.nn.Module):\n",
    "  def __init__(self, base, in_f):\n",
    "    super(FCN, self).__init__()\n",
    "    self.base = base\n",
    "    self.h1 = Head(in_f, 1)\n",
    "  \n",
    "  def forward(self, x):\n",
    "    x = self.base(x)\n",
    "    return self.h1(x)\n",
    "\n",
    "net = []\n",
    "model = FCN(model, 2048)\n",
    "model = model.cuda()\n",
    "model.load_state_dict(torch.load('../input/deepfake-kernel-data/model_50epochs_lr0001_patience5_factor01_batchsize32.pth')) # new, updated\n",
    "net.append(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9-Dbhc5ormWU"
   },
   "source": [
    "### Prediction Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BAGkli9ormWW"
   },
   "outputs": [],
   "source": [
    "def predict_on_video(video_path, batch_size):\n",
    "    try:\n",
    "        # Find the faces for N frames in the video.\n",
    "        faces = face_extractor.process_video(video_path)\n",
    "\n",
    "        # Only look at one face per frame.\n",
    "        face_extractor.keep_only_best_face(faces)\n",
    "        \n",
    "        if len(faces) > 0:\n",
    "            # NOTE: When running on the CPU, the batch size must be fixed\n",
    "            # or else memory usage will blow up. (Bug in PyTorch?)\n",
    "            x = np.zeros((batch_size, input_size, input_size, 3), dtype=np.uint8)\n",
    "\n",
    "            # If we found any faces, prepare them for the model.\n",
    "            n = 0\n",
    "            for frame_data in faces:\n",
    "                for face in frame_data[\"faces\"]:\n",
    "                    # Resize to the model's required input size.\n",
    "                    # We keep the aspect ratio intact and add zero\n",
    "                    # padding if necessary.                    \n",
    "                    resized_face = isotropically_resize_image(face, input_size)\n",
    "                    resized_face = make_square_image(resized_face)\n",
    "\n",
    "                    if n < batch_size:\n",
    "                        x[n] = resized_face\n",
    "                        n += 1\n",
    "                    else:\n",
    "                        print(\"WARNING: have %d faces but batch size is %d\" % (n, batch_size))\n",
    "                    \n",
    "                    # Test time augmentation: horizontal flips.\n",
    "                    # TODO: not sure yet if this helps or not\n",
    "                    #x[n] = cv2.flip(resized_face, 1)\n",
    "                    #n += 1\n",
    "\n",
    "            if n > 0:\n",
    "                x = torch.tensor(x, device=gpu).float()\n",
    "\n",
    "                # Preprocess the images.\n",
    "                x = x.permute((0, 3, 1, 2))\n",
    "\n",
    "                for i in range(len(x)):\n",
    "                    x[i] = normalize_transform(x[i] / 255.)\n",
    "#                     x[i] = x[i] / 255.\n",
    "\n",
    "                # Make a prediction, then take the average.\n",
    "                with torch.no_grad():\n",
    "                    y_pred = model(x)\n",
    "                    y_pred = torch.sigmoid(y_pred.squeeze())\n",
    "                    return y_pred[:n].mean().item()\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Prediction error on video %s: %s\" % (video_path, str(e)))\n",
    "\n",
    "    return 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1lGZgCeBrmWd"
   },
   "outputs": [],
   "source": [
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "def predict_on_video_set(videos, num_workers):\n",
    "    def process_file(i):\n",
    "        filename = videos[i]\n",
    "        y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)\n",
    "        return y_pred\n",
    "\n",
    "    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n",
    "        predictions = ex.map(process_file, range(len(videos)))\n",
    "\n",
    "    return list(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T2MdQ16qrmWn"
   },
   "outputs": [],
   "source": [
    "speed_test = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fbAnStwdrmWt",
    "outputId": "6fd47f14-bf2b-4f43-85ab-1be403df3a04"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed 21.324115 sec. Average per video: 4.264823 sec.\n"
     ]
    }
   ],
   "source": [
    "if speed_test:\n",
    "    start_time = time.time()\n",
    "    speedtest_videos = test_videos[:5]\n",
    "    predictions = predict_on_video_set(speedtest_videos, num_workers=4)\n",
    "    elapsed = time.time() - start_time\n",
    "    print(\"Elapsed %f sec. Average per video: %f sec.\" % (elapsed, elapsed / len(speedtest_videos)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T7T64Dv5rmW0",
    "outputId": "f43f1d2f-4e8b-45b8-f471-66a4a06786fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 47min 58s, sys: 1min 45s, total: 49min 44s\n",
      "Wall time: 26min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model.eval()\n",
    "predictions = predict_on_video_set(test_videos, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uAE0a9I6rmW8"
   },
   "outputs": [],
   "source": [
    "submission_df_xception = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\n",
    "submission_df_xception.to_csv(\"submission_xception.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uemx31kQrmXD",
    "outputId": "804233f6-fa74-481e-80b9-b6e869248b4a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aassnaulhq.mp4</td>\n",
       "      <td>0.514175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aayfryxljh.mp4</td>\n",
       "      <td>0.004443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>acazlolrpz.mp4</td>\n",
       "      <td>0.725788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adohdulfwb.mp4</td>\n",
       "      <td>0.060911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ahjnxtiamx.mp4</td>\n",
       "      <td>0.880215</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         filename     label\n",
       "0  aassnaulhq.mp4  0.514175\n",
       "1  aayfryxljh.mp4  0.004443\n",
       "2  acazlolrpz.mp4  0.725788\n",
       "3  adohdulfwb.mp4  0.060911\n",
       "4  ahjnxtiamx.mp4  0.880215"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_df_resnext.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LNJ4e7S9rmXK",
    "outputId": "227fdd77-3102-4885-eea5-0761c9a342af"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aassnaulhq.mp4</td>\n",
       "      <td>0.977654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aayfryxljh.mp4</td>\n",
       "      <td>0.016241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>acazlolrpz.mp4</td>\n",
       "      <td>0.800488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>adohdulfwb.mp4</td>\n",
       "      <td>0.006942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ahjnxtiamx.mp4</td>\n",
       "      <td>0.825627</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         filename     label\n",
       "0  aassnaulhq.mp4  0.977654\n",
       "1  aayfryxljh.mp4  0.016241\n",
       "2  acazlolrpz.mp4  0.800488\n",
       "3  adohdulfwb.mp4  0.006942\n",
       "4  ahjnxtiamx.mp4  0.825627"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_df_xception.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iW0nUgcOrmXQ"
   },
   "source": [
    "## Ensemble of Resnext & Xception\n",
    "\n",
    "- Resnext single model public score: 0.46441\n",
    "- Xception single model public score: 0.50480"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HFhJkv1-rmXU"
   },
   "outputs": [],
   "source": [
    "submission_df = pd.DataFrame({\"filename\": test_videos})\n",
    "submission_df[\"label\"] = 0.5*submission_df_resnext[\"label\"] + 0.5*submission_df_xception[\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nPdoWj_trmXb"
   },
   "outputs": [],
   "source": [
    "submission_df.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0qPXnNhtrmXh"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "name": "DeepFake Detection Kaggle Challenge (1/2).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}