EmbeddingsNN_PyTorch_ResNet_TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011369,
     "end_time": "2021-02-19T02:48:51.945181",
     "exception": false,
     "start_time": "2021-02-19T02:48:51.933812",
     "status": "completed"
    },
    "tags": []
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01003,
     "end_time": "2021-02-19T02:48:51.965799",
     "exception": false,
     "start_time": "2021-02-19T02:48:51.955769",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Pytorch Embedding NN code:\n",
    "https://www.kaggle.com/sapthrishi007/jane-fastai-embedding-smoothnn5-300x5/notebook?scriptVersionId=53603141&select=Jane_EmbNN5_auc_400_400_400.pth\n",
    "\n",
    "Apart from the 130 featres, it also creates feature embeddings from their tags in features.csv file.\n",
    "\n",
    "## PyTorch train code:\n",
    "https://www.kaggle.com/a763337092/neural-network-starter-pytorch-version\n",
    "https://www.kaggle.com/a763337092/pytorch-resnet-starter-training\n",
    "\n",
    "## TensorFlow training code:\n",
    "https://www.kaggle.com/code1110/jane-street-with-keras-nn-overfit\n",
    "\n",
    "## Blending PyTorch and TensorFlow code:\n",
    "https://www.kaggle.com/a763337092/blending-tensorflow-and-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:48:51.993518Z",
     "iopub.status.busy": "2021-02-19T02:48:51.992626Z",
     "iopub.status.idle": "2021-02-19T02:48:52.016598Z",
     "shell.execute_reply": "2021-02-19T02:48:52.015794Z"
    },
    "papermill": {
     "duration": 0.040769,
     "end_time": "2021-02-19T02:48:52.016851",
     "exception": false,
     "start_time": "2021-02-19T02:48:51.976082",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/jane-street-market-prediction/example_sample_submission.csv\n",
      "/kaggle/input/jane-street-market-prediction/features.csv\n",
      "/kaggle/input/jane-street-market-prediction/example_test.csv\n",
      "/kaggle/input/jane-street-market-prediction/train.csv\n",
      "/kaggle/input/jane-street-market-prediction/janestreet/competition.cpython-37m-x86_64-linux-gnu.so\n",
      "/kaggle/input/jane-street-market-prediction/janestreet/__init__.py\n",
      "/kaggle/input/embedd9203/Jane_EmbNN5_auc_400_400_400.pth\n",
      "/kaggle/input/embedd9203/submission.csv\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/__results__.html\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/submission.csv\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/__resultx__.html\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/__notebook__.ipynb\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/model.h5\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/__output__.json\n",
      "/kaggle/input/jane-street-with-keras-nn-overfit/custom.css\n",
      "/kaggle/input/mlp012003weights/online_model4.pth\n",
      "/kaggle/input/mlp012003weights/online_model2.pth\n",
      "/kaggle/input/mlp012003weights/online_model1.pth\n",
      "/kaggle/input/mlp012003weights/f_mean_online.npy\n",
      "/kaggle/input/mlp012003weights/online_model3.pth\n",
      "/kaggle/input/mlp012003weights/online_model0.pth\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.010313,
     "end_time": "2021-02-19T02:48:52.038548",
     "exception": false,
     "start_time": "2021-02-19T02:48:52.028235",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## PyTorch ResNet part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-02-19T02:48:52.085128Z",
     "iopub.status.busy": "2021-02-19T02:48:52.079385Z",
     "iopub.status.idle": "2021-02-19T02:48:54.475963Z",
     "shell.execute_reply": "2021-02-19T02:48:54.474973Z"
    },
    "papermill": {
     "duration": 2.426885,
     "end_time": "2021-02-19T02:48:54.476118",
     "exception": false,
     "start_time": "2021-02-19T02:48:52.049233",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from collections import namedtuple\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings (\"ignore\")\n",
    "\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "DATA_PATH = '../input/jane-street-market-prediction/'\n",
    "\n",
    "NFOLDS = 5\n",
    "\n",
    "TRAIN = False\n",
    "CACHE_PATH = '../input/mlp012003weights'\n",
    "\n",
    "def save_pickle(dic, save_path):\n",
    "    with open(save_path, 'wb') as f:\n",
    "    # with gzip.open(save_path, 'wb') as f:\n",
    "        pickle.dump(dic, f)\n",
    "\n",
    "def load_pickle(load_path):\n",
    "    with open(load_path, 'rb') as f:\n",
    "    # with gzip.open(load_path, 'rb') as f:\n",
    "        message_dict = pickle.load(f)\n",
    "    return message_dict\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "##### Making features\n",
    "all_feat_cols = [col for col in feat_cols]\n",
    "all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])\n",
    "\n",
    "##### Model&Data fnc\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))\n",
    "        self.dropout0 = nn.Dropout(0.2)\n",
    "\n",
    "        dropout_rate = 0.2\n",
    "        hidden_size = 256\n",
    "        self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout1 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout2 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout3 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm4 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout4 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))\n",
    "\n",
    "        self.Relu = nn.ReLU(inplace=True)\n",
    "        self.PReLU = nn.PReLU()\n",
    "        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n",
    "        # self.GeLU = nn.GELU()\n",
    "        self.RReLU = nn.RReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.batch_norm0(x)\n",
    "        x = self.dropout0(x)\n",
    "\n",
    "        x1 = self.dense1(x)\n",
    "        x1 = self.batch_norm1(x1)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x1 = self.LeakyReLU(x1)\n",
    "        x1 = self.dropout1(x1)\n",
    "\n",
    "        x = torch.cat([x, x1], 1)\n",
    "\n",
    "        x2 = self.dense2(x)\n",
    "        x2 = self.batch_norm2(x2)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x2 = self.LeakyReLU(x2)\n",
    "        x2 = self.dropout2(x2)\n",
    "\n",
    "        x = torch.cat([x1, x2], 1)\n",
    "\n",
    "        x3 = self.dense3(x)\n",
    "        x3 = self.batch_norm3(x3)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x3 = self.LeakyReLU(x3)\n",
    "        x3 = self.dropout3(x3)\n",
    "\n",
    "        x = torch.cat([x2, x3], 1)\n",
    "\n",
    "        x4 = self.dense4(x)\n",
    "        x4 = self.batch_norm4(x4)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x4 = self.LeakyReLU(x4)\n",
    "        x4 = self.dropout4(x4)\n",
    "\n",
    "        x = torch.cat([x3, x4], 1)\n",
    "\n",
    "        x = self.dense5(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "if True:\n",
    "    # device = torch.device(\"cuda:0\")\n",
    "    device = torch.device(\"cpu\")\n",
    "    \n",
    "    model_list = []\n",
    "    tmp = np.zeros(len(feat_cols))\n",
    "    for _fold in range(NFOLDS):\n",
    "        torch.cuda.empty_cache()\n",
    "        model = Model()\n",
    "        model.to(device)\n",
    "        model_weights = f\"{CACHE_PATH}/online_model{_fold}.pth\"\n",
    "        model.load_state_dict(torch.load(model_weights, map_location=torch.device('cpu')))\n",
    "        model.eval()\n",
    "        model_list.append(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.010488,
     "end_time": "2021-02-19T02:48:54.497704",
     "exception": false,
     "start_time": "2021-02-19T02:48:54.487216",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## TensorFlow part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:48:54.524656Z",
     "iopub.status.busy": "2021-02-19T02:48:54.524024Z",
     "iopub.status.idle": "2021-02-19T02:48:55.263620Z",
     "shell.execute_reply": "2021-02-19T02:48:55.262899Z"
    },
    "papermill": {
     "duration": 0.755286,
     "end_time": "2021-02-19T02:48:55.263789",
     "exception": false,
     "start_time": "2021-02-19T02:48:54.508503",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__notebook__.ipynb  __results__.html  custom.css  submission.csv\r\n",
      "__output__.json     __resultx__.html  model.h5\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/jane-street-with-keras-nn-overfit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:48:55.304533Z",
     "iopub.status.busy": "2021-02-19T02:48:55.303717Z",
     "iopub.status.idle": "2021-02-19T02:49:01.080039Z",
     "shell.execute_reply": "2021-02-19T02:49:01.079441Z"
    },
    "papermill": {
     "duration": 5.804847,
     "end_time": "2021-02-19T02:49:01.080186",
     "exception": false,
     "start_time": "2021-02-19T02:48:55.275339",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.losses import BinaryCrossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices\n",
    "\n",
    "\n",
    "SEED = 1111\n",
    "\n",
    "np.random.seed(SEED)\n",
    "\n",
    "# fit\n",
    "def create_mlp(\n",
    "    num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate\n",
    "):\n",
    "\n",
    "    inp = tf.keras.layers.Input(shape=(num_columns,))\n",
    "    x = tf.keras.layers.BatchNormalization()(inp)\n",
    "    x = tf.keras.layers.Dropout(dropout_rates[0])(x)\n",
    "    for i in range(len(hidden_units)):\n",
    "        x = tf.keras.layers.Dense(hidden_units[i])(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.Activation(tf.keras.activations.swish)(x)\n",
    "        x = tf.keras.layers.Dropout(dropout_rates[i + 1])(x)\n",
    "    \n",
    "    x = tf.keras.layers.Dense(num_labels)(x)\n",
    "    out = tf.keras.layers.Activation(\"sigmoid\")(x)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=inp, outputs=out)\n",
    "    model.compile(\n",
    "        optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),\n",
    "        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),\n",
    "        metrics=tf.keras.metrics.AUC(name=\"AUC\"),\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "epochs = 200\n",
    "batch_size = 4096\n",
    "hidden_units = [160, 160, 160]\n",
    "dropout_rates = [0.2, 0.2, 0.2, 0.2]\n",
    "label_smoothing = 1e-2\n",
    "learning_rate = 1e-3\n",
    "\n",
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(SEED)\n",
    "clf = create_mlp(\n",
    "    len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate\n",
    "    )\n",
    "clf.load_weights('../input/jane-street-with-keras-nn-overfit/model.h5')\n",
    "\n",
    "tf_models = [clf]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.01134,
     "end_time": "2021-02-19T02:49:01.103607",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.092267",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# EmbeddingsNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:49:01.149967Z",
     "iopub.status.busy": "2021-02-19T02:49:01.144791Z",
     "iopub.status.idle": "2021-02-19T02:49:01.157756Z",
     "shell.execute_reply": "2021-02-19T02:49:01.157148Z"
    },
    "papermill": {
     "duration": 0.042764,
     "end_time": "2021-02-19T02:49:01.157904",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.115140",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "N_FEAT_TAGS = 29    # No of tags in features.csv\n",
    "DEVICE = device\n",
    "N_FEATURES = 130\n",
    "THREE_HIDDEN_LAYERS = [400, 400, 400]\n",
    "\n",
    "class FFN (nn.Module):\n",
    "    \n",
    "    def __init__(self, inputCount=130, outputCount=5, hiddenLayerCounts=[150, 150, 150], \n",
    "                 drop_prob=0.2, nonlin=nn.SiLU (), isOpAct=False):\n",
    "        \n",
    "        super(FFN, self).__init__()\n",
    "        \n",
    "        self.nonlin     = nonlin\n",
    "        self.dropout    = nn.Dropout (drop_prob)\n",
    "        self.batchnorm0 = nn.BatchNorm1d (inputCount)\n",
    "        self.dense1     = nn.Linear (inputCount, hiddenLayerCounts[0])\n",
    "        self.batchnorm1 = nn.BatchNorm1d (hiddenLayerCounts[0])\n",
    "        self.dense2     = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])\n",
    "        self.batchnorm2 = nn.BatchNorm1d (hiddenLayerCounts[1])\n",
    "        self.dense3     = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])\n",
    "        self.batchnorm3 = nn.BatchNorm1d (hiddenLayerCounts[2])        \n",
    "        self.outDense   = None\n",
    "        if outputCount > 0:\n",
    "            self.outDense   = nn.Linear (hiddenLayerCounts[-1], outputCount)\n",
    "        self.outActivtn = None\n",
    "        if isOpAct:\n",
    "            if outputCount == 1 or outputCount == 2:\n",
    "                self.outActivtn = nn.Sigmoid ()\n",
    "            elif outputCount > 0:\n",
    "                self.outActivtn = nn.Softmax (dim=-1)\n",
    "        return\n",
    "\n",
    "    def forward (self, X):\n",
    "        \n",
    "        # X = self.dropout (self.batchnorm0 (X))\n",
    "        X = self.batchnorm0 (X)\n",
    "        X = self.dropout (self.nonlin (self.batchnorm1 (self.dense1 (X))))\n",
    "        X = self.dropout (self.nonlin (self.batchnorm2 (self.dense2 (X))))\n",
    "        X = self.dropout (self.nonlin (self.batchnorm3 (self.dense3 (X))))\n",
    "        if self.outDense:\n",
    "            X = self.outDense (X)\n",
    "        if self.outActivtn:\n",
    "            X = self.outActivtn (X)\n",
    "        return X\n",
    "    \n",
    "    \n",
    "class Emb_NN_Model (nn.Module):\n",
    "    \n",
    "    def __init__(self, three_hidden_layers=THREE_HIDDEN_LAYERS, embed_dim=(N_FEAT_TAGS), csv_file='../input/jane-street-market-prediction/features.csv'):\n",
    "        \n",
    "        super (Emb_NN_Model, self).__init__()\n",
    "        global N_FEAT_TAGS\n",
    "        N_FEAT_TAGS = 29\n",
    "        \n",
    "        # store the features to tags mapping as a datframe tdf, feature_i mapping is in tdf[i, :]\n",
    "        dtype = {'tag_0' : 'int8'}\n",
    "        for i in range (1, 29):\n",
    "            k = 'tag_' + str (i)\n",
    "            dtype[k] = 'int8'\n",
    "        t_df = pd.read_csv (csv_file, usecols=range (1,N_FEAT_TAGS+1), dtype=dtype)\n",
    "        t_df['tag_29'] = np.array ([1] + ([0] * (t_df.shape[0]-1)) ).astype ('int8')\n",
    "        self.features_tag_matrix = torch.tensor (t_df.to_numpy ())\n",
    "        N_FEAT_TAGS += 1\n",
    "        \n",
    "        # print ('self.features_tag_matrix =', self.features_tag_matrix)\n",
    "        \n",
    "        # embeddings for the tags. Each feature is taken a an embedding which is an avg. of its' tag embeddings\n",
    "        self.embed_dim     = embed_dim\n",
    "        self.tag_embedding = nn.Embedding (N_FEAT_TAGS+1, embed_dim) # create a special tag if not known tag for any feature\n",
    "        self.tag_weights   = nn.Linear (N_FEAT_TAGS, 1)\n",
    "        \n",
    "        drop_prob          = 0.5\n",
    "        self.ffn           = FFN (inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(three_hidden_layers[0]+embed_dim), (three_hidden_layers[1]+embed_dim), (three_hidden_layers[2]+embed_dim)], drop_prob=drop_prob)\n",
    "        self.outDense      = nn.Linear (three_hidden_layers[2]+embed_dim, 5)\n",
    "        return\n",
    "    \n",
    "    def features2emb (self):\n",
    "        \"\"\"\n",
    "        idx : int feature index 0 to N_FEATURES-1 (129)\n",
    "        \"\"\"\n",
    "        \n",
    "        all_tag_idxs = torch.LongTensor (np.arange (N_FEAT_TAGS)) #.to (DEVICE)              # (29,)\n",
    "        tag_bools    = self.features_tag_matrix                                # (130, 29)\n",
    "        # print ('tag_bools.shape =', tag_bools.size())\n",
    "        f_emb        = self.tag_embedding (all_tag_idxs).repeat (130, 1, 1)    #;print ('1. f_emb =', f_emb) # (29, 7) * (130, 1, 1) = (130, 29, 7)\n",
    "        # print ('f_emb.shape =', f_emb.size())\n",
    "        f_emb        = f_emb * tag_bools[:, :, None]                           #;print ('2. f_emb =', f_emb) # (130, 29, 7) * (130, 29, 1) = (130, 29, 7)\n",
    "        # print ('f_emb.shape =', f_emb.size())\n",
    "        \n",
    "        # Take avg. of all the present tag's embeddings to get the embedding for a feature\n",
    "        s = torch.sum (tag_bools, dim=1)                                       # (130,)\n",
    "        # print ('s =', s)              \n",
    "        f_emb = torch.sum (f_emb, dim=-2) / s[:, None]                         # (130, 7)\n",
    "        # print ('f_emb =', f_emb)        \n",
    "        # print ('f_emb.shape =', f_emb.shape)\n",
    "        \n",
    "        # take a linear combination of the present tag's embeddings\n",
    "        # f_emb = f_emb.permute (0, 2, 1)                                        # (130, 7, 29)\n",
    "        # f_emb = self.tag_weights (f_emb)                      #;print ('3. f_emb =', f_emb)                 # (130, 7, 1)\n",
    "        # f_emb = torch.squeeze (f_emb, dim=-1)                 #;print ('4. f_emb =', f_emb)                 # (130, 7)\n",
    "        return f_emb\n",
    "    \n",
    "    def forward (self, cat_featrs, features):\n",
    "        \"\"\"\n",
    "        when you call `model (x ,y, z, ...)` then this method is invoked\n",
    "        \"\"\"\n",
    "        \n",
    "        cat_featrs = None\n",
    "        features   = features.view (-1, N_FEATURES)\n",
    "        f_emb      = self.features2emb ()                                #;print ('5. f_emb =', f_emb); print ('6. features =', features) # (130, 7)\n",
    "        # print ('features.shape =', features.shape, 'f_emb.shape =', f_emb.shape)\n",
    "        features_2 = torch.matmul (features, f_emb)                      #;print ('7. features =', features) # (1, 130) * (130, 7) = (1, 7)\n",
    "        # print ('features.shape =', features.shape)\n",
    "        \n",
    "        # Concatenate the two features (features + their embeddings)\n",
    "        features   = torch.hstack ((features, features_2))        \n",
    "        \n",
    "        x          = self.ffn (features)                               #;print ('8. x.shape = ', x.shape, 'x =', x)   # (1, 7) -> (1, 7)\n",
    "        # x        = self.layer_normal (x + features)                  #;print ('9. x.shape = ', x.shape, 'x =', x)   # (1, 7) -> (1, 2)\n",
    "        out_logits = self.outDense (x)                                 #;print ('10. out_logits.shape = ', out_logits.shape, 'out_logits =', out_logits)        \n",
    "        # return sigmoid probs\n",
    "        # out_probs = F.sigmoid (out_logits)\n",
    "        return out_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:49:01.188644Z",
     "iopub.status.busy": "2021-02-19T02:49:01.187874Z",
     "iopub.status.idle": "2021-02-19T02:49:01.313152Z",
     "shell.execute_reply": "2021-02-19T02:49:01.312429Z"
    },
    "papermill": {
     "duration": 0.143567,
     "end_time": "2021-02-19T02:49:01.313298",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.169731",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "embNN_model = Emb_NN_Model ()\n",
    "\n",
    "try:\n",
    "    embNN_model.load_state_dict (torch.load (\"../input/embedd9203/Jane_EmbNN5_auc_400_400_400.pth\"))\n",
    "except:\n",
    "    embNN_model.load_state_dict (torch.load (\"../input/embedd9203/Jane_EmbNN5_auc_400_400_400.pth\", map_location='cpu'))\n",
    "    \n",
    "embNN_model = embNN_model.eval ()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011313,
     "end_time": "2021-02-19T02:49:01.336547",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.325234",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:49:01.364943Z",
     "iopub.status.busy": "2021-02-19T02:49:01.364265Z",
     "iopub.status.idle": "2021-02-19T02:49:01.387796Z",
     "shell.execute_reply": "2021-02-19T02:49:01.387204Z"
    },
    "papermill": {
     "duration": 0.039582,
     "end_time": "2021-02-19T02:49:01.387939",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.348357",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import janestreet\n",
    "env = janestreet.make_env()\n",
    "env_iter = env.iter_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:49:01.427507Z",
     "iopub.status.busy": "2021-02-19T02:49:01.426727Z",
     "iopub.status.idle": "2021-02-19T02:54:22.460185Z",
     "shell.execute_reply": "2021-02-19T02:54:22.457986Z"
    },
    "papermill": {
     "duration": 321.059141,
     "end_time": "2021-02-19T02:54:22.460343",
     "exception": false,
     "start_time": "2021-02-19T02:49:01.401202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15219it [05:21, 47.41it/s]\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "\n",
    "    for (test_df, pred_df) in tqdm(env_iter):\n",
    "        if test_df['weight'].item() > 0:\n",
    "            x_tt = test_df.loc[:, feat_cols].values\n",
    "            if np.isnan(x_tt.sum()):\n",
    "                x_tt = np.nan_to_num(x_tt) + np.isnan(x_tt) * f_mean\n",
    "\n",
    "            cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]\n",
    "            cross_1_2 = x_tt[:, 1] / (x_tt[:, 2] + 1e-5)\n",
    "            feature_inp = np.concatenate((\n",
    "                x_tt,\n",
    "                np.array(cross_41_42_43).reshape(x_tt.shape[0], 1),\n",
    "                np.array(cross_1_2).reshape(x_tt.shape[0], 1),\n",
    "            ), axis=1)\n",
    "\n",
    "            # torch_pred\n",
    "            torch_pred = np.zeros((1, len(target_cols)))\n",
    "            for model in model_list:\n",
    "                torch_pred += model(torch.tensor(feature_inp, dtype=torch.float).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS\n",
    "            torch_pred = np.median(torch_pred)\n",
    "            \n",
    "            # tf_pred\n",
    "            tf_pred = np.median(np.mean([model(x_tt, training = False).numpy() for model in tf_models],axis=0))\n",
    "            \n",
    "            # torch embedding_NN pred\n",
    "            x_tt    = torch.tensor (x_tt).float ().view (-1, 130)\n",
    "            embnn_p = np.median (torch.sigmoid (embNN_model (None, x_tt)).detach ().cpu ().numpy ().reshape ((-1, 5)), axis=1)   # not logits, actually sigmoid probabilities\n",
    "            \n",
    "            # avg\n",
    "            pred_pr = torch_pred*0.41 + tf_pred*0.41 + embnn_p*0.18\n",
    "            \n",
    "            pred_df.action = np.where (pred_pr >= 0.4918, 1, 0).astype (int)\n",
    "        else:\n",
    "            pred_df.action = 0\n",
    "        env.predict(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:54:24.199683Z",
     "iopub.status.busy": "2021-02-19T02:54:24.198796Z",
     "iopub.status.idle": "2021-02-19T02:54:24.202010Z",
     "shell.execute_reply": "2021-02-19T02:54:24.202510Z"
    },
    "papermill": {
     "duration": 0.898499,
     "end_time": "2021-02-19T02:54:24.202696",
     "exception": false,
     "start_time": "2021-02-19T02:54:23.304197",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done !\n"
     ]
    }
   ],
   "source": [
    "print (\"Done !\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.872358,
     "end_time": "2021-02-19T02:54:25.922124",
     "exception": false,
     "start_time": "2021-02-19T02:54:25.049766",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 341.707624,
   "end_time": "2021-02-19T02:54:28.081132",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-19T02:48:46.373508",
   "version": "2.2.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}