Blending_TensorFlow_and_PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-02-19T02:47:33.947524Z",
     "iopub.status.busy": "2021-02-19T02:47:33.868769Z",
     "iopub.status.idle": "2021-02-19T02:47:40.583714Z",
     "shell.execute_reply": "2021-02-19T02:47:40.583168Z"
    },
    "papermill": {
     "duration": 6.749315,
     "end_time": "2021-02-19T02:47:40.583841",
     "exception": false,
     "start_time": "2021-02-19T02:47:33.834526",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import log_loss, roc_auc_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F\n",
    "\n",
    "pd.set_option('display.max_columns', 100)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "DATA_PATH = '../input/jane-street-market-prediction/'\n",
    "\n",
    "NFOLDS = 5\n",
    "\n",
    "TRAIN = False\n",
    "CACHE_PATH = '../input/mlp012003weights'\n",
    "\n",
    "def save_pickle(dic, save_path):\n",
    "    with open(save_path, 'wb') as f:\n",
    "    # with gzip.open(save_path, 'wb') as f:\n",
    "        pickle.dump(dic, f)\n",
    "\n",
    "def load_pickle(load_path):\n",
    "    with open(load_path, 'rb') as f:\n",
    "    # with gzip.open(load_path, 'rb') as f:\n",
    "        message_dict = pickle.load(f)\n",
    "    return message_dict\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "##### Making features\n",
    "all_feat_cols = [col for col in feat_cols]\n",
    "all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])\n",
    "\n",
    "class EarlyStopping:\n",
    "    def __init__(self, patience=7, mode=\"max\", delta=0.001):\n",
    "        self.patience = patience\n",
    "        self.counter = 0\n",
    "        self.mode = mode\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        self.delta = delta\n",
    "        if self.mode == \"min\":\n",
    "            self.val_score = np.Inf\n",
    "        else:\n",
    "            self.val_score = -np.Inf\n",
    "\n",
    "    def __call__(self, epoch_score, model, model_path):\n",
    "\n",
    "        if self.mode == \"min\":\n",
    "            score = -1.0 * epoch_score\n",
    "        else:\n",
    "            score = np.copy(epoch_score)\n",
    "\n",
    "        if self.best_score is None:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(epoch_score, model, model_path)\n",
    "        elif score < self.best_score: #  + self.delta\n",
    "            self.counter += 1\n",
    "            # print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "        else:\n",
    "            self.best_score = score\n",
    "            # ema.apply_shadow()\n",
    "            self.save_checkpoint(epoch_score, model, model_path)\n",
    "            # ema.restore()\n",
    "            self.counter = 0\n",
    "\n",
    "    def save_checkpoint(self, epoch_score, model, model_path):\n",
    "        if epoch_score not in [-np.inf, np.inf, -np.nan, np.nan]:\n",
    "            # print('Validation score improved ({} --> {}). Saving model!'.format(self.val_score, epoch_score))\n",
    "            # if not DEBUG:\n",
    "            torch.save(model.state_dict(), model_path)\n",
    "        self.val_score = epoch_score\n",
    "\n",
    "def seed_everything(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything(seed=42)\n",
    "\n",
    "feat_cols = [f'feature_{i}' for i in range(130)]\n",
    "\n",
    "if TRAIN:\n",
    "    train = train.loc[train.date > 85].reset_index(drop=True)\n",
    "\n",
    "    train['action'] = (train['resp'] > 0).astype('int')\n",
    "    train['action_1'] = (train['resp_1'] > 0).astype('int')\n",
    "    train['action_2'] = (train['resp_2'] > 0).astype('int')\n",
    "    train['action_3'] = (train['resp_3'] > 0).astype('int')\n",
    "    train['action_4'] = (train['resp_4'] > 0).astype('int')\n",
    "    valid = train.loc[(train.date >= 450) & (train.date < 500)].reset_index(drop=True)\n",
    "    train = train.loc[train.date < 450].reset_index(drop=True)\n",
    "target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']\n",
    "\n",
    "if TRAIN:\n",
    "    df = pd.concat([train[feat_cols], valid[feat_cols]]).reset_index(drop=True)\n",
    "    f_mean = df.mean()\n",
    "    f_mean = f_mean.values\n",
    "    np.save(f'{CACHE_PATH}/f_mean_online.npy', f_mean)\n",
    "\n",
    "    train.fillna(df.mean(), inplace=True)\n",
    "    valid.fillna(df.mean(), inplace=True)\n",
    "else:\n",
    "    f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')\n",
    "\n",
    "##### Making features\n",
    "# https://www.kaggle.com/lucasmorin/running-algos-fe-for-fast-inference/data\n",
    "# eda:https://www.kaggle.com/carlmcbrideellis/jane-street-eda-of-day-0-and-feature-importance\n",
    "# his example:https://www.kaggle.com/gracewan/plot-model\n",
    "\n",
    "def fillna_npwhere_njit(array, values):\n",
    "    if np.isnan(array.sum()):\n",
    "        array = np.where(np.isnan(array), values, array)\n",
    "    return array\n",
    "\n",
    "class RunningEWMean:\n",
    "    def __init__(self, WIN_SIZE=20, n_size=1, lt_mean=None):\n",
    "        if lt_mean is not None:\n",
    "            self.s = lt_mean\n",
    "        else:\n",
    "            self.s = np.zeros(n_size)\n",
    "        self.past_value = np.zeros(n_size)\n",
    "        self.alpha = 2 / (WIN_SIZE + 1)\n",
    "\n",
    "    def clear(self):\n",
    "        self.s = 0\n",
    "\n",
    "    def push(self, x):\n",
    "\n",
    "        x = fillna_npwhere_njit(x, self.past_value)\n",
    "        self.past_value = x\n",
    "        self.s = self.alpha * x + (1 - self.alpha) * self.s\n",
    "\n",
    "    def get_mean(self):\n",
    "        return self.s\n",
    "\n",
    "if TRAIN:\n",
    "    all_feat_cols = [col for col in feat_cols]\n",
    "\n",
    "    train['cross_41_42_43'] = train['feature_41'] + train['feature_42'] + train['feature_43']\n",
    "    train['cross_1_2'] = train['feature_1'] / (train['feature_2'] + 1e-5)\n",
    "    valid['cross_41_42_43'] = valid['feature_41'] + valid['feature_42'] + valid['feature_43']\n",
    "    valid['cross_1_2'] = valid['feature_1'] / (valid['feature_2'] + 1e-5)\n",
    "\n",
    "    all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])\n",
    "\n",
    "##### Model&Data fnc\n",
    "class SmoothBCEwLogits(_WeightedLoss):\n",
    "    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n",
    "        super().__init__(weight=weight, reduction=reduction)\n",
    "        self.smoothing = smoothing\n",
    "        self.weight = weight\n",
    "        self.reduction = reduction\n",
    "\n",
    "    @staticmethod\n",
    "    def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):\n",
    "        assert 0 <= smoothing < 1\n",
    "        with torch.no_grad():\n",
    "            targets = targets * (1.0 - smoothing) + 0.5 * smoothing\n",
    "        return targets\n",
    "\n",
    "    def forward(self, inputs, targets):\n",
    "        targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),\n",
    "            self.smoothing)\n",
    "        loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)\n",
    "\n",
    "        if  self.reduction == 'sum':\n",
    "            loss = loss.sum()\n",
    "        elif  self.reduction == 'mean':\n",
    "            loss = loss.mean()\n",
    "\n",
    "        return loss\n",
    "\n",
    "class MarketDataset:\n",
    "    def __init__(self, df):\n",
    "        self.features = df[all_feat_cols].values\n",
    "\n",
    "        self.label = df[target_cols].values.reshape(-1, len(target_cols))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.label)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            'features': torch.tensor(self.features[idx], dtype=torch.float),\n",
    "            'label': torch.tensor(self.label[idx], dtype=torch.float)\n",
    "        }\n",
    "\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))\n",
    "        self.dropout0 = nn.Dropout(0.2)\n",
    "\n",
    "        dropout_rate = 0.2\n",
    "        hidden_size = 256\n",
    "        self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm1 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout1 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)\n",
    "        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout2 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout3 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)\n",
    "        self.batch_norm4 = nn.BatchNorm1d(hidden_size)\n",
    "        self.dropout4 = nn.Dropout(dropout_rate)\n",
    "\n",
    "        self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))\n",
    "\n",
    "        self.Relu = nn.ReLU(inplace=True)\n",
    "        self.PReLU = nn.PReLU()\n",
    "        self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)\n",
    "        # self.GeLU = nn.GELU()\n",
    "        self.RReLU = nn.RReLU()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.batch_norm0(x)\n",
    "        x = self.dropout0(x)\n",
    "\n",
    "        x1 = self.dense1(x)\n",
    "        x1 = self.batch_norm1(x1)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x1 = self.LeakyReLU(x1)\n",
    "        x1 = self.dropout1(x1)\n",
    "\n",
    "        x = torch.cat([x, x1], 1)\n",
    "\n",
    "        x2 = self.dense2(x)\n",
    "        x2 = self.batch_norm2(x2)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x2 = self.LeakyReLU(x2)\n",
    "        x2 = self.dropout2(x2)\n",
    "\n",
    "        x = torch.cat([x1, x2], 1)\n",
    "\n",
    "        x3 = self.dense3(x)\n",
    "        x3 = self.batch_norm3(x3)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x3 = self.LeakyReLU(x3)\n",
    "        x3 = self.dropout3(x3)\n",
    "\n",
    "        x = torch.cat([x2, x3], 1)\n",
    "\n",
    "        x4 = self.dense4(x)\n",
    "        x4 = self.batch_norm4(x4)\n",
    "        # x = F.relu(x)\n",
    "        # x = self.PReLU(x)\n",
    "        x4 = self.LeakyReLU(x4)\n",
    "        x4 = self.dropout4(x4)\n",
    "\n",
    "        x = torch.cat([x3, x4], 1)\n",
    "\n",
    "        x = self.dense5(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "if True:\n",
    "    device = torch.device(\"cuda:0\")\n",
    "\n",
    "    model_list = []\n",
    "    tmp = np.zeros(len(feat_cols))\n",
    "    for _fold in range(NFOLDS):\n",
    "        torch.cuda.empty_cache()\n",
    "        model = Model()\n",
    "        model.to(device)\n",
    "        model_weights = f\"{CACHE_PATH}/online_model{_fold}.pth\"\n",
    "        model.load_state_dict(torch.load(model_weights))\n",
    "        model.eval()\n",
    "        model_list.append(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.004354,
     "end_time": "2021-02-19T02:47:40.593533",
     "exception": false,
     "start_time": "2021-02-19T02:47:40.589179",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## tensorflow part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:47:40.607607Z",
     "iopub.status.busy": "2021-02-19T02:47:40.606951Z",
     "iopub.status.idle": "2021-02-19T02:47:41.282421Z",
     "shell.execute_reply": "2021-02-19T02:47:41.281945Z"
    },
    "papermill": {
     "duration": 0.684441,
     "end_time": "2021-02-19T02:47:41.282562",
     "exception": false,
     "start_time": "2021-02-19T02:47:40.598121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__notebook__.ipynb  __results__.html  custom.css  submission.csv\r\n",
      "__output__.json     __resultx__.html  model.h5\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/jane-street-with-keras-nn-overfit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:47:41.310570Z",
     "iopub.status.busy": "2021-02-19T02:47:41.306187Z",
     "iopub.status.idle": "2021-02-19T02:47:48.373047Z",
     "shell.execute_reply": "2021-02-19T02:47:48.371949Z"
    },
    "papermill": {
     "duration": 7.084838,
     "end_time": "2021-02-19T02:47:48.373166",
     "exception": false,
     "start_time": "2021-02-19T02:47:41.288328",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation\n",
    "from tensorflow.keras.models import Model, Sequential\n",
    "from tensorflow.keras.losses import BinaryCrossentropy\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.layers.experimental.preprocessing import Normalization\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from random import choices\n",
    "\n",
    "\n",
    "SEED = 1111\n",
    "\n",
    "np.random.seed(SEED)\n",
    "\n",
    "# fit\n",
    "def create_mlp(\n",
    "    num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate\n",
    "):\n",
    "\n",
    "    inp = tf.keras.layers.Input(shape=(num_columns,))\n",
    "    x = tf.keras.layers.BatchNormalization()(inp)\n",
    "    x = tf.keras.layers.Dropout(dropout_rates[0])(x)\n",
    "    for i in range(len(hidden_units)):\n",
    "        x = tf.keras.layers.Dense(hidden_units[i])(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = tf.keras.layers.Activation(tf.keras.activations.swish)(x)\n",
    "        x = tf.keras.layers.Dropout(dropout_rates[i + 1])(x)\n",
    "    \n",
    "    x = tf.keras.layers.Dense(num_labels)(x)\n",
    "    out = tf.keras.layers.Activation(\"sigmoid\")(x)\n",
    "\n",
    "    model = tf.keras.models.Model(inputs=inp, outputs=out)\n",
    "    model.compile(\n",
    "        optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),\n",
    "        loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),\n",
    "        metrics=tf.keras.metrics.AUC(name=\"AUC\"),\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "epochs = 205\n",
    "batch_size = 4096\n",
    "hidden_units = [160, 160, 160]\n",
    "dropout_rates = [0.2, 0.2, 0.2, 0.2]\n",
    "label_smoothing = 1e-2\n",
    "learning_rate = 1e-2\n",
    "\n",
    "tf.keras.backend.clear_session()\n",
    "tf.random.set_seed(SEED)\n",
    "clf = create_mlp(\n",
    "    len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate\n",
    "    )\n",
    "clf.load_weights('../input/jane-street-with-keras-nn-overfit/model.h5')\n",
    "\n",
    "tf_models = [clf]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-02-19T02:47:48.397238Z",
     "iopub.status.busy": "2021-02-19T02:47:48.396637Z",
     "iopub.status.idle": "2021-02-19T02:52:24.943619Z",
     "shell.execute_reply": "2021-02-19T02:52:24.942891Z"
    },
    "papermill": {
     "duration": 276.565233,
     "end_time": "2021-02-19T02:52:24.943785",
     "exception": false,
     "start_time": "2021-02-19T02:47:48.378552",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15219it [04:35, 55.17it/s] \n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    import janestreet\n",
    "    env = janestreet.make_env()\n",
    "    env_iter = env.iter_test()\n",
    "\n",
    "    for (test_df, pred_df) in tqdm(env_iter):\n",
    "        if test_df['weight'].item() > 0:\n",
    "            x_tt = test_df.loc[:, feat_cols].values\n",
    "            if np.isnan(x_tt.sum()):\n",
    "                x_tt = np.nan_to_num(x_tt) + np.isnan(x_tt) * f_mean\n",
    "\n",
    "            cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]\n",
    "            cross_1_2 = x_tt[:, 1] / (x_tt[:, 2] + 1e-5)\n",
    "            feature_inp = np.concatenate((\n",
    "                x_tt,\n",
    "                np.array(cross_41_42_43).reshape(x_tt.shape[0], 1),\n",
    "                np.array(cross_1_2).reshape(x_tt.shape[0], 1),\n",
    "            ), axis=1)\n",
    "\n",
    "            # torch_pred\n",
    "            torch_pred = np.zeros((1, len(target_cols)))\n",
    "            for model in model_list:\n",
    "                torch_pred += model(torch.tensor(feature_inp, dtype=torch.float).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS\n",
    "            torch_pred = np.median(torch_pred)\n",
    "            \n",
    "            # tf_pred\n",
    "            tf_pred = np.median(np.mean([model(x_tt, training = False).numpy() for model in tf_models],axis=0))\n",
    "            \n",
    "            # avg\n",
    "            pred = torch_pred * 0.5 + tf_pred * 0.5\n",
    "            \n",
    "            pred_df.action = np.where(pred >= 0.4913, 1, 0).astype(int)\n",
    "        else:\n",
    "            pred_df.action = 0\n",
    "        env.predict(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.846544,
     "end_time": "2021-02-19T02:52:26.472211",
     "exception": false,
     "start_time": "2021-02-19T02:52:25.625667",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "papermill": {
   "duration": 298.871984,
   "end_time": "2021-02-19T02:52:28.888160",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-02-19T02:47:30.016176",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}