Text Classification-USE-localhost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../Pics/MLSb-T.png\" width=\"160\">\n",
    "<br><br>\n",
    "<center><u><H1>Text Classification with Universal Sentence Encoder-localhost</H1></u></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.log_device_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using C:\\Users\\map25\\AppData\\Local\\Temp\\tfhub_modules to cache modules.\n",
      "INFO:tensorflow:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/3'.\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 50.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 100.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 140.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 180.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 220.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 260.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 300.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 330.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 360.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 400.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 440.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 480.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 520.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 570.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 610.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 620.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 660.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 680.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 690.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 720.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 760.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder-large/3: 780.00MB\n",
      "INFO:tensorflow:Downloaded https://tfhub.dev/google/universal-sentence-encoder-large/3, Total size: 810.60MB\n",
      "INFO:tensorflow:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/3'.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "module_url = \"https://tfhub.dev/google/universal-sentence-encoder-large/3\"\n",
    "# Import the Universal Sentence Encoder's TF Hub module\n",
    "embed = hub.Module(module_url)\n",
    "\n",
    "# Compute a representation for each message, showing various lengths supported.\n",
    "messages = [\"That band rocks!\", \"That song is really cool.\"]\n",
    "\n",
    "with tf.Session() as session:\n",
    "  session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
    "  message_embeddings = session.run(embed(messages))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50000 entries, 0 to 49999\n",
      "Data columns (total 2 columns):\n",
      "review       50000 non-null object\n",
      "sentiment    50000 non-null object\n",
      "dtypes: object(2)\n",
      "memory usage: 781.3+ KB\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('../data/movie_reviews.csv.bz2', compression='bz2')\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  One of the other reviewers has mentioned that ...          1\n",
       "1  A wonderful little production. <br /><br />The...          1\n",
       "2  I thought this was a wonderful way to spend ti...          1\n",
       "3  Basically there's a family where a little boy ...          0\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...          1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['sentiment'] = [1 if sentiment == 'positive' else 0 for sentiment in data['sentiment'].values]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((30000,), (5000,), (15000,))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = data['review'].values\n",
    "sentiments = data['sentiment'].values\n",
    "\n",
    "train_reviews = reviews[:30000]\n",
    "train_sentiments = sentiments[:30000]\n",
    "\n",
    "val_reviews = reviews[30000:35000]\n",
    "val_sentiments = sentiments[30000:35000]\n",
    "\n",
    "test_reviews = reviews[35000:]\n",
    "test_sentiments = sentiments[35000:]\n",
    "train_reviews.shape, val_reviews.shape, test_reviews.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import contractions\n",
    "from bs4 import BeautifulSoup\n",
    "import unicodedata\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_html_tags(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    [s.extract() for s in soup(['iframe', 'script'])]\n",
    "    stripped_text = soup.get_text()\n",
    "    stripped_text = re.sub(r'[\\r|\\n|\\r\\n]+', '\\n', stripped_text)\n",
    "    return stripped_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_accented_chars(text):\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_contractions(text):\n",
    "    return contractions.fix(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_special_characters(text, remove_digits=False):\n",
    "    pattern = r'[^a-zA-Z0-9\\s]' if not remove_digits else r'[^a-zA-Z\\s]'\n",
    "    text = re.sub(pattern, '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_process_document(document):\n",
    "    \n",
    "    # strip HTML\n",
    "    document = strip_html_tags(document)\n",
    "    \n",
    "    # lower case\n",
    "    document = document.lower()\n",
    "    \n",
    "    # remove extra newlines (often might be present in really noisy text)\n",
    "    document = document.translate(document.maketrans(\"\\n\\t\\r\", \"   \"))\n",
    "    \n",
    "    # remove accented characters\n",
    "    document = remove_accented_chars(document)\n",
    "    \n",
    "    # expand contractions    \n",
    "    document = expand_contractions(document)\n",
    "               \n",
    "    # remove special characters and\\or digits    \n",
    "    # insert spaces between special characters to isolate them    \n",
    "    special_char_pattern = re.compile(r'([{.(-)!}])')\n",
    "    document = special_char_pattern.sub(\" \\\\1 \", document)\n",
    "    document = remove_special_characters(document, remove_digits=True)  \n",
    "        \n",
    "    # remove extra whitespace\n",
    "    document = re.sub(' +', ' ', document)\n",
    "    document = document.strip()\n",
    "    \n",
    "    return document\n",
    "\n",
    "pre_process_corpus = np.vectorize(pre_process_document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_reviews = pre_process_corpus(train_reviews)\n",
    "val_reviews = pre_process_corpus(val_reviews)\n",
    "test_reviews = pre_process_corpus(test_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training input on the whole training set with no limit on training epochs.\n",
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    {'sentence': train_reviews}, train_sentiments, \n",
    "    batch_size=256, num_epochs=None, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prediction on the whole training set.\n",
    "predict_train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    {'sentence': train_reviews}, train_sentiments, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prediction on the whole validation set.\n",
    "predict_val_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    {'sentence': val_reviews}, val_sentiments, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prediction on the test set.\n",
    "predict_test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    {'sentence': test_reviews}, test_sentiments, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder/2'.\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 48.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 68.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 108.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 128.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 178.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 188.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 238.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 278.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 318.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 358.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 408.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 448.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 488.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 528.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 568.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 618.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 658.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 678.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 718.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 758.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 788.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 828.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 868.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 908.00MB\n",
      "INFO:tensorflow:Downloading https://tfhub.dev/google/universal-sentence-encoder/2: 958.00MB\n",
      "INFO:tensorflow:Downloaded https://tfhub.dev/google/universal-sentence-encoder/2, Total size: 993.27MB\n",
      "INFO:tensorflow:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder/2'.\n"
     ]
    }
   ],
   "source": [
    "embedding_feature = hub.text_embedding_column(\n",
    "    key='sentence', \n",
    "    module_spec=\"https://tfhub.dev/google/universal-sentence-encoder/2\",\n",
    "    trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\map25\\\\AppData\\\\Local\\\\Temp\\\\tmpiuajx4tq', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x0000014A933F0D68>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "dnn = tf.estimator.DNNClassifier(\n",
    "          hidden_units=[512, 128],\n",
    "          feature_columns=[embedding_feature],\n",
    "          n_classes=2,\n",
    "          activation_fn=tf.nn.relu,\n",
    "          dropout=0.1,\n",
    "          optimizer=tf.train.AdagradOptimizer(learning_rate=0.005))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 0\n",
      "WARNING:tensorflow:From C:\\Anaconda36\\envs\\tf111_gpu\\lib\\site-packages\\tensorflow\\python\\estimator\\inputs\\queues\\feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:From C:\\Anaconda36\\envs\\tf111_gpu\\lib\\site-packages\\tensorflow\\python\\estimator\\inputs\\queues\\feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "WARNING:tensorflow:From C:\\Anaconda36\\envs\\tf111_gpu\\lib\\site-packages\\tensorflow\\python\\training\\monitored_session.py:804: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 178.04242, step = 0\n",
      "INFO:tensorflow:Saving checkpoints for 100 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 95.46022.\n",
      "Train Time (s): 70.1469657421112\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-03:59:03\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-03:59:59\n",
      "INFO:tensorflow:Saving dict for global step 100: accuracy = 0.8493, accuracy_baseline = 0.5005, auc = 0.9282116, auc_precision_recall = 0.928475, average_loss = 0.34376281, global_step = 100, label/mean = 0.5005, loss = 43.884617, precision = 0.861513, prediction/mean = 0.4868925, recall = 0.83276725\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 100: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-100\n",
      "Eval Metrics (Train): {'accuracy': 0.8493, 'accuracy_baseline': 0.5005, 'auc': 0.9282116, 'auc_precision_recall': 0.928475, 'average_loss': 0.34376281, 'label/mean': 0.5005, 'loss': 43.884617, 'precision': 0.861513, 'prediction/mean': 0.4868925, 'recall': 0.83276725, 'global_step': 100}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:00:05\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:00:31\n",
      "INFO:tensorflow:Saving dict for global step 100: accuracy = 0.8462, accuracy_baseline = 0.505, auc = 0.9246989, auc_precision_recall = 0.92106164, average_loss = 0.3505095, global_step = 100, label/mean = 0.495, loss = 43.813686, precision = 0.8445073, prediction/mean = 0.49225673, recall = 0.8448485\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 100: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-100\n",
      "Eval Metrics (Validation): {'accuracy': 0.8462, 'accuracy_baseline': 0.505, 'auc': 0.9246989, 'auc_precision_recall': 0.92106164, 'average_loss': 0.3505095, 'label/mean': 0.495, 'loss': 43.813686, 'precision': 0.8445073, 'prediction/mean': 0.49225673, 'recall': 0.8448485, 'global_step': 100}\n",
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 100\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 100 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 98.4632, step = 100\n",
      "INFO:tensorflow:Saving checkpoints for 200 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 98.53465.\n",
      "Train Time (s): 76.0923364162445\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:01:50\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:02:52\n",
      "INFO:tensorflow:Saving dict for global step 200: accuracy = 0.854, accuracy_baseline = 0.5005, auc = 0.9326798, auc_precision_recall = 0.9327926, average_loss = 0.33372602, global_step = 200, label/mean = 0.5005, loss = 42.60332, precision = 0.84066886, prediction/mean = 0.5222536, recall = 0.8739261\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 200: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-200\n",
      "Eval Metrics (Train): {'accuracy': 0.854, 'accuracy_baseline': 0.5005, 'auc': 0.9326798, 'auc_precision_recall': 0.9327926, 'average_loss': 0.33372602, 'label/mean': 0.5005, 'loss': 42.60332, 'precision': 0.84066886, 'prediction/mean': 0.5222536, 'recall': 0.8739261, 'global_step': 200}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:02:55\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:03:22\n",
      "INFO:tensorflow:Saving dict for global step 200: accuracy = 0.8448, accuracy_baseline = 0.505, auc = 0.9277709, auc_precision_recall = 0.9238656, average_loss = 0.34711793, global_step = 200, label/mean = 0.495, loss = 43.38974, precision = 0.819481, prediction/mean = 0.52800876, recall = 0.88040406\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 200: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-200\n",
      "Eval Metrics (Validation): {'accuracy': 0.8448, 'accuracy_baseline': 0.505, 'auc': 0.9277709, 'auc_precision_recall': 0.9238656, 'average_loss': 0.34711793, 'label/mean': 0.495, 'loss': 43.38974, 'precision': 0.819481, 'prediction/mean': 0.52800876, 'recall': 0.88040406, 'global_step': 200}\n",
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 200\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-200\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 200 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 89.417015, step = 200\n",
      "INFO:tensorflow:Saving checkpoints for 300 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 96.99166.\n",
      "Train Time (s): 76.68918061256409\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:04:42\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-300\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:05:48\n",
      "INFO:tensorflow:Saving dict for global step 300: accuracy = 0.8571333, accuracy_baseline = 0.5005, auc = 0.934535, auc_precision_recall = 0.93439317, average_loss = 0.32713658, global_step = 300, label/mean = 0.5005, loss = 41.76212, precision = 0.86073565, prediction/mean = 0.4985642, recall = 0.8524808\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 300: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-300\n",
      "Eval Metrics (Train): {'accuracy': 0.8571333, 'accuracy_baseline': 0.5005, 'auc': 0.934535, 'auc_precision_recall': 0.93439317, 'average_loss': 0.32713658, 'label/mean': 0.5005, 'loss': 41.76212, 'precision': 0.86073565, 'prediction/mean': 0.4985642, 'recall': 0.8524808, 'global_step': 300}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:05:51\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-300\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:06:25\n",
      "INFO:tensorflow:Saving dict for global step 300: accuracy = 0.85, accuracy_baseline = 0.505, auc = 0.92916447, auc_precision_recall = 0.9246455, average_loss = 0.33946192, global_step = 300, label/mean = 0.495, loss = 42.43274, precision = 0.84376246, prediction/mean = 0.5031992, recall = 0.85535353\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 300: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-300\n",
      "Eval Metrics (Validation): {'accuracy': 0.85, 'accuracy_baseline': 0.505, 'auc': 0.92916447, 'auc_precision_recall': 0.9246455, 'average_loss': 0.33946192, 'label/mean': 0.495, 'loss': 42.43274, 'precision': 0.84376246, 'prediction/mean': 0.5031992, 'recall': 0.85535353, 'global_step': 300}\n",
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 300\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-300\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 300 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 79.00784, step = 300\n",
      "INFO:tensorflow:Saving checkpoints for 400 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 68.74791.\n",
      "Train Time (s): 85.92826247215271\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:07:54\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-400\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:09:04\n",
      "INFO:tensorflow:Saving dict for global step 400: accuracy = 0.85873336, accuracy_baseline = 0.5005, auc = 0.9361063, auc_precision_recall = 0.93587583, average_loss = 0.325451, global_step = 400, label/mean = 0.5005, loss = 41.546932, precision = 0.8444231, prediction/mean = 0.52275807, recall = 0.8798535\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 400: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-400\n",
      "Eval Metrics (Train): {'accuracy': 0.85873336, 'accuracy_baseline': 0.5005, 'auc': 0.9361063, 'auc_precision_recall': 0.93587583, 'average_loss': 0.325451, 'label/mean': 0.5005, 'loss': 41.546932, 'precision': 0.8444231, 'prediction/mean': 0.52275807, 'recall': 0.8798535, 'global_step': 400}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:09:07\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:09:44\n",
      "INFO:tensorflow:Saving dict for global step 400: accuracy = 0.849, accuracy_baseline = 0.505, auc = 0.9301559, auc_precision_recall = 0.9257468, average_loss = 0.34161478, global_step = 400, label/mean = 0.495, loss = 42.70185, precision = 0.82428354, prediction/mean = 0.52780706, recall = 0.8832323\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 400: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-400\n",
      "Eval Metrics (Validation): {'accuracy': 0.849, 'accuracy_baseline': 0.505, 'auc': 0.9301559, 'auc_precision_recall': 0.9257468, 'average_loss': 0.34161478, 'label/mean': 0.495, 'loss': 42.70185, 'precision': 0.82428354, 'prediction/mean': 0.52780706, 'recall': 0.8832323, 'global_step': 400}\n",
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 400\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-400\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 400 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 99.33322, step = 400\n",
      "INFO:tensorflow:Saving checkpoints for 500 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 81.631165.\n",
      "Train Time (s): 88.1827085018158\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:11:16\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:12:32\n",
      "INFO:tensorflow:Saving dict for global step 500: accuracy = 0.86113334, accuracy_baseline = 0.5005, auc = 0.9375029, auc_precision_recall = 0.9373319, average_loss = 0.32048842, global_step = 500, label/mean = 0.5005, loss = 40.913414, precision = 0.85322654, prediction/mean = 0.5139927, recall = 0.8726607\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 500: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-500\n",
      "Eval Metrics (Train): {'accuracy': 0.86113334, 'accuracy_baseline': 0.5005, 'auc': 0.9375029, 'auc_precision_recall': 0.9373319, 'average_loss': 0.32048842, 'label/mean': 0.5005, 'loss': 40.913414, 'precision': 0.85322654, 'prediction/mean': 0.5139927, 'recall': 0.8726607, 'global_step': 500}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:12:35\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:13:16\n",
      "INFO:tensorflow:Saving dict for global step 500: accuracy = 0.8544, accuracy_baseline = 0.505, auc = 0.9311021, auc_precision_recall = 0.92677355, average_loss = 0.3371372, global_step = 500, label/mean = 0.495, loss = 42.142147, precision = 0.837911, prediction/mean = 0.518596, recall = 0.8751515\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 500: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-500\n",
      "Eval Metrics (Validation): {'accuracy': 0.8544, 'accuracy_baseline': 0.505, 'auc': 0.9311021, 'auc_precision_recall': 0.92677355, 'average_loss': 0.3371372, 'label/mean': 0.495, 'loss': 42.142147, 'precision': 0.837911, 'prediction/mean': 0.518596, 'recall': 0.8751515, 'global_step': 500}\n",
      "\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Training for step = 500\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 500 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:loss = 82.547325, step = 500\n",
      "INFO:tensorflow:Saving checkpoints for 600 into C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 89.24525.\n",
      "Train Time (s): 91.32514190673828\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:14:50\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:16:02\n",
      "INFO:tensorflow:Saving dict for global step 600: accuracy = 0.8633, accuracy_baseline = 0.5005, auc = 0.938951, auc_precision_recall = 0.93868655, average_loss = 0.31627437, global_step = 600, label/mean = 0.5005, loss = 40.375454, precision = 0.87051874, prediction/mean = 0.4934998, recall = 0.85387945\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n",
      "Eval Metrics (Train): {'accuracy': 0.8633, 'accuracy_baseline': 0.5005, 'auc': 0.938951, 'auc_precision_recall': 0.93868655, 'average_loss': 0.31627437, 'label/mean': 0.5005, 'loss': 40.375454, 'precision': 0.87051874, 'prediction/mean': 0.4934998, 'recall': 0.85387945, 'global_step': 600}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:16:05\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:16:45\n",
      "INFO:tensorflow:Saving dict for global step 600: accuracy = 0.8564, accuracy_baseline = 0.505, auc = 0.9321009, auc_precision_recall = 0.9275272, average_loss = 0.3323834, global_step = 600, label/mean = 0.495, loss = 41.547924, precision = 0.85323685, prediction/mean = 0.49773434, recall = 0.8573737\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n",
      "Eval Metrics (Validation): {'accuracy': 0.8564, 'accuracy_baseline': 0.505, 'auc': 0.9321009, 'auc_precision_recall': 0.9275272, 'average_loss': 0.3323834, 'label/mean': 0.495, 'loss': 41.547924, 'precision': 0.85323685, 'prediction/mean': 0.49773434, 'recall': 0.8573737, 'global_step': 600}\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "TOTAL_STEPS = 500\n",
    "STEP_SIZE = 100\n",
    "for step in range(0, TOTAL_STEPS+1, STEP_SIZE):\n",
    "    print()\n",
    "    print('-'*100)\n",
    "    print('Training for step =', step)\n",
    "    start_time = time.time()\n",
    "    dnn.train(input_fn=train_input_fn, steps=STEP_SIZE)\n",
    "    elapsed_time = time.time() - start_time\n",
    "    print('Train Time (s):', elapsed_time)\n",
    "    print('Eval Metrics (Train):', dnn.evaluate(input_fn=predict_train_input_fn))\n",
    "    print('Eval Metrics (Validation):', dnn.evaluate(input_fn=predict_val_input_fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:16:49\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:18:05\n",
      "INFO:tensorflow:Saving dict for global step 600: accuracy = 0.8633, accuracy_baseline = 0.5005, auc = 0.938951, auc_precision_recall = 0.93868655, average_loss = 0.31627437, global_step = 600, label/mean = 0.5005, loss = 40.375454, precision = 0.87051874, prediction/mean = 0.4934998, recall = 0.85387945\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.8633,\n",
       " 'accuracy_baseline': 0.5005,\n",
       " 'auc': 0.938951,\n",
       " 'auc_precision_recall': 0.93868655,\n",
       " 'average_loss': 0.31627437,\n",
       " 'label/mean': 0.5005,\n",
       " 'loss': 40.375454,\n",
       " 'precision': 0.87051874,\n",
       " 'prediction/mean': 0.4934998,\n",
       " 'recall': 0.85387945,\n",
       " 'global_step': 600}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dnn.evaluate(input_fn=predict_train_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "WARNING:tensorflow:Trapezoidal rule is known to produce incorrect PR-AUCs; please switch to \"careful_interpolation\" instead.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-01-12-04:18:09\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-01-12-04:19:03\n",
      "INFO:tensorflow:Saving dict for global step 600: accuracy = 0.85973334, accuracy_baseline = 0.5006667, auc = 0.93549514, auc_precision_recall = 0.93464446, average_loss = 0.32488343, global_step = 600, label/mean = 0.5006667, loss = 41.29874, precision = 0.8667571, prediction/mean = 0.4959114, recall = 0.8505992\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 600: C:\\Users\\map25\\AppData\\Local\\Temp\\tmpiuajx4tq\\model.ckpt-600\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.85973334,\n",
       " 'accuracy_baseline': 0.5006667,\n",
       " 'auc': 0.93549514,\n",
       " 'auc_precision_recall': 0.93464446,\n",
       " 'average_loss': 0.32488343,\n",
       " 'label/mean': 0.5006667,\n",
       " 'loss': 41.29874,\n",
       " 'precision': 0.8667571,\n",
       " 'prediction/mean': 0.4959114,\n",
       " 'recall': 0.8505992,\n",
       " 'global_step': 600}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dnn.evaluate(input_fn=predict_test_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1.11_gpu",
   "language": "python",
   "name": "tf111_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}