Chapter-3 Hierarchy of TensorFlow/3-3High-level_API_Demonstration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3-3 High-level API: Demonstration\n",
    "\n",
    "The examples below use high-level APIs in TensorFlow to implement a linear regression model.\n",
    "\n",
    "Typically, the high-level APIs are providing the class interfaces for `tf.keras.models`.\n",
    "\n",
    "There are three ways of modeling using APIs of Keras: sequential modeling using `Sequential()` function, arbitrary modeling using API functions, and customized modeling by inheriting base class `Model`.\n",
    "\n",
    "Here we are demonstrating using `Sequential()` function and customized modeling by inheriting base class `Model`, respectively.\n",
    "\n",
    "\n",
    "### 1. Use `Sequential()` function to create a model sequencially (for the beginners)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import models,layers,optimizers\n",
    "\n",
    "# Number of samples\n",
    "n = 800\n",
    "\n",
    "# Generate testing dataset\n",
    "X = tf.random.uniform([n,2],minval=-10,maxval=10) \n",
    "w0 = tf.constant([[2.0],[-1.0]])\n",
    "b0 = tf.constant(3.0)\n",
    "\n",
    "Y = X@w0 + b0 + tf.random.normal([n,1],mean = 0.0,stddev= 2.0)  # @ represents matrix multiplication; add Gaussian noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 1)                 3         \n",
      "=================================================================\n",
      "Total params: 3\n",
      "Trainable params: 3\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "\n",
    "linear = models.Sequential()\n",
    "linear.add(layers.Dense(1,input_shape =(2,)))\n",
    "linear.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 800 samples\n",
      "Epoch 1/200\n",
      "800/800 [==============================] - 0s 256us/sample - loss: 45.4000 - mae: 5.4908\n",
      "Epoch 2/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 41.9666 - mae: 5.2756\n",
      "Epoch 3/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 38.7465 - mae: 5.0686\n",
      "Epoch 4/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 35.8251 - mae: 4.8681\n",
      "Epoch 5/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 33.0641 - mae: 4.6745\n",
      "Epoch 6/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 30.5758 - mae: 4.4926\n",
      "Epoch 7/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 28.2652 - mae: 4.3157\n",
      "Epoch 8/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 26.1634 - mae: 4.1476\n",
      "Epoch 9/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 24.2196 - mae: 3.9884\n",
      "Epoch 10/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 22.4602 - mae: 3.8422\n",
      "Epoch 11/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 20.8793 - mae: 3.7056\n",
      "Epoch 12/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 19.4025 - mae: 3.5734\n",
      "Epoch 13/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 18.1003 - mae: 3.4484\n",
      "Epoch 14/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 16.8978 - mae: 3.3362\n",
      "Epoch 15/200\n",
      "800/800 [==============================] - 0s 43us/sample - loss: 15.8403 - mae: 3.2295\n",
      "Epoch 16/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 14.8489 - mae: 3.1292\n",
      "Epoch 17/200\n",
      "800/800 [==============================] - 0s 51us/sample - loss: 13.9815 - mae: 3.0380\n",
      "Epoch 18/200\n",
      "800/800 [==============================] - 0s 48us/sample - loss: 13.1848 - mae: 2.9539\n",
      "Epoch 19/200\n",
      "800/800 [==============================] - 0s 51us/sample - loss: 12.4808 - mae: 2.8760\n",
      "Epoch 20/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 11.8449 - mae: 2.8052\n",
      "Epoch 21/200\n",
      "800/800 [==============================] - 0s 53us/sample - loss: 11.2641 - mae: 2.7369\n",
      "Epoch 22/200\n",
      "800/800 [==============================] - 0s 50us/sample - loss: 10.7635 - mae: 2.6756\n",
      "Epoch 23/200\n",
      "800/800 [==============================] - 0s 53us/sample - loss: 10.2921 - mae: 2.6182\n",
      "Epoch 24/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 9.8764 - mae: 2.5654\n",
      "Epoch 25/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 9.5053 - mae: 2.5161\n",
      "Epoch 26/200\n",
      "800/800 [==============================] - 0s 53us/sample - loss: 9.1670 - mae: 2.4727\n",
      "Epoch 27/200\n",
      "800/800 [==============================] - 0s 53us/sample - loss: 8.8610 - mae: 2.4329\n",
      "Epoch 28/200\n",
      "800/800 [==============================] - 0s 54us/sample - loss: 8.5859 - mae: 2.3972\n",
      "Epoch 29/200\n",
      "800/800 [==============================] - 0s 50us/sample - loss: 8.3357 - mae: 2.3609\n",
      "Epoch 30/200\n",
      "800/800 [==============================] - 0s 48us/sample - loss: 8.1072 - mae: 2.3289\n",
      "Epoch 31/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 7.9036 - mae: 2.2969\n",
      "Epoch 32/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 7.7077 - mae: 2.2677\n",
      "Epoch 33/200\n",
      "800/800 [==============================] - 0s 49us/sample - loss: 7.5329 - mae: 2.2407\n",
      "Epoch 34/200\n",
      "800/800 [==============================] - 0s 54us/sample - loss: 7.3701 - mae: 2.2146\n",
      "Epoch 35/200\n",
      "800/800 [==============================] - 0s 50us/sample - loss: 7.2179 - mae: 2.1901\n",
      "Epoch 36/200\n",
      "800/800 [==============================] - 0s 64us/sample - loss: 7.0760 - mae: 2.1663\n",
      "Epoch 37/200\n",
      "800/800 [==============================] - 0s 50us/sample - loss: 6.9437 - mae: 2.1442\n",
      "Epoch 38/200\n",
      "800/800 [==============================] - 0s 46us/sample - loss: 6.8178 - mae: 2.1225\n",
      "Epoch 39/200\n",
      "800/800 [==============================] - 0s 48us/sample - loss: 6.7003 - mae: 2.1017\n",
      "Epoch 40/200\n",
      "800/800 [==============================] - 0s 45us/sample - loss: 6.5877 - mae: 2.0820\n",
      "Epoch 41/200\n",
      "800/800 [==============================] - 0s 47us/sample - loss: 6.4821 - mae: 2.0628\n",
      "Epoch 42/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 6.3825 - mae: 2.0441\n",
      "Epoch 43/200\n",
      "800/800 [==============================] - 0s 55us/sample - loss: 6.2827 - mae: 2.0265\n",
      "Epoch 44/200\n",
      "800/800 [==============================] - 0s 59us/sample - loss: 6.1893 - mae: 2.0095\n",
      "Epoch 45/200\n",
      "800/800 [==============================] - 0s 61us/sample - loss: 6.1026 - mae: 1.9938\n",
      "Epoch 46/200\n",
      "800/800 [==============================] - 0s 59us/sample - loss: 6.0187 - mae: 1.9782\n",
      "Epoch 47/200\n",
      "800/800 [==============================] - 0s 74us/sample - loss: 5.9344 - mae: 1.9632\n",
      "Epoch 48/200\n",
      "800/800 [==============================] - 0s 58us/sample - loss: 5.8526 - mae: 1.9486\n",
      "Epoch 49/200\n",
      "800/800 [==============================] - 0s 47us/sample - loss: 5.7764 - mae: 1.9350\n",
      "Epoch 50/200\n",
      "800/800 [==============================] - 0s 45us/sample - loss: 5.7020 - mae: 1.9217\n",
      "Epoch 51/200\n",
      "800/800 [==============================] - 0s 45us/sample - loss: 5.6317 - mae: 1.9096\n",
      "Epoch 52/200\n",
      "800/800 [==============================] - 0s 44us/sample - loss: 5.5630 - mae: 1.8966\n",
      "Epoch 53/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 5.4940 - mae: 1.8849\n",
      "Epoch 54/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 5.4278 - mae: 1.8728\n",
      "Epoch 55/200\n",
      "800/800 [==============================] - 0s 44us/sample - loss: 5.3657 - mae: 1.8619\n",
      "Epoch 56/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 5.3059 - mae: 1.8509\n",
      "Epoch 57/200\n",
      "800/800 [==============================] - 0s 47us/sample - loss: 5.2466 - mae: 1.8396\n",
      "Epoch 58/200\n",
      "800/800 [==============================] - 0s 51us/sample - loss: 5.1897 - mae: 1.8294\n",
      "Epoch 59/200\n",
      "800/800 [==============================] - 0s 46us/sample - loss: 5.1358 - mae: 1.8199\n",
      "Epoch 60/200\n",
      "800/800 [==============================] - 0s 41us/sample - loss: 5.0823 - mae: 1.8102\n",
      "Epoch 61/200\n",
      "800/800 [==============================] - 0s 52us/sample - loss: 5.0324 - mae: 1.8010\n",
      "Epoch 62/200\n",
      "800/800 [==============================] - 0s 49us/sample - loss: 4.9827 - mae: 1.7925\n",
      "Epoch 63/200\n",
      "800/800 [==============================] - 0s 43us/sample - loss: 4.9370 - mae: 1.7839\n",
      "Epoch 64/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 4.8905 - mae: 1.7758\n",
      "Epoch 65/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 4.8464 - mae: 1.7675\n",
      "Epoch 66/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 4.8025 - mae: 1.7593\n",
      "Epoch 67/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 4.7627 - mae: 1.7523\n",
      "Epoch 68/200\n",
      "800/800 [==============================] - 0s 40us/sample - loss: 4.7247 - mae: 1.7452\n",
      "Epoch 69/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 4.6850 - mae: 1.7378\n",
      "Epoch 70/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 4.6503 - mae: 1.7315\n",
      "Epoch 71/200\n",
      "800/800 [==============================] - 0s 40us/sample - loss: 4.6136 - mae: 1.7247\n",
      "Epoch 72/200\n",
      "800/800 [==============================] - 0s 37us/sample - loss: 4.5818 - mae: 1.7188\n",
      "Epoch 73/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 4.5496 - mae: 1.7126\n",
      "Epoch 74/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.5190 - mae: 1.7068\n",
      "Epoch 75/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.4897 - mae: 1.7016\n",
      "Epoch 76/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 4.4624 - mae: 1.6963\n",
      "Epoch 77/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.4350 - mae: 1.6921\n",
      "Epoch 78/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.4107 - mae: 1.6878\n",
      "Epoch 79/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.3839 - mae: 1.6828\n",
      "Epoch 80/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 4.3602 - mae: 1.6786\n",
      "Epoch 81/200\n",
      "800/800 [==============================] - 0s 37us/sample - loss: 4.3393 - mae: 1.6756\n",
      "Epoch 82/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800/800 [==============================] - 0s 39us/sample - loss: 4.3183 - mae: 1.6719\n",
      "Epoch 83/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 4.2970 - mae: 1.6685\n",
      "Epoch 84/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 4.2778 - mae: 1.6649\n",
      "Epoch 85/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 4.2592 - mae: 1.6614\n",
      "Epoch 86/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 4.2421 - mae: 1.6584\n",
      "Epoch 87/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 4.2279 - mae: 1.6561\n",
      "Epoch 88/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.2110 - mae: 1.6528\n",
      "Epoch 89/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 4.1959 - mae: 1.6501\n",
      "Epoch 90/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.1818 - mae: 1.6477\n",
      "Epoch 91/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.1692 - mae: 1.6458\n",
      "Epoch 92/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.1558 - mae: 1.6431\n",
      "Epoch 93/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.1453 - mae: 1.6414\n",
      "Epoch 94/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.1347 - mae: 1.6398\n",
      "Epoch 95/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.1228 - mae: 1.6374\n",
      "Epoch 96/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.1133 - mae: 1.6361\n",
      "Epoch 97/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.1052 - mae: 1.6348\n",
      "Epoch 98/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.0968 - mae: 1.6333\n",
      "Epoch 99/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 4.0891 - mae: 1.6319\n",
      "Epoch 100/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 4.0810 - mae: 1.6305\n",
      "Epoch 101/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0742 - mae: 1.6293\n",
      "Epoch 102/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 4.0678 - mae: 1.6284\n",
      "Epoch 103/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 4.0636 - mae: 1.6279\n",
      "Epoch 104/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.0570 - mae: 1.6263\n",
      "Epoch 105/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0527 - mae: 1.6264\n",
      "Epoch 106/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0466 - mae: 1.6245\n",
      "Epoch 107/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.0401 - mae: 1.6236\n",
      "Epoch 108/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0362 - mae: 1.6232\n",
      "Epoch 109/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0331 - mae: 1.6230\n",
      "Epoch 110/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0300 - mae: 1.6223\n",
      "Epoch 111/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.0280 - mae: 1.6222\n",
      "Epoch 112/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 4.0226 - mae: 1.6213\n",
      "Epoch 113/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0205 - mae: 1.6208\n",
      "Epoch 114/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 4.0194 - mae: 1.6210\n",
      "Epoch 115/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0152 - mae: 1.6202\n",
      "Epoch 116/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.0138 - mae: 1.6204\n",
      "Epoch 117/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.0127 - mae: 1.6199\n",
      "Epoch 118/200\n",
      "800/800 [==============================] - 0s 44us/sample - loss: 4.0095 - mae: 1.6197\n",
      "Epoch 119/200\n",
      "800/800 [==============================] - 0s 41us/sample - loss: 4.0075 - mae: 1.6194\n",
      "Epoch 120/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 4.0047 - mae: 1.6188\n",
      "Epoch 121/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.0050 - mae: 1.6187\n",
      "Epoch 122/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.0027 - mae: 1.6188\n",
      "Epoch 123/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 4.0018 - mae: 1.6186\n",
      "Epoch 124/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 4.0016 - mae: 1.6191\n",
      "Epoch 125/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 4.0015 - mae: 1.6191\n",
      "Epoch 126/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 4.0007 - mae: 1.6188\n",
      "Epoch 127/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 4.0014 - mae: 1.6191\n",
      "Epoch 128/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9998 - mae: 1.6187\n",
      "Epoch 129/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9988 - mae: 1.6185\n",
      "Epoch 130/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9990 - mae: 1.6190\n",
      "Epoch 131/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9959 - mae: 1.6182\n",
      "Epoch 132/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9968 - mae: 1.6179\n",
      "Epoch 133/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9955 - mae: 1.6175\n",
      "Epoch 134/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9977 - mae: 1.6191\n",
      "Epoch 135/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9964 - mae: 1.6178\n",
      "Epoch 136/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 3.9980 - mae: 1.6190\n",
      "Epoch 137/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 3.9952 - mae: 1.6183\n",
      "Epoch 138/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 3.9942 - mae: 1.6182\n",
      "Epoch 139/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 3.9934 - mae: 1.6181\n",
      "Epoch 140/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9986 - mae: 1.6198\n",
      "Epoch 141/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9964 - mae: 1.6182\n",
      "Epoch 142/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9935 - mae: 1.6178\n",
      "Epoch 143/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9934 - mae: 1.6179\n",
      "Epoch 144/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9944 - mae: 1.6183\n",
      "Epoch 145/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 3.9946 - mae: 1.6182\n",
      "Epoch 146/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9937 - mae: 1.6181\n",
      "Epoch 147/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9942 - mae: 1.6181\n",
      "Epoch 148/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9933 - mae: 1.6182\n",
      "Epoch 149/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9940 - mae: 1.6182\n",
      "Epoch 150/200\n",
      "800/800 [==============================] - 0s 28us/sample - loss: 3.9932 - mae: 1.6179\n",
      "Epoch 151/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9928 - mae: 1.6182\n",
      "Epoch 152/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9942 - mae: 1.6183\n",
      "Epoch 153/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9935 - mae: 1.6184\n",
      "Epoch 154/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9962 - mae: 1.6192\n",
      "Epoch 155/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9936 - mae: 1.6184\n",
      "Epoch 156/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9944 - mae: 1.6182\n",
      "Epoch 157/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9940 - mae: 1.6188\n",
      "Epoch 158/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9931 - mae: 1.6188\n",
      "Epoch 159/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9929 - mae: 1.6183\n",
      "Epoch 160/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9942 - mae: 1.6189\n",
      "Epoch 161/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9950 - mae: 1.6191\n",
      "Epoch 162/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 3.9957 - mae: 1.6190\n",
      "Epoch 163/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9942 - mae: 1.6185\n",
      "Epoch 164/200\n",
      "800/800 [==============================] - 0s 34us/sample - loss: 3.9939 - mae: 1.6181\n",
      "Epoch 165/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9931 - mae: 1.6187\n",
      "Epoch 166/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9953 - mae: 1.6190\n",
      "Epoch 167/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9955 - mae: 1.6192\n",
      "Epoch 168/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9951 - mae: 1.6183\n",
      "Epoch 169/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9948 - mae: 1.6189\n",
      "Epoch 170/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9937 - mae: 1.6183\n",
      "Epoch 171/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9954 - mae: 1.6189\n",
      "Epoch 172/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9947 - mae: 1.6187\n",
      "Epoch 173/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9938 - mae: 1.6186\n",
      "Epoch 174/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9958 - mae: 1.6191\n",
      "Epoch 175/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9971 - mae: 1.6190\n",
      "Epoch 176/200\n",
      "800/800 [==============================] - 0s 35us/sample - loss: 3.9929 - mae: 1.6184\n",
      "Epoch 177/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 3.9952 - mae: 1.6189\n",
      "Epoch 178/200\n",
      "800/800 [==============================] - 0s 32us/sample - loss: 3.9962 - mae: 1.6195\n",
      "Epoch 179/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9924 - mae: 1.6182\n",
      "Epoch 180/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9956 - mae: 1.6192\n",
      "Epoch 181/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9933 - mae: 1.6182\n",
      "Epoch 182/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9939 - mae: 1.6186\n",
      "Epoch 183/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9934 - mae: 1.6185\n",
      "Epoch 184/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9950 - mae: 1.6183\n",
      "Epoch 185/200\n",
      "800/800 [==============================] - 0s 31us/sample - loss: 3.9959 - mae: 1.6190\n",
      "Epoch 186/200\n",
      "800/800 [==============================] - 0s 30us/sample - loss: 3.9982 - mae: 1.6190\n",
      "Epoch 187/200\n",
      "800/800 [==============================] - 0s 29us/sample - loss: 3.9939 - mae: 1.6187\n",
      "Epoch 188/200\n",
      "800/800 [==============================] - 0s 33us/sample - loss: 3.9937 - mae: 1.6191\n",
      "Epoch 189/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 3.9961 - mae: 1.6192\n",
      "Epoch 190/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 3.9939 - mae: 1.6190\n",
      "Epoch 191/200\n",
      "800/800 [==============================] - 0s 41us/sample - loss: 3.9948 - mae: 1.6186\n",
      "Epoch 192/200\n",
      "800/800 [==============================] - 0s 42us/sample - loss: 3.9960 - mae: 1.6186\n",
      "Epoch 193/200\n",
      "800/800 [==============================] - 0s 47us/sample - loss: 3.9957 - mae: 1.6188\n",
      "Epoch 194/200\n",
      "800/800 [==============================] - 0s 43us/sample - loss: 3.9945 - mae: 1.6193\n",
      "Epoch 195/200\n",
      "800/800 [==============================] - 0s 40us/sample - loss: 3.9946 - mae: 1.6184\n",
      "Epoch 196/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 3.9934 - mae: 1.6184\n",
      "Epoch 197/200\n",
      "800/800 [==============================] - 0s 39us/sample - loss: 3.9927 - mae: 1.6184\n",
      "Epoch 198/200\n",
      "800/800 [==============================] - 0s 38us/sample - loss: 3.9931 - mae: 1.6185\n",
      "Epoch 199/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 3.9936 - mae: 1.6190\n",
      "Epoch 200/200\n",
      "800/800 [==============================] - 0s 36us/sample - loss: 3.9940 - mae: 1.6184\n",
      "w =  [[1.99828291]\n",
      " [-1.02342927]]\n"
     ]
    }
   ],
   "source": [
    "### Training with fit method\n",
    "\n",
    "linear.compile(optimizer=\"adam\",loss=\"mse\",metrics=[\"mae\"])\n",
    "linear.fit(X,Y,batch_size = 20,epochs = 200)  \n",
    "\n",
    "tf.print(\"w = \",linear.layers[0].kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Customized modeling by inheriting base class `Model` (for the experts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import models,layers,optimizers,losses,metrics\n",
    "\n",
    "\n",
    "# Print time stamps\n",
    "@tf.function\n",
    "def printbar():\n",
    "    ts = tf.timestamp()\n",
    "    today_ts = ts%(24*60*60)\n",
    "\n",
    "    hour = tf.cast(today_ts//3600+8,tf.int32)%tf.constant(24)\n",
    "    minite = tf.cast((today_ts%3600)//60,tf.int32)\n",
    "    second = tf.cast(tf.floor(today_ts%60),tf.int32)\n",
    "    \n",
    "    def timeformat(m):\n",
    "        if tf.strings.length(tf.strings.format(\"{}\",m))==1:\n",
    "            return(tf.strings.format(\"0{}\",m))\n",
    "        else:\n",
    "            return(tf.strings.format(\"{}\",m))\n",
    "    \n",
    "    timestring = tf.strings.join([timeformat(hour),timeformat(minite),\n",
    "                timeformat(second)],separator = \":\")\n",
    "    tf.print(\"==========\"*8,end = \"\")\n",
    "    tf.print(timestring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of samples\n",
    "n = 800\n",
    "\n",
    "# Generate testing dataset\n",
    "X = tf.random.uniform([n,2],minval=-10,maxval=10) \n",
    "w0 = tf.constant([[2.0],[-1.0]])\n",
    "b0 = tf.constant(3.0)\n",
    "\n",
    "Y = X@w0 + b0 + tf.random.normal([n,1],mean = 0.0,stddev= 2.0)  # @ represents matrix multiplication; add Gaussian noise\n",
    "\n",
    "ds_train = tf.data.Dataset.from_tensor_slices((X[0:n*3//4,:],Y[0:n*3//4,:])) \\\n",
    "     .shuffle(buffer_size = 1000).batch(20) \\\n",
    "     .prefetch(tf.data.experimental.AUTOTUNE) \\\n",
    "     .cache()\n",
    "\n",
    "ds_valid = tf.data.Dataset.from_tensor_slices((X[n*3//4:,:],Y[n*3//4:,:])) \\\n",
    "     .shuffle(buffer_size = 1000).batch(20) \\\n",
    "     .prefetch(tf.data.experimental.AUTOTUNE) \\\n",
    "     .cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"my_model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                multiple                  3         \n",
      "=================================================================\n",
      "Total params: 3\n",
      "Trainable params: 3\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "\n",
    "class MyModel(models.Model):\n",
    "    def __init__(self):\n",
    "        super(MyModel, self).__init__()\n",
    "        \n",
    "    def build(self,input_shape):\n",
    "        self.dense1 = layers.Dense(1)   \n",
    "        super(MyModel,self).build(input_shape)\n",
    "    \n",
    "    def call(self, x):\n",
    "        y = self.dense1(x)\n",
    "        return(y)\n",
    "\n",
    "model = MyModel()\n",
    "model.build(input_shape =(None,2))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================16:31:13\n",
      "Epoch=100,Loss:100.84388,MAE:7.94069195,Valid Loss:97.2003555,Valid MAE:7.996737\n",
      "w= [[1.41411638]\n",
      " [-1.02094865]]\n",
      "b= [1.80114055]\n",
      "\n",
      "================================================================================16:31:20\n",
      "Epoch=200,Loss:53.1216393,MAE:4.89377928,Valid Loss:51.0633812,Valid MAE:4.90537262\n",
      "w= [[1.9891212]\n",
      " [-1.01212275]]\n",
      "b= [2.89756393]\n",
      "\n",
      "================================================================================16:31:27\n",
      "Epoch=300,Loss:36.6023254,MAE:3.76881385,Valid Loss:35.1779671,Valid MAE:3.78182697\n",
      "w= [[1.9923569]\n",
      " [-1.01145089]]\n",
      "b= [2.98073745]\n",
      "\n",
      "================================================================================16:31:34\n",
      "Epoch=400,Loss:28.3633537,MAE:3.20791626,Valid Loss:27.2563629,Valid MAE:3.22140265\n",
      "w= [[1.99243522]\n",
      " [-1.01144516]]\n",
      "b= [2.98090124]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Customizing training loop (tutorial for the experts)\n",
    "\n",
    "\n",
    "optimizer = optimizers.Adam()\n",
    "loss_func = losses.MeanSquaredError()\n",
    "\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_metric = tf.keras.metrics.MeanAbsoluteError(name='train_mae')\n",
    "\n",
    "valid_loss = tf.keras.metrics.Mean(name='valid_loss')\n",
    "valid_metric = tf.keras.metrics.MeanAbsoluteError(name='valid_mae')\n",
    "\n",
    "\n",
    "@tf.function\n",
    "def train_step(model, features, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(features)\n",
    "        loss = loss_func(labels, predictions)\n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "\n",
    "    train_loss.update_state(loss)\n",
    "    train_metric.update_state(labels, predictions)\n",
    "\n",
    "@tf.function\n",
    "def valid_step(model, features, labels):\n",
    "    predictions = model(features)\n",
    "    batch_loss = loss_func(labels, predictions)\n",
    "    valid_loss.update_state(batch_loss)\n",
    "    valid_metric.update_state(labels, predictions)\n",
    "    \n",
    "\n",
    "@tf.function\n",
    "def train_model(model,ds_train,ds_valid,epochs):\n",
    "    for epoch in tf.range(1,epochs+1):\n",
    "        for features, labels in ds_train:\n",
    "            train_step(model,features,labels)\n",
    "\n",
    "        for features, labels in ds_valid:\n",
    "            valid_step(model,features,labels)\n",
    "\n",
    "        logs = 'Epoch={},Loss:{},MAE:{},Valid Loss:{},Valid MAE:{}'\n",
    "        \n",
    "        if  epoch%100 ==0:\n",
    "            printbar()\n",
    "            tf.print(tf.strings.format(logs,\n",
    "            (epoch,train_loss.result(),train_metric.result(),valid_loss.result(),valid_metric.result())))\n",
    "            tf.print(\"w=\",model.layers[0].kernel)\n",
    "            tf.print(\"b=\",model.layers[0].bias)\n",
    "            tf.print(\"\")\n",
    "        \n",
    "        train_loss.reset_states()\n",
    "        valid_loss.reset_states()\n",
    "        train_metric.reset_states()\n",
    "        valid_metric.reset_states()\n",
    "\n",
    "train_model(model,ds_train,ds_valid,400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}