demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import deepos\n",
    "import deepops as dp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Give your tensor a cool name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make Tensors\n",
    "a = dp.Tensor([1.0]*100000, name = \"leo\")\n",
    "b = dp.Tensor([2.0]*100000, name = \"brad pitt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "leo\n",
      "brad pitt\n"
     ]
    }
   ],
   "source": [
    "print(a.name)\n",
    "print(b.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forward Operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add: Tensor(  shape: (100000,), numpy: ([3. 3. 3. ... 3. 3. 3.], dtype=float32), device: cpu) Mul: Tensor(  shape: (100000,), numpy: ([2. 2. 2. ... 2. 2. 2.], dtype=float32), device: cpu)\n"
     ]
    }
   ],
   "source": [
    "# Tensor Operations on CPU\n",
    "print(\"Add:\", a+b, \"Mul:\", a*b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tensor( name: leo,  shape: (100000,), numpy: ([1. 1. 1. ... 1. 1. 1.], dtype=float32), device: cpu)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Default Device is cpu\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tensor( name: brad pitt,  shape: (100000,), numpy: (<pycuda._driver.DeviceAllocation object at 0x7fcd4c6ab710>, dtype=float32), device: gpu)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# throw these Tensors in our precious GPU\n",
    "a.device('gpu') # i.e gpu:0\n",
    "b.device('gpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add them on GPU\n",
    "addition = a.add(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tensor(  shape: (100000,), numpy: (<pycuda._driver.DeviceAllocation object at 0x7fcd3c044170>, dtype=float32), device: gpu)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print the Addition\n",
    "addition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tensor(  shape: (100000,), numpy: (<pycuda._driver.DeviceAllocation object at 0x7fcd3c099bc0>, dtype=float32), device: gpu)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adds beautifully with itself!\n",
    "a.add(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(  shape: (100000,), numpy: (<pycuda._driver.DeviceAllocation object at 0x7fcd3c044ad0>, dtype=float32), device: gpu)\n"
     ]
    }
   ],
   "source": [
    "print(a + a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(  shape: (100000,), numpy: (<pycuda._driver.DeviceAllocation object at 0x7fcd3c044b70>, dtype=float32), device: gpu)\n"
     ]
    }
   ],
   "source": [
    "print(a * b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Tensor(  shape: (100000,), numpy: ([1. 1. 1. ... 1. 1. 1.], dtype=float32), device: cpu)'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repr(a.cpu())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BackPropogration is real now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tensor = dp.Tensor\n",
    "a1 = Tensor([1.0, 3.0, 1.0])\n",
    "b1 = Tensor([7.0, 3.0, 5.0])\n",
    "\n",
    "a2 = Tensor([4.0, 3.0, 1.0])\n",
    "a3 = Tensor([3.0, 3.0, 1.0])\n",
    "a4 = Tensor([7.0, 1.0, 6.0])\n",
    "b2 = Tensor([1.0, 21.0, 12.0])\n",
    "\n",
    "c = a1 * b1 + a3\n",
    "d = a2 * b2 + a4\n",
    "out = c * d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate gradients in style!!\n",
    "out.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# output gradients\n",
    "out.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradients for Tensor(  shape: (3,), numpy: ([1. 3. 1.], dtype=float32), device: cpu) tensor is : [ 77. 192.  90.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([4. 3. 1.], dtype=float32), device: cpu) tensor is : [ 10. 252.  72.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([3. 3. 1.], dtype=float32), device: cpu) tensor is : [11. 64. 18.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([7. 1. 6.], dtype=float32), device: cpu) tensor is : [10. 12.  6.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([7. 3. 5.], dtype=float32), device: cpu) tensor is : [ 11. 192.  18.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([ 1. 21. 12.], dtype=float32), device: cpu) tensor is : [40. 36.  6.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([10. 12.  6.], dtype=float32), device: cpu) tensor is : [11. 64. 18.]\n",
      "Gradients for Tensor(  shape: (3,), numpy: ([11. 64. 18.], dtype=float32), device: cpu) tensor is : [10. 12.  6.]\n"
     ]
    }
   ],
   "source": [
    "# Gradients\n",
    "_ = [print(f\"Gradients for {tensor} tensor is : {tensor.grad}\") for tensor in [a1,a2,a3,a4,b1,b2,c,d]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network is REAL!!."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lets Train our Stupid Little Model on some Stupid Little data :P"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0 loss 0.48186054825782776\n",
      "step 1 loss 0.29423007369041443\n",
      "step 2 loss 0.2337043136358261\n",
      "step 3 loss 0.16823597252368927\n",
      "step 4 loss 0.1794496327638626\n",
      "step 5 loss 0.15764467418193817\n",
      "step 6 loss 0.14568139612674713\n",
      "step 7 loss 0.1508547067642212\n",
      "step 8 loss 0.15446768701076508\n",
      "step 9 loss 0.13757599890232086\n",
      "step 10 loss 0.1331002116203308\n",
      "step 11 loss 0.12326835840940475\n",
      "step 12 loss 0.11941482126712799\n",
      "step 13 loss 0.1102059856057167\n",
      "step 14 loss 0.134320467710495\n",
      "step 15 loss 0.12006167322397232\n",
      "step 16 loss 0.1174803152680397\n",
      "step 17 loss 0.10730060189962387\n",
      "step 18 loss 0.09436262398958206\n",
      "step 19 loss 0.10174205154180527\n",
      "step 20 loss 0.10223479568958282\n",
      "step 21 loss 0.10556267946958542\n",
      "step 22 loss 0.10153289139270782\n",
      "step 23 loss 0.08809555321931839\n",
      "step 24 loss 0.10135357826948166\n",
      "step 25 loss 0.09899930655956268\n",
      "step 26 loss 0.0787389874458313\n",
      "step 27 loss 0.07799404114484787\n",
      "step 28 loss 0.07733466476202011\n",
      "step 29 loss 0.08798769116401672\n",
      "step 30 loss 0.06664631515741348\n",
      "step 31 loss 0.07865214347839355\n",
      "step 32 loss 0.07092539221048355\n",
      "step 33 loss 0.08017434179782867\n",
      "step 34 loss 0.07024269551038742\n",
      "step 35 loss 0.06159163638949394\n",
      "step 36 loss 0.0724535658955574\n",
      "step 37 loss 0.06795867532491684\n",
      "step 38 loss 0.05718230828642845\n",
      "step 39 loss 0.05364077910780907\n",
      "step 40 loss 0.06007293611764908\n",
      "step 41 loss 0.05277714878320694\n",
      "step 42 loss 0.05168500542640686\n",
      "step 43 loss 0.057570215314626694\n",
      "step 44 loss 0.05312574654817581\n",
      "step 45 loss 0.053406525403261185\n",
      "step 46 loss 0.05535206198692322\n",
      "step 47 loss 0.052276935428380966\n",
      "step 48 loss 0.05159563943743706\n",
      "step 49 loss 0.044345904141664505\n",
      "step 50 loss 0.04689960181713104\n",
      "step 51 loss 0.04907463490962982\n",
      "step 52 loss 0.05000928044319153\n",
      "step 53 loss 0.03969946503639221\n",
      "step 54 loss 0.04294203966856003\n",
      "step 55 loss 0.04305502772331238\n",
      "step 56 loss 0.04324258118867874\n",
      "step 57 loss 0.03692266717553139\n",
      "step 58 loss 0.04383039101958275\n",
      "step 59 loss 0.04361964389681816\n",
      "step 60 loss 0.036073338240385056\n",
      "step 61 loss 0.03825995326042175\n",
      "step 62 loss 0.04355667158961296\n",
      "step 63 loss 0.04123234376311302\n",
      "step 64 loss 0.03777093067765236\n",
      "step 65 loss 0.03758279234170914\n",
      "step 66 loss 0.038237880915403366\n",
      "step 67 loss 0.0375782772898674\n",
      "step 68 loss 0.032137431204319\n",
      "step 69 loss 0.030973423272371292\n",
      "step 70 loss 0.029178893193602562\n",
      "step 71 loss 0.03208909556269646\n",
      "step 72 loss 0.025956396013498306\n",
      "step 73 loss 0.025505011901259422\n",
      "step 74 loss 0.0262820553034544\n",
      "step 75 loss 0.022566834464669228\n",
      "step 76 loss 0.02899237349629402\n",
      "step 77 loss 0.035363756120204926\n",
      "step 78 loss 0.02410350926220417\n",
      "step 79 loss 0.04039918631315231\n",
      "step 80 loss 0.025140060111880302\n",
      "step 81 loss 0.029584109783172607\n",
      "step 82 loss 0.022026291117072105\n",
      "step 83 loss 0.031007347628474236\n",
      "step 84 loss 0.01793220452964306\n",
      "step 85 loss 0.02234676480293274\n",
      "step 86 loss 0.027585169300436974\n",
      "step 87 loss 0.02716807834804058\n",
      "step 88 loss 0.01876663975417614\n",
      "step 89 loss 0.024117838591337204\n",
      "step 90 loss 0.023909173905849457\n",
      "step 91 loss 0.018369650468230247\n",
      "step 92 loss 0.02398281916975975\n",
      "step 93 loss 0.024672510102391243\n",
      "step 94 loss 0.019519027322530746\n",
      "step 95 loss 0.02902217209339142\n",
      "step 96 loss 0.018936801701784134\n",
      "step 97 loss 0.016689423471689224\n",
      "step 98 loss 0.02161199413239956\n",
      "step 99 loss 0.01570112444460392\n",
      "step 100 loss 0.022218264639377594\n",
      "step 101 loss 0.018666034564375877\n",
      "step 102 loss 0.020992562174797058\n",
      "step 103 loss 0.022280743345618248\n",
      "step 104 loss 0.025737112388014793\n",
      "step 105 loss 0.021744826808571815\n",
      "step 106 loss 0.019860900938510895\n",
      "step 107 loss 0.016686031594872475\n",
      "step 108 loss 0.023608019575476646\n",
      "step 109 loss 0.023236721754074097\n",
      "step 110 loss 0.014859572052955627\n",
      "step 111 loss 0.020036738365888596\n",
      "step 112 loss 0.015608767978847027\n",
      "step 113 loss 0.017156777903437614\n",
      "step 114 loss 0.012617374770343304\n",
      "step 115 loss 0.012803110294044018\n",
      "step 116 loss 0.02021579071879387\n",
      "step 117 loss 0.014571517705917358\n",
      "step 118 loss 0.014603391289710999\n",
      "step 119 loss 0.014182818122208118\n",
      "step 120 loss 0.012017136439681053\n",
      "step 121 loss 0.01801035925745964\n",
      "step 122 loss 0.017381442710757256\n",
      "step 123 loss 0.017767850309610367\n",
      "step 124 loss 0.0144557049497962\n",
      "step 125 loss 0.013199244625866413\n",
      "step 126 loss 0.023823127150535583\n",
      "step 127 loss 0.015939272940158844\n",
      "step 128 loss 0.01371939666569233\n",
      "step 129 loss 0.01587378792464733\n",
      "step 130 loss 0.016100041568279266\n",
      "step 131 loss 0.017429189756512642\n",
      "step 132 loss 0.01231172401458025\n",
      "step 133 loss 0.013607230968773365\n",
      "step 134 loss 0.017913931980729103\n",
      "step 135 loss 0.012705321423709393\n",
      "step 136 loss 0.011423214338719845\n",
      "step 137 loss 0.01412280648946762\n",
      "step 138 loss 0.01071477122604847\n",
      "step 139 loss 0.01692202500998974\n",
      "step 140 loss 0.011943284422159195\n",
      "step 141 loss 0.013610114343464375\n",
      "step 142 loss 0.014099299907684326\n",
      "step 143 loss 0.010663812048733234\n",
      "step 144 loss 0.013744344003498554\n",
      "step 145 loss 0.014180428348481655\n",
      "step 146 loss 0.01414776872843504\n",
      "step 147 loss 0.016579221934080124\n",
      "step 148 loss 0.01282048225402832\n",
      "step 149 loss 0.01107644010335207\n",
      "step 150 loss 0.011422394774854183\n",
      "step 151 loss 0.011514252983033657\n",
      "step 152 loss 0.011016491800546646\n",
      "step 153 loss 0.008458088152110577\n",
      "step 154 loss 0.00980455707758665\n",
      "step 155 loss 0.010104513727128506\n",
      "step 156 loss 0.008958166465163231\n",
      "step 157 loss 0.010128037072718143\n",
      "step 158 loss 0.016158880665898323\n",
      "step 159 loss 0.006740364711731672\n",
      "step 160 loss 0.009508366696536541\n",
      "step 161 loss 0.011971882544457912\n",
      "step 162 loss 0.006417565047740936\n",
      "step 163 loss 0.011078565381467342\n",
      "step 164 loss 0.009322290308773518\n",
      "step 165 loss 0.009856118820607662\n",
      "step 166 loss 0.008627837523818016\n",
      "step 167 loss 0.009005378000438213\n",
      "step 168 loss 0.008816676214337349\n",
      "step 169 loss 0.00916528981178999\n",
      "step 170 loss 0.006837039720267057\n",
      "step 171 loss 0.006848090793937445\n",
      "step 172 loss 0.008214923553168774\n",
      "step 173 loss 0.012607412412762642\n",
      "step 174 loss 0.010786968283355236\n",
      "step 175 loss 0.014188592322170734\n",
      "step 176 loss 0.008018712513148785\n",
      "step 177 loss 0.007952558808028698\n",
      "step 178 loss 0.007323178928345442\n",
      "step 179 loss 0.007701800670474768\n",
      "step 180 loss 0.009349947795271873\n",
      "step 181 loss 0.009052546694874763\n",
      "step 182 loss 0.006221238989382982\n",
      "step 183 loss 0.006983480881899595\n",
      "step 184 loss 0.009304254315793514\n",
      "step 185 loss 0.007890350185334682\n",
      "step 186 loss 0.007236185949295759\n",
      "step 187 loss 0.010519965551793575\n",
      "step 188 loss 0.008806920610368252\n",
      "step 189 loss 0.009237841702997684\n",
      "step 190 loss 0.011994058266282082\n",
      "step 191 loss 0.006725033279508352\n",
      "step 192 loss 0.00851482804864645\n",
      "step 193 loss 0.003962506074458361\n",
      "step 194 loss 0.006335321813821793\n",
      "step 195 loss 0.007939282804727554\n",
      "step 196 loss 0.008657854050397873\n",
      "step 197 loss 0.007353919092565775\n",
      "step 198 loss 0.00656516570597887\n",
      "step 199 loss 0.010016491636633873\n",
      "step 200 loss 0.012127879075706005\n",
      "step 201 loss 0.005871002562344074\n",
      "step 202 loss 0.00536871375516057\n",
      "step 203 loss 0.0078404750674963\n",
      "step 204 loss 0.01194809004664421\n",
      "step 205 loss 0.010773464106023312\n",
      "step 206 loss 0.0091770114377141\n",
      "step 207 loss 0.004371661227196455\n",
      "step 208 loss 0.00561621505767107\n",
      "step 209 loss 0.00876613985747099\n",
      "step 210 loss 0.0053319744765758514\n",
      "step 211 loss 0.009258688427507877\n",
      "step 212 loss 0.005270653869956732\n",
      "step 213 loss 0.006187940016388893\n",
      "step 214 loss 0.005611987318843603\n",
      "step 215 loss 0.006373608950525522\n",
      "step 216 loss 0.005385959055274725\n",
      "step 217 loss 0.00660913810133934\n",
      "step 218 loss 0.006383187603205442\n",
      "step 219 loss 0.006587942596524954\n",
      "step 220 loss 0.004773195367306471\n",
      "step 221 loss 0.006616340018808842\n",
      "step 222 loss 0.010015738196671009\n",
      "step 223 loss 0.010027708485722542\n",
      "step 224 loss 0.010482779704034328\n",
      "step 225 loss 0.005060449242591858\n",
      "step 226 loss 0.004485534969717264\n",
      "step 227 loss 0.007630572654306889\n",
      "step 228 loss 0.003889108542352915\n",
      "step 229 loss 0.005120021291077137\n",
      "step 230 loss 0.005828004330396652\n",
      "step 231 loss 0.006027442868798971\n",
      "step 232 loss 0.004768471233546734\n",
      "step 233 loss 0.005026430822908878\n",
      "step 234 loss 0.008067016489803791\n",
      "step 235 loss 0.008617733605206013\n",
      "step 236 loss 0.005007588770240545\n",
      "step 237 loss 0.008011551573872566\n",
      "step 238 loss 0.006128623150289059\n",
      "step 239 loss 0.006782311946153641\n",
      "step 240 loss 0.005221766885370016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 241 loss 0.008275728672742844\n",
      "step 242 loss 0.004847466014325619\n",
      "step 243 loss 0.008311986923217773\n",
      "step 244 loss 0.008607068099081516\n",
      "step 245 loss 0.005021723452955484\n",
      "step 246 loss 0.005260996520519257\n",
      "step 247 loss 0.009359478950500488\n",
      "step 248 loss 0.00547941867262125\n",
      "step 249 loss 0.006907690316438675\n",
      "step 250 loss 0.005300631280988455\n",
      "step 251 loss 0.006272898055613041\n",
      "step 252 loss 0.004466293379664421\n",
      "step 253 loss 0.008529753424227238\n",
      "step 254 loss 0.006640049163252115\n",
      "step 255 loss 0.005097014829516411\n",
      "step 256 loss 0.00406254967674613\n",
      "step 257 loss 0.0050222971476614475\n",
      "step 258 loss 0.009896256029605865\n",
      "step 259 loss 0.004756319336593151\n",
      "step 260 loss 0.0033140971790999174\n",
      "step 261 loss 0.0058510019443929195\n",
      "step 262 loss 0.003263678401708603\n",
      "step 263 loss 0.007941587828099728\n",
      "step 264 loss 0.0037235193885862827\n",
      "step 265 loss 0.004746223334223032\n",
      "step 266 loss 0.006233767606317997\n",
      "step 267 loss 0.0051253014244139194\n",
      "step 268 loss 0.0032064635306596756\n",
      "step 269 loss 0.004412891808897257\n",
      "step 270 loss 0.005417747888714075\n",
      "step 271 loss 0.006864684168249369\n",
      "step 272 loss 0.005934314336627722\n",
      "step 273 loss 0.004477646201848984\n",
      "step 274 loss 0.00467973155900836\n",
      "step 275 loss 0.005198272410780191\n",
      "step 276 loss 0.0033159423619508743\n",
      "step 277 loss 0.0045433309860527515\n",
      "step 278 loss 0.0046102446503937244\n",
      "step 279 loss 0.005723667796701193\n",
      "step 280 loss 0.0043038963340222836\n",
      "step 281 loss 0.003594031324610114\n",
      "step 282 loss 0.007027001585811377\n",
      "step 283 loss 0.006326676812022924\n",
      "step 284 loss 0.006002307869493961\n",
      "step 285 loss 0.0050974260084331036\n",
      "step 286 loss 0.006932910531759262\n",
      "step 287 loss 0.004181816708296537\n",
      "step 288 loss 0.0038985670544207096\n",
      "step 289 loss 0.005171845201402903\n",
      "step 290 loss 0.005378539673984051\n",
      "step 291 loss 0.008989361114799976\n",
      "step 292 loss 0.0031545564997941256\n",
      "step 293 loss 0.003824968356639147\n",
      "step 294 loss 0.002936083357781172\n",
      "step 295 loss 0.0043373410589993\n",
      "step 296 loss 0.0027525420300662518\n",
      "step 297 loss 0.00442894222214818\n",
      "step 298 loss 0.005647654645144939\n",
      "step 299 loss 0.0018770917085930705\n",
      "step 300 loss 0.004769538529217243\n",
      "step 301 loss 0.0038765401113778353\n",
      "step 302 loss 0.00568340765312314\n",
      "step 303 loss 0.004873588681221008\n",
      "step 304 loss 0.0041623362340033054\n",
      "step 305 loss 0.003575081704184413\n",
      "step 306 loss 0.004178933799266815\n",
      "step 307 loss 0.003924079239368439\n",
      "step 308 loss 0.003383629024028778\n",
      "step 309 loss 0.0047119916416704655\n",
      "step 310 loss 0.0043400973081588745\n",
      "step 311 loss 0.004333104472607374\n",
      "step 312 loss 0.003534626914188266\n",
      "step 313 loss 0.002518213354051113\n",
      "step 314 loss 0.004340834449976683\n",
      "step 315 loss 0.0032517171930521727\n",
      "step 316 loss 0.007322787772864103\n",
      "step 317 loss 0.006180246826261282\n",
      "step 318 loss 0.0033659618347883224\n",
      "step 319 loss 0.0032819986809045076\n",
      "step 320 loss 0.003389394376426935\n",
      "step 321 loss 0.0029705860652029514\n",
      "step 322 loss 0.002976431278511882\n",
      "step 323 loss 0.00591618986800313\n",
      "step 324 loss 0.005310820881277323\n",
      "step 325 loss 0.004638656973838806\n",
      "step 326 loss 0.0034186237026005983\n",
      "step 327 loss 0.00398485129699111\n",
      "step 328 loss 0.003626373829320073\n",
      "step 329 loss 0.003630660939961672\n",
      "step 330 loss 0.005972468759864569\n",
      "step 331 loss 0.0037279699463397264\n",
      "step 332 loss 0.0027695337776094675\n",
      "step 333 loss 0.005779305938631296\n",
      "step 334 loss 0.003093323204666376\n",
      "step 335 loss 0.003212939715012908\n",
      "step 336 loss 0.0032115865033119917\n",
      "step 337 loss 0.00576289277523756\n",
      "step 338 loss 0.004163335543125868\n",
      "step 339 loss 0.005634629167616367\n",
      "step 340 loss 0.005686149932444096\n",
      "step 341 loss 0.002769807120785117\n",
      "step 342 loss 0.006738197058439255\n",
      "step 343 loss 0.0041080666705966\n",
      "step 344 loss 0.004181112628430128\n",
      "step 345 loss 0.003043847158551216\n",
      "step 346 loss 0.002727631013840437\n",
      "step 347 loss 0.004023885354399681\n",
      "step 348 loss 0.006057631690055132\n",
      "step 349 loss 0.0068441289477050304\n",
      "step 350 loss 0.003170835319906473\n",
      "step 351 loss 0.0030782469548285007\n",
      "step 352 loss 0.005297626834362745\n",
      "step 353 loss 0.0025015645660459995\n",
      "step 354 loss 0.002772912150248885\n",
      "step 355 loss 0.0031869851518422365\n",
      "step 356 loss 0.005676278378814459\n",
      "step 357 loss 0.003642599331215024\n",
      "step 358 loss 0.003203329164534807\n",
      "step 359 loss 0.0028966686222702265\n",
      "step 360 loss 0.0059000044129788876\n",
      "step 361 loss 0.0024992204271256924\n",
      "step 362 loss 0.0027677020989358425\n",
      "step 363 loss 0.002009434625506401\n",
      "step 364 loss 0.003611850319430232\n",
      "step 365 loss 0.0022886490914970636\n",
      "step 366 loss 0.004939247854053974\n",
      "step 367 loss 0.006554365158081055\n",
      "step 368 loss 0.0051462966948747635\n",
      "step 369 loss 0.0026319273747503757\n",
      "step 370 loss 0.0030816190410405397\n",
      "step 371 loss 0.002460159594193101\n",
      "step 372 loss 0.002866613445803523\n",
      "step 373 loss 0.004012309946119785\n",
      "step 374 loss 0.004735546186566353\n",
      "step 375 loss 0.004545924253761768\n",
      "step 376 loss 0.0026900521479547024\n",
      "step 377 loss 0.0032578841783106327\n",
      "step 378 loss 0.0019734736997634172\n",
      "step 379 loss 0.0026041497476398945\n",
      "step 380 loss 0.003291188972070813\n",
      "step 381 loss 0.004880488850176334\n",
      "step 382 loss 0.004622470121830702\n",
      "step 383 loss 0.004457266069948673\n",
      "step 384 loss 0.004236349370330572\n",
      "step 385 loss 0.002112405840307474\n",
      "step 386 loss 0.004096488002687693\n",
      "step 387 loss 0.001978542422875762\n",
      "step 388 loss 0.002084361622110009\n",
      "step 389 loss 0.0029263622127473354\n",
      "step 390 loss 0.004825606010854244\n",
      "step 391 loss 0.0032850808929651976\n",
      "step 392 loss 0.0026443602982908487\n",
      "step 393 loss 0.0023076494690030813\n",
      "step 394 loss 0.002589862560853362\n",
      "step 395 loss 0.0020051216706633568\n",
      "step 396 loss 0.0021209632977843285\n",
      "step 397 loss 0.0038493729662150145\n",
      "step 398 loss 0.004112515598535538\n",
      "step 399 loss 0.0021558506414294243\n",
      "step 400 loss 0.00206829234957695\n",
      "step 401 loss 0.0033494941890239716\n",
      "step 402 loss 0.002883764449506998\n",
      "step 403 loss 0.0038199410773813725\n",
      "step 404 loss 0.004234971944242716\n",
      "step 405 loss 0.002301206346601248\n",
      "step 406 loss 0.006639597471803427\n",
      "step 407 loss 0.0024455960374325514\n",
      "step 408 loss 0.0055376882664859295\n",
      "step 409 loss 0.0021980353631079197\n",
      "step 410 loss 0.004165640566498041\n",
      "step 411 loss 0.0023621453437954187\n",
      "step 412 loss 0.002632994670420885\n",
      "step 413 loss 0.0017829801654443145\n",
      "step 414 loss 0.004392986185848713\n",
      "step 415 loss 0.004873895552009344\n",
      "step 416 loss 0.0019902989733964205\n",
      "step 417 loss 0.0036833558697253466\n",
      "step 418 loss 0.002550940727815032\n",
      "step 419 loss 0.0037840858567506075\n",
      "step 420 loss 0.0019136341288685799\n",
      "step 421 loss 0.0025703790597617626\n",
      "step 422 loss 0.0034768569748848677\n",
      "step 423 loss 0.00233802804723382\n",
      "step 424 loss 0.002733499277383089\n",
      "step 425 loss 0.0022585075348615646\n",
      "step 426 loss 0.005630287807434797\n",
      "step 427 loss 0.0014780667843297124\n",
      "step 428 loss 0.0026875168550759554\n",
      "step 429 loss 0.0025692370254546404\n",
      "step 430 loss 0.002146865241229534\n",
      "step 431 loss 0.0029724673368036747\n",
      "step 432 loss 0.0031517744064331055\n",
      "step 433 loss 0.003039764240384102\n",
      "step 434 loss 0.002140916185453534\n",
      "step 435 loss 0.002470770850777626\n",
      "step 436 loss 0.002009682124480605\n",
      "step 437 loss 0.005091399885714054\n",
      "step 438 loss 0.001803098595701158\n",
      "step 439 loss 0.002996160415932536\n",
      "step 440 loss 0.0021231272257864475\n",
      "step 441 loss 0.004516237881034613\n",
      "step 442 loss 0.003017774550244212\n",
      "step 443 loss 0.0046716793440282345\n",
      "step 444 loss 0.00446179648861289\n",
      "step 445 loss 0.002476820955052972\n",
      "step 446 loss 0.0025025412905961275\n",
      "step 447 loss 0.0016462680650874972\n",
      "step 448 loss 0.0044575645588338375\n",
      "step 449 loss 0.0011898191878572106\n",
      "step 450 loss 0.0030038077384233475\n",
      "step 451 loss 0.0036889740731567144\n",
      "step 452 loss 0.0018566437065601349\n",
      "step 453 loss 0.0023923469707369804\n",
      "step 454 loss 0.002526402473449707\n",
      "step 455 loss 0.0026971199549734592\n",
      "step 456 loss 0.0016967917326837778\n",
      "step 457 loss 0.005850355606526136\n",
      "step 458 loss 0.0022987171541899443\n",
      "step 459 loss 0.0037817792035639286\n",
      "step 460 loss 0.0019402295583859086\n",
      "step 461 loss 0.0034613553434610367\n",
      "step 462 loss 0.0023725300561636686\n",
      "step 463 loss 0.0033660014159977436\n",
      "step 464 loss 0.001968093914911151\n",
      "step 465 loss 0.0017540563130751252\n",
      "step 466 loss 0.00234873965382576\n",
      "step 467 loss 0.00486487103626132\n",
      "step 468 loss 0.004048142582178116\n",
      "step 469 loss 0.0037279229145497084\n",
      "step 470 loss 0.0020432777237147093\n",
      "step 471 loss 0.0015098549192771316\n",
      "step 472 loss 0.0016288614133372903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 473 loss 0.0028883132617920637\n",
      "step 474 loss 0.0023484413977712393\n",
      "step 475 loss 0.0017243943875655532\n",
      "step 476 loss 0.003344489261507988\n",
      "step 477 loss 0.002041997155174613\n",
      "step 478 loss 0.003407067386433482\n",
      "step 479 loss 0.002036064164713025\n",
      "step 480 loss 0.003694357117637992\n",
      "step 481 loss 0.0018295006593689322\n",
      "step 482 loss 0.0035901423543691635\n",
      "step 483 loss 0.00264605856500566\n",
      "step 484 loss 0.0026865568943321705\n",
      "step 485 loss 0.0011393893510103226\n",
      "step 486 loss 0.0015593772986903787\n",
      "step 487 loss 0.0020182584412395954\n",
      "step 488 loss 0.002012305660173297\n",
      "step 489 loss 0.004488702397793531\n",
      "step 490 loss 0.0012824125587940216\n",
      "step 491 loss 0.00353572447784245\n",
      "step 492 loss 0.0011939507676288486\n",
      "step 493 loss 0.0013915677554905415\n",
      "step 494 loss 0.0011070610489696264\n",
      "step 495 loss 0.0017902288818731904\n",
      "step 496 loss 0.0024349496234208345\n",
      "step 497 loss 0.0014325309311971068\n",
      "step 498 loss 0.002332422649487853\n",
      "step 499 loss 0.002455966081470251\n",
      "step 500 loss 0.0011731934500858188\n",
      "step 501 loss 0.0012976530706509948\n",
      "step 502 loss 0.005289546679705381\n",
      "step 503 loss 0.0021511593367904425\n",
      "step 504 loss 0.0013748928904533386\n",
      "step 505 loss 0.004161388613283634\n",
      "step 506 loss 0.0014073317870497704\n",
      "step 507 loss 0.0026517834048718214\n",
      "step 508 loss 0.003503666026517749\n",
      "step 509 loss 0.0010002729250118136\n",
      "step 510 loss 0.002706544939428568\n",
      "step 511 loss 0.002595872851088643\n",
      "step 512 loss 0.0025769073981791735\n",
      "step 513 loss 0.0017658831784501672\n",
      "step 514 loss 0.0015897032571956515\n",
      "step 515 loss 0.0011056644143536687\n",
      "step 516 loss 0.0018248759442940354\n",
      "step 517 loss 0.0038701393641531467\n",
      "step 518 loss 0.001707181567326188\n",
      "step 519 loss 0.0012476579286158085\n",
      "step 520 loss 0.0014170933282002807\n",
      "step 521 loss 0.0021832638885825872\n",
      "step 522 loss 0.0011807003756985068\n",
      "step 523 loss 0.003005451988428831\n",
      "step 524 loss 0.002002895809710026\n",
      "step 525 loss 0.0022284481674432755\n",
      "step 526 loss 0.002243136055767536\n",
      "step 527 loss 0.0024445715826004744\n",
      "step 528 loss 0.0019442824413999915\n",
      "step 529 loss 0.0013759598368778825\n",
      "step 530 loss 0.0024933991953730583\n",
      "step 531 loss 0.00321892648935318\n",
      "step 532 loss 0.002457692753523588\n",
      "step 533 loss 0.0020232570823282003\n",
      "step 534 loss 0.0016775205731391907\n",
      "step 535 loss 0.0037277997471392155\n",
      "step 536 loss 0.001406882656738162\n",
      "step 537 loss 0.0014406718546524644\n",
      "step 538 loss 0.0015714900800958276\n",
      "step 539 loss 0.0045417905785143375\n",
      "step 540 loss 0.0019981972873210907\n",
      "step 541 loss 0.0014406454283744097\n",
      "step 542 loss 0.001217038370668888\n",
      "step 543 loss 0.0021880310960114002\n",
      "step 544 loss 0.0014072919730097055\n",
      "step 545 loss 0.0018192495917901397\n",
      "step 546 loss 0.002622729167342186\n",
      "step 547 loss 0.0027663246728479862\n",
      "step 548 loss 0.0023748355451971292\n",
      "step 549 loss 0.0010784025071188807\n",
      "step 550 loss 0.0028989636339247227\n",
      "step 551 loss 0.001058970927260816\n",
      "step 552 loss 0.0012231149012222886\n",
      "step 553 loss 0.0012611948186531663\n",
      "step 554 loss 0.0018799424869939685\n",
      "step 555 loss 0.0016926892567425966\n",
      "step 556 loss 0.0010872783605009317\n",
      "step 557 loss 0.0033300244249403477\n",
      "step 558 loss 0.0028571547009050846\n",
      "step 559 loss 0.0010761431185528636\n",
      "step 560 loss 0.003017156384885311\n",
      "step 561 loss 0.0028145022224634886\n",
      "step 562 loss 0.003856989089399576\n",
      "step 563 loss 0.0016713442746549845\n",
      "step 564 loss 0.001315099187195301\n",
      "step 565 loss 0.0012047101045027375\n",
      "step 566 loss 0.0015790433390066028\n",
      "step 567 loss 0.0014820731012150645\n",
      "step 568 loss 0.0011466442374512553\n",
      "step 569 loss 0.001347456593066454\n",
      "step 570 loss 0.003449968993663788\n",
      "step 571 loss 0.0018754786578938365\n",
      "step 572 loss 0.0006379940314218402\n",
      "step 573 loss 0.0015309391310438514\n",
      "step 574 loss 0.001378854620270431\n",
      "step 575 loss 0.001100224326364696\n",
      "step 576 loss 0.002198777860030532\n",
      "step 577 loss 0.002013972494751215\n",
      "step 578 loss 0.002061528852209449\n",
      "step 579 loss 0.0038730662781745195\n",
      "step 580 loss 0.001521390862762928\n",
      "step 581 loss 0.002243863185867667\n",
      "step 582 loss 0.0019887778908014297\n",
      "step 583 loss 0.0015998805174604058\n",
      "step 584 loss 0.0011624302715063095\n",
      "step 585 loss 0.002780044684186578\n",
      "step 586 loss 0.002005942864343524\n",
      "step 587 loss 0.0033732063602656126\n",
      "step 588 loss 0.0012192537542432547\n",
      "step 589 loss 0.00321243517100811\n",
      "step 590 loss 0.002812484046444297\n",
      "step 591 loss 0.0011825879337266088\n",
      "step 592 loss 0.0014655609847977757\n",
      "step 593 loss 0.0017853897297754884\n",
      "step 594 loss 0.0017031278694048524\n",
      "step 595 loss 0.0012759703677147627\n",
      "step 596 loss 0.0028324120212346315\n",
      "step 597 loss 0.0020948313176631927\n",
      "step 598 loss 0.0029343036003410816\n",
      "step 599 loss 0.0024725361727178097\n",
      "step 600 loss 0.0011160180438309908\n",
      "step 601 loss 0.002760018454864621\n",
      "step 602 loss 0.0021578921005129814\n",
      "step 603 loss 0.0017077474622055888\n",
      "step 604 loss 0.0026047504507005215\n",
      "step 605 loss 0.0012014728272333741\n",
      "step 606 loss 0.0016294853994622827\n",
      "step 607 loss 0.0008467066218145192\n",
      "step 608 loss 0.001434344332665205\n",
      "step 609 loss 0.002052796771749854\n",
      "step 610 loss 0.0031813785899430513\n",
      "step 611 loss 0.001805545180104673\n",
      "step 612 loss 0.0015123728662729263\n",
      "step 613 loss 0.0011511251796036959\n",
      "step 614 loss 0.0011271268595010042\n",
      "step 615 loss 0.0013980951625853777\n",
      "step 616 loss 0.0015806806040927768\n",
      "step 617 loss 0.0012781968107447028\n",
      "step 618 loss 0.0029182452708482742\n",
      "step 619 loss 0.0009989937534555793\n",
      "step 620 loss 0.0019547040574252605\n",
      "step 621 loss 0.0011025354033336043\n",
      "step 622 loss 0.0034659733064472675\n",
      "step 623 loss 0.0014464932028204203\n",
      "step 624 loss 0.0031156190671026707\n",
      "step 625 loss 0.0029660330619663\n",
      "step 626 loss 0.0016380645101889968\n",
      "step 627 loss 0.00086741748964414\n",
      "step 628 loss 0.0026165114250034094\n",
      "step 629 loss 0.0014897907385602593\n",
      "step 630 loss 0.001190199051052332\n",
      "step 631 loss 0.0009476404520682991\n",
      "step 632 loss 0.0014916704967617989\n",
      "step 633 loss 0.001651265542022884\n",
      "step 634 loss 0.0011405262630432844\n",
      "step 635 loss 0.0008328375406563282\n",
      "step 636 loss 0.0009816293604671955\n",
      "step 637 loss 0.0014499493408948183\n",
      "step 638 loss 0.001228940556757152\n",
      "step 639 loss 0.0015281200176104903\n",
      "step 640 loss 0.0009961130563169718\n",
      "step 641 loss 0.002373129827901721\n",
      "step 642 loss 0.0011159926652908325\n",
      "step 643 loss 0.0014792829751968384\n",
      "step 644 loss 0.002867571311071515\n",
      "step 645 loss 0.0007205841830000281\n",
      "step 646 loss 0.001378286280669272\n",
      "step 647 loss 0.0007947424310259521\n",
      "step 648 loss 0.0029483370017260313\n",
      "step 649 loss 0.000987283536233008\n",
      "step 650 loss 0.0032139942049980164\n",
      "step 651 loss 0.0013113657478243113\n",
      "step 652 loss 0.001854155445471406\n",
      "step 653 loss 0.00279806274920702\n",
      "step 654 loss 0.0017249524826183915\n",
      "step 655 loss 0.0016061330679804087\n",
      "step 656 loss 0.003257437376305461\n",
      "step 657 loss 0.0012851356295868754\n",
      "step 658 loss 0.002942682011052966\n",
      "step 659 loss 0.0017687827348709106\n",
      "step 660 loss 0.0018275972688570619\n",
      "step 661 loss 0.0013461780035868287\n",
      "step 662 loss 0.0010107069974765182\n",
      "step 663 loss 0.003291067900136113\n",
      "step 664 loss 0.0015264273388311267\n",
      "step 665 loss 0.0010418167803436518\n",
      "step 666 loss 0.00152198551222682\n",
      "step 667 loss 0.0017045466229319572\n",
      "step 668 loss 0.00156668690033257\n",
      "step 669 loss 0.0011325537925586104\n",
      "step 670 loss 0.0012692748568952084\n",
      "step 671 loss 0.002141492208465934\n",
      "step 672 loss 0.001311770873144269\n",
      "step 673 loss 0.0015669063432142138\n",
      "step 674 loss 0.0014220550656318665\n",
      "step 675 loss 0.0006317070219665766\n",
      "step 676 loss 0.0007523812237195671\n",
      "step 677 loss 0.0011772600701078773\n",
      "step 678 loss 0.0033840567339211702\n",
      "step 679 loss 0.0012077954597771168\n",
      "step 680 loss 0.0009729089215397835\n",
      "step 681 loss 0.0018726339330896735\n",
      "step 682 loss 0.0012296390486881137\n",
      "step 683 loss 0.0009437149274162948\n",
      "step 684 loss 0.0013786029303446412\n",
      "step 685 loss 0.001072658458724618\n",
      "step 686 loss 0.001734562567435205\n",
      "step 687 loss 0.0030294875614345074\n",
      "step 688 loss 0.0010307923657819629\n",
      "step 689 loss 0.001926611759699881\n",
      "step 690 loss 0.0008090133196674287\n",
      "step 691 loss 0.0007435866864398122\n",
      "step 692 loss 0.003460060805082321\n",
      "step 693 loss 0.0016185472486540675\n",
      "step 694 loss 0.0023374350275844336\n",
      "step 695 loss 0.003366874298080802\n",
      "step 696 loss 0.0011942556593567133\n",
      "step 697 loss 0.0014446842251345515\n",
      "step 698 loss 0.002352742711082101\n",
      "step 699 loss 0.0007758885622024536\n",
      "step 700 loss 0.001338080852292478\n",
      "step 701 loss 0.0009794769575819373\n",
      "step 702 loss 0.001465003821067512\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 703 loss 0.0012025512987747788\n",
      "step 704 loss 0.0012929168296977878\n",
      "step 705 loss 0.0016354777617380023\n",
      "step 706 loss 0.0014324423391371965\n",
      "step 707 loss 0.0009942324832081795\n",
      "step 708 loss 0.0009897196432575583\n",
      "step 709 loss 0.0028924604412168264\n",
      "step 710 loss 0.0015552144031971693\n",
      "step 711 loss 0.0010427499655634165\n",
      "step 712 loss 0.0030470348428934813\n",
      "step 713 loss 0.0009158325265161693\n",
      "step 714 loss 0.0008029171149246395\n",
      "step 715 loss 0.0023273031692951918\n",
      "step 716 loss 0.0018045175820589066\n",
      "step 717 loss 0.0014983590226620436\n",
      "step 718 loss 0.0008336193859577179\n",
      "step 719 loss 0.0012598865432664752\n",
      "step 720 loss 0.001057985587976873\n",
      "step 721 loss 0.0005290746921673417\n",
      "step 722 loss 0.0017961105331778526\n",
      "step 723 loss 0.0012908641947433352\n",
      "step 724 loss 0.0027661530766636133\n",
      "step 725 loss 0.0009474988910369575\n",
      "step 726 loss 0.002121723024174571\n",
      "step 727 loss 0.0014666341012343764\n",
      "step 728 loss 0.0011013713665306568\n",
      "step 729 loss 0.0010586486896499991\n",
      "step 730 loss 0.0025987261906266212\n",
      "step 731 loss 0.0009043163736350834\n",
      "step 732 loss 0.00116145599167794\n",
      "step 733 loss 0.0014526908053085208\n",
      "step 734 loss 0.0023112681228667498\n",
      "step 735 loss 0.0010176211362704635\n",
      "step 736 loss 0.0011062578996643424\n",
      "step 737 loss 0.0016508601838722825\n",
      "step 738 loss 0.0011062589474022388\n",
      "step 739 loss 0.0012263592798262835\n",
      "step 740 loss 0.0008185015176422894\n",
      "step 741 loss 0.00086137157632038\n",
      "step 742 loss 0.0011096730595454574\n",
      "step 743 loss 0.0018820938421413302\n",
      "step 744 loss 0.0009676184854470193\n",
      "step 745 loss 0.0008591741207055748\n",
      "step 746 loss 0.0019114190945401788\n",
      "step 747 loss 0.0012088434305042028\n",
      "step 748 loss 0.000821754161734134\n",
      "step 749 loss 0.0008120153215713799\n",
      "step 750 loss 0.0006993184215389192\n",
      "step 751 loss 0.0007222936255857348\n",
      "step 752 loss 0.003487332258373499\n",
      "step 753 loss 0.002661443082615733\n",
      "step 754 loss 0.0015276175690814853\n",
      "step 755 loss 0.0011345886159688234\n",
      "step 756 loss 0.0008104738080874085\n",
      "step 757 loss 0.0031031270045787096\n",
      "step 758 loss 0.0010373714612796903\n",
      "step 759 loss 0.002936952980235219\n",
      "step 760 loss 0.0026010069996118546\n",
      "step 761 loss 0.0007110388833098114\n",
      "step 762 loss 0.0011391423176974058\n",
      "step 763 loss 0.0008191452361643314\n",
      "step 764 loss 0.0026971499901264906\n",
      "step 765 loss 0.0008766446262598038\n",
      "step 766 loss 0.0012958128936588764\n",
      "step 767 loss 0.0013451485428959131\n",
      "step 768 loss 0.0007049597334116697\n",
      "step 769 loss 0.0020141375716775656\n",
      "step 770 loss 0.0015293002361431718\n",
      "step 771 loss 0.0012679988285526633\n",
      "step 772 loss 0.0013967074919492006\n",
      "step 773 loss 0.0015074522234499454\n",
      "step 774 loss 0.0013913452858105302\n",
      "step 775 loss 0.0006940928869880736\n",
      "step 776 loss 0.000960984849371016\n",
      "step 777 loss 0.001202882849611342\n",
      "step 778 loss 0.0018236648757010698\n",
      "step 779 loss 0.0009224516688846052\n",
      "step 780 loss 0.002623368753120303\n",
      "step 781 loss 0.00103910977486521\n",
      "step 782 loss 0.0020020592492073774\n",
      "step 783 loss 0.0006837546243332326\n",
      "step 784 loss 0.0011302308412268758\n",
      "step 785 loss 0.001624705153517425\n",
      "step 786 loss 0.0014481589896604419\n",
      "step 787 loss 0.001065403688699007\n",
      "step 788 loss 0.0012237699702382088\n",
      "step 789 loss 0.0010379839222878218\n",
      "step 790 loss 0.0007514987955801189\n",
      "step 791 loss 0.0009682727395556867\n",
      "step 792 loss 0.0016626695869490504\n",
      "step 793 loss 0.001515976618975401\n",
      "step 794 loss 0.002143572084605694\n",
      "step 795 loss 0.0005362762021832168\n",
      "step 796 loss 0.0009881053119897842\n",
      "step 797 loss 0.0008552718791179359\n",
      "step 798 loss 0.0010003822389990091\n",
      "step 799 loss 0.002088592853397131\n",
      "step 800 loss 0.000486534321680665\n",
      "step 801 loss 0.0007359625305980444\n",
      "step 802 loss 0.0007010574918240309\n",
      "step 803 loss 0.0014144768938422203\n",
      "step 804 loss 0.0009312574984505773\n",
      "step 805 loss 0.0008016102947294712\n",
      "step 806 loss 0.0008660883759148419\n",
      "step 807 loss 0.001230196445249021\n",
      "step 808 loss 0.001563593395985663\n",
      "step 809 loss 0.0010777873685583472\n",
      "step 810 loss 0.0008789274143055081\n",
      "step 811 loss 0.0014063107082620263\n",
      "step 812 loss 0.0007869858527556062\n",
      "step 813 loss 0.0005665904027409852\n",
      "step 814 loss 0.0011140200076624751\n",
      "step 815 loss 0.0025038504973053932\n",
      "step 816 loss 0.0010253972141072154\n",
      "step 817 loss 0.002151680178940296\n",
      "step 818 loss 0.0009943697368726134\n",
      "step 819 loss 0.0005929911858402193\n",
      "step 820 loss 0.0008047983283177018\n",
      "step 821 loss 0.0008765640086494386\n",
      "step 822 loss 0.0009240622748620808\n",
      "step 823 loss 0.001682079629972577\n",
      "step 824 loss 0.0005024916608817875\n",
      "step 825 loss 0.003598687006160617\n",
      "step 826 loss 0.00238749454729259\n",
      "step 827 loss 0.0004328889772295952\n",
      "step 828 loss 0.0010598202934488654\n",
      "step 829 loss 0.001041723764501512\n",
      "step 830 loss 0.001130752731114626\n",
      "step 831 loss 0.0013018789468333125\n",
      "step 832 loss 0.0005471035256050527\n",
      "step 833 loss 0.0005165647016838193\n",
      "step 834 loss 0.002325275680050254\n",
      "step 835 loss 0.0009519350714981556\n",
      "step 836 loss 0.0008259023888967931\n",
      "step 837 loss 0.0005779304192401469\n",
      "step 838 loss 0.0019242827547714114\n",
      "step 839 loss 0.0009088875958696008\n",
      "step 840 loss 0.0010560634545981884\n",
      "step 841 loss 0.0014458938967436552\n",
      "step 842 loss 0.0019046509405598044\n",
      "step 843 loss 0.00041688448982313275\n",
      "step 844 loss 0.0018943467875942588\n",
      "step 845 loss 0.0009802501881495118\n",
      "step 846 loss 0.0011936265509575605\n",
      "step 847 loss 0.0010281311115249991\n",
      "step 848 loss 0.0008686492219567299\n",
      "step 849 loss 0.0007908915285952389\n",
      "step 850 loss 0.001995656406506896\n",
      "step 851 loss 0.0009315955685451627\n",
      "step 852 loss 0.0006134665454737842\n",
      "step 853 loss 0.0004761580494232476\n",
      "step 854 loss 0.001525915926322341\n",
      "step 855 loss 0.0012093344703316689\n",
      "step 856 loss 0.0012024465249851346\n",
      "step 857 loss 0.0008773435838520527\n",
      "step 858 loss 0.000655791605822742\n",
      "step 859 loss 0.0024468470364809036\n",
      "step 860 loss 0.0008425400010310113\n",
      "step 861 loss 0.0021859908010810614\n",
      "step 862 loss 0.0013833044795319438\n",
      "step 863 loss 0.001085396739654243\n",
      "step 864 loss 0.001387617434374988\n",
      "step 865 loss 0.0014878431102260947\n",
      "step 866 loss 0.0008798906928859651\n",
      "step 867 loss 0.0007430875557474792\n",
      "step 868 loss 0.002279960550367832\n",
      "step 869 loss 0.00048400493687950075\n",
      "step 870 loss 0.0024682634975761175\n",
      "step 871 loss 0.0009506719070486724\n",
      "step 872 loss 0.001395872444845736\n",
      "step 873 loss 0.001655291998758912\n",
      "step 874 loss 0.0014557753456756473\n",
      "step 875 loss 0.0007795049459673464\n",
      "step 876 loss 0.0011110655032098293\n",
      "step 877 loss 0.0009135747677646577\n",
      "step 878 loss 0.0012993182754144073\n",
      "step 879 loss 0.000667076965328306\n",
      "step 880 loss 0.0008075052755884826\n",
      "step 881 loss 0.0025746887549757957\n",
      "step 882 loss 0.0012773985508829355\n",
      "step 883 loss 0.00039442634442821145\n",
      "step 884 loss 0.00048540832358412445\n",
      "step 885 loss 0.001361951814033091\n",
      "step 886 loss 0.000628235749900341\n",
      "step 887 loss 0.0005716880550608039\n",
      "step 888 loss 0.0006988360546529293\n",
      "step 889 loss 0.0009265590342693031\n",
      "step 890 loss 0.0009554166463203728\n",
      "step 891 loss 0.001226272783242166\n",
      "step 892 loss 0.0021005382295697927\n",
      "step 893 loss 0.0005691400147043169\n",
      "step 894 loss 0.0012606383534148335\n",
      "step 895 loss 0.0009954586857929826\n",
      "step 896 loss 0.002045871689915657\n",
      "step 897 loss 0.0021359168458729982\n",
      "step 898 loss 0.0006564191426150501\n",
      "step 899 loss 0.0008728205575607717\n",
      "step 900 loss 0.0009184465743601322\n",
      "step 901 loss 0.00048730309936217964\n",
      "step 902 loss 0.0008505303412675858\n",
      "step 903 loss 0.0008513603243045509\n",
      "step 904 loss 0.0006218897760845721\n",
      "step 905 loss 0.0005638272850774229\n",
      "step 906 loss 0.0009307609288953245\n",
      "step 907 loss 0.0004585850110743195\n",
      "step 908 loss 0.0007579894154332578\n",
      "step 909 loss 0.0006606436218135059\n",
      "step 910 loss 0.0027463578153401613\n",
      "step 911 loss 0.00088049954501912\n",
      "step 912 loss 0.0008022194961085916\n",
      "step 913 loss 0.00042666899389587343\n",
      "step 914 loss 0.0007011715788394213\n",
      "step 915 loss 0.0009633050649426877\n",
      "step 916 loss 0.0012162295170128345\n",
      "step 917 loss 0.0019923378713428974\n",
      "step 918 loss 0.0011121837887912989\n",
      "step 919 loss 0.0005607664934359491\n",
      "step 920 loss 0.0014000297524034977\n",
      "step 921 loss 0.0010984811233356595\n",
      "step 922 loss 0.0006378066027536988\n",
      "step 923 loss 0.000715999398380518\n",
      "step 924 loss 0.0009290989837609231\n",
      "step 925 loss 0.0009495560079813004\n",
      "step 926 loss 0.0007504602544941008\n",
      "step 927 loss 0.0010456214658915997\n",
      "step 928 loss 0.0020673281978815794\n",
      "step 929 loss 0.0007564010447822511\n",
      "step 930 loss 0.0010950510622933507\n",
      "step 931 loss 0.0023236963897943497\n",
      "step 932 loss 0.0006390601629391313\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 933 loss 0.00099033466540277\n",
      "step 934 loss 0.0005110401543788612\n",
      "step 935 loss 0.000967922795098275\n",
      "step 936 loss 0.0011489596217870712\n",
      "step 937 loss 0.002355084288865328\n",
      "step 938 loss 0.0011668786173686385\n",
      "step 939 loss 0.0005797470803372562\n",
      "step 940 loss 0.00139764288906008\n",
      "step 941 loss 0.000688600295688957\n",
      "step 942 loss 0.0007843392086215317\n",
      "step 943 loss 0.0008766269311308861\n",
      "step 944 loss 0.0006658518686890602\n",
      "step 945 loss 0.0006179582560434937\n",
      "step 946 loss 0.0021130056120455265\n",
      "step 947 loss 0.002067610854282975\n",
      "step 948 loss 0.0013243526918813586\n",
      "step 949 loss 0.001180299324914813\n",
      "step 950 loss 0.0010904271621257067\n",
      "step 951 loss 0.001049092854373157\n",
      "step 952 loss 0.0006422174046747386\n",
      "step 953 loss 0.0011146697215735912\n",
      "step 954 loss 0.0009790546027943492\n",
      "step 955 loss 0.0020292389672249556\n",
      "step 956 loss 0.0006011283840052783\n",
      "step 957 loss 0.0011117904214188457\n",
      "step 958 loss 0.0011747190728783607\n",
      "step 959 loss 0.0006692494498565793\n",
      "step 960 loss 0.002185995690524578\n",
      "step 961 loss 0.001937749795615673\n",
      "step 962 loss 0.000775714055635035\n",
      "step 963 loss 0.0009747269796207547\n",
      "step 964 loss 0.0005382904200814664\n",
      "step 965 loss 0.0019900696352124214\n",
      "step 966 loss 0.0007575526251457632\n",
      "step 967 loss 0.0010933352168649435\n",
      "step 968 loss 0.0007908193510957062\n",
      "step 969 loss 0.0010277539258822799\n",
      "step 970 loss 0.00045748805860057473\n",
      "step 971 loss 0.000530113116838038\n",
      "step 972 loss 0.0008814508910290897\n",
      "step 973 loss 0.0009818898979574442\n",
      "step 974 loss 0.0010835472494363785\n",
      "step 975 loss 0.0016618501394987106\n",
      "step 976 loss 0.0005933413631282747\n",
      "step 977 loss 0.0011849455768242478\n",
      "step 978 loss 0.000546945899259299\n",
      "step 979 loss 0.0015036340337246656\n",
      "step 980 loss 0.000695213268045336\n",
      "step 981 loss 0.0009232346201315522\n",
      "step 982 loss 0.0008396813063882291\n",
      "step 983 loss 0.0008489178726449609\n",
      "step 984 loss 0.0007107672863639891\n",
      "step 985 loss 0.0010599916568025947\n",
      "step 986 loss 0.0009041199809871614\n",
      "step 987 loss 0.0019889387767761946\n",
      "step 988 loss 0.0011052970075979829\n",
      "step 989 loss 0.0006630923016928136\n",
      "step 990 loss 0.0008315460872836411\n",
      "step 991 loss 0.0009385090670548379\n",
      "step 992 loss 0.0006585692171938717\n",
      "step 993 loss 0.0005253256531432271\n",
      "step 994 loss 0.0007199629326350987\n",
      "step 995 loss 0.0006851113284938037\n",
      "step 996 loss 0.000888007169123739\n",
      "step 997 loss 0.0005667057703249156\n",
      "step 998 loss 0.00044074529432691634\n",
      "step 999 loss 0.0007509422139264643\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "\n",
    "import deepops as dp\n",
    "from deepops.model import Model\n",
    "\n",
    "class StupidLittleModel(Model):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.dense1 = dp.layers.Dense(4, 16, activation=\"relu\", name=\"dense1\")\n",
    "        self.dense2 = dp.layers.Dense(16, 16, activation=\"relu\", name=\"dense1\")\n",
    "        self.dense3 = dp.layers.Dense(16, 1, activation=\"sigmoid\", name=\"dense2\")\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.dense1(x)\n",
    "        x = self.dense2(x)\n",
    "        x = self.dense3(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "train_X, train_y = datasets.load_iris(return_X_y=True)\n",
    "X, y = np.asarray(train_X[:100]), np.asarray(train_y[:100])\n",
    "yi = np.argwhere(y <= 1)\n",
    "y = np.reshape(y[yi], (-1))\n",
    "X = np.reshape(X[yi], (y.shape[0], -1))\n",
    "X = (X - X.min()) / (X.max() - X.min())\n",
    "X, y = np.asarray(X, np.float32), np.asarray(y, np.float32)\n",
    "\n",
    "\n",
    "def loss(y, ypred):\n",
    "    # MSE\n",
    "    _loss = [(yb - ypb) * (yb - ypb) for yb, ypb in zip(y, ypred)]\n",
    "    return sum(_loss) * dp.Tensor(1 / len(yb))\n",
    "\n",
    "\n",
    "sequential = StupidLittleModel()\n",
    "batch_size = 20\n",
    "\n",
    "\n",
    "for steps in range(1000):\n",
    "    ri = np.random.permutation(X.shape[0])[:batch_size]\n",
    "    xb = X[ri]\n",
    "    yb = y[ri]\n",
    "    xb = [list(map(dp.Tensor, x)) for x in xb]\n",
    "    # forward\n",
    "    y_pred_b = list(map(sequential.forward, xb))\n",
    "    yb = [dp.Tensor(y) for y in yb]\n",
    "    total_loss = loss(yb, y_pred_b)\n",
    "    # backward\n",
    "    sequential.init_backward()\n",
    "    total_loss.backward()\n",
    "    # mini optimizer\n",
    "    learning_rate = 0.1\n",
    "    for p in sequential.parameters():\n",
    "        p._data -= learning_rate * p.grad\n",
    "    if steps % 1 == 0:\n",
    "        print(f\"step {steps} loss {total_loss.data}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Yay Loss is decreasing..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Production Ready using serve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:     Started server process [157341]\n",
      "INFO:     Waiting for application startup.\n",
      "INFO:     Application startup complete.\n",
      "INFO:     Uvicorn running on http://127.0.0.1:5000 (Press CTRL+C to quit)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:     127.0.0.1:53434 - \"GET /predict/ HTTP/1.1\" 405 Method Not Allowed\n"
     ]
    }
   ],
   "source": [
    "import deepops as dp\n",
    "from deepops.serve import Serve, JSONResponse\n",
    "\n",
    "\n",
    "async def model(x):\n",
    "    ret = x ** 2\n",
    "    return ret\n",
    "\n",
    "\n",
    "async def predict(input):\n",
    "    try:\n",
    "        input = dp.Tensor([input])\n",
    "        ret = await model(input)\n",
    "        response = {\n",
    "            \"success\": True,\n",
    "            \"data\": {\"output\": ret.data.tolist()},\n",
    "        }\n",
    "        return response\n",
    "\n",
    "    except Exception as e:\n",
    "        error_response = {\n",
    "            \"success\": False,\n",
    "            \"message\": str(e),\n",
    "        }\n",
    "        return JSONResponse(status_code=500, content=error_response)\n",
    "\n",
    "\n",
    "serve = Serve(port=5000, host=\"127.0.0.1\")\n",
    "serve(routes={\"/predict/\": (predict, \"post\")})\n",
    "serve.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}