Visualizing attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "sys.path.insert(0, 'code/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shortcuts:\n",
    "\n",
    "* [Location based attention](#Location-based-attention)\n",
    "* [Content based attention (dot)](#Content-based-attention-(dot))\n",
    "* [Pointer networks](#Pointer-networks)\n",
    "* [Content based attention (general)](#Content-based-attention-(general))\n",
    "* [Content based attention (concatenation)](#Content-based-attention---concatenation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Location based attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import location_based_attention as attn_local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.02"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_before = []\n",
    "for _ in range(100):\n",
    "    accuracy_before.append(attn_local.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 0: 2.30559754\n",
      "Accuracy in last 100 iterations: 2/10\n",
      "Loss at iteration 100: 2.30545354\n",
      "Accuracy in last 100 iterations: 112/1000\n",
      "Loss at iteration 200: 2.24659204\n",
      "Accuracy in last 100 iterations: 137/1000\n",
      "Loss at iteration 300: 2.21842623\n",
      "Accuracy in last 100 iterations: 169/1000\n",
      "Loss at iteration 400: 2.20180821\n",
      "Accuracy in last 100 iterations: 183/1000\n",
      "Loss at iteration 500: 1.90386105\n",
      "Accuracy in last 100 iterations: 214/1000\n",
      "Loss at iteration 600: 1.78373599\n",
      "Accuracy in last 100 iterations: 246/1000\n",
      "Loss at iteration 700: 1.96263576\n",
      "Accuracy in last 100 iterations: 274/1000\n",
      "Loss at iteration 800: 1.93740916\n",
      "Accuracy in last 100 iterations: 289/1000\n",
      "Loss at iteration 900: 2.07588053\n",
      "Accuracy in last 100 iterations: 299/1000\n",
      "Loss at iteration 1000: 1.78730392\n",
      "Accuracy in last 100 iterations: 332/1000\n",
      "Loss at iteration 1100: 1.80938125\n",
      "Accuracy in last 100 iterations: 356/1000\n",
      "Loss at iteration 1200: 1.73751223\n",
      "Accuracy in last 100 iterations: 339/1000\n",
      "Loss at iteration 1300: 1.82436061\n",
      "Accuracy in last 100 iterations: 395/1000\n",
      "Loss at iteration 1400: 1.61098075\n",
      "Accuracy in last 100 iterations: 379/1000\n",
      "Loss at iteration 1500: 1.40805471\n",
      "Accuracy in last 100 iterations: 420/1000\n",
      "Loss at iteration 1600: 1.43845057\n",
      "Accuracy in last 100 iterations: 415/1000\n",
      "Loss at iteration 1700: 1.40505934\n",
      "Accuracy in last 100 iterations: 446/1000\n",
      "Loss at iteration 1800: 1.33910954\n",
      "Accuracy in last 100 iterations: 437/1000\n",
      "Loss at iteration 1900: 1.63417983\n",
      "Accuracy in last 100 iterations: 453/1000\n",
      "Loss at iteration 2000: 1.42373812\n",
      "Accuracy in last 100 iterations: 453/1000\n",
      "Loss at iteration 2100: 1.46320689\n",
      "Accuracy in last 100 iterations: 463/1000\n",
      "Loss at iteration 2200: 1.65325856\n",
      "Accuracy in last 100 iterations: 466/1000\n",
      "Loss at iteration 2300: 1.25374675\n",
      "Accuracy in last 100 iterations: 463/1000\n",
      "Loss at iteration 2400: 1.60208285\n",
      "Accuracy in last 100 iterations: 469/1000\n",
      "Loss at iteration 2500: 1.35115921\n",
      "Accuracy in last 100 iterations: 522/1000\n",
      "Loss at iteration 2600: 1.13749921\n",
      "Accuracy in last 100 iterations: 503/1000\n",
      "Loss at iteration 2700: 1.49125373\n",
      "Accuracy in last 100 iterations: 534/1000\n",
      "Loss at iteration 2800: 1.20728588\n",
      "Accuracy in last 100 iterations: 555/1000\n",
      "Loss at iteration 2900: 0.98236835\n",
      "Accuracy in last 100 iterations: 538/1000\n",
      "Loss at iteration 3000: 0.90428191\n",
      "Accuracy in last 100 iterations: 601/1000\n",
      "Loss at iteration 3100: 1.05156589\n",
      "Accuracy in last 100 iterations: 597/1000\n",
      "Loss at iteration 3200: 0.94743174\n",
      "Accuracy in last 100 iterations: 589/1000\n",
      "Loss at iteration 3300: 1.16366506\n",
      "Accuracy in last 100 iterations: 617/1000\n",
      "Loss at iteration 3400: 0.82022285\n",
      "Accuracy in last 100 iterations: 603/1000\n",
      "Loss at iteration 3500: 0.98822105\n",
      "Accuracy in last 100 iterations: 629/1000\n",
      "Loss at iteration 3600: 0.84412938\n",
      "Accuracy in last 100 iterations: 664/1000\n",
      "Loss at iteration 3700: 0.97133863\n",
      "Accuracy in last 100 iterations: 656/1000\n",
      "Loss at iteration 3800: 1.05217123\n",
      "Accuracy in last 100 iterations: 662/1000\n",
      "Loss at iteration 3900: 0.76742536\n",
      "Accuracy in last 100 iterations: 679/1000\n",
      "Loss at iteration 4000: 0.90013677\n",
      "Accuracy in last 100 iterations: 666/1000\n",
      "Loss at iteration 4100: 0.81588584\n",
      "Accuracy in last 100 iterations: 671/1000\n",
      "Loss at iteration 4200: 0.70959526\n",
      "Accuracy in last 100 iterations: 670/1000\n",
      "Loss at iteration 4300: 0.70043606\n",
      "Accuracy in last 100 iterations: 667/1000\n",
      "Loss at iteration 4400: 1.01007175\n",
      "Accuracy in last 100 iterations: 672/1000\n",
      "Loss at iteration 4500: 0.72127384\n",
      "Accuracy in last 100 iterations: 688/1000\n",
      "Loss at iteration 4600: 0.87561959\n",
      "Accuracy in last 100 iterations: 697/1000\n",
      "Loss at iteration 4700: 0.63995773\n",
      "Accuracy in last 100 iterations: 708/1000\n",
      "Loss at iteration 4800: 0.80067575\n",
      "Accuracy in last 100 iterations: 706/1000\n",
      "Loss at iteration 4900: 0.68041193\n",
      "Accuracy in last 100 iterations: 706/1000\n",
      "Loss at iteration 5000: 0.75838995\n",
      "Accuracy in last 100 iterations: 717/1000\n",
      "Loss at iteration 5100: 0.88755178\n",
      "Accuracy in last 100 iterations: 708/1000\n",
      "Loss at iteration 5200: 0.65884054\n",
      "Accuracy in last 100 iterations: 726/1000\n",
      "Loss at iteration 5300: 0.66782200\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 5400: 0.61087811\n",
      "Accuracy in last 100 iterations: 756/1000\n",
      "Loss at iteration 5500: 0.92021257\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 5600: 0.85352975\n",
      "Accuracy in last 100 iterations: 716/1000\n",
      "Loss at iteration 5700: 0.76592386\n",
      "Accuracy in last 100 iterations: 724/1000\n",
      "Loss at iteration 5800: 0.72508132\n",
      "Accuracy in last 100 iterations: 720/1000\n",
      "Loss at iteration 5900: 0.56885755\n",
      "Accuracy in last 100 iterations: 719/1000\n",
      "Loss at iteration 6000: 0.54582834\n",
      "Accuracy in last 100 iterations: 729/1000\n",
      "Loss at iteration 6100: 0.65906203\n",
      "Accuracy in last 100 iterations: 754/1000\n",
      "Loss at iteration 6200: 0.56890994\n",
      "Accuracy in last 100 iterations: 767/1000\n",
      "Loss at iteration 6300: 0.75197494\n",
      "Accuracy in last 100 iterations: 750/1000\n",
      "Loss at iteration 6400: 0.59859997\n",
      "Accuracy in last 100 iterations: 788/1000\n",
      "Loss at iteration 6500: 0.85826331\n",
      "Accuracy in last 100 iterations: 739/1000\n",
      "Loss at iteration 6600: 0.50331712\n",
      "Accuracy in last 100 iterations: 780/1000\n",
      "Loss at iteration 6700: 0.57058346\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 6800: 0.65556347\n",
      "Accuracy in last 100 iterations: 759/1000\n",
      "Loss at iteration 6900: 0.50375068\n",
      "Accuracy in last 100 iterations: 765/1000\n",
      "Loss at iteration 7000: 0.94682944\n",
      "Accuracy in last 100 iterations: 774/1000\n",
      "Loss at iteration 7100: 0.65079170\n",
      "Accuracy in last 100 iterations: 776/1000\n",
      "Loss at iteration 7200: 0.57243693\n",
      "Accuracy in last 100 iterations: 776/1000\n",
      "Loss at iteration 7300: 1.06277919\n",
      "Accuracy in last 100 iterations: 758/1000\n",
      "Loss at iteration 7400: 0.48616233\n",
      "Accuracy in last 100 iterations: 768/1000\n",
      "Loss at iteration 7500: 0.55883360\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 7600: 0.67237043\n",
      "Accuracy in last 100 iterations: 777/1000\n",
      "Loss at iteration 7700: 0.80347174\n",
      "Accuracy in last 100 iterations: 769/1000\n",
      "Loss at iteration 7800: 0.46838337\n",
      "Accuracy in last 100 iterations: 791/1000\n",
      "Loss at iteration 7900: 0.67832935\n",
      "Accuracy in last 100 iterations: 797/1000\n",
      "Loss at iteration 8000: 0.62961733\n",
      "Accuracy in last 100 iterations: 784/1000\n",
      "Loss at iteration 8100: 0.62890887\n",
      "Accuracy in last 100 iterations: 817/1000\n",
      "Loss at iteration 8200: 0.68466038\n",
      "Accuracy in last 100 iterations: 782/1000\n",
      "Loss at iteration 8300: 0.66456926\n",
      "Accuracy in last 100 iterations: 799/1000\n",
      "Loss at iteration 8400: 0.37668625\n",
      "Accuracy in last 100 iterations: 810/1000\n",
      "Loss at iteration 8500: 0.43897042\n",
      "Accuracy in last 100 iterations: 827/1000\n",
      "Loss at iteration 8600: 0.52832878\n",
      "Accuracy in last 100 iterations: 792/1000\n",
      "Loss at iteration 8700: 0.36448988\n",
      "Accuracy in last 100 iterations: 802/1000\n",
      "Loss at iteration 8800: 0.68408668\n",
      "Accuracy in last 100 iterations: 793/1000\n",
      "Loss at iteration 8900: 0.64995134\n",
      "Accuracy in last 100 iterations: 816/1000\n",
      "Loss at iteration 9000: 0.48359528\n",
      "Accuracy in last 100 iterations: 804/1000\n",
      "Loss at iteration 9100: 0.47490281\n",
      "Accuracy in last 100 iterations: 843/1000\n",
      "Loss at iteration 9200: 0.63184226\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 9300: 0.36137405\n",
      "Accuracy in last 100 iterations: 797/1000\n",
      "Loss at iteration 9400: 0.57296145\n",
      "Accuracy in last 100 iterations: 826/1000\n",
      "Loss at iteration 9500: 0.56750244\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 9600: 0.54281032\n",
      "Accuracy in last 100 iterations: 820/1000\n",
      "Loss at iteration 9700: 0.59747660\n",
      "Accuracy in last 100 iterations: 804/1000\n",
      "Loss at iteration 9800: 0.49534717\n",
      "Accuracy in last 100 iterations: 825/1000\n",
      "Loss at iteration 9900: 0.38559279\n",
      "Accuracy in last 100 iterations: 808/1000\n",
      "Loss at iteration 0: 0.58328950\n",
      "Accuracy in last 100 iterations: 9/10\n",
      "Loss at iteration 100: 0.59902561\n",
      "Accuracy in last 100 iterations: 837/1000\n",
      "Loss at iteration 200: 0.36039716\n",
      "Accuracy in last 100 iterations: 834/1000\n",
      "Loss at iteration 300: 0.41009465\n",
      "Accuracy in last 100 iterations: 818/1000\n",
      "Loss at iteration 400: 0.41658074\n",
      "Accuracy in last 100 iterations: 828/1000\n",
      "Loss at iteration 500: 0.44776243\n",
      "Accuracy in last 100 iterations: 840/1000\n",
      "Loss at iteration 600: 0.42964906\n",
      "Accuracy in last 100 iterations: 860/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 700: 0.41740829\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 800: 0.96235526\n",
      "Accuracy in last 100 iterations: 828/1000\n",
      "Loss at iteration 900: 0.49132332\n",
      "Accuracy in last 100 iterations: 812/1000\n",
      "Loss at iteration 1000: 0.30895481\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 1100: 0.47231206\n",
      "Accuracy in last 100 iterations: 834/1000\n",
      "Loss at iteration 1200: 0.32228851\n",
      "Accuracy in last 100 iterations: 861/1000\n",
      "Loss at iteration 1300: 0.32089457\n",
      "Accuracy in last 100 iterations: 831/1000\n",
      "Loss at iteration 1400: 0.49965572\n",
      "Accuracy in last 100 iterations: 826/1000\n",
      "Loss at iteration 1500: 0.28005728\n",
      "Accuracy in last 100 iterations: 850/1000\n",
      "Loss at iteration 1600: 0.47767496\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 1700: 0.25161570\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 1800: 0.34356087\n",
      "Accuracy in last 100 iterations: 837/1000\n",
      "Loss at iteration 1900: 0.48398370\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 2000: 0.33357745\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 2100: 0.26212859\n",
      "Accuracy in last 100 iterations: 840/1000\n",
      "Loss at iteration 2200: 0.24852796\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 2300: 0.40727106\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 2400: 0.52311766\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 2500: 0.23940463\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 2600: 0.41918761\n",
      "Accuracy in last 100 iterations: 853/1000\n",
      "Loss at iteration 2700: 0.40972155\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 2800: 0.35100350\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 2900: 0.27429241\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 3000: 0.32534137\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 3100: 0.35902023\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 3200: 0.25939044\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 3300: 0.70177627\n",
      "Accuracy in last 100 iterations: 861/1000\n",
      "Loss at iteration 3400: 0.59245807\n",
      "Accuracy in last 100 iterations: 825/1000\n",
      "Loss at iteration 3500: 0.46056399\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 3600: 0.34437233\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 3700: 0.37292361\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 3800: 0.39935192\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 3900: 0.41084951\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 4000: 0.33351117\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 4100: 0.22165485\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 4200: 0.19281168\n",
      "Accuracy in last 100 iterations: 889/1000\n",
      "Loss at iteration 4300: 0.41460258\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 4400: 0.41025051\n",
      "Accuracy in last 100 iterations: 872/1000\n",
      "Loss at iteration 4500: 0.54100728\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 4600: 0.28479189\n",
      "Accuracy in last 100 iterations: 892/1000\n",
      "Loss at iteration 4700: 0.55408609\n",
      "Accuracy in last 100 iterations: 844/1000\n",
      "Loss at iteration 4800: 0.50079614\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 4900: 0.24465576\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 5000: 0.39561158\n",
      "Accuracy in last 100 iterations: 858/1000\n",
      "Loss at iteration 5100: 0.24463162\n",
      "Accuracy in last 100 iterations: 849/1000\n",
      "Loss at iteration 5200: 0.80595207\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 5300: 0.21636124\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 5400: 0.58302128\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 5500: 0.25754881\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 5600: 0.30707997\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 5700: 0.28787270\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 5800: 0.18489757\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 5900: 0.21557283\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 6000: 0.35301909\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 6100: 0.55499744\n",
      "Accuracy in last 100 iterations: 865/1000\n",
      "Loss at iteration 6200: 0.50460380\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 6300: 0.12623844\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 6400: 0.39194623\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 6500: 0.20315179\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 6600: 0.19170299\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 6700: 0.48732218\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 6800: 0.18778630\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 6900: 0.29148254\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 7000: 0.24031696\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 7100: 0.25854668\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 7200: 0.33001429\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 7300: 0.29371971\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 7400: 0.19105788\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 7500: 0.22571278\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 7600: 0.52807760\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 7700: 0.38635349\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 7800: 0.28612718\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 7900: 0.12382326\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 8000: 0.24871603\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 8100: 0.34846240\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 8200: 0.77273095\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 8300: 0.26368824\n",
      "Accuracy in last 100 iterations: 893/1000\n",
      "Loss at iteration 8400: 0.21532479\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 8500: 0.25540894\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 8600: 0.13051386\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 8700: 0.51728439\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 8800: 0.30842915\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 8900: 0.69527721\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 9000: 0.16408339\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 9100: 0.35445720\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 9200: 0.26213676\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 9300: 0.51599848\n",
      "Accuracy in last 100 iterations: 902/1000\n",
      "Loss at iteration 9400: 0.14700165\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 9500: 0.40868837\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 9600: 0.43112746\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 9700: 0.12372942\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 9800: 0.33331615\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 9900: 0.12654909\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 0: 0.34260684\n",
      "Accuracy in last 100 iterations: 9/10\n",
      "Loss at iteration 100: 0.42451707\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 200: 0.63250512\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 300: 0.25889921\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 400: 0.33603334\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 500: 0.23027501\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 600: 0.50417584\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 700: 0.35049915\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 800: 0.89666462\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 900: 0.23988418\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 1000: 0.08002906\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 1100: 0.07164583\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 1200: 0.30756006\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 1300: 0.16367102\n",
      "Accuracy in last 100 iterations: 919/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1400: 0.33859554\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 1500: 0.10567121\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 1600: 0.27578241\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 1700: 0.12059097\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 1800: 0.22412291\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 1900: 0.74952769\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 2000: 0.38149667\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 2100: 0.30613551\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 2200: 0.25475416\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 2300: 0.36291188\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 2400: 0.25762624\n",
      "Accuracy in last 100 iterations: 902/1000\n",
      "Loss at iteration 2500: 0.51584321\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 2600: 0.24641418\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 2700: 0.14283124\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 2800: 0.31422657\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 2900: 0.20875970\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 3000: 0.09451494\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 3100: 0.24819832\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 3200: 0.13300237\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 3300: 0.14991012\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 3400: 0.32055092\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 3500: 0.09977045\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 3600: 0.20349999\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 3700: 0.19475403\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 3800: 0.16326265\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 3900: 0.19523735\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 4000: 0.42731756\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 4100: 0.30201301\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 4200: 0.49247783\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 4300: 0.12086649\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 4400: 0.61593425\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 4500: 0.37893376\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 4600: 0.35563153\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 4700: 0.43611726\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 4800: 0.20782837\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 4900: 0.07950401\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 5000: 0.60804850\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 5100: 0.14459372\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 5200: 0.08387280\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 5300: 0.13618450\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 5400: 0.11115141\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 5500: 0.17142430\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 5600: 0.14966574\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 5700: 0.15896435\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 5800: 0.21736899\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 5900: 0.05994730\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 6000: 0.07979164\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 6100: 0.14323536\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 6200: 0.23573041\n",
      "Accuracy in last 100 iterations: 942/1000\n",
      "Loss at iteration 6300: 0.22143622\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 6400: 0.22155218\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 6500: 0.09955359\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 6600: 0.21424341\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 6700: 0.20504418\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 6800: 0.19223890\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 6900: 0.22097349\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 7000: 0.08857679\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 7100: 0.51268810\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 7200: 0.07691918\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 7300: 0.21782342\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 7400: 0.25203362\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 7500: 0.22677431\n",
      "Accuracy in last 100 iterations: 947/1000\n",
      "Loss at iteration 7600: 0.40575433\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 7700: 0.08399034\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 7800: 0.15494537\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 7900: 0.11544981\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 8000: 0.36937475\n",
      "Accuracy in last 100 iterations: 956/1000\n",
      "Loss at iteration 8100: 0.31267539\n",
      "Accuracy in last 100 iterations: 951/1000\n",
      "Loss at iteration 8200: 0.09319238\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 8300: 0.37540102\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 8400: 0.21700907\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 8500: 0.17993531\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 8600: 0.51136458\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 8700: 0.63973010\n",
      "Accuracy in last 100 iterations: 950/1000\n",
      "Loss at iteration 8800: 0.30766767\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 8900: 0.49472150\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 9000: 0.15091220\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 9100: 0.05346079\n",
      "Accuracy in last 100 iterations: 967/1000\n",
      "Loss at iteration 9200: 0.33431950\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 9300: 0.05048098\n",
      "Accuracy in last 100 iterations: 944/1000\n",
      "Loss at iteration 9400: 0.15478162\n",
      "Accuracy in last 100 iterations: 946/1000\n",
      "Loss at iteration 9500: 0.22071305\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 9600: 0.07456694\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 9700: 0.16895466\n",
      "Accuracy in last 100 iterations: 948/1000\n",
      "Loss at iteration 9800: 0.11946192\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 9900: 0.28497583\n",
      "Accuracy in last 100 iterations: 913/1000\n"
     ]
    }
   ],
   "source": [
    "for _ in range(3):\n",
    "    attn_local.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.27"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_after = []\n",
    "for _ in range(1000):\n",
    "    accuracy_after.append(attn_local.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sequence, correct_sequence, softmax_input, accurate, attentions = attn_local.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([2, 8, 0, 3, 4, 1, 5, 6, 7, 9]),\n",
       " tensor([2, 8, 0, 3, 4, 1, 5, 6, 7, 9]),\n",
       " tensor(10))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_input.max(1)[1], correct_sequence, accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.02500517, 0.12677486, 0.13308652, 0.08586222, 0.10319298,\n",
       "        0.14779598, 0.09860509, 0.0859079 , 0.10656961, 0.08719966],\n",
       "       [0.01423313, 0.11882726, 0.11295556, 0.10472406, 0.11599562,\n",
       "        0.13844173, 0.08856905, 0.11621656, 0.10641565, 0.08362138],\n",
       "       [0.02127565, 0.05874003, 0.07832192, 0.13570651, 0.15106952,\n",
       "        0.14978892, 0.1067994 , 0.10858371, 0.11416762, 0.07554669],\n",
       "       [0.02618607, 0.08232325, 0.08272905, 0.12541908, 0.17987938,\n",
       "        0.16198073, 0.10002308, 0.10356933, 0.08874743, 0.04914261],\n",
       "       [0.01671478, 0.07781966, 0.10784865, 0.1304249 , 0.1699889 ,\n",
       "        0.13715528, 0.09333012, 0.11139732, 0.11071782, 0.04460258],\n",
       "       [0.03682594, 0.06414368, 0.08341841, 0.12646766, 0.17094119,\n",
       "        0.16492839, 0.10167723, 0.09689867, 0.11545953, 0.03923926],\n",
       "       [0.02557763, 0.09875353, 0.09843216, 0.10449509, 0.16217881,\n",
       "        0.15319407, 0.12292464, 0.11429644, 0.07950535, 0.04064223],\n",
       "       [0.02441961, 0.0956674 , 0.10414512, 0.11259831, 0.13491026,\n",
       "        0.15571435, 0.12677293, 0.12275998, 0.09440198, 0.02861011],\n",
       "       [0.02459221, 0.09543185, 0.09325962, 0.1455247 , 0.13809988,\n",
       "        0.14088765, 0.13085201, 0.11922892, 0.09160922, 0.02051399],\n",
       "       [0.02945588, 0.07813129, 0.09695145, 0.15871905, 0.13344808,\n",
       "        0.13612477, 0.10858984, 0.11808057, 0.10101046, 0.03948865]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([a[0,0].detach().numpy() for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(x):\n",
    "    return (x - x.min())/(x.max() - x.min())\n",
    "\n",
    "np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])\n",
    "\n",
    "attention_normalized = np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x122cf0048>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAC8pJREFUeJzt3cuLXvUdx/HPJzOdZJI2V4sht3phiB0CxTAU24gLddHWUjcRokSom2zqLQqi3fgPSGkXpTiYdlOpi+giiGgLSReiBMeopMlEiZdqxlyRSSQYkyHfLmYKUZN5zji/n2eeL+8XCJnx5OuXMG/PeZ45c+KIEICc5rW9AIB6CBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxHprDLVd5fa4wcHBGmPV09NTfOapU6eKz5SkBQsWVJm7fPnyKnNPnjxZZe6FCxeKz+zv7y8+U5LOnTtXfOb4+LjOnj3rTse5xq2qtQLfv39/jbFavHhx8Zk7duwoPlOSBgYGqszdunVrlbnDw8NV5o6NjRWfuWHDhuIzJWl0dLT4zKefflpjY2MdA+cSHUiMwIHECBxIjMCBxAgcSIzAgcQaBW77F7bftX3Y9uO1lwJQRsfAbfdI+rOkX0oalHS37Tp3nAAoqskZ/KeSDkfEBxFxXtJzku6suxaAEpoEvlrSJ5d8fGTqc19he5vtEdsjpZYDMDtN7kW/3O1w37gVNSKGJQ1L9W5VBTAzTc7gRyStveTjNZI+rbMOgJKaBP6GpAHb19ruk7RF0q66awEooeMlekRM2L5f0iuSeiT9NSIOVN8MwKw1+nnwiHhJ0kuVdwFQGHeyAYkROJAYgQOJETiQGIEDiVV5qmotvb111j1x4kTxmTfffHPxmZJ09dVXV5nbbTZv3lx8Zq0nwNZ4qOe8ec3OzZzBgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHEuuqpqo888kiVuevXry8+c/v27cVnStK6deuqzL148WKVuUuWLKky97333is+c9GiRcVnStLhw4eLz/zyyy8bHccZHEiMwIHECBxIjMCBxAgcSIzAgcQ6Bm57re09tkdtH7D90HexGIDZa/J98AlJj0bEPts/kPSm7X9FxMHKuwGYpY5n8Ig4GhH7pn79uaRRSatrLwZg9mb0Gtz2NZJulLS3xjIAymp8q6rt70t6XtLDEXHmMv9+m6RtBXcDMEuNArf9PU3G/WxEvHC5YyJiWNLw1PFRbEMA31qTd9EtaYek0Yj4Q/2VAJTS5DX4Jkn3SrrV9ttT//yq8l4ACuh4iR4Rr0ryd7ALgMK4kw1IjMCBxAgcSIzAgcQIHEisqx66eNVVV1WZ+9lnnxWf+eqrrxafKUn33HNPlbljY2NV5r7zzjtV5s6fP7/4zGPHjhWfKUm7d+8uPvPMmW/cTHpZnMGBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcS66qmq/f39VeZu3Lix+MyVK1cWn1nT0aNHq8zt6+urMnfdunXFZ65du7b4TEl67bXXis+cN6/ZuZkzOJAYgQOJETiQGIEDiRE4kBiBA4kROJBY48Bt99h+y/aLNRcCUM5MzuAPSRqttQiA8hoFbnuNpDskPVN3HQAlNT2D/1HSY5IuXukA29tsj9geKbIZgFnrGLjtX0s6ERFvTndcRAxHxFBEDBXbDsCsNDmDb5L0G9sfSXpO0q22/151KwBFdAw8Ip6IiDURcY2kLZJ2R8TW6psBmDW+Dw4kNqOfB4+If0v6d5VNABTHGRxIjMCBxAgcSIzAgcQIHEisq56qum/fvipzz5w5U3zmihUris+saXBwsMrcXbt2VZm7f//+4jMHBgaKz5SkgwcPFp85MTHR6DjO4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kRuBAYl31VNW77rqrytyIKD7z2LFjxWfW1PQpnTO1adOmKnPHx8eLz9yzZ0/xmZK0cuXK4jOPHz/e6DjO4EBiBA4kRuBAYgQOJEbgQGIEDiTWKHDbS23vtH3I9qjtn9VeDMDsNf0++J8kvRwRm233SVpYcScAhXQM3PZiSbdI+q0kRcR5SefrrgWghCaX6NdJOinpb7bfsv2M7UWV9wJQQJPAeyVtlPSXiLhR0llJj3/9INvbbI/YHim8I4BvqUngRyQdiYi9Ux/v1GTwXxERwxExFBFDJRcE8O11DDwijkn6xPb6qU/dJulg1a0AFNH0XfQHJD079Q76B5Luq7cSgFIaBR4Rb0vi0hvoMtzJBiRG4EBiBA4kRuBAYgQOJEbgQGJd9VRV21XmLl++vPjMpUuXFp8pSYcOHaoy94Ybbqgyd9myZVXm9vf3F595/fXXF58pSadOnSo+s+lTcDmDA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJBYlYcu2tb8+fOLz12yZEnxmZJ07ty54jM3b95cfKYknT59usrcL774osrcVatWVZl7/Pjx4jNrPMhRqvP11RRncCAxAgcSI3AgMQIHEiNwIDECBxIjcCCxRoHb3m77gO3/2P6H7QW1FwMwex0Dt71a0oOShiJig6QeSVtqLwZg9ppeovdK6rfdK2mhpE/rrQSglI6BR8SYpKckfSzpqKTTEfHPrx9ne5vtEdsjEVF+UwAz1uQSfZmkOyVdK2mVpEW2t379uIgYjoihiBiyXX5TADPW5BL9dkkfRsTJiLgg6QVJP6+7FoASmgT+saSbbC/05Kn5NkmjddcCUEKT1+B7Je2UtE/S/qnfM1x5LwAFNPp58Ih4UtKTlXcBUBh3sgGJETiQGIEDiRE4kBiBA4lVeapqRFR5kuTrr79efKYkDQwMVJlbw/vvv19lbl9fX5W5e/furTJ3xYoVxWdOTEwUnynV2XV8fLzRcZzBgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHEHBHlh9onJf23waFXSTpVfIF6umnfbtpV6q5958KuP4qIH3Y6qErgTdkeiYih1haYoW7at5t2lbpr327alUt0IDECBxJrO/Dhlv/7M9VN+3bTrlJ37ds1u7b6GhxAXW2fwQFU1Frgtn9h+13bh20/3tYendhea3uP7VHbB2w/1PZOTdjusf2W7Rfb3mU6tpfa3mn70NSf8c/a3mk6trdPfR38x/Y/bC9oe6fptBK47R5Jf5b0S0mDku62PdjGLg1MSHo0In4s6SZJv5vDu17qIUmjbS/RwJ8kvRwRN0j6iebwzrZXS3pQ0lBEbJDUI2lLu1tNr60z+E8lHY6IDyLivKTnJN3Z0i7TioijEbFv6tefa/ILcHW7W03P9hpJd0h6pu1dpmN7saRbJO2QpIg4HxHN/l7c9vRK6rfdK2mhpE9b3mdabQW+WtInl3x8RHM8GkmyfY2kGyXV+Uuvy/mjpMckXWx7kQ6uk3RS0t+mXk48Y3tR20tdSUSMSXpK0seSjko6HRH/bHer6bUVuC/zuTn9dr7t70t6XtLDEXGm7X2uxPavJZ2IiDfb3qWBXkkbJf0lIm6UdFbSXH4/ZpkmrzSvlbRK0iLbW9vdanptBX5E0tpLPl6jOXypY/t7moz72Yh4oe19Otgk6Te2P9LkS59bbf+93ZWu6IikIxHx/yuinZoMfq66XdKHEXEyIi5IekHSz1veaVptBf6GpAHb19ru0+QbFbta2mVatq3J14ijEfGHtvfpJCKeiIg1EXGNJv9cd0fEnDzLRMQxSZ/YXj/1qdskHWxxpU4+lnST7YVTXxe3aQ6/KShNXiJ95yJiwvb9kl7R5DuRf42IA23s0sAmSfdK2m/77anP/T4iXmpxp0wekPTs1P/oP5B0X8v7XFFE7LW9U9I+TX535S3N8bvauJMNSIw72YDECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxI7H9cO47zfRpzpQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(attention_normalized, 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 5, 0, 3, 4, 6, 7, 8, 1, 9]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Content based attention (dot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import content_based_dot_attention as attn_dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.01"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_before = []\n",
    "for _ in range(100):\n",
    "    accuracy_before.append(attn_dot.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 0: 2.30426860\n",
      "Accuracy in last 100 iterations: 1/10\n",
      "Loss at iteration 100: 2.28844380\n",
      "Accuracy in last 100 iterations: 100/1000\n",
      "Loss at iteration 200: 2.31006098\n",
      "Accuracy in last 100 iterations: 134/1000\n",
      "Loss at iteration 300: 2.06813049\n",
      "Accuracy in last 100 iterations: 142/1000\n",
      "Loss at iteration 400: 2.04553890\n",
      "Accuracy in last 100 iterations: 175/1000\n",
      "Loss at iteration 500: 2.12632918\n",
      "Accuracy in last 100 iterations: 177/1000\n",
      "Loss at iteration 600: 1.99214339\n",
      "Accuracy in last 100 iterations: 204/1000\n",
      "Loss at iteration 700: 2.09486055\n",
      "Accuracy in last 100 iterations: 236/1000\n",
      "Loss at iteration 800: 2.04562330\n",
      "Accuracy in last 100 iterations: 254/1000\n",
      "Loss at iteration 900: 2.15844464\n",
      "Accuracy in last 100 iterations: 241/1000\n",
      "Loss at iteration 1000: 1.87164283\n",
      "Accuracy in last 100 iterations: 266/1000\n",
      "Loss at iteration 1100: 1.77767849\n",
      "Accuracy in last 100 iterations: 292/1000\n",
      "Loss at iteration 1200: 2.12895489\n",
      "Accuracy in last 100 iterations: 285/1000\n",
      "Loss at iteration 1300: 1.65386462\n",
      "Accuracy in last 100 iterations: 309/1000\n",
      "Loss at iteration 1400: 1.94212842\n",
      "Accuracy in last 100 iterations: 326/1000\n",
      "Loss at iteration 1500: 1.68327546\n",
      "Accuracy in last 100 iterations: 333/1000\n",
      "Loss at iteration 1600: 1.76358378\n",
      "Accuracy in last 100 iterations: 347/1000\n",
      "Loss at iteration 1700: 1.61120284\n",
      "Accuracy in last 100 iterations: 384/1000\n",
      "Loss at iteration 1800: 1.63799059\n",
      "Accuracy in last 100 iterations: 378/1000\n",
      "Loss at iteration 1900: 1.72075820\n",
      "Accuracy in last 100 iterations: 372/1000\n",
      "Loss at iteration 2000: 1.76050091\n",
      "Accuracy in last 100 iterations: 391/1000\n",
      "Loss at iteration 2100: 1.48831689\n",
      "Accuracy in last 100 iterations: 408/1000\n",
      "Loss at iteration 2200: 1.49673963\n",
      "Accuracy in last 100 iterations: 407/1000\n",
      "Loss at iteration 2300: 1.43961275\n",
      "Accuracy in last 100 iterations: 453/1000\n",
      "Loss at iteration 2400: 1.28305006\n",
      "Accuracy in last 100 iterations: 428/1000\n",
      "Loss at iteration 2500: 1.38300729\n",
      "Accuracy in last 100 iterations: 451/1000\n",
      "Loss at iteration 2600: 1.47508872\n",
      "Accuracy in last 100 iterations: 463/1000\n",
      "Loss at iteration 2700: 1.49841785\n",
      "Accuracy in last 100 iterations: 479/1000\n",
      "Loss at iteration 2800: 1.10186648\n",
      "Accuracy in last 100 iterations: 492/1000\n",
      "Loss at iteration 2900: 1.29417300\n",
      "Accuracy in last 100 iterations: 471/1000\n",
      "Loss at iteration 3000: 1.36506820\n",
      "Accuracy in last 100 iterations: 478/1000\n",
      "Loss at iteration 3100: 1.29852462\n",
      "Accuracy in last 100 iterations: 501/1000\n",
      "Loss at iteration 3200: 1.10437357\n",
      "Accuracy in last 100 iterations: 521/1000\n",
      "Loss at iteration 3300: 1.08731341\n",
      "Accuracy in last 100 iterations: 540/1000\n",
      "Loss at iteration 3400: 1.06317401\n",
      "Accuracy in last 100 iterations: 537/1000\n",
      "Loss at iteration 3500: 1.32042909\n",
      "Accuracy in last 100 iterations: 527/1000\n",
      "Loss at iteration 3600: 1.14607704\n",
      "Accuracy in last 100 iterations: 553/1000\n",
      "Loss at iteration 3700: 1.39010739\n",
      "Accuracy in last 100 iterations: 565/1000\n",
      "Loss at iteration 3800: 0.85770577\n",
      "Accuracy in last 100 iterations: 587/1000\n",
      "Loss at iteration 3900: 1.17323327\n",
      "Accuracy in last 100 iterations: 603/1000\n",
      "Loss at iteration 4000: 0.93283069\n",
      "Accuracy in last 100 iterations: 603/1000\n",
      "Loss at iteration 4100: 1.25485599\n",
      "Accuracy in last 100 iterations: 576/1000\n",
      "Loss at iteration 4200: 1.12896097\n",
      "Accuracy in last 100 iterations: 603/1000\n",
      "Loss at iteration 4300: 0.83579379\n",
      "Accuracy in last 100 iterations: 608/1000\n",
      "Loss at iteration 4400: 1.20907688\n",
      "Accuracy in last 100 iterations: 606/1000\n",
      "Loss at iteration 4500: 0.87520772\n",
      "Accuracy in last 100 iterations: 641/1000\n",
      "Loss at iteration 4600: 1.50305104\n",
      "Accuracy in last 100 iterations: 613/1000\n",
      "Loss at iteration 4700: 0.88585341\n",
      "Accuracy in last 100 iterations: 640/1000\n",
      "Loss at iteration 4800: 0.85920155\n",
      "Accuracy in last 100 iterations: 662/1000\n",
      "Loss at iteration 4900: 0.78381968\n",
      "Accuracy in last 100 iterations: 662/1000\n",
      "Loss at iteration 5000: 0.75558603\n",
      "Accuracy in last 100 iterations: 695/1000\n",
      "Loss at iteration 5100: 0.81053954\n",
      "Accuracy in last 100 iterations: 689/1000\n",
      "Loss at iteration 5200: 0.84877414\n",
      "Accuracy in last 100 iterations: 659/1000\n",
      "Loss at iteration 5300: 1.38961422\n",
      "Accuracy in last 100 iterations: 657/1000\n",
      "Loss at iteration 5400: 0.76013321\n",
      "Accuracy in last 100 iterations: 669/1000\n",
      "Loss at iteration 5500: 0.89919984\n",
      "Accuracy in last 100 iterations: 671/1000\n",
      "Loss at iteration 5600: 0.69289219\n",
      "Accuracy in last 100 iterations: 721/1000\n",
      "Loss at iteration 5700: 0.77164876\n",
      "Accuracy in last 100 iterations: 693/1000\n",
      "Loss at iteration 5800: 1.12241435\n",
      "Accuracy in last 100 iterations: 704/1000\n",
      "Loss at iteration 5900: 0.86906689\n",
      "Accuracy in last 100 iterations: 723/1000\n",
      "Loss at iteration 6000: 0.93848741\n",
      "Accuracy in last 100 iterations: 660/1000\n",
      "Loss at iteration 6100: 0.69857293\n",
      "Accuracy in last 100 iterations: 737/1000\n",
      "Loss at iteration 6200: 0.96328229\n",
      "Accuracy in last 100 iterations: 718/1000\n",
      "Loss at iteration 6300: 0.59627020\n",
      "Accuracy in last 100 iterations: 721/1000\n",
      "Loss at iteration 6400: 0.69889420\n",
      "Accuracy in last 100 iterations: 742/1000\n",
      "Loss at iteration 6500: 0.69700664\n",
      "Accuracy in last 100 iterations: 740/1000\n",
      "Loss at iteration 6600: 0.74116230\n",
      "Accuracy in last 100 iterations: 740/1000\n",
      "Loss at iteration 6700: 0.57909769\n",
      "Accuracy in last 100 iterations: 732/1000\n",
      "Loss at iteration 6800: 0.67406869\n",
      "Accuracy in last 100 iterations: 711/1000\n",
      "Loss at iteration 6900: 0.90506804\n",
      "Accuracy in last 100 iterations: 768/1000\n",
      "Loss at iteration 7000: 0.65843326\n",
      "Accuracy in last 100 iterations: 750/1000\n",
      "Loss at iteration 7100: 0.62942165\n",
      "Accuracy in last 100 iterations: 740/1000\n",
      "Loss at iteration 7200: 0.74291813\n",
      "Accuracy in last 100 iterations: 768/1000\n",
      "Loss at iteration 7300: 0.45570523\n",
      "Accuracy in last 100 iterations: 788/1000\n",
      "Loss at iteration 7400: 0.51765966\n",
      "Accuracy in last 100 iterations: 769/1000\n",
      "Loss at iteration 7500: 0.76422828\n",
      "Accuracy in last 100 iterations: 760/1000\n",
      "Loss at iteration 7600: 0.57607061\n",
      "Accuracy in last 100 iterations: 765/1000\n",
      "Loss at iteration 7700: 0.52149016\n",
      "Accuracy in last 100 iterations: 774/1000\n",
      "Loss at iteration 7800: 0.54467499\n",
      "Accuracy in last 100 iterations: 760/1000\n",
      "Loss at iteration 7900: 0.79659724\n",
      "Accuracy in last 100 iterations: 781/1000\n",
      "Loss at iteration 8000: 0.49094754\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 8100: 0.43145475\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 8200: 0.82935417\n",
      "Accuracy in last 100 iterations: 782/1000\n",
      "Loss at iteration 8300: 0.52845997\n",
      "Accuracy in last 100 iterations: 784/1000\n",
      "Loss at iteration 8400: 0.43075505\n",
      "Accuracy in last 100 iterations: 801/1000\n",
      "Loss at iteration 8500: 0.54459924\n",
      "Accuracy in last 100 iterations: 795/1000\n",
      "Loss at iteration 8600: 0.69082898\n",
      "Accuracy in last 100 iterations: 773/1000\n",
      "Loss at iteration 8700: 0.62104785\n",
      "Accuracy in last 100 iterations: 779/1000\n",
      "Loss at iteration 8800: 0.77401578\n",
      "Accuracy in last 100 iterations: 767/1000\n",
      "Loss at iteration 8900: 0.95238668\n",
      "Accuracy in last 100 iterations: 795/1000\n",
      "Loss at iteration 9000: 0.49944073\n",
      "Accuracy in last 100 iterations: 820/1000\n",
      "Loss at iteration 9100: 0.52913415\n",
      "Accuracy in last 100 iterations: 761/1000\n",
      "Loss at iteration 9200: 0.46819657\n",
      "Accuracy in last 100 iterations: 774/1000\n",
      "Loss at iteration 9300: 0.64458090\n",
      "Accuracy in last 100 iterations: 776/1000\n",
      "Loss at iteration 9400: 0.64222455\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 9500: 0.48819679\n",
      "Accuracy in last 100 iterations: 813/1000\n",
      "Loss at iteration 9600: 0.53960484\n",
      "Accuracy in last 100 iterations: 807/1000\n",
      "Loss at iteration 9700: 0.67292595\n",
      "Accuracy in last 100 iterations: 788/1000\n",
      "Loss at iteration 9800: 0.64877689\n",
      "Accuracy in last 100 iterations: 791/1000\n",
      "Loss at iteration 9900: 0.37999979\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 0: 0.61031640\n",
      "Accuracy in last 100 iterations: 7/10\n",
      "Loss at iteration 100: 0.56207836\n",
      "Accuracy in last 100 iterations: 810/1000\n",
      "Loss at iteration 200: 0.56661665\n",
      "Accuracy in last 100 iterations: 814/1000\n",
      "Loss at iteration 300: 0.69333714\n",
      "Accuracy in last 100 iterations: 809/1000\n",
      "Loss at iteration 400: 0.60255504\n",
      "Accuracy in last 100 iterations: 808/1000\n",
      "Loss at iteration 500: 0.34181017\n",
      "Accuracy in last 100 iterations: 835/1000\n",
      "Loss at iteration 600: 0.67199695\n",
      "Accuracy in last 100 iterations: 793/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 700: 0.52225518\n",
      "Accuracy in last 100 iterations: 831/1000\n",
      "Loss at iteration 800: 0.46359435\n",
      "Accuracy in last 100 iterations: 815/1000\n",
      "Loss at iteration 900: 0.66224509\n",
      "Accuracy in last 100 iterations: 824/1000\n",
      "Loss at iteration 1000: 0.47919297\n",
      "Accuracy in last 100 iterations: 844/1000\n",
      "Loss at iteration 1100: 0.57544416\n",
      "Accuracy in last 100 iterations: 820/1000\n",
      "Loss at iteration 1200: 0.45685250\n",
      "Accuracy in last 100 iterations: 800/1000\n",
      "Loss at iteration 1300: 0.57029414\n",
      "Accuracy in last 100 iterations: 818/1000\n",
      "Loss at iteration 1400: 0.65509033\n",
      "Accuracy in last 100 iterations: 803/1000\n",
      "Loss at iteration 1500: 0.52400881\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 1600: 0.50158513\n",
      "Accuracy in last 100 iterations: 834/1000\n",
      "Loss at iteration 1700: 0.42724234\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 1800: 0.43560082\n",
      "Accuracy in last 100 iterations: 828/1000\n",
      "Loss at iteration 1900: 0.53793609\n",
      "Accuracy in last 100 iterations: 853/1000\n",
      "Loss at iteration 2000: 0.38761300\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 2100: 0.49786791\n",
      "Accuracy in last 100 iterations: 834/1000\n",
      "Loss at iteration 2200: 0.65663421\n",
      "Accuracy in last 100 iterations: 841/1000\n",
      "Loss at iteration 2300: 0.57080257\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 2400: 0.34158367\n",
      "Accuracy in last 100 iterations: 816/1000\n",
      "Loss at iteration 2500: 0.35592836\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 2600: 0.91992569\n",
      "Accuracy in last 100 iterations: 832/1000\n",
      "Loss at iteration 2700: 0.41561618\n",
      "Accuracy in last 100 iterations: 850/1000\n",
      "Loss at iteration 2800: 0.39435253\n",
      "Accuracy in last 100 iterations: 865/1000\n",
      "Loss at iteration 2900: 0.37831631\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 3000: 0.28887638\n",
      "Accuracy in last 100 iterations: 892/1000\n",
      "Loss at iteration 3100: 0.76559150\n",
      "Accuracy in last 100 iterations: 862/1000\n",
      "Loss at iteration 3200: 0.30992284\n",
      "Accuracy in last 100 iterations: 861/1000\n",
      "Loss at iteration 3300: 0.35981354\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 3400: 0.23029880\n",
      "Accuracy in last 100 iterations: 829/1000\n",
      "Loss at iteration 3500: 0.57519931\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 3600: 0.34511223\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 3700: 0.50889069\n",
      "Accuracy in last 100 iterations: 817/1000\n",
      "Loss at iteration 3800: 0.27792257\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 3900: 0.28200465\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 4000: 0.34298688\n",
      "Accuracy in last 100 iterations: 853/1000\n",
      "Loss at iteration 4100: 0.31283721\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 4200: 0.23591128\n",
      "Accuracy in last 100 iterations: 846/1000\n",
      "Loss at iteration 4300: 0.24365549\n",
      "Accuracy in last 100 iterations: 849/1000\n",
      "Loss at iteration 4400: 0.35887975\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 4500: 0.33255106\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 4600: 0.50061649\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 4700: 0.38821530\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 4800: 0.23453188\n",
      "Accuracy in last 100 iterations: 889/1000\n",
      "Loss at iteration 4900: 0.35804844\n",
      "Accuracy in last 100 iterations: 879/1000\n",
      "Loss at iteration 5000: 0.34238118\n",
      "Accuracy in last 100 iterations: 858/1000\n",
      "Loss at iteration 5100: 0.26076898\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 5200: 0.21986660\n",
      "Accuracy in last 100 iterations: 846/1000\n",
      "Loss at iteration 5300: 0.26596275\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 5400: 0.46078762\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 5500: 0.20962009\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 5600: 0.38887089\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 5700: 0.34139881\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 5800: 0.39204121\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 5900: 0.54907769\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 6000: 0.44373816\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 6100: 0.26737586\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 6200: 0.33604163\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 6300: 0.33985671\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 6400: 0.39101228\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 6500: 0.27186006\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 6600: 0.25382385\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 6700: 0.32251590\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 6800: 0.18932323\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 6900: 0.18611507\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 7000: 0.25772843\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 7100: 0.22983904\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 7200: 0.28308144\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 7300: 0.37585697\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 7400: 0.19593887\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 7500: 0.38364410\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 7600: 0.75102979\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 7700: 0.38582811\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 7800: 0.19467005\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 7900: 0.47639862\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 8000: 0.50254762\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 8100: 0.21684532\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 8200: 0.79026592\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 8300: 0.21849570\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 8400: 0.24720006\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 8500: 0.27946788\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 8600: 0.29527241\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 8700: 0.52134609\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 8800: 0.23111658\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 8900: 0.18068066\n",
      "Accuracy in last 100 iterations: 880/1000\n",
      "Loss at iteration 9000: 0.11021171\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 9100: 0.17180414\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 9200: 0.68191957\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 9300: 0.54785681\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 9400: 0.20756149\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 9500: 0.22143078\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 9600: 0.29437131\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 9700: 0.11600532\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 9800: 0.45130739\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 9900: 0.50591534\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 0: 0.43888998\n",
      "Accuracy in last 100 iterations: 8/10\n",
      "Loss at iteration 100: 0.17310190\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 200: 0.10988178\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 300: 0.35720474\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 400: 0.15357618\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 500: 0.55782378\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 600: 0.28980693\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 700: 0.45402089\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 800: 0.39977035\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 900: 0.22765465\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 1000: 0.11582494\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 1100: 0.13536663\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 1200: 0.13467188\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 1300: 0.19857445\n",
      "Accuracy in last 100 iterations: 906/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1400: 0.19422288\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 1500: 0.25596395\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 1600: 0.57701194\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 1700: 0.44495326\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 1800: 0.18556948\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 1900: 0.23801985\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 2000: 0.13537665\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 2100: 0.12306385\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 2200: 0.23225126\n",
      "Accuracy in last 100 iterations: 892/1000\n",
      "Loss at iteration 2300: 0.42526332\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 2400: 0.10016946\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 2500: 0.20608358\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 2600: 0.21780267\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 2700: 0.07937107\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 2800: 0.50701272\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 2900: 0.17276955\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 3000: 0.21527243\n",
      "Accuracy in last 100 iterations: 891/1000\n",
      "Loss at iteration 3100: 0.37925404\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 3200: 0.10787086\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 3300: 0.15004483\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 3400: 0.26841575\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 3500: 0.11267719\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 3600: 0.51063645\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 3700: 0.12277851\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 3800: 0.19061013\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 3900: 0.18639517\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 4000: 0.12377720\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 4100: 0.74198884\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 4200: 0.16565236\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 4300: 0.15035287\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 4400: 0.29607105\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 4500: 0.23987404\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 4600: 0.08768864\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 4700: 0.09454212\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 4800: 0.09614706\n",
      "Accuracy in last 100 iterations: 928/1000\n",
      "Loss at iteration 4900: 0.19799443\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 5000: 0.34996796\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 5100: 0.35075054\n",
      "Accuracy in last 100 iterations: 902/1000\n",
      "Loss at iteration 5200: 0.11464195\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 5300: 0.21743932\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 5400: 0.25113592\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 5500: 0.27330789\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 5600: 0.15556097\n",
      "Accuracy in last 100 iterations: 931/1000\n",
      "Loss at iteration 5700: 0.24184999\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 5800: 0.42055291\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 5900: 0.29793826\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 6000: 0.56165838\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 6100: 0.20933485\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 6200: 0.26751909\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 6300: 0.44018134\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 6400: 0.13385591\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 6500: 0.28532267\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 6600: 0.25214654\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 6700: 0.33064285\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 6800: 0.14163437\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 6900: 0.11354876\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 7000: 0.20507249\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 7100: 0.42504272\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 7200: 0.25947315\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 7300: 0.59830463\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 7400: 0.14563064\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 7500: 0.08865061\n",
      "Accuracy in last 100 iterations: 949/1000\n",
      "Loss at iteration 7600: 0.26540375\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 7700: 0.08142767\n",
      "Accuracy in last 100 iterations: 948/1000\n",
      "Loss at iteration 7800: 0.28021002\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 7900: 0.09654455\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 8000: 0.06730652\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 8100: 0.11108828\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 8200: 0.09012432\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 8300: 0.35469073\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 8400: 0.16717348\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 8500: 0.17310515\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 8600: 0.06228304\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 8700: 0.21360493\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 8800: 0.22725877\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 8900: 0.33174238\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 9000: 0.08504839\n",
      "Accuracy in last 100 iterations: 944/1000\n",
      "Loss at iteration 9100: 0.09787598\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 9200: 0.48831949\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 9300: 0.26149425\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 9400: 0.31413716\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 9500: 0.11273547\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 9600: 0.19020681\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 9700: 0.06561384\n",
      "Accuracy in last 100 iterations: 942/1000\n",
      "Loss at iteration 9800: 0.18413515\n",
      "Accuracy in last 100 iterations: 949/1000\n",
      "Loss at iteration 9900: 0.10886355\n",
      "Accuracy in last 100 iterations: 944/1000\n"
     ]
    }
   ],
   "source": [
    "for _ in range(3):\n",
    "    attn_dot.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.097"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_after = []\n",
    "for _ in range(1000):\n",
    "    accuracy_after.append(attn_dot.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sequence, correct_sequence, softmax_input, accurate, attentions = attn_local.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([1, 0, 8, 9, 2, 3, 7, 5, 6, 4]),\n",
       " tensor([1, 0, 8, 9, 2, 3, 7, 5, 6, 4]),\n",
       " tensor(10))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_input.max(1)[1], correct_sequence, accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.02583587, 0.09620975, 0.125464  , 0.08053908, 0.10777347,\n",
       "        0.13319768, 0.10848153, 0.11107256, 0.14066486, 0.07076123],\n",
       "       [0.02584182, 0.08619056, 0.09662391, 0.08750036, 0.1608577 ,\n",
       "        0.13818946, 0.11471014, 0.13490936, 0.12649107, 0.02868561],\n",
       "       [0.05972865, 0.10340665, 0.07586919, 0.11349026, 0.16530794,\n",
       "        0.11994054, 0.1142733 , 0.11866467, 0.09135054, 0.03796829],\n",
       "       [0.04030048, 0.0639444 , 0.09281634, 0.15245914, 0.13917434,\n",
       "        0.12763622, 0.10286935, 0.10457511, 0.08420879, 0.09201584],\n",
       "       [0.03806725, 0.06188602, 0.09038958, 0.14604926, 0.14205602,\n",
       "        0.13809843, 0.10190767, 0.10159109, 0.06856105, 0.11139359],\n",
       "       [0.02625175, 0.09217142, 0.09091506, 0.13272944, 0.15680654,\n",
       "        0.16077399, 0.11008523, 0.10267216, 0.07427777, 0.05331663],\n",
       "       [0.02112081, 0.08698103, 0.09840474, 0.1160111 , 0.14992224,\n",
       "        0.15542722, 0.10493094, 0.12336662, 0.08394917, 0.05988621],\n",
       "       [0.01874015, 0.07773704, 0.07649286, 0.15477352, 0.15368542,\n",
       "        0.15474541, 0.11687383, 0.11070844, 0.08603623, 0.05020712],\n",
       "       [0.01794426, 0.08854837, 0.09714372, 0.14881343, 0.13846424,\n",
       "        0.1582028 , 0.10625657, 0.1008221 , 0.08671172, 0.05709275],\n",
       "       [0.01437061, 0.08152883, 0.09351627, 0.16450404, 0.13568375,\n",
       "        0.16372843, 0.11405055, 0.09192764, 0.09272672, 0.04796318]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([a[0,0].detach().numpy() for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(x):\n",
    "    return (x - x.min())/(x.max() - x.min())\n",
    "\n",
    "np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])\n",
    "\n",
    "attention_normalized = np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x122e00080>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAC85JREFUeJzt3V1onvUZx/Hfb0ljrHG0dQVdW9T5tpXh6AjDrjiwnbCuY57soAOFeWBPVteJMHQnnokHY8wDGdTqKEzmQVtkDNkLrDsYSmisRdtmYqltjOlLfOkLLdK3awfJoLo2z93m//fOc/H9gNDEu5cX4fl6P8+TO3ccEQKQ05faXgBAPQQOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGK9NYbarnJ53M0331xjrK6//vriMz/88MPiMyVpYGCgq+aeOnWqylzbxWfWeBxI0sGDB4vPPHnypD799NOOXwTXuFS1VuCbNm2qMVYrV64sPvP5558vPlOS7r333ipzly9fXmXu0NBQlbl9fX3FZ953333FZ0rSI488UnzmK6+8oomJiY6B8xQdSIzAgcQIHEiMwIHECBxIjMCBxBoFbvsHtt+xvc/2E7WXAlBGx8Bt90h6TtJqSUsl/dT20tqLAZi5Jmfw70jaFxH7I+KMpJclPVB3LQAlNAl8kaT3L/p4bOpzn2F7ne1h28OllgMwM02uRb/U5XD/dylqRGyUtFGqd6kqgCvT5Aw+JmnJRR8vljReZx0AJTUJfIekO2zfartP0lpJf667FoASOj5Fj4hzttdL+pukHkkvRsSe6psBmLFGPw8eEa9KerXyLgAK40o2IDECBxIjcCAxAgcSI3AgsSp3Va1lz5463507evRo8Zlnz54tPlOSVq9eXWXu3r17q8w9duxYlbl33nln8Znbt28vPlOSNm/eXHzmuXPnGh3HGRxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSKzKXVX7+/t12223FZ+7YMGC4jMl6bXXXis+c82aNcVn1rRz584qc0+fPl1l7u7du4vPHBoaKj5TqneH3SY4gwOJETiQGIEDiRE4kBiBA4kROJBYx8BtL7G93faI7T22N3wRiwGYuSbfBz8n6fGI2Gn7eklv2P5HRNT5dZQAiul4Bo+IQxGxc+rPJyWNSFpUezEAM3dFr8Ft3yJpmaQ6l/wAKKrxpaq2ByRtlfTLiDhxiX+/TtI6SZozZ06xBQFcvUZncNtzNBn3SxGx7VLHRMTGiBiMiMGenp6SOwK4Sk3eRbekFySNRMRv668EoJQmZ/AVkh6StNL2rql/flh5LwAFdHwNHhH/luQvYBcAhXElG5AYgQOJETiQGIEDiRE4kJgjovxQu/xQSYODgzXGatWqVcVnPvPMM8VnStLo6GiVuePj41Xmvv7661Xm9vaWv1/oW2+9VXymVOdmoZs3b9bhw4c7fneLMziQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kFj5W1NWVONOmlJ3/T7zhQsXVpk7MDBQZe6uXbuqzK1xN+CxsbHiMyVp9erVxWdu3bq10XGcwYHECBxIjMCBxAgcSIzAgcQIHEiMwIHEGgduu8f2m7b/UnMhAOVcyRl8g6SRWosAKK9R4LYXS1ojaVPddQCU1PQM/jtJv5J04XIH2F5ne9j2cJHNAMxYx8Bt/0jS0Yh4Y7rjImJjRAxGxGCx7QDMSJMz+ApJP7Z9QNLLklba/mPVrQAU0THwiHgyIhZHxC2S1kr6Z0Q8WH0zADPG98GBxK7oB6wj4l+S/lVlEwDFcQYHEiNwIDECBxIjcCAxAgcS66q7qt5///1V5l64cNkrcK/ajh07is+UpLvvvrvK3AULFlSZ29/fX2XuiRMnis+84YYbis+UpKeffrr4zEOHDjU6jjM4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJBYV91VtZYbb7yx+MwjR44UnylJ58+frzK32yxbtqz4zPHx8eIzJenjjz8uPvPdd99tdBxncCAxAgcSI3AgMQIHEiNwIDECBxJrFLjteba32P6P7RHby2svBmDmmn4f/FlJf42In9jukzS34k4ACukYuO0vS/qepJ9JUkSckXSm7loASmjyFP1rkiYk/cH2m7Y32b6u8l4ACmgSeK+kb0v6fUQsk3RK0hOfP8j2OtvDtocL7wjgKjUJfEzSWEQMTX28RZPBf0ZEbIyIwYgYLLkggKvXMfCIOCzpfdt3TX1qlaS9VbcCUETTd9EflfTS1Dvo+yU9XG8lAKU0CjwidkniqTfQZbiSDUiMwIHECBxIjMCBxAgcSIzAgcS66q6qJ06cqDL3k08+KT5z/fr1xWdK0oULF6rMreWmm26qMndiYqL4zGuuuab4TKnO47bp3XU5gwOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWFfddHH+/PlV5s6bN6/4zBo3cpSkjz76qMrc22+/vcrcI0eOVJl7+vTp4jMjovhMSRoYGCg+s6enp9FxnMGBxAgcSIzAgcQIHEiMwIHECBxIjMCBxBoFbvsx23ts77b9J9v9tRcDMHMdA7e9SNIvJA1GxDcl9UhaW3sxADPX9Cl6r6RrbfdKmitpvN5KAErpGHhEfCDpN5JGJR2SdDwi/v7542yvsz1se7j8mgCuRpOn6PMlPSDpVklflXSd7Qc/f1xEbIyIwYgYLL8mgKvR5Cn69yW9FxETEXFW0jZJ3627FoASmgQ+Kuke23NtW9IqSSN11wJQQpPX4EOStkjaKentqb+zsfJeAApo9PPgEfGUpKcq7wKgMK5kAxIjcCAxAgcSI3AgMQIHEuuqu6oeP368yty+vr7iM2vdAfbAgQNV5ta6o2h/f50fPBwdHS0+s9auNR6358+fb3QcZ3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDHXuJum7QlJBxsc+hVJHxZfoJ5u2rebdpW6a9/ZsOvNEbGw00FVAm/K9nBEDLa2wBXqpn27aVepu/btpl15ig4kRuBAYm0HvrHl//6V6qZ9u2lXqbv27ZpdW30NDqCuts/gACpqLXDbP7D9ju19tp9oa49ObC+xvd32iO09tje0vVMTtntsv2n7L23vMh3b82xvsf2fqa/x8rZ3mo7tx6YeB7tt/8l2nd9YWEgrgdvukfScpNWSlkr6qe2lbezSwDlJj0fENyTdI+nns3jXi22QNNL2Eg08K+mvEfF1Sd/SLN7Z9iJJv5A0GBHflNQjaW27W02vrTP4dyTti4j9EXFG0suSHmhpl2lFxKGI2Dn155OafAAuaner6dleLGmNpE1t7zId21+W9D1JL0hSRJyJiGPtbtVRr6RrbfdKmitpvOV9ptVW4IskvX/Rx2Oa5dFIku1bJC2TNNTuJh39TtKvJF1oe5EOviZpQtIfpl5ObLJ9XdtLXU5EfCDpN5JGJR2SdDwi/t7uVtNrK3Bf4nOz+u182wOStkr6ZUScaHufy7H9I0lHI+KNtndpoFfStyX9PiKWSTolaTa/HzNfk880b5X0VUnX2X6w3a2m11bgY5KWXPTxYs3ipzq252gy7pciYlvb+3SwQtKPbR/Q5Euflbb/2O5KlzUmaSwi/veMaIsmg5+tvi/pvYiYiIizkrZJ+m7LO02rrcB3SLrD9q22+zT5RsWfW9plWratydeIIxHx27b36SQinoyIxRFxiya/rv+MiFl5lomIw5Let33X1KdWSdrb4kqdjEq6x/bcqcfFKs3iNwWlyadIX7iIOGd7vaS/afKdyBcjYk8buzSwQtJDkt62vWvqc7+OiFdb3CmTRyW9NPU/+v2SHm55n8uKiCHbWyTt1OR3V97ULL+qjSvZgMS4kg1IjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxP4LISuiQPm9OPIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(attention_normalized, 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 4, 5, 9, 7, 8, 6, 2, 3]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Pointer networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pointer_network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.02"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_before = []\n",
    "for _ in range(100):\n",
    "    accuracy_before.append(pointer_network.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 0: 2.30333447\n",
      "Accuracy in last 100 iterations: 1/10\n",
      "Loss at iteration 100: 2.30174279\n",
      "Accuracy in last 100 iterations: 104/1000\n",
      "Loss at iteration 200: 2.27741671\n",
      "Accuracy in last 100 iterations: 129/1000\n",
      "Loss at iteration 300: 2.31787348\n",
      "Accuracy in last 100 iterations: 142/1000\n",
      "Loss at iteration 400: 1.93306637\n",
      "Accuracy in last 100 iterations: 188/1000\n",
      "Loss at iteration 500: 2.12627888\n",
      "Accuracy in last 100 iterations: 219/1000\n",
      "Loss at iteration 600: 2.07406092\n",
      "Accuracy in last 100 iterations: 231/1000\n",
      "Loss at iteration 700: 1.94512343\n",
      "Accuracy in last 100 iterations: 239/1000\n",
      "Loss at iteration 800: 1.74219537\n",
      "Accuracy in last 100 iterations: 256/1000\n",
      "Loss at iteration 900: 2.06431174\n",
      "Accuracy in last 100 iterations: 281/1000\n",
      "Loss at iteration 1000: 1.84636652\n",
      "Accuracy in last 100 iterations: 300/1000\n",
      "Loss at iteration 1100: 1.82321358\n",
      "Accuracy in last 100 iterations: 317/1000\n",
      "Loss at iteration 1200: 1.53765678\n",
      "Accuracy in last 100 iterations: 344/1000\n",
      "Loss at iteration 1300: 1.89754939\n",
      "Accuracy in last 100 iterations: 337/1000\n",
      "Loss at iteration 1400: 1.56627822\n",
      "Accuracy in last 100 iterations: 354/1000\n",
      "Loss at iteration 1500: 1.76984251\n",
      "Accuracy in last 100 iterations: 384/1000\n",
      "Loss at iteration 1600: 1.55115128\n",
      "Accuracy in last 100 iterations: 358/1000\n",
      "Loss at iteration 1700: 1.65946269\n",
      "Accuracy in last 100 iterations: 373/1000\n",
      "Loss at iteration 1800: 1.76682127\n",
      "Accuracy in last 100 iterations: 392/1000\n",
      "Loss at iteration 1900: 1.88184428\n",
      "Accuracy in last 100 iterations: 406/1000\n",
      "Loss at iteration 2000: 1.43956041\n",
      "Accuracy in last 100 iterations: 402/1000\n",
      "Loss at iteration 2100: 1.54225874\n",
      "Accuracy in last 100 iterations: 426/1000\n",
      "Loss at iteration 2200: 1.79670644\n",
      "Accuracy in last 100 iterations: 433/1000\n",
      "Loss at iteration 2300: 1.63269114\n",
      "Accuracy in last 100 iterations: 437/1000\n",
      "Loss at iteration 2400: 1.43369842\n",
      "Accuracy in last 100 iterations: 439/1000\n",
      "Loss at iteration 2500: 1.49908245\n",
      "Accuracy in last 100 iterations: 447/1000\n",
      "Loss at iteration 2600: 1.37714350\n",
      "Accuracy in last 100 iterations: 461/1000\n",
      "Loss at iteration 2700: 1.36577094\n",
      "Accuracy in last 100 iterations: 471/1000\n",
      "Loss at iteration 2800: 1.38101792\n",
      "Accuracy in last 100 iterations: 486/1000\n",
      "Loss at iteration 2900: 1.43958974\n",
      "Accuracy in last 100 iterations: 448/1000\n",
      "Loss at iteration 3000: 1.49709558\n",
      "Accuracy in last 100 iterations: 467/1000\n",
      "Loss at iteration 3100: 1.49165368\n",
      "Accuracy in last 100 iterations: 485/1000\n",
      "Loss at iteration 3200: 1.41941667\n",
      "Accuracy in last 100 iterations: 482/1000\n",
      "Loss at iteration 3300: 1.33507597\n",
      "Accuracy in last 100 iterations: 497/1000\n",
      "Loss at iteration 3400: 1.42976892\n",
      "Accuracy in last 100 iterations: 504/1000\n",
      "Loss at iteration 3500: 1.34110510\n",
      "Accuracy in last 100 iterations: 493/1000\n",
      "Loss at iteration 3600: 1.38362849\n",
      "Accuracy in last 100 iterations: 513/1000\n",
      "Loss at iteration 3700: 1.43422961\n",
      "Accuracy in last 100 iterations: 535/1000\n",
      "Loss at iteration 3800: 1.39321744\n",
      "Accuracy in last 100 iterations: 523/1000\n",
      "Loss at iteration 3900: 1.23344254\n",
      "Accuracy in last 100 iterations: 518/1000\n",
      "Loss at iteration 4000: 1.10469365\n",
      "Accuracy in last 100 iterations: 515/1000\n",
      "Loss at iteration 4100: 1.22793138\n",
      "Accuracy in last 100 iterations: 532/1000\n",
      "Loss at iteration 4200: 1.00796473\n",
      "Accuracy in last 100 iterations: 516/1000\n",
      "Loss at iteration 4300: 1.26357758\n",
      "Accuracy in last 100 iterations: 554/1000\n",
      "Loss at iteration 4400: 1.39006209\n",
      "Accuracy in last 100 iterations: 549/1000\n",
      "Loss at iteration 4500: 1.20851564\n",
      "Accuracy in last 100 iterations: 535/1000\n",
      "Loss at iteration 4600: 1.31680346\n",
      "Accuracy in last 100 iterations: 513/1000\n",
      "Loss at iteration 4700: 1.17041874\n",
      "Accuracy in last 100 iterations: 560/1000\n",
      "Loss at iteration 4800: 1.29171777\n",
      "Accuracy in last 100 iterations: 576/1000\n",
      "Loss at iteration 4900: 1.11074924\n",
      "Accuracy in last 100 iterations: 584/1000\n",
      "Loss at iteration 5000: 1.00168204\n",
      "Accuracy in last 100 iterations: 541/1000\n",
      "Loss at iteration 5100: 0.84106791\n",
      "Accuracy in last 100 iterations: 572/1000\n",
      "Loss at iteration 5200: 1.56270719\n",
      "Accuracy in last 100 iterations: 578/1000\n",
      "Loss at iteration 5300: 1.02520669\n",
      "Accuracy in last 100 iterations: 579/1000\n",
      "Loss at iteration 5400: 1.12227297\n",
      "Accuracy in last 100 iterations: 587/1000\n",
      "Loss at iteration 5500: 1.03442073\n",
      "Accuracy in last 100 iterations: 588/1000\n",
      "Loss at iteration 5600: 0.90032446\n",
      "Accuracy in last 100 iterations: 563/1000\n",
      "Loss at iteration 5700: 1.11289251\n",
      "Accuracy in last 100 iterations: 596/1000\n",
      "Loss at iteration 5800: 0.90258074\n",
      "Accuracy in last 100 iterations: 605/1000\n",
      "Loss at iteration 5900: 1.05670524\n",
      "Accuracy in last 100 iterations: 584/1000\n",
      "Loss at iteration 6000: 0.97131461\n",
      "Accuracy in last 100 iterations: 602/1000\n",
      "Loss at iteration 6100: 1.03752029\n",
      "Accuracy in last 100 iterations: 629/1000\n",
      "Loss at iteration 6200: 0.89790851\n",
      "Accuracy in last 100 iterations: 621/1000\n",
      "Loss at iteration 6300: 0.93001330\n",
      "Accuracy in last 100 iterations: 618/1000\n",
      "Loss at iteration 6400: 0.76034170\n",
      "Accuracy in last 100 iterations: 604/1000\n",
      "Loss at iteration 6500: 1.17207253\n",
      "Accuracy in last 100 iterations: 622/1000\n",
      "Loss at iteration 6600: 1.00601268\n",
      "Accuracy in last 100 iterations: 630/1000\n",
      "Loss at iteration 6700: 1.22028482\n",
      "Accuracy in last 100 iterations: 632/1000\n",
      "Loss at iteration 6800: 0.87812459\n",
      "Accuracy in last 100 iterations: 625/1000\n",
      "Loss at iteration 6900: 0.88812858\n",
      "Accuracy in last 100 iterations: 635/1000\n",
      "Loss at iteration 7000: 0.83367968\n",
      "Accuracy in last 100 iterations: 655/1000\n",
      "Loss at iteration 7100: 0.66200054\n",
      "Accuracy in last 100 iterations: 645/1000\n",
      "Loss at iteration 7200: 0.84945881\n",
      "Accuracy in last 100 iterations: 650/1000\n",
      "Loss at iteration 7300: 1.41595709\n",
      "Accuracy in last 100 iterations: 651/1000\n",
      "Loss at iteration 7400: 1.03761315\n",
      "Accuracy in last 100 iterations: 659/1000\n",
      "Loss at iteration 7500: 0.80814970\n",
      "Accuracy in last 100 iterations: 675/1000\n",
      "Loss at iteration 7600: 1.02405930\n",
      "Accuracy in last 100 iterations: 642/1000\n",
      "Loss at iteration 7700: 1.19118822\n",
      "Accuracy in last 100 iterations: 666/1000\n",
      "Loss at iteration 7800: 0.91512328\n",
      "Accuracy in last 100 iterations: 654/1000\n",
      "Loss at iteration 7900: 0.90320933\n",
      "Accuracy in last 100 iterations: 657/1000\n",
      "Loss at iteration 8000: 0.87056172\n",
      "Accuracy in last 100 iterations: 684/1000\n",
      "Loss at iteration 8100: 0.93051445\n",
      "Accuracy in last 100 iterations: 675/1000\n",
      "Loss at iteration 8200: 0.95555687\n",
      "Accuracy in last 100 iterations: 690/1000\n",
      "Loss at iteration 8300: 1.08350182\n",
      "Accuracy in last 100 iterations: 691/1000\n",
      "Loss at iteration 8400: 0.82585108\n",
      "Accuracy in last 100 iterations: 668/1000\n",
      "Loss at iteration 8500: 0.81405485\n",
      "Accuracy in last 100 iterations: 663/1000\n",
      "Loss at iteration 8600: 0.79010910\n",
      "Accuracy in last 100 iterations: 692/1000\n",
      "Loss at iteration 8700: 0.85135430\n",
      "Accuracy in last 100 iterations: 696/1000\n",
      "Loss at iteration 8800: 1.04974711\n",
      "Accuracy in last 100 iterations: 686/1000\n",
      "Loss at iteration 8900: 0.85113752\n",
      "Accuracy in last 100 iterations: 680/1000\n",
      "Loss at iteration 9000: 1.16012931\n",
      "Accuracy in last 100 iterations: 700/1000\n",
      "Loss at iteration 9100: 0.78306639\n",
      "Accuracy in last 100 iterations: 732/1000\n",
      "Loss at iteration 9200: 0.64046001\n",
      "Accuracy in last 100 iterations: 716/1000\n",
      "Loss at iteration 9300: 0.83498591\n",
      "Accuracy in last 100 iterations: 718/1000\n",
      "Loss at iteration 9400: 0.38902125\n",
      "Accuracy in last 100 iterations: 715/1000\n",
      "Loss at iteration 9500: 0.70835435\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 9600: 0.65144891\n",
      "Accuracy in last 100 iterations: 715/1000\n",
      "Loss at iteration 9700: 0.86602795\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 9800: 0.70874417\n",
      "Accuracy in last 100 iterations: 733/1000\n",
      "Loss at iteration 9900: 0.91454476\n",
      "Accuracy in last 100 iterations: 760/1000\n",
      "Loss at iteration 0: 0.72721398\n",
      "Accuracy in last 100 iterations: 7/10\n",
      "Loss at iteration 100: 0.71734083\n",
      "Accuracy in last 100 iterations: 731/1000\n",
      "Loss at iteration 200: 0.47398415\n",
      "Accuracy in last 100 iterations: 763/1000\n",
      "Loss at iteration 300: 0.71117038\n",
      "Accuracy in last 100 iterations: 750/1000\n",
      "Loss at iteration 400: 0.59000051\n",
      "Accuracy in last 100 iterations: 758/1000\n",
      "Loss at iteration 500: 0.57156998\n",
      "Accuracy in last 100 iterations: 747/1000\n",
      "Loss at iteration 600: 0.64633942\n",
      "Accuracy in last 100 iterations: 777/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 700: 0.69423360\n",
      "Accuracy in last 100 iterations: 779/1000\n",
      "Loss at iteration 800: 0.44024819\n",
      "Accuracy in last 100 iterations: 761/1000\n",
      "Loss at iteration 900: 0.69708550\n",
      "Accuracy in last 100 iterations: 771/1000\n",
      "Loss at iteration 1000: 0.41075617\n",
      "Accuracy in last 100 iterations: 774/1000\n",
      "Loss at iteration 1100: 0.38236457\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 1200: 0.55658674\n",
      "Accuracy in last 100 iterations: 785/1000\n",
      "Loss at iteration 1300: 0.61730278\n",
      "Accuracy in last 100 iterations: 790/1000\n",
      "Loss at iteration 1400: 0.57004654\n",
      "Accuracy in last 100 iterations: 824/1000\n",
      "Loss at iteration 1500: 0.46400499\n",
      "Accuracy in last 100 iterations: 806/1000\n",
      "Loss at iteration 1600: 0.59595829\n",
      "Accuracy in last 100 iterations: 786/1000\n",
      "Loss at iteration 1700: 0.49912876\n",
      "Accuracy in last 100 iterations: 795/1000\n",
      "Loss at iteration 1800: 0.82773703\n",
      "Accuracy in last 100 iterations: 781/1000\n",
      "Loss at iteration 1900: 0.46161413\n",
      "Accuracy in last 100 iterations: 816/1000\n",
      "Loss at iteration 2000: 0.32348344\n",
      "Accuracy in last 100 iterations: 796/1000\n",
      "Loss at iteration 2100: 0.79737985\n",
      "Accuracy in last 100 iterations: 829/1000\n",
      "Loss at iteration 2200: 0.51816046\n",
      "Accuracy in last 100 iterations: 814/1000\n",
      "Loss at iteration 2300: 0.35118586\n",
      "Accuracy in last 100 iterations: 821/1000\n",
      "Loss at iteration 2400: 0.76540560\n",
      "Accuracy in last 100 iterations: 804/1000\n",
      "Loss at iteration 2500: 0.61414158\n",
      "Accuracy in last 100 iterations: 819/1000\n",
      "Loss at iteration 2600: 0.68033296\n",
      "Accuracy in last 100 iterations: 819/1000\n",
      "Loss at iteration 2700: 0.51773471\n",
      "Accuracy in last 100 iterations: 819/1000\n",
      "Loss at iteration 2800: 0.62729007\n",
      "Accuracy in last 100 iterations: 825/1000\n",
      "Loss at iteration 2900: 0.35363215\n",
      "Accuracy in last 100 iterations: 794/1000\n",
      "Loss at iteration 3000: 0.43442398\n",
      "Accuracy in last 100 iterations: 825/1000\n",
      "Loss at iteration 3100: 0.33291516\n",
      "Accuracy in last 100 iterations: 820/1000\n",
      "Loss at iteration 3200: 0.61582029\n",
      "Accuracy in last 100 iterations: 812/1000\n",
      "Loss at iteration 3300: 0.56885397\n",
      "Accuracy in last 100 iterations: 821/1000\n",
      "Loss at iteration 3400: 0.49414960\n",
      "Accuracy in last 100 iterations: 816/1000\n",
      "Loss at iteration 3500: 0.36295539\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 3600: 0.35701734\n",
      "Accuracy in last 100 iterations: 830/1000\n",
      "Loss at iteration 3700: 0.32603160\n",
      "Accuracy in last 100 iterations: 850/1000\n",
      "Loss at iteration 3800: 0.40097111\n",
      "Accuracy in last 100 iterations: 841/1000\n",
      "Loss at iteration 3900: 0.43607998\n",
      "Accuracy in last 100 iterations: 821/1000\n",
      "Loss at iteration 4000: 0.31414455\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 4100: 0.25074282\n",
      "Accuracy in last 100 iterations: 816/1000\n",
      "Loss at iteration 4200: 0.78645980\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 4300: 0.43766180\n",
      "Accuracy in last 100 iterations: 846/1000\n",
      "Loss at iteration 4400: 0.37329286\n",
      "Accuracy in last 100 iterations: 853/1000\n",
      "Loss at iteration 4500: 0.35835066\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 4600: 0.52205014\n",
      "Accuracy in last 100 iterations: 843/1000\n",
      "Loss at iteration 4700: 0.70548850\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 4800: 0.32220560\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 4900: 0.42492047\n",
      "Accuracy in last 100 iterations: 821/1000\n",
      "Loss at iteration 5000: 0.59760123\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 5100: 0.22068682\n",
      "Accuracy in last 100 iterations: 863/1000\n",
      "Loss at iteration 5200: 0.22066550\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 5300: 0.62219077\n",
      "Accuracy in last 100 iterations: 850/1000\n",
      "Loss at iteration 5400: 0.45018655\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 5500: 0.21568327\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 5600: 0.30030569\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 5700: 0.29045373\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 5800: 0.42556587\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 5900: 0.31176084\n",
      "Accuracy in last 100 iterations: 856/1000\n",
      "Loss at iteration 6000: 0.56157595\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 6100: 0.73231786\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 6200: 0.41566667\n",
      "Accuracy in last 100 iterations: 872/1000\n",
      "Loss at iteration 6300: 0.52074701\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 6400: 0.41171741\n",
      "Accuracy in last 100 iterations: 863/1000\n",
      "Loss at iteration 6500: 0.37019271\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 6600: 0.19250521\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 6700: 0.21409540\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 6800: 0.28578848\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 6900: 0.17392740\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 7000: 0.22331944\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 7100: 0.26438269\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 7200: 0.48324308\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 7300: 0.22366925\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 7400: 0.37368259\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 7500: 0.49775043\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 7600: 0.43537205\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 7700: 0.36418334\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 7800: 0.50884169\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 7900: 0.34443220\n",
      "Accuracy in last 100 iterations: 893/1000\n",
      "Loss at iteration 8000: 0.22424741\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 8100: 0.27423054\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 8200: 0.38989377\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 8300: 0.50864589\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 8400: 0.31843704\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 8500: 0.11133905\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 8600: 0.15361276\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 8700: 0.45340905\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 8800: 0.46702996\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 8900: 0.28542286\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 9000: 0.14267965\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 9100: 0.14994244\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 9200: 0.55412787\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 9300: 0.24391790\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 9400: 0.51686704\n",
      "Accuracy in last 100 iterations: 891/1000\n",
      "Loss at iteration 9500: 0.47763005\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 9600: 0.45207113\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 9700: 0.29439345\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 9800: 0.24247822\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 9900: 0.20838375\n",
      "Accuracy in last 100 iterations: 889/1000\n",
      "Loss at iteration 0: 0.24558672\n",
      "Accuracy in last 100 iterations: 8/10\n",
      "Loss at iteration 100: 0.58635610\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 200: 0.34479427\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 300: 0.24014187\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 400: 0.46195215\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 500: 0.63613260\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 600: 0.20311293\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 700: 0.30255398\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 800: 0.45826903\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 900: 0.42710763\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 1000: 0.12247081\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 1100: 0.18494621\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 1200: 0.27114287\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 1300: 0.15163155\n",
      "Accuracy in last 100 iterations: 919/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1400: 0.20649552\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 1500: 0.21002665\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 1600: 0.20861773\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 1700: 0.28529134\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 1800: 0.34874392\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 1900: 0.13836618\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 2000: 0.12529774\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 2100: 0.12930956\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 2200: 0.12383294\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 2300: 0.19011493\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 2400: 0.55007231\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 2500: 0.11470008\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 2600: 0.12319203\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 2700: 0.48430276\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 2800: 0.12734528\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 2900: 0.39146823\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 3000: 0.10986972\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 3100: 0.17142224\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 3200: 0.14689317\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 3300: 0.35483885\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 3400: 0.22206573\n",
      "Accuracy in last 100 iterations: 944/1000\n",
      "Loss at iteration 3500: 0.42585143\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 3600: 0.22529507\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 3700: 0.21022014\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 3800: 0.06769190\n",
      "Accuracy in last 100 iterations: 941/1000\n",
      "Loss at iteration 3900: 0.42629758\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 4000: 0.12471561\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 4100: 0.06708755\n",
      "Accuracy in last 100 iterations: 931/1000\n",
      "Loss at iteration 4200: 0.09405885\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 4300: 0.30832329\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 4400: 0.27542838\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 4500: 0.13630000\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 4600: 0.08014555\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 4700: 0.31884646\n",
      "Accuracy in last 100 iterations: 942/1000\n",
      "Loss at iteration 4800: 0.11330990\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 4900: 0.16055441\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 5000: 0.06746292\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 5100: 0.22298293\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 5200: 0.30389291\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 5300: 0.14778896\n",
      "Accuracy in last 100 iterations: 965/1000\n",
      "Loss at iteration 5400: 0.05980339\n",
      "Accuracy in last 100 iterations: 931/1000\n",
      "Loss at iteration 5500: 0.16403751\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 5600: 0.15425977\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 5700: 0.20757456\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 5800: 0.49358234\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 5900: 0.19168019\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 6000: 0.19493374\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 6100: 0.32931867\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 6200: 0.18515368\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 6300: 0.44092113\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 6400: 0.05656471\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 6500: 0.13901892\n",
      "Accuracy in last 100 iterations: 928/1000\n",
      "Loss at iteration 6600: 0.09844170\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 6700: 0.10783400\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 6800: 0.16317168\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 6900: 0.09669676\n",
      "Accuracy in last 100 iterations: 945/1000\n",
      "Loss at iteration 7000: 0.32610887\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 7100: 0.05979719\n",
      "Accuracy in last 100 iterations: 949/1000\n",
      "Loss at iteration 7200: 0.03985281\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 7300: 0.09537601\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 7400: 0.24104562\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 7500: 0.40995225\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 7600: 0.15041666\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 7700: 0.19896245\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 7800: 0.36178184\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 7900: 0.07943354\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 8000: 0.21279350\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 8100: 0.28385004\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 8200: 0.13339229\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 8300: 0.43648735\n",
      "Accuracy in last 100 iterations: 946/1000\n",
      "Loss at iteration 8400: 0.10404401\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 8500: 0.11590528\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 8600: 0.59795398\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 8700: 0.09546275\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 8800: 0.13680887\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 8900: 0.05807152\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 9000: 0.17967291\n",
      "Accuracy in last 100 iterations: 947/1000\n",
      "Loss at iteration 9100: 0.05474391\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 9200: 0.07935014\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 9300: 0.03500767\n",
      "Accuracy in last 100 iterations: 950/1000\n",
      "Loss at iteration 9400: 0.41310519\n",
      "Accuracy in last 100 iterations: 944/1000\n",
      "Loss at iteration 9500: 0.11875644\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 9600: 0.17292027\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 9700: 0.11549716\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 9800: 0.34902686\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 9900: 0.09578715\n",
      "Accuracy in last 100 iterations: 933/1000\n"
     ]
    }
   ],
   "source": [
    "for _ in range(3):\n",
    "    pointer_network.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.278"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_after = []\n",
    "for _ in range(1000):\n",
    "    accuracy_after.append(pointer_network.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sequence, correct_sequence, softmax_input, accurate, attentions = pointer_network.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([7, 6, 5, 9, 8, 4, 1, 3, 0, 2]),\n",
       " tensor([7, 6, 5, 9, 8, 4, 1, 3, 0, 2]),\n",
       " tensor(10))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_input.max(1)[1], correct_sequence, accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.10259458, 0.06929223, 0.09369805, 0.14922158, 0.12905785,\n",
       "        0.09097019, 0.08032312, 0.08187154, 0.16698276, 0.03598804],\n",
       "       [0.10259458, 0.06929226, 0.09369805, 0.14922152, 0.12905791,\n",
       "        0.09097023, 0.0803231 , 0.08187155, 0.1669828 , 0.03598804],\n",
       "       [0.10259459, 0.06929222, 0.09369807, 0.14922157, 0.12905791,\n",
       "        0.09097017, 0.08032313, 0.08187155, 0.1669828 , 0.03598804],\n",
       "       [0.1025946 , 0.06929226, 0.09369805, 0.14922157, 0.12905796,\n",
       "        0.09097018, 0.08032311, 0.08187154, 0.16698274, 0.03598804],\n",
       "       [0.10259462, 0.06929223, 0.09369803, 0.14922151, 0.12905791,\n",
       "        0.09097017, 0.0803231 , 0.08187155, 0.16698273, 0.03598804],\n",
       "       [0.10259458, 0.06929225, 0.09369806, 0.14922155, 0.12905793,\n",
       "        0.09097021, 0.0803231 , 0.08187155, 0.16698278, 0.03598803],\n",
       "       [0.10259461, 0.06929225, 0.09369807, 0.14922158, 0.12905791,\n",
       "        0.09097021, 0.08032311, 0.08187154, 0.16698276, 0.03598804],\n",
       "       [0.10259458, 0.06929225, 0.09369804, 0.14922158, 0.12905791,\n",
       "        0.09097023, 0.08032313, 0.08187157, 0.1669828 , 0.03598804],\n",
       "       [0.1025946 , 0.06929225, 0.09369805, 0.14922157, 0.12905791,\n",
       "        0.09097021, 0.08032311, 0.08187155, 0.16698277, 0.03598804],\n",
       "       [0.10259459, 0.06929224, 0.09369807, 0.14922151, 0.12905791,\n",
       "        0.09097023, 0.08032309, 0.08187155, 0.1669828 , 0.03598803]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([a[0,0].detach().numpy() for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(x):\n",
    "    return (x - x.min())/(x.max() - x.min())\n",
    "\n",
    "np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])\n",
    "\n",
    "attention_normalized = np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x123038f28>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACdBJREFUeJzt3M+L3Xe9x/HnyxmLyVGJcG8XJsVGELUIpTJIteCideEv7MZFhQq6CYVrrCJIvRv/ARFdiBCqbmztInYhEvwBxoWb4CQtaDoRSvW2sRXjwhgGSgy+72LmQuxt5nzT+X77nXnzfEAhM/3m0xdDnvmec+ZMU1VI6ukNcw+QNB0DlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKmx1SkOXSwWdejQodHPvXLlyuhnAhw4cGD0M2+99dbRzwR4+eWXJzn38uXLk5x77dq1Sc5NMvqZR48eHf1MgLNnz05yblUt/SJMEvihQ4d46KGHRj/39OnTo58JcOedd45+5vHjx0c/E+DChQuTnHvq1KlJzr106dIk566srIx+5uOPPz76mTDNX0ZD+RBdaszApcYMXGrMwKXGDFxqzMClxgYFnuSjSf6Q5Nkkj0w9StI4lgaeZAX4DvAx4A7gM0numHqYpN0bcgf/APBsVT1XVVeBJ4D7p50laQxDAj8MvHDdxxe3P/dvkhxLsp5kfXNzc6x9knZhSOCv9j67//e/Yq2qE1W1VlVri8Vi98sk7dqQwC8Ct1338RHgxWnmSBrTkMB/C7wrydEktwAPAD+ZdpakMSz9abKqupbkC8DPgRXg+1V1fvJlknZt0I+LVtUpYJqfJ5Q0Gd/JJjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSY0sDT3JbktNJNpKcT/Lw6zFM0u6tDrjmGvCVqjqX5C3A2SS/rKpnJt4maZeW3sGr6qWqOrf96yvABnB46mGSdu+mnoMnuR24CzgzxRhJ4xoceJI3Az8GvlRV/3iVf38syXqS9c3NzTE3SnqNBgWe5I1sxf1YVT35atdU1YmqWquqtcViMeZGSa/RkFfRA3wP2Kiqb04/SdJYhtzB7wE+C9yb5Ontfz4+8S5JI1j6bbKq+g2Q12GLpJH5TjapMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKmxwYEnWUnyVJKfTjlI0nhu5g7+MLAx1RBJ4xsUeJIjwCeAR6edI2lMQ+/g3wK+CvzrRhckOZZkPcn65ubmKOMk7c7SwJN8EvhrVZ3d6bqqOlFVa1W1tlgsRhso6bUbcge/B/hUkj8BTwD3JvnhpKskjWJp4FX1tao6UlW3Aw8Av6qqBydfJmnX/D641NjqzVxcVb8Gfj3JEkmj8w4uNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjgwJPcijJySQXkmwk+eDUwyTt3urA674N/KyqPp3kFuDghJskjWRp4EneCnwY+BxAVV0Frk47S9IYhjxEfydwCfhBkqeSPJpkMfEuSSMYEvgq8H7gu1V1F7AJPPLKi5IcS7KeZH1zc3PkmZJeiyGBXwQuVtWZ7Y9PshX8v6mqE1W1VlVri4U3eGkvWBp4Vf0FeCHJu7c/dR/wzKSrJI1i6Kvox4HHtl9Bfw74/HSTJI1lUOBV9TSwNvEWSSPznWxSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjgwJP8uUk55P8PsmPkrxp6mGSdm9p4EkOA18E1qrqfcAK8MDUwyTt3tCH6KvAgSSrwEHgxekmSRrL0sCr6s/AN4DngZeAy1X1i1del+RYkvUk65ubm+MvlXTThjxEfxtwP3AUeDuwSPLgK6+rqhNVtVZVa4vFYvylkm7akIfoHwH+WFWXquqfwJPAh6adJWkMQwJ/Hrg7ycEkAe4DNqadJWkMQ56DnwFOAueA323/nhMT75I0gtUhF1XV14GvT7xF0sh8J5vUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41FiqavxDk0vA/wy49D+Av40+YDr7ae9+2gr7a+9e2PqOqvrPZRdNEvhQSdaram22ATdpP+3dT1thf+3dT1t9iC41ZuBSY3MHfmLm//7N2k9799NW2F97983WWZ+DS5rW3HdwSROaLfAkH03yhyTPJnlkrh3LJLktyekkG0nOJ3l47k1DJFlJ8lSSn869ZSdJDiU5meTC9tf4g3Nv2kmSL2//Ofh9kh8ledPcm3YyS+BJVoDvAB8D7gA+k+SOObYMcA34SlW9F7gb+K89vPV6DwMbc48Y4NvAz6rqPcCd7OHNSQ4DXwTWqup9wArwwLyrdjbXHfwDwLNV9VxVXQWeAO6facuOquqlqjq3/esrbP0BPDzvqp0lOQJ8Anh07i07SfJW4MPA9wCq6mpV/X3eVUutAgeSrAIHgRdn3rOjuQI/DLxw3ccX2ePRACS5HbgLODPvkqW+BXwV+NfcQ5Z4J3AJ+MH204lHkyzmHnUjVfVn4BvA88BLwOWq+sW8q3Y2V+B5lc/t6Zfzk7wZ+DHwpar6x9x7biTJJ4G/VtXZubcMsAq8H/huVd0FbAJ7+fWYt7H1SPMo8HZgkeTBeVftbK7ALwK3XffxEfbwQ50kb2Qr7seq6sm59yxxD/CpJH9i66nPvUl+OO+kG7oIXKyq/3tEdJKt4PeqjwB/rKpLVfVP4EngQzNv2tFcgf8WeFeSo0luYeuFip/MtGVHScLWc8SNqvrm3HuWqaqvVdWRqrqdra/rr6pqT95lquovwAtJ3r39qfuAZ2actMzzwN1JDm7/ubiPPfyiIGw9RHrdVdW1JF8Afs7WK5Hfr6rzc2wZ4B7gs8Dvkjy9/bn/rqpTM27q5Djw2PZf9M8Bn595zw1V1ZkkJ4FzbH135Sn2+LvafCeb1JjvZJMaM3CpMQOXGjNwqTEDlxozcKkxA5caM3Cpsf8FSe8dZlhdqK0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(attention_normalized, 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[8, 6, 9, 7, 5, 2, 1, 0, 4, 3]])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Content based attention (general)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import content_based_general_attention as attn_general"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_before = []\n",
    "for _ in range(100):\n",
    "    accuracy_before.append(attn_general.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 0: 2.30342245\n",
      "Accuracy in last 100 iterations: 1/10\n",
      "Loss at iteration 100: 2.26773119\n",
      "Accuracy in last 100 iterations: 115/1000\n",
      "Loss at iteration 200: 2.30444646\n",
      "Accuracy in last 100 iterations: 136/1000\n",
      "Loss at iteration 300: 2.19670057\n",
      "Accuracy in last 100 iterations: 156/1000\n",
      "Loss at iteration 400: 2.10812354\n",
      "Accuracy in last 100 iterations: 176/1000\n",
      "Loss at iteration 500: 2.12257886\n",
      "Accuracy in last 100 iterations: 197/1000\n",
      "Loss at iteration 600: 1.97918820\n",
      "Accuracy in last 100 iterations: 219/1000\n",
      "Loss at iteration 700: 2.03648329\n",
      "Accuracy in last 100 iterations: 264/1000\n",
      "Loss at iteration 800: 2.01891637\n",
      "Accuracy in last 100 iterations: 272/1000\n",
      "Loss at iteration 900: 1.73257983\n",
      "Accuracy in last 100 iterations: 281/1000\n",
      "Loss at iteration 1000: 1.96025348\n",
      "Accuracy in last 100 iterations: 314/1000\n",
      "Loss at iteration 1100: 1.78275323\n",
      "Accuracy in last 100 iterations: 326/1000\n",
      "Loss at iteration 1200: 1.73934615\n",
      "Accuracy in last 100 iterations: 319/1000\n",
      "Loss at iteration 1300: 1.69216371\n",
      "Accuracy in last 100 iterations: 363/1000\n",
      "Loss at iteration 1400: 1.44014454\n",
      "Accuracy in last 100 iterations: 357/1000\n",
      "Loss at iteration 1500: 1.53146970\n",
      "Accuracy in last 100 iterations: 375/1000\n",
      "Loss at iteration 1600: 2.05448079\n",
      "Accuracy in last 100 iterations: 388/1000\n",
      "Loss at iteration 1700: 1.59382737\n",
      "Accuracy in last 100 iterations: 382/1000\n",
      "Loss at iteration 1800: 1.63169611\n",
      "Accuracy in last 100 iterations: 388/1000\n",
      "Loss at iteration 1900: 1.66749609\n",
      "Accuracy in last 100 iterations: 429/1000\n",
      "Loss at iteration 2000: 1.35672259\n",
      "Accuracy in last 100 iterations: 443/1000\n",
      "Loss at iteration 2100: 1.48041511\n",
      "Accuracy in last 100 iterations: 426/1000\n",
      "Loss at iteration 2200: 1.39822495\n",
      "Accuracy in last 100 iterations: 444/1000\n",
      "Loss at iteration 2300: 1.32365048\n",
      "Accuracy in last 100 iterations: 452/1000\n",
      "Loss at iteration 2400: 1.50747907\n",
      "Accuracy in last 100 iterations: 478/1000\n",
      "Loss at iteration 2500: 1.26036334\n",
      "Accuracy in last 100 iterations: 462/1000\n",
      "Loss at iteration 2600: 1.27009487\n",
      "Accuracy in last 100 iterations: 506/1000\n",
      "Loss at iteration 2700: 1.39205480\n",
      "Accuracy in last 100 iterations: 496/1000\n",
      "Loss at iteration 2800: 1.10879302\n",
      "Accuracy in last 100 iterations: 493/1000\n",
      "Loss at iteration 2900: 1.33619523\n",
      "Accuracy in last 100 iterations: 506/1000\n",
      "Loss at iteration 3000: 1.35105109\n",
      "Accuracy in last 100 iterations: 496/1000\n",
      "Loss at iteration 3100: 1.19542789\n",
      "Accuracy in last 100 iterations: 539/1000\n",
      "Loss at iteration 3200: 1.32820940\n",
      "Accuracy in last 100 iterations: 523/1000\n",
      "Loss at iteration 3300: 1.17173421\n",
      "Accuracy in last 100 iterations: 550/1000\n",
      "Loss at iteration 3400: 1.20037305\n",
      "Accuracy in last 100 iterations: 553/1000\n",
      "Loss at iteration 3500: 1.22587717\n",
      "Accuracy in last 100 iterations: 537/1000\n",
      "Loss at iteration 3600: 0.96238697\n",
      "Accuracy in last 100 iterations: 563/1000\n",
      "Loss at iteration 3700: 1.26000094\n",
      "Accuracy in last 100 iterations: 582/1000\n",
      "Loss at iteration 3800: 0.74143732\n",
      "Accuracy in last 100 iterations: 620/1000\n",
      "Loss at iteration 3900: 1.25894797\n",
      "Accuracy in last 100 iterations: 581/1000\n",
      "Loss at iteration 4000: 0.89237165\n",
      "Accuracy in last 100 iterations: 595/1000\n",
      "Loss at iteration 4100: 0.87491989\n",
      "Accuracy in last 100 iterations: 632/1000\n",
      "Loss at iteration 4200: 0.93631172\n",
      "Accuracy in last 100 iterations: 601/1000\n",
      "Loss at iteration 4300: 1.02095771\n",
      "Accuracy in last 100 iterations: 633/1000\n",
      "Loss at iteration 4400: 0.93548948\n",
      "Accuracy in last 100 iterations: 640/1000\n",
      "Loss at iteration 4500: 0.73972279\n",
      "Accuracy in last 100 iterations: 609/1000\n",
      "Loss at iteration 4600: 1.26885068\n",
      "Accuracy in last 100 iterations: 643/1000\n",
      "Loss at iteration 4700: 0.86254722\n",
      "Accuracy in last 100 iterations: 617/1000\n",
      "Loss at iteration 4800: 0.80972588\n",
      "Accuracy in last 100 iterations: 657/1000\n",
      "Loss at iteration 4900: 0.81690538\n",
      "Accuracy in last 100 iterations: 657/1000\n",
      "Loss at iteration 5000: 0.74332285\n",
      "Accuracy in last 100 iterations: 673/1000\n",
      "Loss at iteration 5100: 0.73853445\n",
      "Accuracy in last 100 iterations: 647/1000\n",
      "Loss at iteration 5200: 0.82163227\n",
      "Accuracy in last 100 iterations: 678/1000\n",
      "Loss at iteration 5300: 0.72679734\n",
      "Accuracy in last 100 iterations: 674/1000\n",
      "Loss at iteration 5400: 1.00347400\n",
      "Accuracy in last 100 iterations: 680/1000\n",
      "Loss at iteration 5500: 0.73103285\n",
      "Accuracy in last 100 iterations: 677/1000\n",
      "Loss at iteration 5600: 0.88505089\n",
      "Accuracy in last 100 iterations: 694/1000\n",
      "Loss at iteration 5700: 0.72902560\n",
      "Accuracy in last 100 iterations: 718/1000\n",
      "Loss at iteration 5800: 0.92583960\n",
      "Accuracy in last 100 iterations: 724/1000\n",
      "Loss at iteration 5900: 0.75050092\n",
      "Accuracy in last 100 iterations: 740/1000\n",
      "Loss at iteration 6000: 0.60889721\n",
      "Accuracy in last 100 iterations: 727/1000\n",
      "Loss at iteration 6100: 0.80499268\n",
      "Accuracy in last 100 iterations: 699/1000\n",
      "Loss at iteration 6200: 0.72809166\n",
      "Accuracy in last 100 iterations: 734/1000\n",
      "Loss at iteration 6300: 1.02738702\n",
      "Accuracy in last 100 iterations: 747/1000\n",
      "Loss at iteration 6400: 0.75950420\n",
      "Accuracy in last 100 iterations: 709/1000\n",
      "Loss at iteration 6500: 0.68151093\n",
      "Accuracy in last 100 iterations: 722/1000\n",
      "Loss at iteration 6600: 0.68720376\n",
      "Accuracy in last 100 iterations: 741/1000\n",
      "Loss at iteration 6700: 0.74148524\n",
      "Accuracy in last 100 iterations: 741/1000\n",
      "Loss at iteration 6800: 0.67242336\n",
      "Accuracy in last 100 iterations: 769/1000\n",
      "Loss at iteration 6900: 0.66465986\n",
      "Accuracy in last 100 iterations: 763/1000\n",
      "Loss at iteration 7000: 0.54150641\n",
      "Accuracy in last 100 iterations: 771/1000\n",
      "Loss at iteration 7100: 0.77868557\n",
      "Accuracy in last 100 iterations: 760/1000\n",
      "Loss at iteration 7200: 0.54315388\n",
      "Accuracy in last 100 iterations: 769/1000\n",
      "Loss at iteration 7300: 0.68764728\n",
      "Accuracy in last 100 iterations: 782/1000\n",
      "Loss at iteration 7400: 0.51279908\n",
      "Accuracy in last 100 iterations: 766/1000\n",
      "Loss at iteration 7500: 0.64254719\n",
      "Accuracy in last 100 iterations: 771/1000\n",
      "Loss at iteration 7600: 0.57138848\n",
      "Accuracy in last 100 iterations: 803/1000\n",
      "Loss at iteration 7700: 0.80416507\n",
      "Accuracy in last 100 iterations: 754/1000\n",
      "Loss at iteration 7800: 0.77662861\n",
      "Accuracy in last 100 iterations: 773/1000\n",
      "Loss at iteration 7900: 0.52392620\n",
      "Accuracy in last 100 iterations: 768/1000\n",
      "Loss at iteration 8000: 0.39395919\n",
      "Accuracy in last 100 iterations: 790/1000\n",
      "Loss at iteration 8100: 1.04152465\n",
      "Accuracy in last 100 iterations: 779/1000\n",
      "Loss at iteration 8200: 0.71937025\n",
      "Accuracy in last 100 iterations: 781/1000\n",
      "Loss at iteration 8300: 0.49642071\n",
      "Accuracy in last 100 iterations: 779/1000\n",
      "Loss at iteration 8400: 0.40617600\n",
      "Accuracy in last 100 iterations: 805/1000\n",
      "Loss at iteration 8500: 0.43525106\n",
      "Accuracy in last 100 iterations: 782/1000\n",
      "Loss at iteration 8600: 0.73094285\n",
      "Accuracy in last 100 iterations: 796/1000\n",
      "Loss at iteration 8700: 0.44543463\n",
      "Accuracy in last 100 iterations: 812/1000\n",
      "Loss at iteration 8800: 0.48850617\n",
      "Accuracy in last 100 iterations: 808/1000\n",
      "Loss at iteration 8900: 0.75204390\n",
      "Accuracy in last 100 iterations: 787/1000\n",
      "Loss at iteration 9000: 0.59020770\n",
      "Accuracy in last 100 iterations: 799/1000\n",
      "Loss at iteration 9100: 0.72049272\n",
      "Accuracy in last 100 iterations: 805/1000\n",
      "Loss at iteration 9200: 0.49160600\n",
      "Accuracy in last 100 iterations: 802/1000\n",
      "Loss at iteration 9300: 0.48970541\n",
      "Accuracy in last 100 iterations: 797/1000\n",
      "Loss at iteration 9400: 0.45901242\n",
      "Accuracy in last 100 iterations: 810/1000\n",
      "Loss at iteration 9500: 0.55068702\n",
      "Accuracy in last 100 iterations: 819/1000\n",
      "Loss at iteration 9600: 0.98233223\n",
      "Accuracy in last 100 iterations: 824/1000\n",
      "Loss at iteration 9700: 0.52279931\n",
      "Accuracy in last 100 iterations: 812/1000\n",
      "Loss at iteration 9800: 0.68569291\n",
      "Accuracy in last 100 iterations: 828/1000\n",
      "Loss at iteration 9900: 0.35688090\n",
      "Accuracy in last 100 iterations: 827/1000\n",
      "Loss at iteration 0: 0.40497142\n",
      "Accuracy in last 100 iterations: 9/10\n",
      "Loss at iteration 100: 0.41003007\n",
      "Accuracy in last 100 iterations: 824/1000\n",
      "Loss at iteration 200: 0.60343492\n",
      "Accuracy in last 100 iterations: 832/1000\n",
      "Loss at iteration 300: 0.33939618\n",
      "Accuracy in last 100 iterations: 815/1000\n",
      "Loss at iteration 400: 0.46731049\n",
      "Accuracy in last 100 iterations: 841/1000\n",
      "Loss at iteration 500: 0.42343909\n",
      "Accuracy in last 100 iterations: 839/1000\n",
      "Loss at iteration 600: 0.28033024\n",
      "Accuracy in last 100 iterations: 835/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 700: 0.76284140\n",
      "Accuracy in last 100 iterations: 813/1000\n",
      "Loss at iteration 800: 0.47293895\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 900: 0.44528484\n",
      "Accuracy in last 100 iterations: 837/1000\n",
      "Loss at iteration 1000: 0.66426647\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 1100: 0.35119995\n",
      "Accuracy in last 100 iterations: 817/1000\n",
      "Loss at iteration 1200: 0.72006685\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 1300: 0.55954564\n",
      "Accuracy in last 100 iterations: 842/1000\n",
      "Loss at iteration 1400: 0.30809802\n",
      "Accuracy in last 100 iterations: 815/1000\n",
      "Loss at iteration 1500: 0.51825374\n",
      "Accuracy in last 100 iterations: 837/1000\n",
      "Loss at iteration 1600: 0.39874333\n",
      "Accuracy in last 100 iterations: 832/1000\n",
      "Loss at iteration 1700: 0.47264877\n",
      "Accuracy in last 100 iterations: 831/1000\n",
      "Loss at iteration 1800: 0.31630525\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 1900: 0.41236225\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 2000: 0.48123702\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 2100: 0.55043066\n",
      "Accuracy in last 100 iterations: 836/1000\n",
      "Loss at iteration 2200: 0.98849660\n",
      "Accuracy in last 100 iterations: 841/1000\n",
      "Loss at iteration 2300: 0.47506642\n",
      "Accuracy in last 100 iterations: 819/1000\n",
      "Loss at iteration 2400: 0.25348514\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 2500: 0.34456921\n",
      "Accuracy in last 100 iterations: 843/1000\n",
      "Loss at iteration 2600: 0.52969193\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 2700: 0.40576202\n",
      "Accuracy in last 100 iterations: 835/1000\n",
      "Loss at iteration 2800: 0.35829800\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 2900: 0.33672196\n",
      "Accuracy in last 100 iterations: 856/1000\n",
      "Loss at iteration 3000: 0.34302020\n",
      "Accuracy in last 100 iterations: 863/1000\n",
      "Loss at iteration 3100: 0.24883084\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 3200: 0.30435252\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 3300: 0.21914139\n",
      "Accuracy in last 100 iterations: 856/1000\n",
      "Loss at iteration 3400: 0.38715142\n",
      "Accuracy in last 100 iterations: 865/1000\n",
      "Loss at iteration 3500: 0.41941375\n",
      "Accuracy in last 100 iterations: 863/1000\n",
      "Loss at iteration 3600: 0.31353173\n",
      "Accuracy in last 100 iterations: 880/1000\n",
      "Loss at iteration 3700: 0.53408086\n",
      "Accuracy in last 100 iterations: 852/1000\n",
      "Loss at iteration 3800: 0.32963815\n",
      "Accuracy in last 100 iterations: 840/1000\n",
      "Loss at iteration 3900: 0.41664609\n",
      "Accuracy in last 100 iterations: 835/1000\n",
      "Loss at iteration 4000: 0.55068696\n",
      "Accuracy in last 100 iterations: 850/1000\n",
      "Loss at iteration 4100: 0.63091910\n",
      "Accuracy in last 100 iterations: 848/1000\n",
      "Loss at iteration 4200: 0.52227002\n",
      "Accuracy in last 100 iterations: 859/1000\n",
      "Loss at iteration 4300: 0.32258925\n",
      "Accuracy in last 100 iterations: 890/1000\n",
      "Loss at iteration 4400: 0.28117150\n",
      "Accuracy in last 100 iterations: 889/1000\n",
      "Loss at iteration 4500: 0.33256549\n",
      "Accuracy in last 100 iterations: 855/1000\n",
      "Loss at iteration 4600: 0.23626614\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 4700: 0.87060547\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 4800: 0.38458714\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 4900: 0.26141587\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 5000: 0.28378314\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 5100: 0.30976874\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 5200: 0.20904541\n",
      "Accuracy in last 100 iterations: 845/1000\n",
      "Loss at iteration 5300: 0.34837031\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 5400: 0.33831173\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 5500: 0.18068047\n",
      "Accuracy in last 100 iterations: 892/1000\n",
      "Loss at iteration 5600: 0.42699227\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 5700: 0.31838140\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 5800: 0.48624906\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 5900: 0.27927899\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 6000: 0.17411347\n",
      "Accuracy in last 100 iterations: 858/1000\n",
      "Loss at iteration 6100: 0.57237047\n",
      "Accuracy in last 100 iterations: 872/1000\n",
      "Loss at iteration 6200: 0.38790196\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 6300: 0.42974395\n",
      "Accuracy in last 100 iterations: 856/1000\n",
      "Loss at iteration 6400: 0.33043212\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 6500: 0.18029256\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 6600: 0.22622594\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 6700: 0.38999686\n",
      "Accuracy in last 100 iterations: 875/1000\n",
      "Loss at iteration 6800: 0.22234926\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 6900: 0.61901677\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 7000: 0.32516688\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 7100: 0.26135725\n",
      "Accuracy in last 100 iterations: 891/1000\n",
      "Loss at iteration 7200: 0.47558013\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 7300: 0.33268365\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 7400: 0.27156591\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 7500: 0.19333729\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 7600: 0.39731708\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 7700: 0.51356107\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 7800: 0.32060164\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 7900: 0.42195973\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 8000: 0.32316130\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 8100: 0.47330943\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 8200: 0.31795007\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 8300: 0.25921267\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 8400: 0.47088009\n",
      "Accuracy in last 100 iterations: 883/1000\n",
      "Loss at iteration 8500: 0.28360367\n",
      "Accuracy in last 100 iterations: 884/1000\n",
      "Loss at iteration 8600: 0.22885275\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 8700: 0.36321658\n",
      "Accuracy in last 100 iterations: 892/1000\n",
      "Loss at iteration 8800: 0.20993944\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 8900: 0.21163473\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 9000: 0.26821560\n",
      "Accuracy in last 100 iterations: 893/1000\n",
      "Loss at iteration 9100: 0.26228935\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 9200: 0.33329314\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 9300: 0.26150838\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 9400: 0.24346586\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 9500: 0.36116236\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 9600: 0.26208305\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 9700: 0.27637926\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 9800: 0.20555377\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 9900: 0.19409776\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 0: 0.28446227\n",
      "Accuracy in last 100 iterations: 10/10\n",
      "Loss at iteration 100: 0.23208185\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 200: 0.41484123\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 300: 0.23017105\n",
      "Accuracy in last 100 iterations: 893/1000\n",
      "Loss at iteration 400: 0.35335812\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 500: 0.31316200\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 600: 0.18322173\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 700: 0.43444142\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 800: 0.19868989\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 900: 0.26816931\n",
      "Accuracy in last 100 iterations: 882/1000\n",
      "Loss at iteration 1000: 0.24806213\n",
      "Accuracy in last 100 iterations: 869/1000\n",
      "Loss at iteration 1100: 0.44770616\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 1200: 0.26388869\n",
      "Accuracy in last 100 iterations: 864/1000\n",
      "Loss at iteration 1300: 0.28860641\n",
      "Accuracy in last 100 iterations: 925/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1400: 0.21556768\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 1500: 0.30342746\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 1600: 0.28520697\n",
      "Accuracy in last 100 iterations: 891/1000\n",
      "Loss at iteration 1700: 0.14761725\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 1800: 0.32201451\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 1900: 0.14645715\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 2000: 0.26170453\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 2100: 0.14873472\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 2200: 0.16916394\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 2300: 0.21385804\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 2400: 0.25315881\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 2500: 0.14264898\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 2600: 0.19340658\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 2700: 0.15795675\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 2800: 0.16978312\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 2900: 0.13504434\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 3000: 0.18509808\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 3100: 0.21027637\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 3200: 0.29861680\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 3300: 0.26968044\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 3400: 0.20921302\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 3500: 0.25857949\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 3600: 0.20469490\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 3700: 0.25168127\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 3800: 0.23451605\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 3900: 0.34140259\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 4000: 0.09988312\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 4100: 0.16693664\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 4200: 0.14705181\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 4300: 0.30526534\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 4400: 0.19893703\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 4500: 0.11661200\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 4600: 0.22365336\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 4700: 0.21349673\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 4800: 0.17181206\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 4900: 0.49634796\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 5000: 0.29153758\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 5100: 0.22963409\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 5200: 0.41114560\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 5300: 0.47433519\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 5400: 0.18838811\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 5500: 0.12375794\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 5600: 0.19334897\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 5700: 0.64096403\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 5800: 0.19778857\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 5900: 0.11670633\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 6000: 0.08418822\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 6100: 0.64649832\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 6200: 0.13167886\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 6300: 0.47356540\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 6400: 0.22612262\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 6500: 0.08226185\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 6600: 0.22086649\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 6700: 0.16420965\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 6800: 0.60821569\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 6900: 0.16713934\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 7000: 0.37205663\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 7100: 0.13523945\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 7200: 0.34277064\n",
      "Accuracy in last 100 iterations: 879/1000\n",
      "Loss at iteration 7300: 0.52593386\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 7400: 0.15628958\n",
      "Accuracy in last 100 iterations: 942/1000\n",
      "Loss at iteration 7500: 0.19833808\n",
      "Accuracy in last 100 iterations: 961/1000\n",
      "Loss at iteration 7600: 0.08827873\n",
      "Accuracy in last 100 iterations: 944/1000\n",
      "Loss at iteration 7700: 0.69758016\n",
      "Accuracy in last 100 iterations: 928/1000\n",
      "Loss at iteration 7800: 0.12976427\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 7900: 0.34135938\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 8000: 0.37828475\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 8100: 0.20757246\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 8200: 0.14417210\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 8300: 0.06553669\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 8400: 0.28839025\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 8500: 0.24243793\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 8600: 0.17170843\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 8700: 0.14645854\n",
      "Accuracy in last 100 iterations: 920/1000\n",
      "Loss at iteration 8800: 0.17973046\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 8900: 0.13900141\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 9000: 0.12985125\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 9100: 0.12307806\n",
      "Accuracy in last 100 iterations: 941/1000\n",
      "Loss at iteration 9200: 0.30365053\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 9300: 0.27683130\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 9400: 0.09665985\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 9500: 0.19632873\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 9600: 0.06510182\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 9700: 0.07247929\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 9800: 0.07060447\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 9900: 0.19613428\n",
      "Accuracy in last 100 iterations: 925/1000\n"
     ]
    }
   ],
   "source": [
    "for _ in range(3):\n",
    "    attn_general.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.262"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_after = []\n",
    "for _ in range(1000):\n",
    "    accuracy_after.append(attn_general.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sequence, correct_sequence, softmax_input, accurate, attentions = attn_general.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([2, 6, 8, 5, 3, 4, 9, 0, 1, 7]),\n",
       " tensor([2, 6, 8, 5, 3, 4, 9, 0, 1, 7]),\n",
       " tensor(10))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_input.max(1)[1], correct_sequence, accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(x):\n",
    "    return (x - x.min())/(x.max() - x.min())\n",
    "\n",
    "np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])\n",
    "\n",
    "attention_normalized = np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1232b7550>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADCdJREFUeJzt3V1onvUZx/Hfzyb2xa740iB9c62l6GQ6OkNxCuLLkHUb82RIK4oK0pOtujEZbicTEY9K2Q7GsNR6srqWdiJzlL2Iq7ADS2NrqTUWpauaNtP6ttYitmmvHSSDztk8d8z/751c+35AaOLt5UV4vr3v58mdJ44IAcjpnLYXAFAPgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWFeNobNmzYqenp7ic7u6qqyrI0eOFJ9pu/hMSVq0aFGVuceOHasy94MPPqgyd2hoqPjMDz/8sPhMSap1t2hEdHyQVSmmp6dHjz76aPG5s2fPLj5Tkh577LHiM2v9ZfTkk09Wmfv8889Xmbt58+Yqc997773iM5955pniMyXp448/rjK3CS7RgcQIHEiMwIHECBxIjMCBxAgcSKxR4La/ZXu/7ddtP1h7KQBldAzc9hRJv5a0XNIVklbavqL2YgDGr8kZfJmk1yPiQESckLRJ0q111wJQQpPA50l664yPB0Y+919sr7LdZ7vv6NGjpfYDMA5NAv+s+13/5+baiFgXEb0R0Ttr1qzxbwZg3JoEPiBpwRkfz5d0uM46AEpqEvhOSUtsL7J9rqQVkv5Qdy0AJXT8kaeIGLL9Q0l/ljRF0oaI2Fd9MwDj1uhnGiNim6RtlXcBUBh3sgGJETiQGIEDiRE4kBiBA4lVeWfA48ePa+fOncXnrlmzpvhMSdq7d2/xmYcP17kXaM6cOVXmLl++vMrc3bt3V5l71113FZ954MCB4jMlqa+vr8rcJjiDA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJVXtX1R07dhSf+8gjjxSfKUkPPfRQ8ZkPPPBA8ZmSNDg4WGXuDTfcUGXuwYMHq8zdvHlz8Zl79uwpPrNtnMGBxAgcSIzAgcQIHEiMwIHECBxIrGPgthfY/pvtftv7bN//RSwGYPyafB98SNJPImKX7S9JetH2XyPilcq7ARinjmfwiBiMiF0jfz4mqV/SvNqLARi/MT0Ht71Q0lJJ5W9TA1Bc41tVbc+U9HtJP4qIo5/x71dJWiVJU6dOLbYggM+v0RncdreG494YEU991jERsS4ieiOit7u7u+SOAD6nJq+iW9LjkvojYm39lQCU0uQMfp2kOyXdZPulkX++XXkvAAV0fA4eEX+X5C9gFwCFcScbkBiBA4kROJAYgQOJETiQmCOi+NBp06bFggULis997bXXis+UpIULFxafuX79+uIzJemFF16oMvfdd9+tMnfjxo1V5j788MPFZ65evbr4TEk6depUlbkR0fG7W5zBgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHEqryrand3d8yePbv43Ntuu634TEnat29f8ZnPPvts8ZmSdOONN1aZe/nll1eZu3///ipzDx06VHzm4OBg8ZmSdOzYsSpzeVdV4P8cgQOJETiQGIEDiRE4kBiBA4kROJBY48BtT7G92/Yfay4EoJyxnMHvl9RfaxEA5TUK3PZ8Sd+RVOeXXgOooukZ/JeSfirp9NkOsL3Kdp/tvtOnz3oYgC9Qx8Btf1fSOxHx4mjHRcS6iOiNiN5zzuG1O2AiaFLidZK+Z/ugpE2SbrL926pbASiiY+AR8bOImB8RCyWtkPRcRNxRfTMA48a1NJBY11gOjojtkrZX2QRAcZzBgcQIHEiMwIHECBxIjMCBxMb0KnpTF154oVauXFl87tq1a4vPlKTFixcXn3nJJZcUnylJF198cZW5y5YtqzJ3+/btVebWeHxt2LCh+Eyp3ruqNsEZHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIzBFRfOjMmTPjyiuvLD53zpw5xWdK0tNPP1185vTp04vPlKTu7u4qc++9994qc7ds2VJl7htvvFF85kUXXVR8piS9//77VeZGhDsdwxkcSIzAgcQIHEiMwIHECBxIjMCBxBoFbvt821ttv2q73/Y3ai8GYPya/nbRX0n6U0R83/a5kmZU3AlAIR0Dtz1L0vWS7pakiDgh6UTdtQCU0OQS/VJJRyQ9YXu37fW2z6u8F4ACmgTeJenrkn4TEUslHZf04KcPsr3Kdp/tvpMnTxZeE8Dn0STwAUkDEbFj5OOtGg7+v0TEuojojYjeWvdLAxibjoFHxD8lvWX7spFP3SzplapbASii6avoqyVtHHkF/YCke+qtBKCURoFHxEuSeivvAqAw7mQDEiNwIDECBxIjcCAxAgcSI3AgsabfBx+ToaGhau8kWcPVV19dfOa2bduKz5Sknp6eKnOXLFlSZW5XV5WHmE6fPl185kcffVR8Zts4gwOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWJV3xJsxY4auuuqq4nO3bNlSfKYkzZ07t/jMW265pfhMSVq6dGmVuYsXL64yd8+ePVXmbtq0qfhM28Vnto0zOJAYgQOJETiQGIEDiRE4kBiBA4kROJBYo8Bt/9j2Ptsv2/6d7Wm1FwMwfh0Dtz1P0n2SeiPiq5KmSFpRezEA49f0Er1L0nTbXZJmSDpcbyUApXQMPCIOSVoj6U1Jg5L+FRF/+fRxtlfZ7rPd98knn5TfFMCYNblEv0DSrZIWSZor6Tzbd3z6uIhYFxG9EdE7derU8psCGLMml+jflPSPiDgSESclPSXp2rprASihSeBvSrrG9gwP/7jNzZL6664FoIQmz8F3SNoqaZekvSP/zbrKewEooNHPg0fELyT9ovIuAArjTjYgMQIHEiNwIDECBxIjcCCxKu+qevz4ce3cubP43BUr6vyMy9tvv118ZldXlS+tTp06VWXu7bffXmXuwMBAlbk19r377ruLz2wbZ3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwIDFHRPmh9hFJbzQ4dLakd4svUM9k2ncy7SpNrn0nwq5fjoieTgdVCbwp230R0dvaAmM0mfadTLtKk2vfybQrl+hAYgQOJNZ24Ota/v+P1WTadzLtKk2ufSfNrq0+BwdQV9tncAAVtRa47W/Z3m/7ddsPtrVHJ7YX2P6b7X7b+2zf3/ZOTdieYnu37T+2vctobJ9ve6vtV0e+xt9oe6fR2P7xyOPgZdu/sz2t7Z1G00rgtqdI+rWk5ZKukLTS9hVt7NLAkKSfRMRXJF0j6QcTeNcz3S+pv+0lGviVpD9FxOWSvqYJvLPteZLuk9QbEV+VNEVSnV95W0hbZ/Blkl6PiAMRcULSJkm3trTLqCJiMCJ2jfz5mIYfgPPa3Wp0tudL+o6k9W3vMhrbsyRdL+lxSYqIExHxYbtbddQlabrtLkkzJB1ueZ9RtRX4PElvnfHxgCZ4NJJke6GkpZJ2tLtJR7+U9FNJp9tepINLJR2R9MTI04n1ts9re6mziYhDktZIelPSoKR/RcRf2t1qdG0F7s/43IR+Od/2TEm/l/SjiDja9j5nY/u7kt6JiBfb3qWBLklfl/SbiFgq6bikifx6zAUavtJcJGmupPNs39HuVqNrK/ABSQvO+Hi+JvClju1uDce9MSKeanufDq6T9D3bBzX81Ocm279td6WzGpA0EBH/uSLaquHgJ6pvSvpHRByJiJOSnpJ0bcs7jaqtwHdKWmJ7ke1zNfxCxR9a2mVUtq3h54j9EbG27X06iYifRcT8iFio4a/rcxExIc8yEfFPSW/ZvmzkUzdLeqXFlTp5U9I1tmeMPC5u1gR+UVAavkT6wkXEkO0fSvqzhl+J3BAR+9rYpYHrJN0paa/tl0Y+9/OI2NbiTpmslrRx5C/6A5LuaXmfs4qIHba3Stql4e+u7NYEv6uNO9mAxLiTDUiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHE/g29acTqm/y46AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(attention_normalized, 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[7, 8, 0, 4, 5, 3, 1, 9, 2, 6]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Content based attention - concatenation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import content_based_concat_attention as attn_concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_before = []\n",
    "for _ in range(100):\n",
    "    accuracy_before.append(attn_concat.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_before)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 0: 2.30491900\n",
      "Accuracy in last 100 iterations: 1/10\n",
      "Loss at iteration 100: 2.28978395\n",
      "Accuracy in last 100 iterations: 113/1000\n",
      "Loss at iteration 200: 2.30278301\n",
      "Accuracy in last 100 iterations: 126/1000\n",
      "Loss at iteration 300: 2.35006380\n",
      "Accuracy in last 100 iterations: 156/1000\n",
      "Loss at iteration 400: 2.24085283\n",
      "Accuracy in last 100 iterations: 185/1000\n",
      "Loss at iteration 500: 1.91915226\n",
      "Accuracy in last 100 iterations: 208/1000\n",
      "Loss at iteration 600: 2.21466279\n",
      "Accuracy in last 100 iterations: 219/1000\n",
      "Loss at iteration 700: 2.01521349\n",
      "Accuracy in last 100 iterations: 251/1000\n",
      "Loss at iteration 800: 2.09983015\n",
      "Accuracy in last 100 iterations: 244/1000\n",
      "Loss at iteration 900: 2.11935568\n",
      "Accuracy in last 100 iterations: 247/1000\n",
      "Loss at iteration 1000: 2.04693079\n",
      "Accuracy in last 100 iterations: 259/1000\n",
      "Loss at iteration 1100: 1.94330001\n",
      "Accuracy in last 100 iterations: 268/1000\n",
      "Loss at iteration 1200: 1.87062323\n",
      "Accuracy in last 100 iterations: 313/1000\n",
      "Loss at iteration 1300: 1.75163579\n",
      "Accuracy in last 100 iterations: 303/1000\n",
      "Loss at iteration 1400: 1.68175149\n",
      "Accuracy in last 100 iterations: 313/1000\n",
      "Loss at iteration 1500: 1.89253199\n",
      "Accuracy in last 100 iterations: 314/1000\n",
      "Loss at iteration 1600: 1.87828600\n",
      "Accuracy in last 100 iterations: 321/1000\n",
      "Loss at iteration 1700: 1.91299593\n",
      "Accuracy in last 100 iterations: 343/1000\n",
      "Loss at iteration 1800: 1.75422978\n",
      "Accuracy in last 100 iterations: 358/1000\n",
      "Loss at iteration 1900: 1.69822180\n",
      "Accuracy in last 100 iterations: 348/1000\n",
      "Loss at iteration 2000: 1.59665966\n",
      "Accuracy in last 100 iterations: 368/1000\n",
      "Loss at iteration 2100: 1.56245613\n",
      "Accuracy in last 100 iterations: 372/1000\n",
      "Loss at iteration 2200: 1.68558717\n",
      "Accuracy in last 100 iterations: 379/1000\n",
      "Loss at iteration 2300: 1.55360222\n",
      "Accuracy in last 100 iterations: 380/1000\n",
      "Loss at iteration 2400: 1.59330285\n",
      "Accuracy in last 100 iterations: 397/1000\n",
      "Loss at iteration 2500: 1.71676278\n",
      "Accuracy in last 100 iterations: 415/1000\n",
      "Loss at iteration 2600: 1.64847946\n",
      "Accuracy in last 100 iterations: 417/1000\n",
      "Loss at iteration 2700: 1.47487247\n",
      "Accuracy in last 100 iterations: 416/1000\n",
      "Loss at iteration 2800: 1.48156679\n",
      "Accuracy in last 100 iterations: 410/1000\n",
      "Loss at iteration 2900: 1.69426215\n",
      "Accuracy in last 100 iterations: 452/1000\n",
      "Loss at iteration 3000: 1.40074027\n",
      "Accuracy in last 100 iterations: 452/1000\n",
      "Loss at iteration 3100: 1.53510499\n",
      "Accuracy in last 100 iterations: 475/1000\n",
      "Loss at iteration 3200: 1.47946346\n",
      "Accuracy in last 100 iterations: 459/1000\n",
      "Loss at iteration 3300: 1.35343146\n",
      "Accuracy in last 100 iterations: 468/1000\n",
      "Loss at iteration 3400: 1.26804757\n",
      "Accuracy in last 100 iterations: 485/1000\n",
      "Loss at iteration 3500: 1.47665381\n",
      "Accuracy in last 100 iterations: 468/1000\n",
      "Loss at iteration 3600: 1.42587101\n",
      "Accuracy in last 100 iterations: 468/1000\n",
      "Loss at iteration 3700: 1.38153851\n",
      "Accuracy in last 100 iterations: 494/1000\n",
      "Loss at iteration 3800: 1.57551479\n",
      "Accuracy in last 100 iterations: 482/1000\n",
      "Loss at iteration 3900: 1.14968240\n",
      "Accuracy in last 100 iterations: 519/1000\n",
      "Loss at iteration 4000: 1.33527279\n",
      "Accuracy in last 100 iterations: 504/1000\n",
      "Loss at iteration 4100: 1.47940230\n",
      "Accuracy in last 100 iterations: 508/1000\n",
      "Loss at iteration 4200: 1.11802459\n",
      "Accuracy in last 100 iterations: 513/1000\n",
      "Loss at iteration 4300: 1.09457231\n",
      "Accuracy in last 100 iterations: 503/1000\n",
      "Loss at iteration 4400: 1.19059169\n",
      "Accuracy in last 100 iterations: 521/1000\n",
      "Loss at iteration 4500: 1.14801335\n",
      "Accuracy in last 100 iterations: 517/1000\n",
      "Loss at iteration 4600: 1.16217542\n",
      "Accuracy in last 100 iterations: 540/1000\n",
      "Loss at iteration 4700: 1.23489916\n",
      "Accuracy in last 100 iterations: 570/1000\n",
      "Loss at iteration 4800: 1.21466792\n",
      "Accuracy in last 100 iterations: 565/1000\n",
      "Loss at iteration 4900: 1.26336765\n",
      "Accuracy in last 100 iterations: 569/1000\n",
      "Loss at iteration 5000: 1.20781934\n",
      "Accuracy in last 100 iterations: 578/1000\n",
      "Loss at iteration 5100: 0.96090877\n",
      "Accuracy in last 100 iterations: 568/1000\n",
      "Loss at iteration 5200: 1.23310924\n",
      "Accuracy in last 100 iterations: 571/1000\n",
      "Loss at iteration 5300: 1.05984819\n",
      "Accuracy in last 100 iterations: 571/1000\n",
      "Loss at iteration 5400: 0.96905023\n",
      "Accuracy in last 100 iterations: 578/1000\n",
      "Loss at iteration 5500: 0.85679054\n",
      "Accuracy in last 100 iterations: 593/1000\n",
      "Loss at iteration 5600: 1.28480697\n",
      "Accuracy in last 100 iterations: 582/1000\n",
      "Loss at iteration 5700: 1.05127442\n",
      "Accuracy in last 100 iterations: 607/1000\n",
      "Loss at iteration 5800: 0.82839239\n",
      "Accuracy in last 100 iterations: 577/1000\n",
      "Loss at iteration 5900: 1.04382753\n",
      "Accuracy in last 100 iterations: 578/1000\n",
      "Loss at iteration 6000: 0.92141068\n",
      "Accuracy in last 100 iterations: 614/1000\n",
      "Loss at iteration 6100: 0.87831211\n",
      "Accuracy in last 100 iterations: 606/1000\n",
      "Loss at iteration 6200: 0.81992328\n",
      "Accuracy in last 100 iterations: 617/1000\n",
      "Loss at iteration 6300: 1.14245236\n",
      "Accuracy in last 100 iterations: 600/1000\n",
      "Loss at iteration 6400: 0.97954094\n",
      "Accuracy in last 100 iterations: 649/1000\n",
      "Loss at iteration 6500: 0.92738801\n",
      "Accuracy in last 100 iterations: 622/1000\n",
      "Loss at iteration 6600: 0.94328582\n",
      "Accuracy in last 100 iterations: 628/1000\n",
      "Loss at iteration 6700: 0.98360175\n",
      "Accuracy in last 100 iterations: 614/1000\n",
      "Loss at iteration 6800: 0.82481021\n",
      "Accuracy in last 100 iterations: 636/1000\n",
      "Loss at iteration 6900: 1.19280934\n",
      "Accuracy in last 100 iterations: 647/1000\n",
      "Loss at iteration 7000: 0.97006834\n",
      "Accuracy in last 100 iterations: 647/1000\n",
      "Loss at iteration 7100: 0.90537453\n",
      "Accuracy in last 100 iterations: 646/1000\n",
      "Loss at iteration 7200: 0.94789684\n",
      "Accuracy in last 100 iterations: 672/1000\n",
      "Loss at iteration 7300: 1.48410833\n",
      "Accuracy in last 100 iterations: 673/1000\n",
      "Loss at iteration 7400: 0.85715353\n",
      "Accuracy in last 100 iterations: 661/1000\n",
      "Loss at iteration 7500: 1.03066075\n",
      "Accuracy in last 100 iterations: 692/1000\n",
      "Loss at iteration 7600: 0.73362923\n",
      "Accuracy in last 100 iterations: 646/1000\n",
      "Loss at iteration 7700: 0.49576011\n",
      "Accuracy in last 100 iterations: 685/1000\n",
      "Loss at iteration 7800: 0.85772336\n",
      "Accuracy in last 100 iterations: 675/1000\n",
      "Loss at iteration 7900: 0.92417097\n",
      "Accuracy in last 100 iterations: 705/1000\n",
      "Loss at iteration 8000: 1.13220000\n",
      "Accuracy in last 100 iterations: 692/1000\n",
      "Loss at iteration 8100: 1.00734770\n",
      "Accuracy in last 100 iterations: 678/1000\n",
      "Loss at iteration 8200: 0.90001202\n",
      "Accuracy in last 100 iterations: 697/1000\n",
      "Loss at iteration 8300: 0.53853405\n",
      "Accuracy in last 100 iterations: 694/1000\n",
      "Loss at iteration 8400: 0.65028727\n",
      "Accuracy in last 100 iterations: 717/1000\n",
      "Loss at iteration 8500: 0.68162709\n",
      "Accuracy in last 100 iterations: 715/1000\n",
      "Loss at iteration 8600: 0.72851396\n",
      "Accuracy in last 100 iterations: 691/1000\n",
      "Loss at iteration 8700: 0.64380968\n",
      "Accuracy in last 100 iterations: 718/1000\n",
      "Loss at iteration 8800: 0.94989681\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 8900: 0.55013198\n",
      "Accuracy in last 100 iterations: 734/1000\n",
      "Loss at iteration 9000: 0.65869206\n",
      "Accuracy in last 100 iterations: 698/1000\n",
      "Loss at iteration 9100: 0.97137612\n",
      "Accuracy in last 100 iterations: 772/1000\n",
      "Loss at iteration 9200: 1.02669454\n",
      "Accuracy in last 100 iterations: 735/1000\n",
      "Loss at iteration 9300: 0.69566649\n",
      "Accuracy in last 100 iterations: 749/1000\n",
      "Loss at iteration 9400: 0.41289648\n",
      "Accuracy in last 100 iterations: 736/1000\n",
      "Loss at iteration 9500: 0.51405632\n",
      "Accuracy in last 100 iterations: 741/1000\n",
      "Loss at iteration 9600: 0.87584198\n",
      "Accuracy in last 100 iterations: 750/1000\n",
      "Loss at iteration 9700: 0.97436440\n",
      "Accuracy in last 100 iterations: 778/1000\n",
      "Loss at iteration 9800: 0.51791561\n",
      "Accuracy in last 100 iterations: 766/1000\n",
      "Loss at iteration 9900: 0.67674148\n",
      "Accuracy in last 100 iterations: 740/1000\n",
      "Loss at iteration 0: 0.31863084\n",
      "Accuracy in last 100 iterations: 10/10\n",
      "Loss at iteration 100: 0.74318671\n",
      "Accuracy in last 100 iterations: 733/1000\n",
      "Loss at iteration 200: 0.31517595\n",
      "Accuracy in last 100 iterations: 763/1000\n",
      "Loss at iteration 300: 0.47506228\n",
      "Accuracy in last 100 iterations: 773/1000\n",
      "Loss at iteration 400: 0.60944915\n",
      "Accuracy in last 100 iterations: 778/1000\n",
      "Loss at iteration 500: 0.51358825\n",
      "Accuracy in last 100 iterations: 770/1000\n",
      "Loss at iteration 600: 0.57780379\n",
      "Accuracy in last 100 iterations: 783/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 700: 0.77074766\n",
      "Accuracy in last 100 iterations: 775/1000\n",
      "Loss at iteration 800: 0.50222474\n",
      "Accuracy in last 100 iterations: 794/1000\n",
      "Loss at iteration 900: 0.60948765\n",
      "Accuracy in last 100 iterations: 817/1000\n",
      "Loss at iteration 1000: 0.99139655\n",
      "Accuracy in last 100 iterations: 778/1000\n",
      "Loss at iteration 1100: 0.96068174\n",
      "Accuracy in last 100 iterations: 788/1000\n",
      "Loss at iteration 1200: 0.33649811\n",
      "Accuracy in last 100 iterations: 794/1000\n",
      "Loss at iteration 1300: 0.38193366\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 1400: 0.49591899\n",
      "Accuracy in last 100 iterations: 783/1000\n",
      "Loss at iteration 1500: 0.61563414\n",
      "Accuracy in last 100 iterations: 813/1000\n",
      "Loss at iteration 1600: 0.31038985\n",
      "Accuracy in last 100 iterations: 813/1000\n",
      "Loss at iteration 1700: 0.31561655\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 1800: 0.39372167\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 1900: 0.55202359\n",
      "Accuracy in last 100 iterations: 809/1000\n",
      "Loss at iteration 2000: 0.34098417\n",
      "Accuracy in last 100 iterations: 798/1000\n",
      "Loss at iteration 2100: 0.30510697\n",
      "Accuracy in last 100 iterations: 829/1000\n",
      "Loss at iteration 2200: 0.27169585\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 2300: 0.35513392\n",
      "Accuracy in last 100 iterations: 847/1000\n",
      "Loss at iteration 2400: 0.24581151\n",
      "Accuracy in last 100 iterations: 820/1000\n",
      "Loss at iteration 2500: 0.24231967\n",
      "Accuracy in last 100 iterations: 825/1000\n",
      "Loss at iteration 2600: 0.42262048\n",
      "Accuracy in last 100 iterations: 822/1000\n",
      "Loss at iteration 2700: 0.42403674\n",
      "Accuracy in last 100 iterations: 831/1000\n",
      "Loss at iteration 2800: 0.26114744\n",
      "Accuracy in last 100 iterations: 845/1000\n",
      "Loss at iteration 2900: 0.23321486\n",
      "Accuracy in last 100 iterations: 851/1000\n",
      "Loss at iteration 3000: 0.67040056\n",
      "Accuracy in last 100 iterations: 829/1000\n",
      "Loss at iteration 3100: 0.65921050\n",
      "Accuracy in last 100 iterations: 843/1000\n",
      "Loss at iteration 3200: 0.61134720\n",
      "Accuracy in last 100 iterations: 824/1000\n",
      "Loss at iteration 3300: 0.30097947\n",
      "Accuracy in last 100 iterations: 835/1000\n",
      "Loss at iteration 3400: 0.41850558\n",
      "Accuracy in last 100 iterations: 830/1000\n",
      "Loss at iteration 3500: 0.45618266\n",
      "Accuracy in last 100 iterations: 826/1000\n",
      "Loss at iteration 3600: 0.45137817\n",
      "Accuracy in last 100 iterations: 844/1000\n",
      "Loss at iteration 3700: 0.37528881\n",
      "Accuracy in last 100 iterations: 837/1000\n",
      "Loss at iteration 3800: 0.34443873\n",
      "Accuracy in last 100 iterations: 826/1000\n",
      "Loss at iteration 3900: 0.47065964\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 4000: 0.54010373\n",
      "Accuracy in last 100 iterations: 861/1000\n",
      "Loss at iteration 4100: 0.90744179\n",
      "Accuracy in last 100 iterations: 865/1000\n",
      "Loss at iteration 4200: 0.19896579\n",
      "Accuracy in last 100 iterations: 853/1000\n",
      "Loss at iteration 4300: 0.37246022\n",
      "Accuracy in last 100 iterations: 852/1000\n",
      "Loss at iteration 4400: 0.34768996\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 4500: 0.49801230\n",
      "Accuracy in last 100 iterations: 866/1000\n",
      "Loss at iteration 4600: 0.37550253\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 4700: 0.17514300\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 4800: 0.28477973\n",
      "Accuracy in last 100 iterations: 843/1000\n",
      "Loss at iteration 4900: 0.20754246\n",
      "Accuracy in last 100 iterations: 863/1000\n",
      "Loss at iteration 5000: 0.33713397\n",
      "Accuracy in last 100 iterations: 854/1000\n",
      "Loss at iteration 5100: 0.50826144\n",
      "Accuracy in last 100 iterations: 872/1000\n",
      "Loss at iteration 5200: 0.32257280\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 5300: 0.36457688\n",
      "Accuracy in last 100 iterations: 847/1000\n",
      "Loss at iteration 5400: 0.44556946\n",
      "Accuracy in last 100 iterations: 857/1000\n",
      "Loss at iteration 5500: 0.27426395\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 5600: 0.55537719\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 5700: 0.28175354\n",
      "Accuracy in last 100 iterations: 867/1000\n",
      "Loss at iteration 5800: 0.18466692\n",
      "Accuracy in last 100 iterations: 833/1000\n",
      "Loss at iteration 5900: 0.45125073\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 6000: 0.36491117\n",
      "Accuracy in last 100 iterations: 875/1000\n",
      "Loss at iteration 6100: 0.27548295\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 6200: 0.33296543\n",
      "Accuracy in last 100 iterations: 878/1000\n",
      "Loss at iteration 6300: 0.20521355\n",
      "Accuracy in last 100 iterations: 856/1000\n",
      "Loss at iteration 6400: 0.51593006\n",
      "Accuracy in last 100 iterations: 871/1000\n",
      "Loss at iteration 6500: 0.16886997\n",
      "Accuracy in last 100 iterations: 887/1000\n",
      "Loss at iteration 6600: 0.28222212\n",
      "Accuracy in last 100 iterations: 855/1000\n",
      "Loss at iteration 6700: 0.46200985\n",
      "Accuracy in last 100 iterations: 874/1000\n",
      "Loss at iteration 6800: 0.56856734\n",
      "Accuracy in last 100 iterations: 886/1000\n",
      "Loss at iteration 6900: 0.18471870\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 7000: 0.46280074\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 7100: 0.23348813\n",
      "Accuracy in last 100 iterations: 875/1000\n",
      "Loss at iteration 7200: 0.30596384\n",
      "Accuracy in last 100 iterations: 879/1000\n",
      "Loss at iteration 7300: 0.32141265\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 7400: 0.53931630\n",
      "Accuracy in last 100 iterations: 868/1000\n",
      "Loss at iteration 7500: 0.16728191\n",
      "Accuracy in last 100 iterations: 885/1000\n",
      "Loss at iteration 7600: 0.16556039\n",
      "Accuracy in last 100 iterations: 889/1000\n",
      "Loss at iteration 7700: 0.16057305\n",
      "Accuracy in last 100 iterations: 895/1000\n",
      "Loss at iteration 7800: 0.44647306\n",
      "Accuracy in last 100 iterations: 891/1000\n",
      "Loss at iteration 7900: 0.17456503\n",
      "Accuracy in last 100 iterations: 877/1000\n",
      "Loss at iteration 8000: 0.74347723\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 8100: 0.11574521\n",
      "Accuracy in last 100 iterations: 896/1000\n",
      "Loss at iteration 8200: 0.21230169\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 8300: 0.25604701\n",
      "Accuracy in last 100 iterations: 870/1000\n",
      "Loss at iteration 8400: 0.42745894\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 8500: 0.18858066\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 8600: 0.26706514\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 8700: 0.23350525\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 8800: 0.68253791\n",
      "Accuracy in last 100 iterations: 881/1000\n",
      "Loss at iteration 8900: 0.44272113\n",
      "Accuracy in last 100 iterations: 876/1000\n",
      "Loss at iteration 9000: 0.15828061\n",
      "Accuracy in last 100 iterations: 873/1000\n",
      "Loss at iteration 9100: 0.42523178\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 9200: 0.14057212\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 9300: 0.37657323\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 9400: 0.28731936\n",
      "Accuracy in last 100 iterations: 893/1000\n",
      "Loss at iteration 9500: 0.21108079\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 9600: 0.44796729\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 9700: 0.17186585\n",
      "Accuracy in last 100 iterations: 902/1000\n",
      "Loss at iteration 9800: 0.15373054\n",
      "Accuracy in last 100 iterations: 894/1000\n",
      "Loss at iteration 9900: 0.16247396\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 0: 0.15970139\n",
      "Accuracy in last 100 iterations: 10/10\n",
      "Loss at iteration 100: 0.17037669\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 200: 0.26441193\n",
      "Accuracy in last 100 iterations: 914/1000\n",
      "Loss at iteration 300: 0.25687796\n",
      "Accuracy in last 100 iterations: 903/1000\n",
      "Loss at iteration 400: 0.11237822\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 500: 0.15606113\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 600: 0.15541354\n",
      "Accuracy in last 100 iterations: 888/1000\n",
      "Loss at iteration 700: 0.20026556\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 800: 0.09270744\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 900: 0.14391465\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 1000: 0.20649429\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 1100: 0.15853110\n",
      "Accuracy in last 100 iterations: 900/1000\n",
      "Loss at iteration 1200: 0.15250936\n",
      "Accuracy in last 100 iterations: 904/1000\n",
      "Loss at iteration 1300: 0.36443359\n",
      "Accuracy in last 100 iterations: 907/1000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1400: 0.93679249\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 1500: 0.50631601\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 1600: 0.15184522\n",
      "Accuracy in last 100 iterations: 898/1000\n",
      "Loss at iteration 1700: 0.16795869\n",
      "Accuracy in last 100 iterations: 905/1000\n",
      "Loss at iteration 1800: 0.35315418\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 1900: 0.08675241\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 2000: 0.25390124\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 2100: 0.25954843\n",
      "Accuracy in last 100 iterations: 916/1000\n",
      "Loss at iteration 2200: 0.13572201\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 2300: 0.25778443\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 2400: 0.17485651\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 2500: 0.31829682\n",
      "Accuracy in last 100 iterations: 913/1000\n",
      "Loss at iteration 2600: 0.11687322\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 2700: 0.09260311\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 2800: 0.27300024\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 2900: 0.08089714\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 3000: 0.28405491\n",
      "Accuracy in last 100 iterations: 897/1000\n",
      "Loss at iteration 3100: 0.13763809\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 3200: 0.12764087\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 3300: 0.19163179\n",
      "Accuracy in last 100 iterations: 906/1000\n",
      "Loss at iteration 3400: 0.16012773\n",
      "Accuracy in last 100 iterations: 899/1000\n",
      "Loss at iteration 3500: 0.35534161\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 3600: 0.71957517\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 3700: 0.14973536\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 3800: 0.41252238\n",
      "Accuracy in last 100 iterations: 929/1000\n",
      "Loss at iteration 3900: 0.09035778\n",
      "Accuracy in last 100 iterations: 940/1000\n",
      "Loss at iteration 4000: 0.15236530\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 4100: 0.25269660\n",
      "Accuracy in last 100 iterations: 901/1000\n",
      "Loss at iteration 4200: 0.08326306\n",
      "Accuracy in last 100 iterations: 925/1000\n",
      "Loss at iteration 4300: 0.17970690\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 4400: 0.77311337\n",
      "Accuracy in last 100 iterations: 930/1000\n",
      "Loss at iteration 4500: 0.07962384\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 4600: 0.44417292\n",
      "Accuracy in last 100 iterations: 910/1000\n",
      "Loss at iteration 4700: 0.34920311\n",
      "Accuracy in last 100 iterations: 911/1000\n",
      "Loss at iteration 4800: 0.65246791\n",
      "Accuracy in last 100 iterations: 922/1000\n",
      "Loss at iteration 4900: 0.16850457\n",
      "Accuracy in last 100 iterations: 935/1000\n",
      "Loss at iteration 5000: 0.21815591\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 5100: 0.18773127\n",
      "Accuracy in last 100 iterations: 907/1000\n",
      "Loss at iteration 5200: 0.25855479\n",
      "Accuracy in last 100 iterations: 917/1000\n",
      "Loss at iteration 5300: 0.15772133\n",
      "Accuracy in last 100 iterations: 936/1000\n",
      "Loss at iteration 5400: 0.16649786\n",
      "Accuracy in last 100 iterations: 949/1000\n",
      "Loss at iteration 5500: 0.13705158\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 5600: 0.22431222\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 5700: 0.13220516\n",
      "Accuracy in last 100 iterations: 919/1000\n",
      "Loss at iteration 5800: 0.47263438\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 5900: 0.65719163\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 6000: 0.24450412\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 6100: 0.09883104\n",
      "Accuracy in last 100 iterations: 931/1000\n",
      "Loss at iteration 6200: 0.10417719\n",
      "Accuracy in last 100 iterations: 934/1000\n",
      "Loss at iteration 6300: 0.24865837\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 6400: 0.08734532\n",
      "Accuracy in last 100 iterations: 912/1000\n",
      "Loss at iteration 6500: 0.07823863\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 6600: 0.77017939\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 6700: 0.16433057\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 6800: 0.10281072\n",
      "Accuracy in last 100 iterations: 941/1000\n",
      "Loss at iteration 6900: 0.40549469\n",
      "Accuracy in last 100 iterations: 915/1000\n",
      "Loss at iteration 7000: 0.07288752\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 7100: 0.08258171\n",
      "Accuracy in last 100 iterations: 939/1000\n",
      "Loss at iteration 7200: 0.11798882\n",
      "Accuracy in last 100 iterations: 941/1000\n",
      "Loss at iteration 7300: 0.11632681\n",
      "Accuracy in last 100 iterations: 947/1000\n",
      "Loss at iteration 7400: 0.22138739\n",
      "Accuracy in last 100 iterations: 921/1000\n",
      "Loss at iteration 7500: 0.09295740\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 7600: 0.09713030\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 7700: 0.29095286\n",
      "Accuracy in last 100 iterations: 927/1000\n",
      "Loss at iteration 7800: 0.26684204\n",
      "Accuracy in last 100 iterations: 909/1000\n",
      "Loss at iteration 7900: 0.09701385\n",
      "Accuracy in last 100 iterations: 924/1000\n",
      "Loss at iteration 8000: 0.16285281\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 8100: 0.34438071\n",
      "Accuracy in last 100 iterations: 953/1000\n",
      "Loss at iteration 8200: 0.17931060\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 8300: 0.05412097\n",
      "Accuracy in last 100 iterations: 948/1000\n",
      "Loss at iteration 8400: 0.06040997\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 8500: 0.08856483\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 8600: 0.32753158\n",
      "Accuracy in last 100 iterations: 923/1000\n",
      "Loss at iteration 8700: 0.05845599\n",
      "Accuracy in last 100 iterations: 938/1000\n",
      "Loss at iteration 8800: 0.09872751\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 8900: 0.16384606\n",
      "Accuracy in last 100 iterations: 918/1000\n",
      "Loss at iteration 9000: 0.16121459\n",
      "Accuracy in last 100 iterations: 908/1000\n",
      "Loss at iteration 9100: 0.33184975\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 9200: 0.05212460\n",
      "Accuracy in last 100 iterations: 931/1000\n",
      "Loss at iteration 9300: 0.12684259\n",
      "Accuracy in last 100 iterations: 933/1000\n",
      "Loss at iteration 9400: 0.13573651\n",
      "Accuracy in last 100 iterations: 943/1000\n",
      "Loss at iteration 9500: 0.10604773\n",
      "Accuracy in last 100 iterations: 932/1000\n",
      "Loss at iteration 9600: 0.37078777\n",
      "Accuracy in last 100 iterations: 941/1000\n",
      "Loss at iteration 9700: 0.08130054\n",
      "Accuracy in last 100 iterations: 926/1000\n",
      "Loss at iteration 9800: 0.07077136\n",
      "Accuracy in last 100 iterations: 937/1000\n",
      "Loss at iteration 9900: 0.18114915\n",
      "Accuracy in last 100 iterations: 948/1000\n"
     ]
    }
   ],
   "source": [
    "for _ in range(3):\n",
    "    attn_concat.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.375"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_after = []\n",
    "for _ in range(1000):\n",
    "    accuracy_after.append(attn_concat.test()[-2].item())\n",
    "    \n",
    "np.mean(accuracy_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_sequence, correct_sequence, softmax_input, accurate, attentions = attn_concat.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([4, 2, 7, 6, 2, 3, 5, 0, 9, 8]),\n",
       " tensor([4, 2, 7, 6, 1, 3, 5, 0, 9, 8]),\n",
       " tensor(9))"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_input.max(1)[1], correct_sequence, accurate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(x):\n",
    "    return (x - x.min())/(x.max() - x.min())\n",
    "\n",
    "np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])\n",
    "\n",
    "attention_normalized = np.array([min_max_normalization(a[0,0].detach().numpy()) for a in attentions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x12331dfd0>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACdFJREFUeJzt3M+LXge9x/H3Z2YsmqaiULswKbeVivcWwVYepFpw0brQq9jNXVSocN1k448qgtS78R8Q0YUIoerGYhexCwlFvaAu7iY4TQOajkKp3jZpxNxFNXWTtM/XxcyFWJt5Tpxzema+vF9QyPP05PTDMO+e85xOmqpCUk9rcw+QNB0DlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKmxjSlOura2Vhsb45/6ypUro58T4Kabbhr9nK+++uro5wQ4dOjQJOed6ms71ddhfX199HPecccdo58T4MyZM6Ofc7lcslwus+q4SQLf2NjglltuGf2858+fH/2cAIvFYvRzXrp0afRzAtx1112TnPfChQuTnPfll1+e5LyHDx8e/ZwnT54c/ZwAN9988+jnfOmllwYd5y261JiBS40ZuNSYgUuNGbjUmIFLjQ0KPMlHk/wuybNJHpl6lKRxrAw8yTrwbeBjwJ3Ap5LcOfUwSXs35Ar+AeDZqnquqi4DjwMPTDtL0hiGBH4EeOGq1+d23vs7SY4l2UyyuVwux9onaQ+GBP56P+/6D/8r1qo6XlWLqlqsrfnsTtoPhpR4Drj1qtdHgRenmSNpTEMC/xXw7iS3J7kBeBD48bSzJI1h5Z8mq6pXknwO+CmwDnyvqs5OvkzSng3646JV9STw5MRbJI3Mp2FSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjW2MvAktyb5RZKtJGeTPPxGDJO0dxsDjnkF+HJVnU5yE/BUkv+uqmcm3iZpj1ZewavqQlWd3vn1JWALODL1MEl7d12fwZPcBtwNnJpijKRxDblFByDJYeBHwBer6i+v8/ePAccA1tfXRxso6Z836Aqe5E1sx/1YVT3xesdU1fGqWlTVYm3Nh/PSfjDkKXqA7wJbVfWN6SdJGsuQS+29wKeB+5Kc2fnr3yfeJWkEKz+DV9X/AHkDtkgamR+WpcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgxcaszApcYMXGrMwKXGDFxqzMClxgYHnmQ9ydNJTk45SNJ4rucK/jCwNdUQSeMbFHiSo8DHgUennSNpTEOv4N8EvgIsr3VAkmNJNpNsLpfXPEzSG2hl4Ek+Afypqp7a7biqOl5Vi6parK357E7aD4aUeC/wySR/AB4H7kvyg0lXSRrFysCr6qtVdbSqbgMeBH5eVQ9NvkzSnnkvLTW2cT0HV9UvgV9OskTS6LyCS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjUmIFLjRm41JiBS40ZuNSYgUuNGbjU2KDAk7wtyYkkv02yleSDUw+TtHcbA4/7FvCTqvqPJDcAhybcJGkkKwNP8lbgw8B/AlTVZeDytLMkjWHILfq7gIvA95M8neTRJDdOvEvSCIYEvgG8H/hOVd0N/BV45LUHJTmWZDPJ5nK5HHmmpH/GkMDPAeeq6tTO6xNsB/93qup4VS2qarG25sN5aT9YWWJV/RF4Icl7dt66H3hm0lWSRjH0Kfrngcd2nqA/B3xmukmSxjIo8Ko6Aywm3iJpZH5YlhozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGjNwqTEDlxozcKkxA5caM3CpMQOXGhsUeJIvJTmb5DdJfpjkzVMPk7R3KwNPcgT4ArCoqvcC68CDUw+TtHdDb9E3gLck2QAOAS9ON0nSWFYGXlXnga8DzwMXgD9X1c9ee1ySY0k2k2wul8vxl0q6bkNu0d8OPADcDrwTuDHJQ689rqqOV9WiqhZraz67k/aDISV+BPh9VV2sqivAE8CHpp0laQxDAn8euCfJoSQB7ge2pp0laQxDPoOfAk4Ap4Ff7/ye4xPvkjSCjSEHVdXXgK9NvEXSyHwaJjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjVm4FJjBi41lqoa/6TJReB/Bxx6M/B/ow+YzkHae5C2wsHaux+2/ktVvWPVQZMEPlSSzapazDbgOh2kvQdpKxysvQdpq7foUmMGLjU2d+DHZ/7nX6+DtPcgbYWDtffAbJ31M7ikac19BZc0odkCT/LRJL9L8mySR+basUqSW5P8IslWkrNJHp570xBJ1pM8neTk3Ft2k+RtSU4k+e3O1/iDc2/aTZIv7Xwf/CbJD5O8ee5Nu5kl8CTrwLeBjwF3Ap9KcuccWwZ4BfhyVf0bcA/w2X289WoPA1tzjxjgW8BPqupfgfexjzcnOQJ8AVhU1XuBdeDBeVftbq4r+AeAZ6vquaq6DDwOPDDTll1V1YWqOr3z60tsfwMemXfV7pIcBT4OPDr3lt0keSvwYeC7AFV1uapemnfVShvAW5JsAIeAF2fes6u5Aj8CvHDV63Ps82gAktwG3A2cmnfJSt8EvgIs5x6ywruAi8D3dz5OPJrkxrlHXUtVnQe+DjwPXAD+XFU/m3fV7uYKPK/z3r5+nJ/kMPAj4ItV9Ze591xLkk8Af6qqp+beMsAG8H7gO1V1N/BXYD8/j3k723eatwPvBG5M8tC8q3Y3V+DngFuven2UfXyrk+RNbMf9WFU9MfeeFe4FPpnkD2x/9LkvyQ/mnXRN54BzVfX/d0Qn2A5+v/oI8PuqulhVV4AngA/NvGlXcwX+K+DdSW5PcgPbDyp+PNOWXSUJ258Rt6rqG3PvWaWqvlpVR6vqNra/rj+vqn15lamqPwIvJHnPzlv3A8/MOGmV54F7khza+b64n338UBC2b5HecFX1SpLPAT9l+0nk96rq7BxbBrgX+DTw6yRndt77r6p6csZNnXweeGznX/TPAZ+Zec81VdWpJCeA02z/15Wn2ec/1eZPskmN+ZNsUmMGLjVm4FJjBi41ZuBSYwYuNWbgUmMGLjX2Ny/XHzzWraZxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(attention_normalized, 'gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[7, 4, 1, 5, 0, 6, 3, 2, 9, 8]])"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}