reddit_classification-Copy2-LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(60000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 60 seconds\n"
     ]
    }
   ],
   "source": [
    "%autosave 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "string.ascii_letters + string.digits\n",
    "string.printable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class for the dataset:\n",
    "import torch\n",
    "# class for text-datasets\n",
    "# Jama, NEJM and lancet\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import unicodedata\n",
    "class TextDataset(Dataset):\n",
    "\n",
    "    SPECIAL_CHARS = [None, \"\\n\"]\n",
    "    def __len__(self):\n",
    "        return len(self.total_examples)\n",
    "        pass\n",
    " \n",
    "    # generates tensors for the vocab\n",
    "    # i.e. assigns a number to each character\n",
    "    # Q: possible to do one-pass?\n",
    "    def make_vocab(self):\n",
    "        \n",
    "        self.char2index = {}\n",
    "        self.index2char = {}\n",
    "        index = 0\n",
    "        with open(self.text_file) as file:\n",
    "            for line in file:\n",
    "                for char in line:\n",
    "                    if char not in self.char2index:\n",
    "                        self.char2index[char] = index\n",
    "                        self.index2char[index]  = char\n",
    "                        index+=1\n",
    "        \n",
    "        for char in self.SPECIAL_CHARS:\n",
    "#             print(\"this is the char\")\n",
    "            if char not in self.char2index:\n",
    "                self.char2index[char] = index\n",
    "                self.index2char[index] = char\n",
    "                index += 1\n",
    "        \n",
    "        print(self.char2index[None])\n",
    "        # potentially need to add 2 for the None and \\n characters\n",
    "        self.vocab_size = len(self.char2index)\n",
    "    \n",
    "    # this allows the model to handle all possible strings passed into it!!\n",
    "    def full_vocab(self):\n",
    "        self.char2index = {}\n",
    "        self.index2char = {}\n",
    "        index = 0\n",
    "        all_letters = string.printable \n",
    "        for char in all_letters:\n",
    "            if char not in self.char2index:\n",
    "                self.char2index[char] = index\n",
    "                self.index2char[index]  = char\n",
    "                index+=1\n",
    "                \n",
    "        for char in self.SPECIAL_CHARS:\n",
    "#             print(\"this is the char\")\n",
    "            if char not in self.char2index:\n",
    "                self.char2index[char] = index\n",
    "                self.index2char[index] = char\n",
    "                index += 1\n",
    "                \n",
    "        self.vocab_size = len(self.char2index)   \n",
    "        \n",
    "        \n",
    "    \n",
    "    \n",
    "    def generate_tensor_for_char(self, char):\n",
    "        temp = torch.zeros(1, self.vocab_size)\n",
    "        temp[0][self.char2index[char]] = 1 \n",
    "        return temp\n",
    "\n",
    "#     def generate_char_from_tensor(self, char):\n",
    "#         temp = torch.zeros(self.vocab_size, 1)\n",
    "#         temp[self.char2index[char]][0] = 1 \n",
    "#         return temp\n",
    "    def lineToTensor(self,line):\n",
    "        tensor = torch.zeros(len(line), 1, self.vocab_size)\n",
    "#         print(line)\n",
    "        for li, letter in enumerate(line):\n",
    "            tensor[li][0][self.char2index[letter]] = 1\n",
    "        return tensor\n",
    "    \n",
    "    # wrap the lines!\n",
    "    def train_example_builder(self, line):\n",
    "        tensor = torch.zeros(len(line), 1, self.vocab_size)\n",
    "        for li, letter in enumerate(line):\n",
    "            tensor[li][0][self.char2index[letter]] = 1\n",
    "        return tensor\n",
    "      \n",
    "#     could do more of a common build api\n",
    "    def get_None_tensor(self):\n",
    "        tensor = torch.zeros(1, 1, self.vocab_size)\n",
    "        tensor[0][0][self.char2index[None]] = 1\n",
    "        return tensor\n",
    "    \n",
    "    def list_to_tensor(self, input_list):\n",
    "        tensor = torch.zeros(len(input_list), 1, self.vocab_size)\n",
    "        for elt in input_list:\n",
    "            tensor[li][0][self.char2index[letter]] = 1\n",
    "            \n",
    "#     def get_training_pair(self, line):\n",
    "        \n",
    "        \n",
    "    \n",
    "    def get_new_line_tensor(self):\n",
    "        tensor = torch.zeros(1, 1, self.vocab_size)\n",
    "        tensor[0][0][self.char2index[\"\\n\"]] = 1\n",
    "        return tensor\n",
    "\n",
    "        \n",
    "#         tensor[li][0][self.char2index[letter]] \n",
    "#         temp \n",
    "    def unicodeToAscii(self,s):\n",
    "        return ''.join(\n",
    "            c for c in unicodedata.normalize('NFD', s)\n",
    "            if unicodedata.category(c) != 'Mn'\n",
    "            and c in self.char2index\n",
    "        )\n",
    "\n",
    "#     OK, so given the raw lines of text, then we want to get it! (but it is json, so we need to process it anyways)\n",
    "# \n",
    "    def get_test_split(self):\n",
    "        return self.test_split\n",
    "        pass\n",
    "    def __init__(self, text_files,  convert_to_ascii = True):\n",
    "        self.string = string\n",
    "        self.text_files = text_files\n",
    "\n",
    "        self.training_examples = []\n",
    "\n",
    "        total_inputs = []\n",
    "        total_outputs = []\n",
    "        \n",
    "#         make the vocab\n",
    "        self.full_vocab()\n",
    "    \n",
    "        from tqdm import tqdm\n",
    "        for target_class, file_name in enumerate(tqdm(text_files)):\n",
    "            with open(file_name,  encoding=\"utf8\") as file:\n",
    "                for counter, raw_line in enumerate(file):\n",
    "#                     if counter > 500:\n",
    "#                         break\n",
    "\n",
    "                    line = self.unicodeToAscii(raw_line) if convert_to_ascii else raw_line\n",
    "                    line = line[0:49]\n",
    "                    # zero pad to start\n",
    "\n",
    "\n",
    "    #                 think about how to make this a vector or tensor line! \n",
    "    #                 print()\n",
    "\n",
    "    #                 inputs = self.get_None_tensor() + line_to_tensor(line)\n",
    "    #                 inputs = [None] + [x for x in line]\n",
    "                    inputs = self.lineToTensor([None] + [x for x in line])\n",
    "\n",
    "                    targets = torch.zeros((1,4))  # we need a 0 as well!\n",
    "                    targets[0,target_class] = 1\n",
    "                \n",
    "#                     print(targets)\n",
    "\n",
    "                    # append: will be a list of list (of lines)\n",
    "                    # extend: will be a list of lines\n",
    "                    total_inputs.append(inputs)\n",
    "                    total_outputs.append(targets)\n",
    "\n",
    "        assert len(total_inputs) == len(total_outputs)\n",
    "        \n",
    "        import random\n",
    "        \n",
    "        \n",
    "        # this is now a LIST of lists!\n",
    "        self.total_examples = list(zip(total_inputs, total_outputs))\n",
    "        random.shuffle(self.total_examples)\n",
    "        train_split = self.total_examples[0:32000]\n",
    "        test_split = self.total_examples[32000:-1] #all the rest are test exampels\n",
    "        print(len(train_split))\n",
    "        print(len(test_split))\n",
    "        self.test_split = test_split\n",
    "        self.total_examples = train_split\n",
    "        \n",
    "\n",
    "\n",
    "        #                 for char in line:\n",
    "        # how many chars before the target? or all chars before the target?\n",
    "        # so we simply pair up x with [x+1] i think\n",
    "        # ok, and then we iterate them in the loader?\n",
    "        #                     inputs = [x for x in ]\n",
    "        #                     targets = []\n",
    "\n",
    "        # we should also have some big list of entries \n",
    "\n",
    "        pass\n",
    "    \n",
    "    \n",
    "    # ok, now let us just try adding the transforms\n",
    "    # actually the transforms should take responsibility for all of the vocab stuff\n",
    "    \n",
    "    # number of items in the dataset. This involves a rather large computation!\n",
    "    # we can either do a feed forward cllm in the style of bengio, or we can just \n",
    "    # do the RNN approach (as specified by Sean Robertson in his tutorial)\n",
    "\n",
    "    # the indices should represent examples inside the list\n",
    "    # one strange thing is the following: we need to precompute al the indices earlier, imo, and then simply return the list\n",
    "    # at that element! (perhaps with some transforms!)\n",
    "    def __getitem__(self, index):\n",
    "        return self.total_examples[index]\n",
    "\n",
    "        # build a list, return elements from that list? we need to internally keep track of some index\n",
    "        pass\n",
    "\n",
    "\n",
    "\n",
    "# ok, so it works now!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:17<00:00,  4.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000\n",
      "8028\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "files = [r\"reddit_data/Left-reduced\",r\"reddit_data/Center-reduced\", \n",
    "        r\"reddit_data/Right-reduced\", r\"reddit_data/Alt-reduced\"]\n",
    "abc = TextDataset( files, True)\n",
    "\n",
    "\n",
    "#think about unicode issues\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n"
     ]
    }
   ],
   "source": [
    "print(abc.char2index[None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]],\n",
       "\n",
       "        [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]],\n",
       "\n",
       "        [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]],\n",
       "\n",
       "        [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc.generate_tensor_for_char(\"a\")\n",
    "# abc.index2char[2]\n",
    "abc.lineToTensor(\"abcd\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "for (a,b) in abc:\n",
    "    pass\n",
    "#     this should give two lines/tests\n",
    "#     print(a,b)\n",
    "#     print()\n",
    "    \n",
    "#     realistically, we should also have it as tensors/vectors\n",
    "# finally, we should also have some way where we run the rnn forward and \n",
    "#  iteratoring over the set, does not consume the elements! It is not a genertor! Rather, we simply index into the elements we would like\n",
    "for(a,b) in abc:\n",
    "    pass\n",
    "#     print(\"ok\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_x = [None] + [\"e\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ok, so now let's just do the stuff\n",
    "# the RNN actually makes one-character over predictions! \n",
    "# rnn_forward is a utility function that does a lot of stuff\n",
    "len(char_x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ok, so now we can start building the network that will process these!!\n",
    "# simply: iterate the dataset, and run train on it, do the log likelihood and etc.\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "class RNN(nn.Module):\n",
    "    \n",
    "    \n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(RNN,self).__init__()\n",
    "        self.input_dim = input_size\n",
    "        self.hidden_dim = hidden_size\n",
    "        self.num_layers = 2\n",
    "        self.batch_size=1\n",
    "\n",
    "        # Define the LSTM layer\n",
    "        self.lstm = nn.LSTM(self.input_dim, self.hidden_dim, self.num_layers)\n",
    "\n",
    "        # Define the output layer\n",
    "        self.linear = nn.Linear(self.hidden_dim, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "        \n",
    "    def initHidden(self):\n",
    "        # This is what we'll initialise our hidden state as\n",
    "        return (torch.zeros(self.num_layers,  self.hidden_dim),\n",
    "                torch.zeros(self.num_layers,  self.hidden_dim))\n",
    "\n",
    "    def forward(self, input):\n",
    "        # Forward pass through LSTM layer\n",
    "        # shape of lstm_out: [input_size, batch_size, hidden_dim]\n",
    "        # shape of self.hidden: (a, b), where a and b both \n",
    "        # have shape (num_layers, batch_size, hidden_dim).\n",
    "        lstm_out, self.hidden = self.lstm(input.view(len(input), self.batch_size, -1))\n",
    "        \n",
    "        # Only take the output from the final timetep\n",
    "        # Can pass on the entirety of lstm_out to the next layer if it is a seq2seq prediction\n",
    "        y_pred = self.linear(lstm_out[-1].view(self.batch_size, -1))\n",
    "#         print(y_pred.shape)\n",
    "#         print(y_pred.view(-1).shape)\n",
    "        y_pred = self.softmax(y_pred)\n",
    "        return y_pred\n",
    "                                \n",
    "\n",
    "     \n",
    "    \n",
    "n_hidden = 256\n",
    "\n",
    "\n",
    "rnn = RNN(abc.vocab_size, n_hidden, 4) #since 4 categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]],\n",
       "\n",
       "        [[ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "           0.,  0.]]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc.lineToTensor(\"we\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# WE NEED THIS LINE TO RESET THE RNN!!\n",
    "rnn = RNN(abc.vocab_size, n_hidden, 4)\n",
    "learning_rate = 0.0005 # If you set this too high, it might explode. If too low, it might not learn\n",
    "#  we had to set it a big lower to force convergence\n",
    "\n",
    "import torch.optim as optim\n",
    "optimizer = optim.Adam(rnn.parameters())\n",
    "\n",
    "# returns the loss for a line\n",
    "\n",
    "def train(input_tensor, target_tensor):\n",
    "    \n",
    "        \n",
    "#     print(input_tensor)\n",
    "#     print(input_tensor.shape)\n",
    "\n",
    "#     print(target_tensor)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "#     print (input_tensor)\n",
    "    \n",
    "#     print(\"changed3\")\n",
    "#     print(input_tensor[1])\n",
    "    criterion = nn.NLLLoss()    \n",
    "    hidden = rnn.initHidden()\n",
    "    loss = 0\n",
    "    rnn.zero_grad()\n",
    "#     print(input_tensor.size())\n",
    "#     print(target_tensor[0])\n",
    "#     print(target_tensor)\n",
    "#     print(input_tensor.size()[0])\n",
    "    \n",
    "    \n",
    "    rnn.zero_grad()\n",
    "    lstm_out = rnn(input_tensor)\n",
    "    output = lstm_out\n",
    "    \n",
    "        \n",
    "        \n",
    "        \n",
    "#         find where the one is:\n",
    "# \n",
    "#         torch.tensor([all_categories.index(category)], dtype=torch.long)\n",
    "    \n",
    "    true_target_tensor = torch.tensor([torch.argmax(target_tensor)], dtype=torch.long)\n",
    "    \n",
    "#     true_target_tensor = target_tensor\n",
    "#     print(output)\n",
    "#     print(output.shape)\n",
    "#     print(true_target_tensor)\n",
    "#     print(true_target_tensor.shape)\n",
    "# it needs a single max value interestingly!\n",
    "#     Evaluate the loss on each character!\n",
    "    \n",
    "#     some \"gradient clipping\"; the loss has no effect in this case\n",
    "\n",
    "    loss += criterion(output,true_target_tensor)\n",
    "    loss.backward()\n",
    "        \n",
    "#         expects JUST a single target class:\n",
    "\n",
    "#     print (\"loss for i \" + str(loss.item()))    \n",
    "#         print()\n",
    "# should be do the loss on each thing, or each batch! no, we should accumulate it! \n",
    "    loss_val = loss.item()\n",
    "    if loss_val < 5000000000:\n",
    "        optimizer.step()\n",
    "    else:\n",
    "#         print(\"clipping\")\n",
    "#         print(loss_val)\n",
    "        loss_val = 5\n",
    "    \n",
    "    # Add parameters' gradients to their values, multiplied by learning rate\n",
    "#     for p in rnn.parameters():\n",
    "#         p.data.add_(-learning_rate, p.grad.data)\n",
    "\n",
    "    return output, loss_val\n",
    "\n",
    "# why not run this just on all the tensors!! \n",
    "\n",
    "# then, we just call it with the dataset class, and then just iterate everything!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc.index2char[0]\n",
    "abc.index2char[1]\n",
    "\n",
    "abc.char2index[\"A\"]\n",
    "\n",
    "# ok some weird stuff here...\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for (x,y) in abc:\n",
    "#     print(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 2/32000 [00:00<1:50:01,  4.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 0 is 0.002689204216003418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 501/32000 [01:08<1:12:13,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 500 is 1.4018716850280761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 1002/32000 [02:17<1:11:02,  7.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 1000 is 1.3887336239814758\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 1502/32000 [03:26<1:09:52,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 1500 is 1.3872996892929077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▋         | 2002/32000 [04:36<1:09:02,  7.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 2000 is 1.3862911579608916\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 2502/32000 [05:45<1:07:58,  7.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 2500 is 1.3878173398971558\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 3002/32000 [06:58<1:07:21,  7.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 3000 is 1.3851725051403045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 3502/32000 [08:08<1:06:17,  7.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 3500 is 1.386129989385605\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 4002/32000 [09:22<1:05:32,  7.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 4000 is 1.3838596456050873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 4501/32000 [10:36<1:04:47,  7.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 4500 is 1.3882139940261842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 5001/32000 [12:04<1:05:12,  6.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 5000 is 1.3892502589225768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 5501/32000 [13:37<1:05:38,  6.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 5500 is 1.3880570647716521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 6003/32000 [15:10<1:05:42,  6.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 6000 is 1.3897153835296632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 6501/32000 [16:41<1:05:26,  6.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 6500 is 1.3853685715198516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 7001/32000 [18:16<1:05:15,  6.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 7000 is 1.388749430179596\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 7502/32000 [19:51<1:04:51,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 7500 is 1.386324653148651\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 8001/32000 [21:09<1:03:27,  6.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 8000 is 1.3866511664390564\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 8501/32000 [22:21<1:01:47,  6.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 8500 is 1.388136600255966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 9002/32000 [23:31<1:00:06,  6.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 9000 is 1.3855771267414092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 9502/32000 [24:40<58:25,  6.42it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 9500 is 1.386913557767868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 10002/32000 [25:50<56:49,  6.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 10000 is 1.3852787790298462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 10502/32000 [27:02<55:21,  6.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 10500 is 1.3890872621536254\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 11001/32000 [28:16<53:58,  6.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 11000 is 1.3874887909889222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 11502/32000 [29:27<52:30,  6.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 11500 is 1.3878985636234284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 12002/32000 [30:34<50:57,  6.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 12000 is 1.386782744884491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 12502/32000 [31:44<49:29,  6.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 12500 is 1.3866895258426666\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 13002/32000 [32:56<48:08,  6.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 13000 is 1.3865750880241394\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 13502/32000 [34:06<46:43,  6.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 13500 is 1.3866495928764344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 14002/32000 [35:15<45:19,  6.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 14000 is 1.3870424551963807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 14502/32000 [36:25<43:56,  6.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 14500 is 1.3867470951080323\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 15001/32000 [37:36<42:36,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 15000 is 1.3861834828853608\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 15502/32000 [38:48<41:18,  6.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 15500 is 1.3839298803806306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 16002/32000 [40:07<40:06,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss for 16000 is 1.3916975083351135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 16184/32000 [40:41<39:45,  6.63it/s]"
     ]
    }
   ],
   "source": [
    "# del rnn\n",
    "# rnn = RNN(abc.vocab_size, n_hidden, abc.vocab_size)\n",
    "\n",
    "epoch_loss = 0\n",
    "# loss every k iters\n",
    "# \n",
    "total_loss = 0\n",
    "total_length  = 0\n",
    "epoch_length = 0\n",
    "other_loss = 0\n",
    "\n",
    "from tqdm import tqdm\n",
    "num_epochs = 3\n",
    "for k in range(num_epochs ):\n",
    "    \n",
    "    for i,(x,y) in enumerate(tqdm(abc)):\n",
    "    #     print(x)\n",
    "    #     print(x)\n",
    "    #     print(y)\n",
    "        _, loss = train(x,y)\n",
    "        epoch_loss += loss\n",
    "        epoch_length += x.size()[0]\n",
    "        total_loss += loss\n",
    "        total_length += x.size()[0]\n",
    "        other_loss +=loss\n",
    "#         print(loss)\n",
    "    #     print(i)\n",
    "    #     abc = i\n",
    "        if i % 500 == 0:\n",
    "            print (\"loss for {} is {}\".format(i,other_loss/500))\n",
    "            other_loss = 0\n",
    "#         break\n",
    "\n",
    "    \n",
    "    print(\"epoch {} loss is {}\".format(k, epoch_loss/i))\n",
    "    print(\"per character loss is {}\".format(epoch_loss/epoch_length))\n",
    "    epoch_loss = 0\n",
    "    epoch_length = 0\n",
    "#     ideally: we would be able to compute the quintile losses\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saves the entire model under the given filename\n",
    "def save_model(model,filename):\n",
    "    torch.save(model, filename)\n",
    "    \n",
    "    return\n",
    "\n",
    "# Saves the model parameters only (for maximum usability)\n",
    "\n",
    "def save_model_params(model, filename):\n",
    "\n",
    "    torch.save(model.state_dict(), filename)\n",
    "\n",
    "    return filename\n",
    "\n",
    "#  ideally, I could get per character loss, then I would also be able to get the total expected loss\n",
    "# so: we want to get total loss, then divide it by all the length of the number of examples we saw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model_from_params(filename):\n",
    "    model = RNN(abc.vocab_size, n_hidden, abc.vocab_size)\n",
    "    model.load_state_dict(torch.load(filename))\n",
    "    return model #vs return model.eval()??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"reddit_draft_new_lstm.pt\"\n",
    "\n",
    "if True:\n",
    "\n",
    "    filename = save_model_params(rnn, \"reddit_draft_new_lstm.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if True:\n",
    "    eval_model = load_model_from_params(filename)\n",
    "    eval_model.eval() # this is a mutating operation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# returns the loss for the entire line (requires normalization for comparison)\n",
    "def get_loss_on_line(rnn, line):\n",
    "    input_tensor =  abc.lineToTensor([None] + [x for x in line])\n",
    "    target_tensor = abc.lineToTensor([x for x in line] + [\"\\n\"])\n",
    "    criterion = nn.NLLLoss()    \n",
    "    hidden = rnn.initHidden()\n",
    "    loss = 0\n",
    "    rnn.zero_grad()\n",
    "    \n",
    "    \n",
    "#     print(input_tensor.size())\n",
    "#     print(target_tensor[0])\n",
    "#     print(target_tensor)\n",
    "    for i in range(input_tensor.size()[0]):\n",
    "        output, hidden = rnn(input_tensor[i], hidden)\n",
    "        \n",
    "        \n",
    "        # had to do this simply because of how the NLLL and CrossEntropy are defined\n",
    "        max_tensor = torch.argmax(target_tensor[i])\n",
    "        ind = (torch.argmax(target_tensor[i])).item()\n",
    "        true_target_tensor = torch.tensor([ind], dtype=torch.long)\n",
    "    \n",
    "#     Evaluate the loss on each character!\n",
    "        loss += criterion(output, true_target_tensor)\n",
    "    \n",
    "    return loss.item()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# out = eval_model)\n",
    "\n",
    "# so there's a couple possibilities: we can have jim and ask to predict as well as just get the loss of the example\n",
    "# out = train()\n",
    "# ()\n",
    "#lineToTensor\n",
    "\n",
    "# get it as tensors, then run it, then do loss addition\n",
    "\n",
    "STRING_TO_TEST =  \"David\"\n",
    "print(get_loss_on_line(eval_model, STRING_TO_TEST )/len(STRING_TO_TEST ))\n",
    "\n",
    "# we now want to implement sampling:\n",
    "#  is that useful though? \n",
    "# will it be good?\n",
    "# instead, we can also try and get text for the inputs \n",
    "#  then train on that, then run it on some numerical numbers\n",
    "\n",
    "# visualization would be super useful \n",
    "# for pretty printing, we can colour the terminal output\n",
    "# otherwise, we will need to somehow encode the colour so that it is output to the file\n",
    "# \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  print(\"\\x1b[31m\\\"red\\\"\\x1b[0m\")\n",
    "\n",
    "# if no thresholding, then we should at least like somehow graphically display?\n",
    "# no,... this is a different concern. Don't put frontend and backend together! (or visualization and etc.)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from colorama import Fore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(Fore.RED + \"yoo\")\n",
    "\n",
    "# print(Fore.BLUE + \"haha\")\n",
    "\n",
    "# if on average more than twice as high loss on this sample, then we should flag it for further review\n",
    "THRESHOLD =  1.8332151545300774 * 2\n",
    "\n",
    "\n",
    "# some more powerful model: uses LSTM as well as additional fields and operators!\n",
    "# ok, so now let us just iterate the text file\n",
    "with open(\"data\\\\test-cases.txt\", \"r\") as file:\n",
    "    for line in file:\n",
    "        example_loss = get_loss_on_line(eval_model, line)/len(line )\n",
    "        if (example_loss > THRESHOLD):\n",
    "            print(Fore.RED + line)\n",
    "        else:\n",
    "            print(Fore.BLUE + line)\n",
    "        \n",
    "        print(\"loss\" + str(example_loss))\n",
    "    \n",
    "# perhaps we should train on google news corpus for a little while..\n",
    "# this \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import reuters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"ok\")\n",
    "\n",
    "sentences = reuters.raw().splitlines()\n",
    "# print(reuters.raw())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(type(sentences))\n",
    "print(len(sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# these are not exactly tokenized on sentences; but that is OK!\n",
    "sentences[:100]\n",
    "\n",
    "with open(\"reuters_news.txt\", \"w\") as file:\n",
    "    file.write(reuters.raw())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we need a smaller dataset for development\n",
    "# so, let's cut up the reuters into smaller (for example, load just the first K bytes of the file.)\n",
    "import time\n",
    "\n",
    "def make_proto_dataset(size):\n",
    "    with open(\"reuters_news_{}.txt\".format(size), \"w\") as file:\n",
    "        \n",
    "        \n",
    "        file.write('\\n'.join(reuters.raw().splitlines()[0:size]))\n",
    "    print(\"file created with {} lines\".format(size))\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#     the question now is how many neurons and the architecture to use\n",
    "# we can also look into enhancement with the tqdm inside the object?\n",
    "start = time.time()\n",
    "\n",
    "\n",
    "make_proto_dataset(10000)\n",
    "elapsed = time.time() - start\n",
    "print(\"elapsed time is {}\".format(elapsed ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reuters dataset has umlauts and other non-standard characters, so we should try and account for that! \n",
    "# also: we should try and do a sort of full merge: merge between the train dataset and test dataset vocabs\n",
    "# other than that, we can just apply a regularizer to both sets, to get out the processed data\n",
    "# for now, let's just apply a regularization\n",
    "\n",
    "start = time.time()\n",
    "reuters_dataset = TextDataset( \"reuters_news_10000.txt\", True)\n",
    "elapsed = time.time() - start\n",
    "print(\"elapsed time is {}\".format(elapsed ))\n",
    "\n",
    "# jeez, so it takes more than 8 minutes to create the dataset!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(reuters_dataset))\n",
    "from tqdm import tqdm\n",
    "def general_train_function(dataset):\n",
    "# del rnn\n",
    "# rnn = RNN(abc.vocab_size, n_hidden, abc.vocab_size)\n",
    "\n",
    "    epoch_loss = 0\n",
    "    # loss every k iters\n",
    "    # \n",
    "    loss_per_k = 0\n",
    "    \n",
    "    total_loss = 0\n",
    "    total_length  = 0\n",
    "    epoch_length = 0\n",
    "\n",
    "\n",
    "    num_epochs = 10\n",
    "    \n",
    "    for k in range(num_epochs ):\n",
    "\n",
    "        for i,(x,y) in enumerate(tqdm(dataset)):\n",
    "        #     print(x)\n",
    "        #     print(x)\n",
    "        #     print(y)\n",
    "            _, loss = train(x,y)\n",
    "            epoch_loss += loss\n",
    "            epoch_length += x.size()[0]\n",
    "            total_loss += loss\n",
    "            total_length += x.size()[0]\n",
    "            loss_per_k += loss\n",
    "        #     print(i)\n",
    "        #     abc = i\n",
    "        \n",
    "            # for the first 100 iters, print the loss of every line!\n",
    "#             if i < 100:\n",
    "#                 print (\"loss for {} is {}\".format(i,epoch_loss/(i+1)))\n",
    "                \n",
    "            \n",
    "            if i % 100 == 0 and i != 0:\n",
    "                print (\"loss for {} is {}\".format(i,loss_per_k/100))\n",
    "                loss_per_k = 0\n",
    "                \n",
    "\n",
    "        print(\"epoch {} loss is {}\".format(k, epoch_loss/i))\n",
    "        print(\"per character loss is {}\".format(epoch_loss/epoch_length))\n",
    "        epoch_loss = 0\n",
    "        epoch_length = 0\n",
    "#     ideally: we would be able to compute the quintile losses\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# WE NEED THIS LINE TO RESET THE RNN!!\n",
    "rnn = RNN(reuters_dataset.vocab_size, 512, reuters_dataset.vocab_size)\n",
    "learning_rate = 0.0005 # If you set this too high, it might explode. If too low, it might not learn\n",
    "#  we had to set it a big lower to force convergence\n",
    "\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(rnn.parameters(), lr = 0.0001)\n",
    "\n",
    "# returns the loss for a line\n",
    "\n",
    "def general_train(input_tensor, target_tensor):\n",
    "    \n",
    "        \n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "#     print (input_tensor)\n",
    "    \n",
    "#     print(\"changed3\")\n",
    "#     print(input_tensor[1])\n",
    "    criterion = nn.NLLLoss()    \n",
    "    hidden = rnn.initHidden()\n",
    "    loss = 0\n",
    "    rnn.zero_grad()\n",
    "#     print(input_tensor.size())\n",
    "#     print(target_tensor[0])\n",
    "#     print(target_tensor)\n",
    "    for i in range(input_tensor.size()[0]):\n",
    "        output, hidden = rnn(input_tensor[i], hidden)\n",
    "        \n",
    "        \n",
    "        \n",
    "#         find where the one is:\n",
    "# \n",
    "#         torch.tensor([all_categories.index(category)], dtype=torch.long)\n",
    "        max_tensor = torch.argmax(target_tensor[i])\n",
    "#         print(max_tensor)\n",
    "        ind = (torch.argmax(target_tensor[i])).item()\n",
    "#         print(\"this the target\")\n",
    "#         print(target_tensor[i])\n",
    "#         print(ind)\n",
    "        true_target_tensor = torch.tensor([ind], dtype=torch.long)\n",
    "    \n",
    "#     Evaluate the loss on each character!\n",
    "        loss += criterion(output, true_target_tensor)\n",
    "        \n",
    "#         expects JUST a single target class:\n",
    "\n",
    "#     print (\"loss for i \" + str(loss.item()))    \n",
    "#         print()\n",
    "# should be do the loss on each thing, or each batch! no, we should accumulate it! \n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    # Add parameters' gradients to their values, multiplied by learning rate\n",
    "#     for p in rnn.parameters():\n",
    "#         p.data.add_(-learning_rate, p.grad.data)\n",
    "\n",
    "    return output, loss.item()\n",
    "\n",
    "# why not run this just on all the tensors!! \n",
    "\n",
    "# then, we just call it with the dataset class, and then just iterate everything!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "general_train_function(reuters_dataset)\n",
    "# perhaps: examine the gradient instead!\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# returns the loss for a line, evaluating on a BASIC rnn, and using the provided optimizer\n",
    "def prototype_general_train(input_tensor, target_tensor, basic_rnn, optimizer):\n",
    "    \n",
    "        \n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    criterion = nn.NLLLoss()    \n",
    "    hidden = rnn.initHidden()\n",
    "    loss = 0\n",
    "    basic_rnn.zero_grad()\n",
    "    for i in range(input_tensor.size()[0]):\n",
    "        output, hidden = basic_rnn(input_tensor[i], hidden)\n",
    "        \n",
    "        max_tensor = torch.argmax(target_tensor[i])\n",
    "        ind = (torch.argmax(target_tensor[i])).item()\n",
    "        true_target_tensor = torch.tensor([ind], dtype=torch.long)\n",
    "        #     Evaluate the loss on each character!\n",
    "        loss += criterion(output, true_target_tensor)\n",
    "        \n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "    return output, loss.item()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}