southpark_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from ds_utils.imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seasons = np.arange(1, 20)\n",
    "\n",
    "data_seasons = []\n",
    "for season in seasons:\n",
    "    data_season = pd.read_csv('https://github.com/BobAdamsEE/SouthParkData/raw/master/Season-%d.csv' % season)\n",
    "    data_seasons.append(data_season)\n",
    "\n",
    "data = pd.concat(data_seasons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "School day, school day, teacher's golden ru... Ah, damn it! My little brother's trying to follow me to school again. Zeeponanner. Ike, you can't come to school with me.  Yeah, go home you little dildo. Dude, don't call my brother a dildo! What's a dildo? Well, I don't know...  and I'll bet Cartman doesn't know either! I know what it means! Well, what? I'm not telling you. What's a dildo, Kenny? (It's a giant stick that goes inside the mom's vagina)  He-yeah, that's what Kyle's little brother is all right!  Ow!  Dude, that kicks ass! Yeah, check this one out. Ready, Ike? Kick the baby! Don't kick the baby. Kick the baby.  Whoa, Cartman! Looks like you didn't get much sleep last night. That's 'cause I was having these... bogus nightmares. Really? What about? Well, I dreamt that I was lying in my bed...  in the dark, when all of a sudden this bright blue light filled the room.  Then slowly my bedroom door begin to open,  and the next thing I remember, I was being drug through a hallway.  \n"
     ]
    }
   ],
   "source": [
    "text = ' '.join([line[:-1] for line in data.Line])\n",
    "print(text[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chars = list(set(text))\n",
    "vocab_size = len(chars)+1\n",
    "\n",
    "char_ind = dict((i, c) for i, c in enumerate(chars))\n",
    "ind_char = dict((c, i) for i, c in enumerate(chars))\n",
    "\n",
    "idx = [ind_char[c] for c in text]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN - N chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cs = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c_in_dat = [[idx[i+n] for i in range(0, len(idx)-1-cs, cs)]\n",
    "            for n in range(cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c_out_dat = [idx[i+cs] for i in range(0, len(idx)-1-cs, cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xs = [np.stack(c[:-2]) for c in c_in_dat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = np.stack(c_out_dat[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_fac = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def embedding_input(name, n_in, n_out):\n",
    "    inp = keras.layers.Input(shape=(1,), dtype='int64', name=name+'_in')\n",
    "    emb = keras.layers.Embedding(n_in, n_out, input_length=1, name=name+'_emb')(inp)\n",
    "    return inp, keras.layers.Flatten()(emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "c_ins = [embedding_input('c'+str(n), vocab_size, n_fac) for n in range(cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_hidden = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/Keras-2.0.0-py3.6.egg/keras/legacy/interfaces.py:86: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(256, activation=\"relu\", kernel_initializer=\"identity\")`\n",
      "  '` call to the Keras 2 API: ' + signature)\n"
     ]
    }
   ],
   "source": [
    "dense_in = keras.layers.Dense(n_hidden, activation='relu')\n",
    "dense_hidden = keras.layers.Dense(n_hidden, activation='relu', init='identity')\n",
    "dense_out = keras.layers.Dense(vocab_size, activation='softmax')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hidden = dense_in(c_ins[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel/__main__.py:4: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `sum`, `concatenate`, etc.\n",
      "/opt/conda/lib/python3.6/site-packages/Keras-2.0.0-py3.6.egg/keras/legacy/layers.py:456: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,cs):\n",
    "    c_dense = dense_in(c_ins[i][1])\n",
    "    hidden = dense_hidden(hidden)\n",
    "    hidden = keras.layers.merge([c_dense, hidden])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c_out = dense_out(hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = keras.models.Model([c[0] for c in c_ins], c_out)\n",
    "model.compile(\n",
    "    loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.Adam())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      " 12544/573685 [..............................] - ETA: 229s - loss: 3.1322\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
     ]
    },
    {
     "data": {
      "text/html": [
       "<b>limit_output extension: Maximum message size of 10000 exceeded with 10081 characters</b>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.fit(xs, y, batch_size=64, epochs=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_weight_path = 'models/southpark_rnn_weights.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.save_weights(model_weight_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.load_weights(model_weight_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_next(inp):\n",
    "    idxs = [np.array(ind_char[c])[np.newaxis] for c in inp]\n",
    "    p = model.predict(idxs)\n",
    "    return chars[np.argmax(p)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "and upkeep - so that's about\n",
      ".......... - so the\n",
      "--------------------------------------------------\n",
      " tell us. Eh, Timmih. You kn\n",
      "..........Eh, Timmy\n",
      "--------------------------------------------------\n",
      "ed a lot, and he appears to \n",
      "..........and he an\n",
      "--------------------------------------------------\n",
      "re gonna love... Alabama Man\n",
      "..........ove... An\n",
      "--------------------------------------------------\n",
      " That was your plan to stop \n",
      "..........your plan\n",
      "--------------------------------------------------\n",
      "e. The answer is \"no\", Stanl\n",
      "..........wer is \" \n",
      "--------------------------------------------------\n",
      "ging's almost done. You're g\n",
      "..........ost done \n",
      "--------------------------------------------------\n",
      " somethin' horrible, then yo\n",
      ".......... horrible\n",
      "--------------------------------------------------\n",
      "d Kathie Lee and nobody got \n",
      "..........ee and no\n",
      "--------------------------------------------------\n",
      " the the startup company thi\n",
      "..........tartup ch\n",
      "--------------------------------------------------\n",
      " No. No, it will work! I hav\n",
      "..........t will we\n",
      "--------------------------------------------------\n",
      " positive and gives them hop\n",
      "..........and give \n",
      "--------------------------------------------------\n",
      "cery stores?  And what about\n",
      "..........s?  And t\n",
      "--------------------------------------------------\n",
      "his newscast for an importan\n",
      "..........st for an\n",
      "--------------------------------------------------\n",
      ", Ming Li! Ten dollar? Eight\n",
      ".......... Ten doll\n",
      "--------------------------------------------------\n",
      "  Do you know why young peop\n",
      "..........now why d\n",
      "--------------------------------------------------\n",
      "natomy. Cool! Oh man, this i\n",
      "..........ol! Oh my\n",
      "--------------------------------------------------\n",
      " son, m'kay?  Am I the only \n",
      "..........y?  Am I \n",
      "--------------------------------------------------\n",
      "! The list where you rate th\n",
      ".......... where yo\n",
      "--------------------------------------------------\n",
      "e pee got mixed with the poo\n",
      "..........mixed wit\n",
      "--------------------------------------------------\n",
      "goin' home. Go on and go hom\n",
      "........... Go on y\n",
      "--------------------------------------------------\n",
      " Shut. Up. Shut. Up. Shut. U\n",
      ".......... Shut. Uh\n",
      "--------------------------------------------------\n",
      "don't even understand me! Ye\n",
      ".......... understa\n",
      "--------------------------------------------------\n",
      " off. You need to freeze som\n",
      "..........need to s\n",
      "--------------------------------------------------\n",
      "ant to bag that animal! That\n",
      ".......... that and\n",
      "--------------------------------------------------\n",
      "more! Get out of here!  Amaz\n",
      "..........out of he\n",
      "--------------------------------------------------\n",
      "oing to be- My journey is th\n",
      "..........- My joun\n",
      "--------------------------------------------------\n",
      "ht now! That product changed\n",
      "..........at produc\n",
      "--------------------------------------------------\n",
      " his stupid questions. If yo\n",
      "..........d questio\n",
      "--------------------------------------------------\n",
      "ving the good life while I'm\n",
      "..........ood life.\n",
      "--------------------------------------------------\n",
      ". What?? But Dick, you told \n",
      "..........ut Dick, \n",
      "--------------------------------------------------\n",
      "u get to do the shooting. I \n",
      "..........o the sha\n",
      "--------------------------------------------------\n",
      " \"You weren't looking out fo\n",
      "..........n't look \n",
      "--------------------------------------------------\n",
      "r something.  Ha-a-aaa-haaa-\n",
      "..........g.  Ha-al\n",
      "--------------------------------------------------\n",
      "uble enough as it is.  All r\n",
      "..........h as it w\n",
      "--------------------------------------------------\n",
      "so other people can do norma\n",
      "..........eople can\n",
      "--------------------------------------------------\n",
      "n.  The old ways are dying. \n",
      "..........d ways an\n",
      "--------------------------------------------------\n",
      "trying to become a gentleman\n",
      "..........become an\n",
      "--------------------------------------------------\n",
      "py evening, friends. My name\n",
      ".........., friends\n",
      "--------------------------------------------------\n",
      "nst his will!! Oh-kay, well \n",
      "..........ll!! Oh- \n",
      "--------------------------------------------------\n",
      "Oof.  Oh my! I haven't seen \n",
      "..........y! I have\n",
      "--------------------------------------------------\n",
      "ow...  Flora, you have to pi\n",
      "..........ra, you g\n",
      "--------------------------------------------------\n",
      "looks a lot like your car, H\n",
      "..........t like yo\n",
      "--------------------------------------------------\n",
      "zation it deserves to be! Yo\n",
      "..........deserves \n",
      "--------------------------------------------------\n",
      "ght, that does it!  Hello? T\n",
      "..........does it! \n",
      "--------------------------------------------------\n",
      "  Oh God. What have I done? \n",
      "..........What have\n",
      "--------------------------------------------------\n",
      "rol. Wuzzariah? Ulduhooyah? \n",
      "..........riah? Ull\n",
      "--------------------------------------------------\n",
      "ountry bumpkins! Okay?! Kenn\n",
      "..........pkins! Oh\n",
      "--------------------------------------------------\n",
      "sion from your parents.  Oh,\n",
      "..........your pare\n",
      "--------------------------------------------------\n",
      " wanna stay in my room- Butt\n",
      "..........y in my f\n",
      "--------------------------------------------------\n",
      "Keywords here: Wendy's made \n",
      "..........ere: Wend\n",
      "--------------------------------------------------\n",
      "ty-run world?\" When the smok\n",
      "..........ld?\" Whea\n",
      "--------------------------------------------------\n",
      ". Fuck you! I understand you\n",
      "..........! I under\n",
      "--------------------------------------------------\n",
      "Park. Please, Mom and Dad, p\n",
      "..........se, Mom y\n",
      "--------------------------------------------------\n",
      "a stick up his butthole, and\n",
      ".......... his butt\n",
      "--------------------------------------------------\n",
      "y grandpa gave to me as a pr\n",
      "..........gave to t\n",
      "--------------------------------------------------\n",
      " (Mhm.) Yeah yeah, not bad. \n",
      "..........ah yeah, \n",
      "--------------------------------------------------\n",
      "ers, the boys have made a te\n",
      "..........oys have \n",
      "--------------------------------------------------\n",
      "m feel welcome. I want you a\n",
      "..........come. I d\n",
      "--------------------------------------------------\n",
      "ber, you all start to fight \n",
      "..........ll starte\n",
      "--------------------------------------------------\n",
      "\"it's the Penn State Gazette\n",
      "..........Penn Stan\n",
      "--------------------------------------------------\n",
      "om Jerry Maguire? Uh, yeah, \n",
      "..........aguire? I\n",
      "--------------------------------------------------\n",
      "he hell are you doing?! (Sto\n",
      "..........e you don\n",
      "--------------------------------------------------\n",
      "andy Wendy?! Oh, I didn't wa\n",
      "..........?! Oh, I \n",
      "--------------------------------------------------\n",
      " I just can't do this today.\n",
      "..........n't do th\n",
      "--------------------------------------------------\n",
      "eah! They ruined it! Oh my G\n",
      "..........ruined in\n",
      "--------------------------------------------------\n",
      " you all see? I can't believ\n",
      "..........ee? I can\n",
      "--------------------------------------------------\n",
      " Something big is going on, \n",
      ".......... big is t\n",
      "--------------------------------------------------\n",
      "ng to feel Emo. Wait a minut\n",
      ".......... Emo. Wal\n",
      "--------------------------------------------------\n",
      "s and stay there for six hou\n",
      ".......... there fo\n",
      "--------------------------------------------------\n",
      "hoa-wait, this is all happen\n",
      "..........this is t\n",
      "--------------------------------------------------\n",
      "nd he knows that we put the \n",
      "..........s that wa\n",
      "--------------------------------------------------\n",
      "d plummets into despair the \n",
      ".......... into de \n",
      "--------------------------------------------------\n",
      "ittle compassion, huh? I mea\n",
      "..........assion, t\n",
      "--------------------------------------------------\n",
      "t have to condone what these\n",
      "..........condone t\n",
      "--------------------------------------------------\n",
      "m and dad are home.  My hous\n",
      "..........are home \n",
      "--------------------------------------------------\n",
      "annot do in virtual reality.\n",
      "..........n virtual\n",
      "--------------------------------------------------\n",
      " nothing and EA made all the\n",
      "..........nd EA man\n",
      "--------------------------------------------------\n",
      " be marchng on the town squa\n",
      "..........g on the \n",
      "--------------------------------------------------\n",
      " He was too fast for me. He \n",
      "..........o fast fo\n",
      "--------------------------------------------------\n",
      "Why can't Old Ones die? The \n",
      "..........Old Ones \n",
      "--------------------------------------------------\n",
      " You're alive! Dude, how'd y\n",
      "..........ive! Dude\n",
      "--------------------------------------------------\n",
      " Yeah, who cares? All right,\n",
      ".......... cares? I\n",
      "--------------------------------------------------\n",
      " ourselves. I mean, is there\n",
      "........... I mean,\n",
      "--------------------------------------------------\n",
      "ght back.  Mom, Dad, can I g\n",
      ".......... Mom, Dad\n",
      "--------------------------------------------------\n",
      "ion dollars' worth of coal? \n",
      "..........s' worth \n",
      "--------------------------------------------------\n",
      "only to wind up in the land \n",
      "..........nd up in \n",
      "--------------------------------------------------\n",
      "! Bring them back here! It's\n",
      "..........em back o\n",
      "--------------------------------------------------\n",
      " down there, cowboy. You've \n",
      "..........e, cowbor\n",
      "--------------------------------------------------\n",
      " That's a new low. What is P\n",
      "..........new low. \n",
      "--------------------------------------------------\n",
      "rageous Nurse Gollum! Agghh!\n",
      "..........rse Golli\n",
      "--------------------------------------------------\n",
      "sandwich? Well you're not ex\n",
      "..........Well you \n",
      "--------------------------------------------------\n",
      "s a good thing we got her to\n",
      "..........hing we c\n",
      "--------------------------------------------------\n",
      "lip the... Flip the cop car!\n",
      ".......... Flip the\n",
      "--------------------------------------------------\n",
      "he best! The sleekest! The m\n",
      "..........he sleek \n",
      "--------------------------------------------------\n",
      "size of them! My God, this i\n",
      "..........em! My Go\n",
      "--------------------------------------------------\n",
      "es of cold and dark water. A\n",
      ".......... and dark\n",
      "--------------------------------------------------\n",
      "Eric Cartman his ways are no\n",
      "..........an his wa\n",
      "--------------------------------------------------\n",
      "ea, herpes, chlamydia, HPV, \n",
      ".........., chlamy \n",
      "--------------------------------------------------\n",
      "d and buy one copy for nine \n",
      "..........one copy \n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    n = np.random.choice(len(text))\n",
    "    str_piece = text[n:n + 8]\n",
    "    str_piece_context = text[n - 10:n + 8 + 10]\n",
    "    print(str_piece_context)\n",
    "    print(10 * '.' + str_piece + str(get_next(str_piece)))\n",
    "    print(50 * '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "References:\n",
    "- https://github.com/yang-zhang/courses/blob/master/deeplearning1/nbs/lesson6.ipynb\n",
    "- https://github.com/yang-zhang/courses/blob/master/deeplearning1/nbs/char-rnn.ipynb"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "49px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}