dancegen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from model import vae,decoder\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Reshape\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "from keras.optimizers import adam\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import cv2\n",
    "import numpy as np\n",
    "import mdn\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENCODED_DATA_PATH = '/floyd/input/dancenetweights/data/lv.npy'\n",
    "VAE_PATH = '/floyd/input/dancenetweights/weights/vae_cnn.h5'\n",
    "DANCENET_PATH = '/floyd/input/dancenetweights/weights/gendance.h5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load encoded data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = np.load(ENCODED_DATA_PATH)\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.array(data).reshape(-1,128)\n",
    "scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "scaler = scaler.fit(data)\n",
    "data =  scaler.transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numComponents = 24\n",
    "outputDim = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM + MDN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "inputs = Input(shape=(128,))\n",
    "x = Reshape((1,128))(inputs)\n",
    "x = LSTM(512, return_sequences=True,input_shape=(1,128))(x)\n",
    "x = Dropout(0.40)(x)\n",
    "x = LSTM(512, return_sequences=True)(x)\n",
    "x = Dropout(0.40)(x)\n",
    "x = LSTM(512)(x)\n",
    "x = Dropout(0.40)(x)\n",
    "x = Dense(1000,activation='relu')(x)\n",
    "outputs = mdn.MDN(outputDim, numComponents)(x)\n",
    "model = Model(inputs=inputs,outputs=outputs)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = adam(lr=0.0005)\n",
    "model.compile(loss=mdn.get_mixture_loss_func(outputDim,numComponents),optimizer=opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train = False #change to True to train from scratch\n",
    "\n",
    "if train:\n",
    "    X = data[0:len(data)-1]\n",
    "    Y = data[1:len(data)]\n",
    "    checkpoint = ModelCheckpoint(DANCENET_PATH, monitor='loss', verbose=1, save_best_only=True, mode='auto')\n",
    "    callbacks_list = [checkpoint]\n",
    "    model.fit(X,Y,batch_size=1024, verbose=1, shuffle=False, validation_split=0.20, epochs=10000, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vae.load_weights(VAE_PATH)\n",
    "model.load_weights(DANCENET_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fourcc = cv2.VideoWriter_fourcc(*'mp4v')\n",
    "video = cv2.VideoWriter(\"out.mp4\", fourcc, 30.0, (208, 120))\n",
    "lv_in = data[0]\n",
    "\n",
    "for i in range(500):\n",
    "    input = np.array(lv_in).reshape(1,128)\n",
    "    lv_out = model.predict(input)\n",
    "    shape = np.array(lv_out).shape[1]\n",
    "    lv_out = np.array(lv_out).reshape(shape)\n",
    "    lv_out = mdn.sample_from_output(lv_out,128,numComponents,temp=0.01)\n",
    "    lv_out = scaler.inverse_transform(lv_out)\n",
    "    img = decoder.predict(np.array(lv_out).reshape(1,128))\n",
    "    img = np.array(img).reshape(120,208,1)\n",
    "    img = img * 255\n",
    "    img = np.array(img).astype(\"uint8\")\n",
    "    img = cv2.cvtColor(img,cv2.COLOR_GRAY2RGB)\n",
    "    lv_in = lv_out\n",
    "    video.write(img)\n",
    "video.release()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}