Domestic Audio Classification (Tensorflow).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "seed = 1356\n",
    "raw_data = pd.read_pickle(\"domestic_audio_features.pkl\")\n",
    "\n",
    "afs, labels, _ = zip(*[line.rstrip('\\n').split('\\t') for line in open('DCASE2018-task5-dev.meta/DCASE2018-task5-dev/meta.txt').readlines()])\n",
    "\n",
    "sces = set(labels)\n",
    "sce_int_map = {sce:i+1 for i, sce in enumerate(sces)}\n",
    "int_sce_map = {sce_int_map[i]:i for i in sce_int_map.keys()}\n",
    "\n",
    "# split data such that each label is equally represented\n",
    "bal_data = raw_data.groupby(\"label\")\n",
    "bal_data = bal_data.apply(lambda x: x.sample(bal_data.size().min(), random_state=seed).reset_index(drop=True))\n",
    "y = bal_data[\"label\"]\n",
    "X = bal_data.drop(\"label\", axis=1)\n",
    "\n",
    "# split data into stratified folds for cross validation\n",
    "# training, validation set is in a ratio of 3:1\n",
    "skf = StratifiedKFold(n_splits=4, random_state=seed)\n",
    "\n",
    "# generate folds and pre-process the data\n",
    "# std_train, minmax_train, y_train, std_test, minmax_test, y_test\n",
    "folds = []\n",
    "for train, test in skf.split(X, y):\n",
    "    X_train, X_test = X.values[train], X.values[test]\n",
    "    y_train, y_test = y.values[train], y.values[test]\n",
    "    \n",
    "    # convert y into one-hot-encode (ie. [x x x x x x x x x] where x is 1 if index = label and 0 otherwise)\n",
    "    y_train = np.array([[1 if c == label - 1 else 0 for c in range(len(sces))] for label in y_train])\n",
    "    y_test = np.array([[1 if c == label - 1 else 0 for c in range(len(sces))] for label in y_test])\n",
    "\n",
    "\n",
    "    std_scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "    X_train_std = std_scaler.transform(X_train)\n",
    "    X_test_std = std_scaler.transform(X_test)\n",
    "\n",
    "    minmax_scaler = preprocessing.MinMaxScaler().fit(X_train)\n",
    "    X_train_minmax = minmax_scaler.transform(X_train)\n",
    "    X_test_minmax = minmax_scaler.transform(X_test)\n",
    "    folds.append([X_train_std, X_train_minmax, y_train, X_test_std, X_test_minmax, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\jerem_000\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# 50 features, 9 labels/classes\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "n_features = 50\n",
    "n_classes = 9\n",
    "\n",
    "sd = 1 / np.sqrt(n_features)\n",
    "\n",
    "tfX = tf.placeholder(tf.float32, [None, n_features])\n",
    "tfY = tf.placeholder(tf.float32, [None, n_classes])\n",
    "\n",
    "def ff_neural_model(data, hidden_units = (200, 200)):\n",
    "    \n",
    "    temp_units = (n_features,) + hidden_units\n",
    "    \n",
    "    hidden_layers = [{'weights':tf.Variable(tf.random_normal([temp_units[i], units], seed=seed, mean=0, stddev=sd)),\n",
    "                      'biases':tf.Variable(tf.zeros([units]))} for i, units in enumerate(hidden_units)]\n",
    "    \n",
    "    output_layer = {'weights':tf.Variable(tf.random_normal([hidden_units[-1], n_classes], seed=seed, mean=0, stddev=sd)),\n",
    "                    'biases':tf.Variable(tf.zeros([n_classes]))}\n",
    "    \n",
    "    \n",
    "    res = data\n",
    "    for layer in hidden_layers:\n",
    "        res = tf.add(tf.matmul(res, layer['weights']), layer['biases'])\n",
    "        res = tf.nn.relu(res)\n",
    "    \n",
    "    output = tf.matmul(res, output_layer['weights']) + output_layer['biases']\n",
    "    \n",
    "    return output\n",
    "\n",
    "def train_neural_network(x, xtrain, ytrain, xtest, ytest, batch_size = -1, epochs = 500, show_graphs = False, hidden_units = (200, 200), verbose = False, show_accuracy = False, show_checkpoints = False):\n",
    "    pred = ff_neural_model(x, hidden_units)\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=tfY))\n",
    "    #cost = -tf.reduce_sum(tfY * tf.log(pred))\n",
    "    optimiser = tf.train.AdamOptimizer().minimize(cost)\n",
    "    \n",
    "    training_epochs = epochs\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        training_acc, training_loss = [], []\n",
    "        \n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        \n",
    "        for epoch in range(training_epochs):\n",
    "            epoch_loss = 0\n",
    "            index = 0\n",
    "            while index < len(xtrain):\n",
    "                if batch_size == -1:\n",
    "                    batch_x = np.array(xtrain)\n",
    "                    batch_y = np.array(ytrain)\n",
    "                    \n",
    "                    _, c = sess.run([optimiser, cost], feed_dict={x: batch_x, tfY: batch_y})\n",
    "                    epoch_loss += c\n",
    "                    break\n",
    "                    \n",
    "                batch_x = np.array(xtrain[index:index+batch_size])\n",
    "                batch_y = np.array(ytrain[index:index+batch_size])\n",
    "                \n",
    "                _, c = sess.run([optimiser, cost], feed_dict={x: batch_x, tfY: batch_y})\n",
    "                epoch_loss += c\n",
    "                index += batch_size\n",
    "            \n",
    "            if show_accuracy:\n",
    "                correct = tf.equal(tf.argmax(pred, 1), tf.argmax(tfY, 1))\n",
    "                accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)).eval({x: xtrain, tfY: ytrain})\n",
    "                training_acc.append(accuracy)\n",
    "                \n",
    "            training_loss.append(epoch_loss)\n",
    "            \n",
    "            if show_checkpoints:\n",
    "                if epoch+1 in [int(.25 * epochs), int(.5 * epochs), int(.75 * epochs)]:\n",
    "                    print(\"{}% complete.\".format(int(100*(epoch+1)/epochs)))\n",
    "            \n",
    "            if verbose:\n",
    "                print(\"Epoch {} completed out of {}. Loss: {}.\".format(epoch + 1, training_epochs, epoch_loss), end=' ')\n",
    "                if show_accuracy:\n",
    "                    print(\"Accuracy: {}.\".format(accuracy))\n",
    "                else:\n",
    "                    print()\n",
    "        \n",
    "        correct = tf.equal(tf.argmax(pred, 1), tf.argmax(tfY, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)).eval({x: xtest, tfY: ytest})\n",
    "        \n",
    "        y_pred = sess.run(tf.argmax(pred, 1), feed_dict={x: xtest})\n",
    "        y_true = sess.run(tf.argmax(ytest, 1))\n",
    "        \n",
    "        if show_graphs:\n",
    "            x_axis = np.arange(1,epochs+1,1)\n",
    "            if show_accuracy:\n",
    "                plt.plot(x_axis, 10 * np.array(training_acc), label='Training accuracy (x10)')\n",
    "            plt.plot(x_axis, np.log(training_loss), label='Loss (ln scale)')\n",
    "            plt.legend()\n",
    "        \n",
    "        # print(\"Accuracy: {}\".format(accuracy))\n",
    "        return accuracy, f1_score(y_true, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8783721923828125. F1: 0.8779266275720157\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8614540696144104. F1: 0.8589937276009603\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8888888955116272. F1: 0.8885341339188535\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8760859370231628. F1: 0.8719753138556797\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8888888955116272. F1: 0.8882296998270252\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8683127760887146. F1: 0.8671914987409043\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.873799741268158. F1: 0.8729218971918612\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8637402653694153. F1: 0.8586522500219554\n",
      "\n",
      "Standard accuracy: 0.873799741268158. Minmax accuracy: 0.8637402653694153.\n",
      "Standard f1: 0.8729218971918612. Minmax f1: 0.8586522500219554.\n"
     ]
    }
   ],
   "source": [
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, show_checkpoints=True)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, show_checkpoints=True)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8770004510879517. F1: 0.8775134677927043\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.7032464742660522. F1: 0.697169942282119\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8856881856918335. F1: 0.8844771347720397\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8673982620239258. F1: 0.8657976469165626\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8774576783180237. F1: 0.8774224898397374\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.7540009021759033. F1: 0.7485678349543143\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8838591575622559. F1: 0.8833238520509522\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8221307992935181. F1: 0.8231063540231598\n",
      "\n",
      "Standard accuracy: 0.8838591575622559. Minmax accuracy: 0.8221307992935181.\n",
      "Standard f1: 0.8833238520509522. Minmax f1: 0.8231063540231598.\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, show_checkpoints=True, batch_size=batch_size)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, show_checkpoints=True, batch_size=batch_size)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8673982620239258. F1: 0.8666139668948829\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8267032504081726. F1: 0.8212065913630483\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8733424544334412. F1: 0.8708492375032412\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8390489220619202. F1: 0.8303182456539931\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.87425696849823. F1: 0.8724372035672314\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8427069187164307. F1: 0.8368767988908016\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8587105870246887. F1: 0.8556888402082723\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8303612470626831. F1: 0.8217207277616816\n",
      "\n",
      "Standard accuracy: 0.8587105870246887. Minmax accuracy: 0.8303612470626831.\n",
      "Standard f1: 0.8556888402082723. Minmax f1: 0.8217207277616816.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (100,)\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8783721923828125. F1: 0.8788761898170145\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8372199535369873. F1: 0.83592276306514\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8934613466262817. F1: 0.8935297071769526\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8573388457298279. F1: 0.8534133312247358\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8724279999732971. F1: 0.8726858248142645\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.798811137676239. F1: 0.8043254712573932\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8751714825630188. F1: 0.8748168099157994\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8504801392555237. F1: 0.8455538928310049\n",
      "\n",
      "Standard accuracy: 0.8751714825630188. Minmax accuracy: 0.8504801392555237.\n",
      "Standard f1: 0.8748168099157994. Minmax f1: 0.8455538928310049.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (300,200,100)\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8449931144714355. F1: 0.8431558795353928\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8074988722801208. F1: 0.8024760696456834\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8582533001899719. F1: 0.8555766763789237\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8385916948318481. F1: 0.8263713039180124\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.840877890586853. F1: 0.8387813054214986\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8116140961647034. F1: 0.810572961670882\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8372199535369873. F1: 0.8323723720832561\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8267032504081726. F1: 0.8172426067134482\n",
      "\n",
      "Standard accuracy: 0.8372199535369873. Minmax accuracy: 0.8267032504081726.\n",
      "Standard f1: 0.8323723720832561. Minmax f1: 0.8172426067134482.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (30,30)\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8555098176002502. F1: 0.8530047280081372\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8116140961647034. F1: 0.8052129186496748\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8646547794342041. F1: 0.8609222902614362\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8180155754089355. F1: 0.8082185862529153\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8646547794342041. F1: 0.8620974128917077\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8212162852287292. F1: 0.8130844309509822\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8550525903701782. F1: 0.8517889402051542\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8166437745094299. F1: 0.8079964313064845\n",
      "\n",
      "Standard accuracy: 0.8550525903701782. Minmax accuracy: 0.8166437745094299.\n",
      "Standard f1: 0.8517889402051542. Minmax f1: 0.8079964313064845.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (30,)\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = 100, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8459076285362244. F1: 0.8462269586913286\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8353909254074097. F1: 0.8312388684622563\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8637402653694153. F1: 0.8618886825006729\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8349336981773376. F1: 0.8310005332440329\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8376771807670593. F1: 0.8356741256750165\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8353909254074097. F1: 0.8312705890261576\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8491083383560181. F1: 0.8479915194137353\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8262460231781006. F1: 0.8210639469652707\n",
      "\n",
      "Standard accuracy: 0.8491083383560181. Minmax accuracy: 0.8262460231781006.\n",
      "Standard f1: 0.8479915194137353. Minmax f1: 0.8210639469652707.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (50, 50)\n",
    "batch_size = 64\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = batch_size, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = batch_size, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standard preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8555098176002502. F1: 0.8551502301924327\n",
      "\n",
      "Minmax preprocessing. Fold 1\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8029263615608215. F1: 0.7986002500596542\n",
      "\n",
      "Standard preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8907178640365601. F1: 0.8912407972473478\n",
      "\n",
      "Minmax preprocessing. Fold 2\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.855967104434967. F1: 0.852060841721626\n",
      "\n",
      "Standard preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8692272305488586. F1: 0.8691505590941262\n",
      "\n",
      "Minmax preprocessing. Fold 3\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.7764060497283936. F1: 0.7808524589198673\n",
      "\n",
      "Standard preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8765432238578796. F1: 0.8757781646984388\n",
      "\n",
      "Minmax preprocessing. Fold 4\n",
      "25% complete.\n",
      "50% complete.\n",
      "75% complete.\n",
      "Accuracy: 0.8353909254074097. F1: 0.8329150821793737\n",
      "\n",
      "Standard accuracy: 0.8765432238578796. Minmax accuracy: 0.8353909254074097.\n",
      "Standard f1: 0.8757781646984388. Minmax f1: 0.8329150821793737.\n"
     ]
    }
   ],
   "source": [
    "hidden_units = (200,200)\n",
    "batch_size = 64\n",
    "\n",
    "for i,(train_std, train_minmax, train_y, test_std, test_minmax, test_y) in enumerate(folds):\n",
    "    std_acc, minmax_acc, std_f1, minmax_f1 = [], [], [], []\n",
    "    print(\"Standard preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_std, train_y, test_std, test_y, batch_size = batch_size, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    std_acc += [acc]\n",
    "    std_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "    print(\"Minmax preprocessing. Fold {}\".format(i+1))\n",
    "    acc, f1 = train_neural_network(tfX, train_minmax, train_y, test_minmax, test_y, batch_size = batch_size, show_checkpoints=True, hidden_units=hidden_units)\n",
    "    minmax_acc += [acc]\n",
    "    minmax_f1 += [f1]\n",
    "    print(\"Accuracy: {}. F1: {}\".format(acc, f1))\n",
    "    print()\n",
    "print(\"Standard accuracy: {}. Minmax accuracy: {}.\".format(np.mean(std_acc), np.mean(minmax_acc)))\n",
    "print(\"Standard f1: {}. Minmax f1: {}.\".format(np.mean(std_f1), np.mean(minmax_f1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}