D1.merge_mood_context_location_data_gen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge mood,location, and the primary dataset (sensor data and self-reporetd context) based on the timestamp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import gzip\n",
    "import pandas as pd\n",
    "import os\n",
    "from io import StringIO\n",
    "import glob\n",
    "import datetime\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (15, 10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path1 =r'C:\\Users\\dihla\\Desktop\\ExtraSensory_data\\ExtraSensory.per_uuid_mood_labels\\Mood_csv' # use your path\n",
    "path2 =r'C:\\Users\\dihla\\Desktop\\ExtraSensory_data\\ExtraSensory.per_uuid_features_labels\\primary_csv' # use your path\n",
    "path3 =r'C:\\Users\\dihla\\Desktop\\ExtraSensory_data\\ExtraSensory.per_uuid_absolute_location\\location_csv' # use your path\n",
    "\n",
    "#Read mood file\n",
    "moodFiles = glob.glob(path1 + \"/*.csv\")\n",
    "#Read primary data\n",
    "primFiles = glob.glob(path2 + \"/*.csv\")\n",
    "#Read location\n",
    "locFiles = glob.glob(path3 + \"/*.csv\")\n",
    "\n",
    "dfs_mood = [pd.read_csv(fp) for fp in moodFiles]\n",
    "dfs_prim = [pd.read_csv(fp) for fp in primFiles]\n",
    "dfs_loc = [pd.read_csv(fp) for fp in locFiles]\n",
    "\n",
    "\n",
    "#df = pd.concat(dfs_mood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Person  1\n",
      "Person  2\n",
      "Person  3\n",
      "Person  4\n",
      "Person  5\n",
      "Person  6\n",
      "Person  7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dihla\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\dihla\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:635: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item_labels[indexer[info_axis]]] = value\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Person  9\n",
      "Person  11\n",
      "Person  14\n",
      "Person  16\n",
      "Person  17\n",
      "Person  18\n",
      "Person  19\n",
      "Person  22\n",
      "Person  23\n",
      "Person  24\n",
      "Person  26\n",
      "Person  28\n",
      "Person  29\n",
      "Person  30\n",
      "Person  31\n",
      "Person  32\n",
      "Person  34\n",
      "Person  36\n",
      "Person  37\n",
      "Person  38\n",
      "Person  39\n",
      "Person  41\n",
      "Person  42\n",
      "Person  43\n",
      "Person  44\n",
      "Person  45\n",
      "Person  47\n",
      "Person  50\n",
      "Person  51\n",
      "Person  53\n",
      "Person  56\n",
      "Person  57\n",
      "Person  58\n",
      "Person  59\n",
      "Person  60\n"
     ]
    }
   ],
   "source": [
    "person=1\n",
    "pList=[]\n",
    "list_ = []\n",
    "\n",
    "for mood_f, prim_f, loc_f in  zip(dfs_mood, dfs_prim, dfs_loc):\n",
    "    \n",
    "    mood_data=mood_f.dropna()\n",
    "    count_row = mood_data.shape[0]\n",
    "    mood_data['person'] = np.nan\n",
    "    \n",
    "    #Select persons having at least 100 samples\n",
    "    if mood_data.shape[0]>999:   \n",
    "        pList.append(person) \n",
    "        mood_data.loc[:,'person']=pd.Series(np.repeat(person, count_row), index=mood_data.index)\n",
    "        #Merge data using inner join of timestamps\n",
    "        merged_data=pd.merge(mood_data, prim_f, on='timestamp', how='inner')\n",
    "        merged_data=pd.merge(merged_data, loc_f, on='timestamp', how='inner')\n",
    "        \n",
    "        list_.append(merged_data)\n",
    "        \n",
    "    else:\n",
    "        print('Person ', person)\n",
    "    person=person+1\n",
    "\n",
    "merged_frame = pd.concat(list_, axis = 0, ignore_index = True, sort=False)\n",
    "#merged_frame.to_csv(r'C:\\Users\\dihla\\Desktop\\ExtraSensory_data\\Merged_Mood_Primary_data\\Mood_primary.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63648, 330)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Save the merged file\n",
    "merged_frame.to_csv('data/Mood_primary_loc_18.csv') # Use youe own path\n",
    "merged_frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ACTIVE</th>\n",
       "      <th>AFRAID</th>\n",
       "      <th>ALERT</th>\n",
       "      <th>AMUSED</th>\n",
       "      <th>ANGRY</th>\n",
       "      <th>ASHAMED</th>\n",
       "      <th>ATTENTIVE</th>\n",
       "      <th>BORED</th>\n",
       "      <th>CALM</th>\n",
       "      <th>...</th>\n",
       "      <th>label:OR_standing</th>\n",
       "      <th>label:AT_SCHOOL</th>\n",
       "      <th>label:PHONE_IN_HAND</th>\n",
       "      <th>label:PHONE_IN_BAG</th>\n",
       "      <th>label:PHONE_ON_TABLE</th>\n",
       "      <th>label:WITH_CO-WORKERS</th>\n",
       "      <th>label:WITH_FRIENDS</th>\n",
       "      <th>label_source</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.440094e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>32.880801</td>\n",
       "      <td>-117.234119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.440094e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>32.880789</td>\n",
       "      <td>-117.234164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.440094e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>32.880985</td>\n",
       "      <td>-117.234749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.440094e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>32.880207</td>\n",
       "      <td>-117.234823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.440094e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>32.879510</td>\n",
       "      <td>-117.234436</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 330 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      timestamp  ACTIVE  AFRAID  ALERT  AMUSED  ANGRY  ASHAMED  ATTENTIVE  \\\n",
       "0  1.440094e+09     0.0     0.0    0.0     0.0    0.0      0.0        0.0   \n",
       "1  1.440094e+09     0.0     0.0    0.0     0.0    0.0      0.0        0.0   \n",
       "2  1.440094e+09     0.0     0.0    0.0     0.0    0.0      0.0        0.0   \n",
       "3  1.440094e+09     0.0     0.0    0.0     0.0    0.0      0.0        0.0   \n",
       "4  1.440094e+09     0.0     0.0    0.0     0.0    0.0      0.0        0.0   \n",
       "\n",
       "   BORED  CALM  ...  label:OR_standing  label:AT_SCHOOL  label:PHONE_IN_HAND  \\\n",
       "0    0.0   0.0  ...                0.0              0.0                  NaN   \n",
       "1    0.0   0.0  ...                0.0              0.0                  NaN   \n",
       "2    0.0   0.0  ...                0.0              0.0                  NaN   \n",
       "3    0.0   0.0  ...                0.0              0.0                  NaN   \n",
       "4    0.0   0.0  ...                0.0              0.0                  NaN   \n",
       "\n",
       "   label:PHONE_IN_BAG  label:PHONE_ON_TABLE  label:WITH_CO-WORKERS  \\\n",
       "0                 NaN                   NaN                    NaN   \n",
       "1                 NaN                   NaN                    NaN   \n",
       "2                 NaN                   NaN                    NaN   \n",
       "3                 NaN                   NaN                    NaN   \n",
       "4                 NaN                   NaN                    NaN   \n",
       "\n",
       "   label:WITH_FRIENDS  label_source   latitude   longitude  \n",
       "0                 NaN             2  32.880801 -117.234119  \n",
       "1                 NaN             2  32.880789 -117.234164  \n",
       "2                 NaN             2  32.880985 -117.234749  \n",
       "3                 NaN             2  32.880207 -117.234823  \n",
       "4                 NaN             2  32.879510 -117.234436  \n",
       "\n",
       "[5 rows x 330 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}