Model_Retrained_on_the_Hybrid_Dataset.py

# -*- coding: utf-8 -*-
"""Untitled30.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1OD2mGJqzc5l-Cd9vEUtaRrhZCctM8aUA

# **Real-Time Facial Emotion Recognition Using AI**

# **When Retraining the Model on the Hybrid Dataset**

# **Importing Libraries**
"""

import numpy as np
import pandas as pd
import os
import cv2
import tensorflow as tf
from tensorflow import keras
from keras.models import Sequential
from keras.models import Model
from keras import layers
from keras.utils import load_img, array_to_img, img_to_array
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Conv2D, Dense, BatchNormalization, Activation, Dropout, MaxPooling2D, Flatten
from keras.optimizers import Adam, RMSprop, SGD
from keras import regularizers
from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, EarlyStopping, ReduceLROnPlateau
import datetime
import seaborn as sns
from matplotlib import pyplot as plt
from keras.utils.vis_utils import plot_model
from keras.models import load_model
from keras.utils import np_utils
from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from zipfile import ZipFile
import zipfile
from IPython.display import display, Javascript, Image,clear_output
from google.colab.output import eval_js
from base64 import b64decode, b64encode
import PIL
import io
import html
import time  
from PIL import Image
from time import sleep

"""# **Reading the Iranian Emotional Face Database (IEFDB)**"""

with ZipFile('data.zip', 'r') as zipObj:
   # Extract all the contents of zip file in current directory
   zipObj.extractall()

# Sort list of files in the 'data' directory
data_dir = 'data'
sorted(os.listdir(data_dir))

# Counting the number of images in each subdirectory
total_images = 0
for dir_ in os.listdir(data_dir):
    count = 0
    for f in os.listdir(data_dir + "/" + dir_ + "/"):
            count += 1
            total_images += 1
    print(f"{dir_} has {count} number of images")
    
print(f"\ntotal images are {total_images}")

"""# **Preprocessing the Iranian Emotional Face Database (IEFDB)**"""

for dir_ in os.listdir(data_dir): 
  # Loop through all images in the input directory
  for f in os.listdir(data_dir + "/" + dir_ +"/"):
      # Load the image
      image = cv2.imread(data_dir + "/" + dir_ + "/" + f)
      # Crop the image by removing the parts on the left and right edges
      image = image[:, 250:725] 
      # Resize the image to 48x48 while maintaining the aspect ratio
      image = cv2.resize(image,(48,48), interpolation=cv2.INTER_AREA)
      # Convert the image to grayscale
      gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
      # Save the resized image to the directory
      cv2.imwrite(os.path.join(data_dir, dir_,f), gray)

# Loading image data and labels from subdirectories into NumPy arrays
EMOTIONS = ["angry", "disgust", "fear","happy", "sad", "surprise", "neutral"]
img_arr = np.empty(shape=(total_images,48,48,3))
img_label = np.empty(shape=(total_images))
label_to_text = {}

i = 0
for e, emotion in enumerate(EMOTIONS):
    dir_ = emotion
    label_to_text[e] = dir_
    for f in os.listdir(data_dir + "/" + dir_ + "/"):
        img_arr[i] = cv2.imread(data_dir + "/" + dir_ + "/" + f)
        img_label[i] = e
        i += 1
    print(f"loaded all {dir_} images to numpy arrays")

label_to_text

"""# **Displaying the Iranian Emotional Face Database (IEFDB) Images**"""

fig = plt.figure(1, (10,10))
idx = 0
for k in label_to_text:
    sample_indices = np.random.choice(np.where(img_label==k)[0], size=7, replace=False)
    sample_images = img_arr[sample_indices]
    for img in sample_images:
        idx += 1
        ax = plt.subplot(7,7,idx)
        ax.imshow(img[:,:,0], cmap='gray')
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_title(label_to_text[k])
        plt.tight_layout()

# Converting the categorical labels of an image dataset into one-hot encoded arrays then print the shape of the resulting array
img_label = np_utils.to_categorical(img_label)
img_label.shape

# Scale data and returns the maximum value in the img_arr array after normalization
img_arr = img_arr / 255.
img_arr.max()

"""# **Splitting the Iranian Emotional Face Database (IEFDB) into Train, Test, and Valid Sets**"""

# Split data into 80% training and 20% testing
X_train, X_test, y_train, y_test = train_test_split(img_arr, img_label, shuffle=True, test_size=0.2, random_state=42)

# Split training data into 80% training and 20% validation
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)

# Checking shapes
X_train.shape, X_test.shape, y_train.shape, y_test.shape, X_val.shape, X_test.shape, y_val.shape, y_test.shape

folder_name = 'data'
zip_filename = 'iefdb.zip'

# Create a ZipFile object with write mode, which will create a new .zip file if it doesn't exist, or overwrite an existing one
with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zip:
    # Iterate through all the files in the folder
    for root, dirs, files in os.walk(folder_name):
        for file in files:
            # Create the full file path by joining the root directory and file name
            file_path = os.path.join(root, file)
            # Add the file to the .zip archive using its relative path
            zip.write(file_path, os.path.relpath(file_path, folder_name))

"""# **Reading the Extended Cohn-Kanade Dataset (CK+)**"""

with ZipFile('CK.zip', 'r') as zipObj:
   # Extract all the contents of zip file in current directory
   zipObj.extractall()

# Sort list of files in the 'CK' directory
data_dir2 = 'CK'
sorted(os.listdir(data_dir2))

# Counting the number of images in each subdirectory
total_images2 = 0
for dir_2 in os.listdir(data_dir2):
    count = 0
    for f2 in os.listdir(data_dir2 + "/" + dir_2 + "/"):
            count += 1
            total_images2 += 1
    print(f"{dir_2} has {count} number of images")
    
print(f"\ntotal images are {total_images2}")

"""# **Preprocessing the Extended Cohn-Kanade Dataset (CK+)**"""

for dir_2 in os.listdir(data_dir2):
  # Loop through all images in the input directory
  for f2 in os.listdir(data_dir2 + "/" + dir_2 +"/"):
      # Load the image
      image2 = cv2.imread(data_dir2 + "/" + dir_2 + "/" + f2)
      # Crop the image by removing the parts on the left and right edges
      image2 = image2[:, 50:570]
      # Resize the image to 48x48 while maintaining the aspect ratio
      image2 = cv2.resize(image2,(48,48), interpolation=cv2.INTER_AREA)
      # Convert the image to grayscale
      gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
      # Save the resized image to the directory
      cv2.imwrite(os.path.join(data_dir2, dir_2,f2), gray2)

folder_name = 'CK'
zip_filename = 'ck+.zip'

# Create a ZipFile object with write mode, which will create a new .zip file if it doesn't exist, or overwrite an existing one
with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zip:
    # Iterate through all the files in the folder
    for root, dirs, files in os.walk(folder_name):
        for file in files:
            # Create the full file path by joining the root directory and file name
            file_path = os.path.join(root, file)
            # Add the file to the .zip archive using its relative path
            zip.write(file_path, os.path.relpath(file_path, folder_name))

"""# **Reading the Japanese Female Facial Expression Dataset (JAFFE)**"""

with ZipFile('JAFFE.zip', 'r') as zipObj:
   # Extract all the contents of zip file in current directory
   zipObj.extractall()

# Sort list of files in the 'JAFFE' directory
data_dir3 = 'JAFFE'
sorted(os.listdir(data_dir3))

# Counting the number of images in each subdirectory
total_images3 = 0
for dir_3 in os.listdir(data_dir3):
    count = 0
    for f3 in os.listdir(data_dir3 + "/" + dir_3 + "/"):
            count += 1
            total_images3 += 1
    print(f"{dir_3} has {count} number of images")
    
print(f"\ntotal images are {total_images3}")

"""# **Preprocessing the Japanese Female Facial Expression Dataset (JAFFE)**"""

for dir_3 in os.listdir(data_dir3):
  # Loop through all images in the input directory
  for f3 in os.listdir(data_dir3 + "/" + dir_3 +"/"):
      # Load the image
      image3 = cv2.imread(data_dir3 + "/" + dir_3 + "/" + f3)
      # Crop the image by removing the parts on the left and right edges
      image3 = image3[:, 40:220]
      # Resize the image to 48x48 while maintaining the aspect ratio
      image3 = cv2.resize(image3,(48,48), interpolation=cv2.INTER_AREA)
      # Convert the image to grayscale
      gray3 = cv2.cvtColor(image3, cv2.COLOR_BGR2GRAY)
      # Save the resized image to the directory
      cv2.imwrite(os.path.join(data_dir3, dir_3,f3), gray3)

folder_name = 'JAFFE'
zip_filename = 'jaffe.zip'

# Create a ZipFile object with write mode, which will create a new .zip file if it doesn't exist, or overwrite an existing one
with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zip:
    # Iterate through all the files in the folder
    for root, dirs, files in os.walk(folder_name):
        for file in files:
            # Create the full file path by joining the root directory and file name
            file_path = os.path.join(root, file)
            # Add the file to the .zip archive using its relative path
            zip.write(file_path, os.path.relpath(file_path, folder_name))

"""# **Reading the Hybrid Dataset that Contains the Three Datasets (IEFDB, CK+, JAFFE)**"""

with ZipFile('alldataset.zip', 'r') as zipObj:
   # Extract all the contents of zip file in current directory
   zipObj.extractall()

# Sort list of files in the 'alldataset' directory
data_dir4 = 'alldataset'
sorted(os.listdir(data_dir4))

# Counting the number of images in each subdirectory
total_images4 = 0
for dir_4 in os.listdir(data_dir4):
    count = 0
    for f4 in os.listdir(data_dir4 + "/" + dir_4 + "/"):
            count += 1
            total_images4 += 1
    print(f"{dir_4} has {count} number of images")
    
print(f"\ntotal images are {total_images4}")

"""# **Preprocessing the Hybrid Dataset**"""

# Loading image data and labels from subdirectories into NumPy arrays
EMOTIONS = ["angry", "disgust", "fear","happy", "sad", "surprise", "neutral"]
img_arr2 = np.empty(shape=(total_images4,48,48,3))
img_label2 = np.empty(shape=(total_images4))
label_to_text2 = {}

j = 0
for n, emotion in enumerate(EMOTIONS):
    dir_4 = emotion
    label_to_text2[n] = dir_4
    for f4 in os.listdir(data_dir4 + "/" + dir_4 + "/"):
        img_arr2[j] = cv2.imread(data_dir4 + "/" + dir_4 + "/" + f4)
        img_label2[j] = n
        j += 1
    print(f"loaded all {dir_4} images to numpy arrays")

label_to_text2

"""# **Displaying the Hybrid Dataset Images**"""

fig2 = plt.figure(1, (10,10))
idx = 0
for z in label_to_text2:
    sample_indices2 = np.random.choice(np.where(img_label2==z)[0], size=7, replace=False)
    sample_images2 = img_arr2[sample_indices2]
    for img in sample_images2:
        idx += 1
        ax = plt.subplot(7,7,idx)
        ax.imshow(img[:,:,0], cmap='gray')
        ax.set_xticks([])
        ax.set_yticks([])
        ax.set_title(label_to_text2[z])
        plt.tight_layout()

# Converting the categorical labels of an image dataset into one-hot encoded arrays then print the shape of the resulting array
img_label2 = np_utils.to_categorical(img_label2)
img_label2.shape

# Scale data and returns the maximum value in the img_arr2 array after normalization
img_arr2 = img_arr2 / 255.
img_arr2.max()

# data generator Generate batches of tensor image data with real-time data augmentation
data_generator = ImageDataGenerator(
                        featurewise_center=False,#Set input mean to 0 over the dataset, feature-wise.
                        featurewise_std_normalization=False,# Divide inputs by std of the dataset, feature-wise.
                        rotation_range=10,      # randomly rotate images in the range (degrees, 0 to 180)
                        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
                        height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
                        zoom_range=.1,          # randomly zoom into images
                        horizontal_flip=True)   # randomly flip images

"""# **Splitting the Hybrid Dataset into Train, Test, and Valid Sets**"""

# Split data into 80% training and 20% testing
X_train2, X_test2, y_train2, y_test2 = train_test_split(img_arr2, img_label2, shuffle=True, test_size=0.2, random_state=42)

# Split training data into 80% training and 20% validation
X_train2, X_val2, y_train2, y_val2 = train_test_split(X_train2, y_train2, test_size=0.2, random_state=42)

# Checking shapes
X_train2.shape, X_test2.shape, y_train2.shape, y_test2.shape, X_val2.shape, X_test2.shape, y_val2.shape, y_test2.shape

"""# **Load the Pre-trained Model**"""

# Load the saved model architecture
fermodel = load_model('kaggle_model.h5')
# Gives the information about the architecture and configuration of the neural network.
fermodel.summary()

# Plotting architecture of defined model
keras.utils.plot_model(fermodel, to_file='fermodel.png', show_shapes=True, show_layer_names=True)

"""# **Compiling the Model**"""

batch_size = 70
epochs = 300

# Compiling the model
opt = keras.optimizers.Adam(learning_rate=0.001) #using Adam Optimizer
fermodel.compile(optimizer=opt,loss='categorical_crossentropy', metrics=['accuracy'])

"""# **Evaluating the Model's Loss and Accuracy on the Iranian Emotional Face Testing Dataset Before Retraining the Model**"""

test_model_loss, test_model_accu = fermodel.evaluate(X_test, y_test)
print("test accuracy = {:.2f}".format(test_model_accu*100))

"""# **Evaluating the Model's Loss and Accuracy on the Testing Dataset that Contains the Three Datasets Before Retraining the Model**"""

test_model_loss2, test_model_accu2 = fermodel.evaluate(X_test2, y_test2)
print("test accuracy = {:.2f}".format(test_model_accu2*100))

"""# **Training the Model on the Hybrid Dataset**"""

# Defining three callback functions 'EarlyStopping', 'ReduceLROnPlateau', 'ModelCheckpoint' and creating a list 'callbacks' to hold these functions
early_stop = EarlyStopping('val_loss', patience=30)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
                                  patience=25, min_lr=0.00001,model='auto')
trained_models_path = 'kaggle_model'
model_names = trained_models_path + '.{epoch:02d}-{val_accuracy:.2f}.h5'
model_checkpoint = ModelCheckpoint(model_names, 'val_loss', verbose=1,
                                                    save_best_only=True, mode='max')
callbacks = [model_checkpoint, early_stop, reduce_lr]

# Fitting the CNN model
history2 =fermodel.fit(data_generator.flow(X_train2, y_train2, batch_size),
          steps_per_epoch=len(X_train2) / batch_size,
          epochs=epochs, 
          verbose=1, 
          callbacks=callbacks,
          validation_data=(X_val2,y_val2))

"""# **Evaluating the Model's Loss and Accuracy on the Testing, Training, and Validation Sets for the Hybrid Dataset After Retraining the Model**"""

train_model_loss_new2, train_model_accu_new2 = fermodel.evaluate(X_train2, y_train2)
test_model_loss_new3, test_model_accu_new3 = fermodel.evaluate(X_test2, y_test2)
print("train accuracy = {:.2f} , test accuracy = {:.2f}".format(train_model_accu_new2*100, test_model_accu_new3*100))

val_model_loss_new4, val_model_accu_new4 = fermodel.evaluate(X_val2, y_val2)
print("valid accuracy = {:.2f}".format( val_model_accu_new4*100))

"""# **Evaluating the Model's Loss and Accuracy on the Iranian Emotional Face Testing Dataset After Retraining the Model**"""

test_model_loss_new4, test_model_accu_new4 = fermodel.evaluate(X_test, y_test)
print("test accuracy = {:.2f}".format( test_model_accu_new4*100))

"""# **Save the Retrained Model**"""

# Save best Model
fermodel.save('pretrainmodel.h5')

# Saving model in json format along with weights
fer_json = fermodel.to_json()  
with open("pretrainmodelweight.json", "w") as json_file:  
    json_file.write(fer_json)  
fermodel.save_weights("pretrainmodelweight.h5")

"""# **Plotting Accuracy & Loss**"""

plt.figure(figsize=(14,5))
plt.subplot(1,2,1)
plt.plot(history2.history['accuracy'])
plt.plot(history2.history['val_accuracy'])
plt.title('Model Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend(['train', 'valid'], loc='upper left')

plt.subplot(1,2,2)
plt.plot(history2.history['loss'])
plt.plot(history2.history['val_loss'])
plt.title('Model Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend(['train', 'valid'], loc='upper left')

plt.savefig('history86.png')
plt.show()

"""# **Confusion Matrix for the Iranian Emotional Face Test Set**"""

# Generate predictions on the test data
y_pred = fermodel.predict(X_test)
# Convert predictions to class labels
y_pred_classes = np.argmax(y_pred, axis=1)
# Convert true labels to class labels
y_test_classes = np.argmax(y_test, axis=1)

# Compute the confusion matrix
conf_matrix = confusion_matrix(y_test_classes, y_pred_classes)
print("FER confusion matrix")
print(conf_matrix,'\n')
# Plot the confusion matrix as an image
plt.imshow(conf_matrix, cmap=plt.cm.Blues)

# Add title and axis labels
plt.title("FER Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")

# Add tick marks and class labels on the axes
tick_marks = np.arange(7)
class_labels = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]
plt.xticks(tick_marks, class_labels, rotation=50)
plt.yticks(tick_marks, class_labels)

# Add data labels in each cell
threshold = conf_matrix.max() / 2.
for i, j in np.ndindex(conf_matrix.shape):
    plt.text(j, i, conf_matrix[i, j], ha="center", va="center", color="white" if conf_matrix[i, j] > threshold else "black")

# Add color bar
plt.colorbar()
# Save the plot to a file
plt.savefig("confusion_matrix77.png")
# Show the plot
plt.show()

"""# **Confusion Matrix for the Hybrid Test Set**"""

# Generate predictions on the test data
y_pred2 = fermodel.predict(X_test2)
# Convert predictions to class labels
y_pred_classes2 = np.argmax(y_pred2, axis=1)
# Convert true labels to class labels
y_test_classes2 = np.argmax(y_test2, axis=1)

# Compute the confusion matrix
conf_matrix2 = confusion_matrix(y_test_classes2, y_pred_classes2)
print("FER confusion matrix")
print(conf_matrix2,'\n')
# Plot the confusion matrix as an image
plt.imshow(conf_matrix2, cmap=plt.cm.Blues)

# Add title and axis labels
plt.title("FER Confusion Matrix")
plt.xlabel("Predicted Labels")
plt.ylabel("True Labels")

# Add tick marks and class labels on the axes
tick_marks2 = np.arange(7)
class_labels = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]
plt.xticks(tick_marks2, class_labels, rotation=50)
plt.yticks(tick_marks2, class_labels)

# Add data labels in each cell
threshold = conf_matrix2.max() / 2.
for i, j in np.ndindex(conf_matrix2.shape):
    plt.text(j, i, conf_matrix2[i, j], ha="center", va="center", color="white" if conf_matrix2[i, j] > threshold else "black")

# Add color bar
plt.colorbar()
# Save the plot to a file
plt.savefig("confusion_matrix67.png")
# Show the plot
plt.show()

"""# **Classification Report for the Iranian Emotional Face Test Set**"""

report = classification_report(y_test_classes, y_pred_classes)
print(report)

"""# **Classification Report for the Hybrid Test Set**"""

report2 = classification_report(y_test_classes2, y_pred_classes2)
print(report2)

"""# **Load our Retrained Model**"""

# Load the pre-trained model architecture
fer_model = load_model('pretrainmodel.h5')
fer_model.summary()

# Load the pre-trained model weights
fer_model.load_weights('pretrainmodelweight.h5')

"""# **Face Detection**"""

# Reads an image file then print the shape of the resulting array
frame = cv2.imread("f.jpg")
frame.shape

# Convert the color space of the image data from BGR to RGB then displaying it
plt.imshow(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

# Loads a Haar cascade classifier for detecting frontal faces in images
faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

# Converts the color image stored in the frame variable from the BGR color space to the grayscale color space
grayimg = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
grayimg.shape

# Detect all faces in the gray image
faces = faceCascade.detectMultiScale(grayimg,1.1,4)
# Iterate over all faces detected
for x,y,w,h in faces:
  # Extract the region of interest of the face in gray image
  roi_gray = grayimg[y:y+h, x:x+w]
  # Extract the region of interest of the face in color image
  roi_color = frame[y:y+h, x:x+w]
  # Draw a rectangle around the detected face in the color image
  cv2.rectangle(frame, (x,y), (x+w, y+h), (0,255,0), 2)
  # Detect faces within the detected face region in gray image
  facess = faceCascade.detectMultiScale(roi_gray)
  # Check if any faces are detected in the detected face region
  if len(facess) == 0:
    print("Face not detected")
  else:
    # Iterate over all faces detected within the detected face region
    for(ex,ey,ew,eh) in facess:
      # Extract the region of interest of the detected face in color image
      face_roi = roi_color[ey: ey+eh, ex:ex + ew]

# Convert the color space of the image from BGR to RGB then displays the input image after the face detection algorithm has been applied to it
plt.imshow(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

# Displays the image of the extracted face region after converting the color space from BGR to RGB
plt.imshow(cv2.cvtColor(face_roi, cv2.COLOR_BGR2RGB))

# Resize the image to (48,48) using OpenCV
final_img = cv2.resize(face_roi, (48,48))
# Expand the dimensions of the image by adding an extra axis at position 0
# This is done because the model expects an input of shape (batch_size, height, width, channels)
final_img = np.expand_dims(final_img,axis= 0) 
# Normalize the pixel values of the image to be in the range [0,1]
final_img = final_img/255.0

"""# **Test the Facial Emotion Recognition Model Using an Input Image**"""

# Predicting emotion of detected face using fer_model
Predictions = fer_model.predict(final_img)
# Accessing the first element of the array since only one image is predicted
Predictions[0]

# Get the emotion label with the highest probability
emotion_label = np.argmax(Predictions)
# List of emotions
emotions = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"]
print("Emotion label detected: ", emotion_label)
# Get the emotion with the highest probability
detected_emotion = emotions[emotion_label]
print("Emotion detected: ", detected_emotion)

"""# **Real-Time Face Emotion Detection Using Local Webcam**"""

# function to convert the JavaScript object into an OpenCV image
def js_to_image(js_reply):
  """
  Params:
          js_reply: JavaScript object containing image from webcam
  Returns:
          img: OpenCV BGR image
  """
  # decode base64 image
  image_bytes = b64decode(js_reply.split(',')[1])
  # convert bytes to numpy array
  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)
  # decode numpy array into OpenCV BGR image
  img = cv2.imdecode(jpg_as_np, flags=1)

  return img

# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream
def bbox_to_bytes(bbox_array):
  """
  Params:
          bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.
  Returns:
        bytes: Base64 image byte string
  """
  # convert array into PIL image
  bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')
  iobuf = io.BytesIO()
  # format bbox into png for return
  bbox_PIL.save(iobuf, format='png')
  # format return string
  bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))

  return bbox_bytes

# initialize the Haar Cascade face detection model
face_cascade = cv2.CascadeClassifier(cv2.samples.findFile(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml'))

# JavaScript to properly create our live video stream using our webcam as input
def video_stream():
  js = Javascript('''
    var video;
    var div = null;
    var stream;
    var captureCanvas;
    var imgElement;
    var labelElement;
    
    var pendingResolve = null;
    var shutdown = false;
    
    function removeDom() {
       stream.getVideoTracks()[0].stop();
       video.remove();
       div.remove();
       video = null;
       div = null;
       stream = null;
       imgElement = null;
       captureCanvas = null;
       labelElement = null;
    }
    
    function onAnimationFrame() {
      if (!shutdown) {
        window.requestAnimationFrame(onAnimationFrame);
      }
      if (pendingResolve) {
        var result = "";
        if (!shutdown) {
          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);
          result = captureCanvas.toDataURL('image/jpeg', 0.8)
        }
        var lp = pendingResolve;
        pendingResolve = null;
        lp(result);
      }
    }
    
    async function createDom() {
      if (div !== null) {
        return stream;
      }

      div = document.createElement('div');
      div.style.border = '2px solid black';
      div.style.padding = '3px';
      div.style.width = '100%';
      div.style.maxWidth = '600px';
      document.body.appendChild(div);
      
      const modelOut = document.createElement('div');
      modelOut.innerHTML = "Status:";
      labelElement = document.createElement('span');
      labelElement.innerText = 'No data';
      labelElement.style.fontWeight = 'bold';
      modelOut.appendChild(labelElement);
      div.appendChild(modelOut);
           
      video = document.createElement('video');
      video.style.display = 'block';
      video.width = div.clientWidth - 6;
      video.setAttribute('playsinline', '');
      video.onclick = () => { shutdown = true; };
      stream = await navigator.mediaDevices.getUserMedia(
          {video: { facingMode: "environment"}});
      div.appendChild(video);

      imgElement = document.createElement('img');
      imgElement.style.position = 'absolute';
      imgElement.style.zIndex = 1;
      imgElement.onclick = () => { shutdown = true; };
      div.appendChild(imgElement);
      
      const instruction = document.createElement('div');
      instruction.innerHTML = 
          '' +
          'When finished, click here or on the video to stop this demo';
      div.appendChild(instruction);
      instruction.onclick = () => { shutdown = true; };
      
      video.srcObject = stream;
      await video.play();

      captureCanvas = document.createElement('canvas');
      captureCanvas.width = 640; //video.videoWidth;
      captureCanvas.height = 480; //video.videoHeight;
      window.requestAnimationFrame(onAnimationFrame);
      
      return stream;
    }
    async function stream_frame(label, imgData) {
      if (shutdown) {
        removeDom();
        shutdown = false;
        return '';
      }

      var preCreate = Date.now();
      stream = await createDom();
      
      var preShow = Date.now();
      if (label != "") {
        labelElement.innerHTML = label;
      }
            
      if (imgData != "") {
        var videoRect = video.getClientRects()[0];
        imgElement.style.top = videoRect.top + "px";
        imgElement.style.left = videoRect.left + "px";
        imgElement.style.width = videoRect.width + "px";
        imgElement.style.height = videoRect.height + "px";
        imgElement.src = imgData;
      }
      
      var preCapture = Date.now();
      var result = await new Promise(function(resolve, reject) {
        pendingResolve = resolve;
      });
      shutdown = false;
      
      return {'create': preShow - preCreate, 
              'show': preCapture - preShow, 
              'capture': Date.now() - preCapture,
              'img': result};
    }
    ''')

  display(js)
  
def video_frame(label, bbox):
  data = eval_js('stream_frame("{}", "{}")'.format(label, bbox))
  return data

# Defining function to detect face
def face_detect():
  
# start streaming video from webcam
  video_stream()
# label for video
  label_html = 'Capturing...'
# initialze bounding box to empty
  bbox = ''
  count = 0
  face_classifier = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml') # Face Detection
  classifier =load_model('pretrainmodel.h5')  # Load model
  emotion_labels = ['Angry','Disgust','Fear','Happy','Sad','Surprise','Neutral'] 

  while True:
    js_reply = video_frame(label_html, bbox)
    if not js_reply:
      break

    # convert JS response to OpenCV Image
    img = js_to_image(js_reply["img"])

    # create transparent overlay for bounding box
    bbox_array = np.zeros([480,640,4], dtype=np.uint8)

    # convert the image to RGB format for face detection
    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    # get face region coordinates
    faces = face_cascade.detectMultiScale(rgb)
    # get face bounding box for overlay
    for (x,y,w,h) in faces:
      bbox_array = cv2.rectangle(bbox_array,(x,y),(x+w,y+h),(0,255,0),2)
      roi_rgb = rgb[y:y+h,x:x+w]
      roi_rgb = cv2.resize(roi_rgb,(48,48),interpolation=cv2.INTER_AREA)  # Face Cropping for prediction
      if np.sum([roi_rgb])!=0:
        roi = roi_rgb.astype('float')/255.0
        roi = img_to_array(roi)
        roi = np.expand_dims(roi,axis=0) # reshaping the cropped face image for prediction
        prediction = classifier.predict(roi)[0]   # Prediction
        label=emotion_labels[prediction.argmax()]
        label_position = (x,y)
        cv2.putText(bbox_array,label,label_position,cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,0),2) # Text Adding
      else:
        cv2.putText(bbox_array,'No Faces',(30,80),cv2.FONT_HERSHEY_SIMPLEX,1,(0,255,0),2)

    bbox_array[:,:,3] = (bbox_array.max(axis = 2) > 0 ).astype(int) * 255
    # convert overlay of bbox into bytes
    bbox_bytes = bbox_to_bytes(bbox_array)
    # update bbox so next frame gets new overlay
    bbox = bbox_bytes

# Finally detecting the face with live webcam
face_detect()

"""# **TF Lite Model**"""

# convert keras model to tflite 
def get_file_size(file_path):
    size = os.path.getsize(file_path)
    return size

def convert_bytes(size, unit=None):
    if unit == "KB":
        return print('File size: ' + str(round(size / 1024, 3)) + ' Kilobytes')
    elif unit == "MB":
        return print('File size: ' + str(round(size / (1024 * 1024), 3)) + ' Megabytes')
    else:
        return print('File size: ' + str(size) + ' bytes')

TF_LITE_MODEL_FILE_NAME = "tf_lite_model.tflite"
# Convert the model     
tf_lite_converter = tf.lite.TFLiteConverter.from_keras_model(fer_model)
tflite_model = tf_lite_converter.convert()

# Save the model
tflite_model_name = TF_LITE_MODEL_FILE_NAME
open(tflite_model_name, "wb").write(tflite_model)

convert_bytes(get_file_size(TF_LITE_MODEL_FILE_NAME), "KB")

"""# **Check Input Tensor Shape**"""

interpreter = tf.lite.Interpreter(model_path = TF_LITE_MODEL_FILE_NAME)
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
print("Input Shape:", input_details[0]['shape'])
print("Input Type:", input_details[0]['dtype'])
print("Output Shape:", output_details[0]['shape'])
print("Output Type:", output_details[0]['dtype'])