Face Classification By Python Using CNN in Google Colab

For more details and information, you can watch the YouTube video on Face Classification by Python using CNN in Google Colab.

Video URL : https://youtu.be/wDDGw7U5hPo

Overview

In this tutorial, we will be implementing on how the machines are able to read Human emotions by Convolution Neural Network. We will be trying to detect a facial expression in this article. For this time, you need to install python on your native PC and camera is needed.

What is Facial Emotion Recognition ?

It is the process of detecting human emotions from facial expressions.

We will be using a datasets of 2013 from the web. This datasets contains data of 48x48 pixels images having different facial expressions.

7 Universal Expressions

1. Angry
2. Disgust
3. Fear
4. Happy
5. Sad
6. Surprise
7. Neutral

The Files path on my Desktop:

The files contained:

The following is the code:

//this statement function will bring python3 into python2.6 plus
import keras
from _future_ import print_function
//It will multiply a single image into multiple images by rescaling, flipping, turning etc.
from keras.proprocessing.image import ImageDataGenerator
//importing modules and these models can be used for prediction and feature extraction and fine tuning
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation,Flatten,BatchNormalization
from keras.layers import Conv2D,MaxPooling2D
//using operating system dependent functionality
import os
//for 7 different expression
num_classes = 7
//setting the dimensions of images
img_rows, img_cols = 48, 48
//setting batch size to 32 since it will train 32 images at a time
batch_size = 32
//location of train directory and validation directory 
train_data_dir = r'C:\Users\user\Desktop\emotion_classifier\train'
validation_data_dir = r'C:\Users\user\Desktop\emotion_classifier\validation'
//block of train data generation which generates many images from a single image which will help in training machine and improve their accuracy
train_datagen = ImageDataGenerator(
                    reScale=1./255,
                    rotation_range=30,
                    shear_range=0.3,
                    zoom_range=0.3,
                    width_shift_range=0.4,
                    height_shift_range=0.4,
                    horizontal_flip=True,
                    fill_mode='nearest')
//rescaling of image because for validation it is not necessary to have many images of same image
validation_datagen = ImageDataGenerator(rescale=1./255)
//this block of codes will refer to the root directory of our files directory for train directory, color grayscale for the machine to reading the emotion classification, target size is same for the dimensions of image given above, the categorical means the images are categorized into 7 categories and the images will be shuffled before training those images
train_generator = train_datagen.flow_from_directory(
                    train_data_dir,
                    color_mode='grayscale',
                    traget_size=(img_rows, img_cols),
                    batch_size=batch_size,
                    class_mode='categorical',
                    shuffle=True)
//this block of codes will be same as for train directory but in this it will refer to the root directory of our validation directory
validation_generator = validation_datagen.flow_from_directory(
                            validation_data_dir,
                            color_mode='grayscale',
                            traget_size=(img_rows, img_cols),
                            batch_size=batch_size,
                            class_mode='categorical',
                            shuffle=True)
//sequential API that allows us to create model layers by layer model = sequential()
//these are the Convolution Neural Network in sequential blocks model.add(Conv2D(32,(3,3),padding='same',kernel_initializer='he_normal',input_shape=(img_rows,img_cols,1)))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Conv2D(32,(3,3),padding='same',kernel_initializer='he_normal',input_shape=(img_rows,img_cols,1)))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0,2))
model.add(Conv2D(64,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Conv2D(64,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0,2))
model.add(Conv2D(128,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Conv2D(128,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0,2))
model.add(Conv2D(256,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Conv2D(256,(3,3),padding='same',kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0,2))
//this block of codes will flatten the network where it will not have a matrix 
model.add(Flatten())
model.add(Dense(64,kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
//The above block of codes and the below block of codes will be fully connected network having 64 neuron having dropout of 50%
model.add(Dense(64,kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
//these will specify the number of neuron to be used model.add(Dense(num_classes,kernel_initializer='he_normal'))
model.add(Activation('softmax'))
//saving and printing the models 
print(model.summary())

We can see the total params, trainable params and non-trainable params.

The following codes are for training of models:

//importing optimizer 
from keras.optimizers import RMSprop,SGD,Adam
from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau
//checking checkpoint for selecting the best model for the emotion detection and save the best model with minimum validation loss having the name written on it in the current working folder
checkpoint = ModelCheckpoint('datacollect.h5',
                             monitor='val_loss',
                             mode='min',
                             save_best_only=True,
                             verbose=1)
//earlystop will be applied for where there is no improvement in validation loss upto 3 epochs
earlystop = EarlyStopping(monitor='val_loss',
                          min_delta=0,
                          patience=3,
                          verbose=1,
                          restore_best_weights=True)
//reduce_lr will also monitor the validation loss 
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                              factor=0.2,
                              patience=3,
                              verbose=1,
                              min_delta=0.0001)
callbacks = [earlystop, checkpoint, reduce_lr]
model.compile(loss='categorical_crossentropy',
              optimizer = Adam(lr=0.001),
              metrics=['accuracy'])
//these are the sample inside the class to train our machine and sample for validation which we can get from the class during our first execution of the code
nb_train_samples = 28789
nb_validation_samples = 3589
epoches=25
//This last block of codes will train our codes with the given parameters where it will fit train generator for training epochs to 25, callbacks will call the earlystop, checkpoint and reduce_lr and validation data will be equal to validation generator which was created on the first execution 
history=model.fit_generator(
                train_generator,
                steps_per_epoch=nb_train_samples//batch_size,
                callbacks=callbacks,
                validation_data=validation_generator,
                validation_steps = nb_validation_samples//batch_size)

Lastly, the below given codes is used for Capturing the Face using the camera in your computer.

//importing all the important packages or modules for face detection
from keras.models import load_model
from time import sleep
from keras.preprocessing.image import img_to_array
from keras.preprocessing import image
import cv2
import numpy as np
//face_classifier will import an XML file of cascade which is used for face detection
face_classifier = cv2.CascadeClassifier(r'C:\Users\user\Desktop\emotion_classifier\haarcascade_frontalface_default.xml')
//import the train data file from the current working directory 
classifier = load_model(r'C:\Users\user\Desktop\emotion_classifier\datacollect.h5')
class_labels = ['Angry', 'Disgust', 'Fear', 'Happy', 'Neutral', 'Sad', 'Surprise']
cap = cv2.VideoCapture(0)
//These blocks of codes is for face detection in a box will be drawn your face on the monitor, if your face is detected
while True:
    ret, frame = cap.read()
    labels = []
    gray = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
    faces = face_classifier.detectMultiScale(gray,1.3,5)
    
    for (x,y,w,h) in faces:
        cv2.rectangle(frame,(x,y),(x+w,y+h),(255,0,0),2)
        roi_gray = gray[y:y+h,x:x+w]
        roi_gray = cv2.resize(roi_gray,(48,48),interpolation=cv2.INTER_AREA)
if np.sum([roi_gray])!=0:
            roi = roi_gray.astype('float')/255.0
            roi = img_to_array(roi)
            roi = np.expand_dims(roi,axis=0)
            
            preds = classifier.predict(roi)[0]
            label=class_labels[preds.argmax()]
            label_position = (x,y)
            cv2.putText(frame,label,label_position,cv2.FONT_HERSHEY_SIMPLEX,2,(0,255,0),3)
        else:
            cv2.putText(frame,'No Face Found',(20,60),cv2.FONT_HERSHEY_SIMPLEX,2,(0,255,0),3)
if cv2.waitKey(1) & 0xFF == ord('q'):
        break
//A new window will be opened for face detection and emotion detection , so this will close the window if pressed on closed button
cap.release()
cv2.destroyAllWindows()