Snapchat-like Augmented Reality Filters

Have you ever wondered how snapchat makes those cool augmented reality filters? I’ve always been fascinated by augmented reality myself. So I started hacking together my own crude version of it. Augmented reality basically involves superimposing a computer-generated image in a live video feed of the real world. To achieve this you need to detect areas of interest and warp the computer generated image around it.

My project lets you try on virtual pairs of sunglasses or masks. To achieve this I utilized python, Dlib, OpenCV, Scipy, and Numpy. The pipeline involves opening a live webcam feed, detecting keypoints on faces in the feed, warping a png image of sunglasses to match the face, rotating the png with face movements, and blending the two images together so they look like one real image.

It’s a crude hack, but it will get you started in augmented reality programming. If you have questions leave them in the comments below. Enjoy.

#!/usr/bin/python

import dlib
import cv2
import numpy as np
from scipy import ndimage

video_capture = cv2.VideoCapture(0)
glasses = cv2.imread("images/sunglasses.png", -1)
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

#Resize an image to a certain width
def resize(img, width):
    r = float(width) / img.shape[1]
    dim = (width, int(img.shape[0] * r))
    img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA)
    return img

#Combine an image that has a transparency alpha channel
def blend_transparent(face_img, sunglasses_img):

    overlay_img = sunglasses_img[:,:,:3]
    overlay_mask = sunglasses_img[:,:,3:]
    
    background_mask = 255 - overlay_mask

    overlay_mask = cv2.cvtColor(overlay_mask, cv2.COLOR_GRAY2BGR)
    background_mask = cv2.cvtColor(background_mask, cv2.COLOR_GRAY2BGR)

    face_part = (face_img * (1 / 255.0)) * (background_mask * (1 / 255.0))
    overlay_part = (overlay_img * (1 / 255.0)) * (overlay_mask * (1 / 255.0))

    return np.uint8(cv2.addWeighted(face_part, 255.0, overlay_part, 255.0, 0.0))

#Find the angle between two points
def angle_between(point_1, point_2):
    angle_1 = np.arctan2(*point_1[::-1])
    angle_2 = np.arctan2(*point_2[::-1])
    return np.rad2deg((angle_1 - angle_2) % (2 * np.pi))


#Start main program
while True:

    ret, img = video_capture.read()
    img = resize(img, 700)
    img_copy = img.copy()
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    try:
        # detect faces
        dets = detector(gray, 1)

        #find face box bounding points
        for d in dets:

            x = d.left()
            y = d.top()
            w = d.right()
            h = d.bottom()

        dlib_rect = dlib.rectangle(x, y, w, h)

        ##############   Find facial landmarks   ##############
        detected_landmarks = predictor(gray, dlib_rect).parts()

        landmarks = np.matrix([[p.x, p.y] for p in detected_landmarks])

        for idx, point in enumerate(landmarks):
            pos = (point[0, 0], point[0, 1])
            if idx == 0:
                eye_left = pos
            elif idx == 16:
                eye_right = pos

            try:
                #cv2.line(img_copy, eye_left, eye_right, color=(0, 255, 255))
                degree = np.rad2deg(np.arctan2(eye_left[0] - eye_right[0], eye_left[1] - eye_right[1]))

            except:
                pass

        ##############   Resize and rotate glasses   ##############

        #Translate facial object based on input object.

        eye_center = (eye_left[1] + eye_right[1]) / 2

        #Sunglasses translation
        glass_trans = int(.2 * (eye_center - y))

        #Funny tanslation
        #glass_trans = int(-.3 * (eye_center - y ))

        # Mask translation
        #glass_trans = int(-.8 * (eye_center - y))


        # resize glasses to width of face and blend images
        face_width = w - x

        # resize_glasses
        glasses_resize = resize(glasses, face_width)

        # Rotate glasses based on angle between eyes
        yG, xG, cG = glasses_resize.shape
        glasses_resize_rotated = ndimage.rotate(glasses_resize, (degree+90))
        glass_rec_rotated = ndimage.rotate(img[y + glass_trans:y + yG + glass_trans, x:w], (degree+90))


        #blending with rotation
        h5, w5, s5 = glass_rec_rotated.shape
        rec_resize = img_copy[y + glass_trans:y + h5 + glass_trans, x:x + w5]
        blend_glass3 = blend_transparent(rec_resize , glasses_resize_rotated)
        img_copy[y + glass_trans:y + h5 + glass_trans, x:x+w5 ] = blend_glass3
        cv2.imshow('Output', img_copy)

    except:
        cv2.imshow('Output', img_copy)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break