计算机动画作业:图像morphing

本学期选了计算机动画课程，第一次作业是图像morphing， 本来打算选择基于四边网格的morphing， 但因为要用到曲面插值，感觉比较麻烦，因此使用基于三角网格的face morphing。

一、总体方案

1、检测人脸特征点，可以检测人脸81个特征点：https://github.com/codenik/shape_predictor_81_face_landmarks
2、基于81个面部特征点 + 图像边界的8个点 ， 共拿到每张图片的 89个点
3、对两张图像的采样点进行Delaunay三角剖分
4、根据时间t，分别计算原图像 -> t 时刻的原图像， 目标图像 -> t时刻的目标图像
5、根据时间t， 融合 t 时刻的原图像 和 t时刻的目标图像
难点：三角形区域仿射变换：使用OpenCV将一个三角形仿射变换到另一个三角形

二、具体实现

1、main.py

import sys, os, cv2, time
from skimage import io
import numpy as np
from util import face_detection_triangle
from util import img_tri_affine

src_img_path = 'images/dst.png'
dst_img_path = 'images/src.png'
src_img = cv2.imread(src_img_path)
dst_img = cv2.imread(dst_img_path)

src_points, src_tri = face_detection_triangle(src_img, 'images/src_tri.jpg')
dst_points, dst_tri = face_detection_triangle(dst_img, 'images/dst_tri.jpg')

tcnt = 50
t = 0
src_img_t = 255 * np.ones(src_img.shape, dtype = src_img.dtype)
dst_img_t = 255 * np.ones(dst_img.shape, dtype = dst_img.dtype)

for i in range(50, 50+tcnt):
    for j in range(src_tri.shape[0]) : #src_tri.shape[0]
        src_p0 = src_points[src_tri[j, 0]]
        src_p1 = src_points[src_tri[j, 1]]
        src_p2 = src_points[src_tri[j, 2]]
        src_tri_points = np.float32([[src_p0[0], src_p0[1]], [src_p1[0], src_p1[1]], [src_p2[0], src_p2[1]]])

        dst_p0 = dst_points[src_tri[j, 0]]
        dst_p1 = dst_points[src_tri[j, 1]]
        dst_p2 = dst_points[src_tri[j, 2]]
        dst_tri_points = np.float32([[dst_p0[0], dst_p0[1]], [dst_p1[0], dst_p1[1]], [dst_p2[0], dst_p2[1]]])

        mid_tri_points = src_tri_points * (1-t) + dst_tri_points * t
        img_tri_affine(src_img, src_img_t, src_tri_points, mid_tri_points)
        img_tri_affine(dst_img, dst_img_t, dst_tri_points, mid_tri_points)

    res_image_t = src_img_t[0:651, :, :] * (1-t)/255 + dst_img_t[0:651, :, :] * t/255
    cv2.imshow('morphing-res', res_image_t)
    cv2.waitKey(50)
    t += 1.0 / tcnt
    
cv2.waitKey(0)

2、util.py

import sys, os, dlib, glob, cv2
from skimage import io
import numpy as np
from scipy.spatial import Delaunay
import matplotlib.pyplot as plt

def face_detection_triangle(image, path_out):
    predictor_path = 'face_landmarks_predictor.dat'
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor(predictor_path)

    frame = image
    dets = detector(frame, 0)
    points = np.zeros((81+8, 2))

    for k, d in enumerate(dets):
        shape = predictor(frame, d)
        landmarks = np.matrix([[p.x, p.y] for p in shape.parts()])
        for num in range(shape.num_parts):
            cv2.circle(frame, (shape.parts()[num].x, shape.parts()[num].y), 3, (255, 0, 0), -1)
            points[num][0] = int(shape.parts()[num].x)
            points[num][1] = int(shape.parts()[num].y)

    height = frame.shape[0] 
    width = frame.shape[1]
    points[81] = [4, 4]
    points[82] = [width-4, 4]
    points[83] = [4, height-4]
    points[84] = [width-4, height-4]
    points[85] = [int(width/2), 4]
    points[86] = [width-4, int(height/2)]
    points[87] = [4, int(height/2)]
    points[88] = [int(width/2), height-4]

    tri = Delaunay(points)
    color = (255, 0, 0)
    
    for i in range(tri.simplices.shape[0]):
        p0 = points[tri.simplices[i, 0]]
        p1 = points[tri.simplices[i, 1]]
        p2 = points[tri.simplices[i, 2]]
        cv2.line(frame, (int(p0[0]), int(p0[1])), (int(p1[0]), int(p1[1])), color)
        cv2.line(frame, (int(p0[0]), int(p0[1])), (int(p2[0]), int(p2[1])), color)
        cv2.line(frame, (int(p1[0]), int(p1[1])), (int(p2[0]), int(p2[1])), color)

    cv2.imwrite(path_out, frame)
    return points, tri.simplices

# Warps and alpha blends triangular regions from img1 and img2 to img
def img_tri_affine(img1, img2, tri1, tri2) :
    # Find bounding rectangle for each triangle
    r1 = cv2.boundingRect(tri1)
    r2 = cv2.boundingRect(tri2)
    
    # Offset points by left top corner of the respective rectangles
    tri1Cropped = []
    tri2Cropped = []
    
    for i in range(0, 3):
        tri1Cropped.append(((tri1[i][0] - r1[0]),(tri1[i][1] - r1[1])))
        tri2Cropped.append(((tri2[i][0] - r2[0]),(tri2[i][1] - r2[1])))
 
    # Crop input image
    img1Cropped = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
 
    # Given a pair of triangles, find the affine transform.
    warpMat = cv2.getAffineTransform( np.float32(tri1Cropped), np.float32(tri2Cropped) )
    
    # Apply the Affine Transform just found to the src image
    img2Cropped = cv2.warpAffine( img1Cropped, warpMat, (r2[2], r2[3]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )
 
    # Get mask by filling triangle
    mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
    
    cv2.fillConvexPoly(mask, np.int32(tri2Cropped), (1.0, 1.0, 1.0), 16, 0);
    img2Cropped = img2Cropped * mask
    
    # Copy triangular region of the rectangular patch to the output image
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]]  * ( (1.0, 1.0, 1.0) - mask )  
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]]  + img2Cropped