計算機動畫作業:圖像morphing

本學期選了計算機動畫課程，第一次作業是圖像morphing， 本來打算選擇基於四邊網格的morphing， 但因爲要用到曲面插值，感覺比較麻煩，因此使用基於三角網格的face morphing。

一、總體方案

1、檢測人臉特徵點，可以檢測人臉81個特徵點：https://github.com/codenik/shape_predictor_81_face_landmarks
2、基於81個面部特徵點 + 圖像邊界的8個點 ， 共拿到每張圖片的 89個點
3、對兩張圖像的採樣點進行Delaunay三角剖分
4、根據時間t，分別計算原圖像 -> t 時刻的原圖像， 目標圖像 -> t時刻的目標圖像
5、根據時間t， 融合 t 時刻的原圖像 和 t時刻的目標圖像
難點：三角形區域仿射變換：使用OpenCV將一個三角形仿射變換到另一個三角形

二、具體實現

1、main.py

import sys, os, cv2, time
from skimage import io
import numpy as np
from util import face_detection_triangle
from util import img_tri_affine

src_img_path = 'images/dst.png'
dst_img_path = 'images/src.png'
src_img = cv2.imread(src_img_path)
dst_img = cv2.imread(dst_img_path)

src_points, src_tri = face_detection_triangle(src_img, 'images/src_tri.jpg')
dst_points, dst_tri = face_detection_triangle(dst_img, 'images/dst_tri.jpg')

tcnt = 50
t = 0
src_img_t = 255 * np.ones(src_img.shape, dtype = src_img.dtype)
dst_img_t = 255 * np.ones(dst_img.shape, dtype = dst_img.dtype)

for i in range(50, 50+tcnt):
    for j in range(src_tri.shape[0]) : #src_tri.shape[0]
        src_p0 = src_points[src_tri[j, 0]]
        src_p1 = src_points[src_tri[j, 1]]
        src_p2 = src_points[src_tri[j, 2]]
        src_tri_points = np.float32([[src_p0[0], src_p0[1]], [src_p1[0], src_p1[1]], [src_p2[0], src_p2[1]]])

        dst_p0 = dst_points[src_tri[j, 0]]
        dst_p1 = dst_points[src_tri[j, 1]]
        dst_p2 = dst_points[src_tri[j, 2]]
        dst_tri_points = np.float32([[dst_p0[0], dst_p0[1]], [dst_p1[0], dst_p1[1]], [dst_p2[0], dst_p2[1]]])

        mid_tri_points = src_tri_points * (1-t) + dst_tri_points * t
        img_tri_affine(src_img, src_img_t, src_tri_points, mid_tri_points)
        img_tri_affine(dst_img, dst_img_t, dst_tri_points, mid_tri_points)

    res_image_t = src_img_t[0:651, :, :] * (1-t)/255 + dst_img_t[0:651, :, :] * t/255
    cv2.imshow('morphing-res', res_image_t)
    cv2.waitKey(50)
    t += 1.0 / tcnt
    
cv2.waitKey(0)

2、util.py

import sys, os, dlib, glob, cv2
from skimage import io
import numpy as np
from scipy.spatial import Delaunay
import matplotlib.pyplot as plt

def face_detection_triangle(image, path_out):
    predictor_path = 'face_landmarks_predictor.dat'
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor(predictor_path)

    frame = image
    dets = detector(frame, 0)
    points = np.zeros((81+8, 2))

    for k, d in enumerate(dets):
        shape = predictor(frame, d)
        landmarks = np.matrix([[p.x, p.y] for p in shape.parts()])
        for num in range(shape.num_parts):
            cv2.circle(frame, (shape.parts()[num].x, shape.parts()[num].y), 3, (255, 0, 0), -1)
            points[num][0] = int(shape.parts()[num].x)
            points[num][1] = int(shape.parts()[num].y)

    height = frame.shape[0] 
    width = frame.shape[1]
    points[81] = [4, 4]
    points[82] = [width-4, 4]
    points[83] = [4, height-4]
    points[84] = [width-4, height-4]
    points[85] = [int(width/2), 4]
    points[86] = [width-4, int(height/2)]
    points[87] = [4, int(height/2)]
    points[88] = [int(width/2), height-4]

    tri = Delaunay(points)
    color = (255, 0, 0)
    
    for i in range(tri.simplices.shape[0]):
        p0 = points[tri.simplices[i, 0]]
        p1 = points[tri.simplices[i, 1]]
        p2 = points[tri.simplices[i, 2]]
        cv2.line(frame, (int(p0[0]), int(p0[1])), (int(p1[0]), int(p1[1])), color)
        cv2.line(frame, (int(p0[0]), int(p0[1])), (int(p2[0]), int(p2[1])), color)
        cv2.line(frame, (int(p1[0]), int(p1[1])), (int(p2[0]), int(p2[1])), color)

    cv2.imwrite(path_out, frame)
    return points, tri.simplices

# Warps and alpha blends triangular regions from img1 and img2 to img
def img_tri_affine(img1, img2, tri1, tri2) :
    # Find bounding rectangle for each triangle
    r1 = cv2.boundingRect(tri1)
    r2 = cv2.boundingRect(tri2)
    
    # Offset points by left top corner of the respective rectangles
    tri1Cropped = []
    tri2Cropped = []
    
    for i in range(0, 3):
        tri1Cropped.append(((tri1[i][0] - r1[0]),(tri1[i][1] - r1[1])))
        tri2Cropped.append(((tri2[i][0] - r2[0]),(tri2[i][1] - r2[1])))
 
    # Crop input image
    img1Cropped = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
 
    # Given a pair of triangles, find the affine transform.
    warpMat = cv2.getAffineTransform( np.float32(tri1Cropped), np.float32(tri2Cropped) )
    
    # Apply the Affine Transform just found to the src image
    img2Cropped = cv2.warpAffine( img1Cropped, warpMat, (r2[2], r2[3]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )
 
    # Get mask by filling triangle
    mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
    
    cv2.fillConvexPoly(mask, np.int32(tri2Cropped), (1.0, 1.0, 1.0), 16, 0);
    img2Cropped = img2Cropped * mask
    
    # Copy triangular region of the rectangular patch to the output image
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]]  * ( (1.0, 1.0, 1.0) - mask )  
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]]  + img2Cropped