【火眼金睛】用PWC光流法來訓練Deepfakes換臉檢測模型

這段時間，一直在搞這個光流法，從一開始完全不知道，再到現在搞出點東西，畢竟是畢設相關，以此作爲紀念

方法

數據集處理

這裏我使用的celeb-df數據集

face.py找到人臉

首先在github上下載facenet，裏面有一個detect_face.py文件，要把align標爲source dir，pycharm才能找到

然後在這個基礎上編寫檢測臉部的代碼 face.py

"""
@Time    : 2020/3/6 5:17
@Author  : cs
@content: 
"""
import re

from scipy import misc
import tensorflow as tf
import detect_face
import cv2
import matplotlib.pyplot as plt
import imageio
import os
# def startGetFace():

#傳入一張圖的地址，返回圖中的人臉座標和原圖
def getFace(imagePath):
    minsize = 20  # minimum size of face
    threshold = [0.6, 0.7, 0.7]  # three steps's threshold
    factor = 0.709  # scale factor
    gpu_memory_fraction = 1.0

    print('Creating networks and loading parameters')

    with tf.Graph().as_default():
        # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_memory_fraction)
        # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        sess = tf.Session(config=config)
        with sess.as_default():
            pnet, rnet, onet = detect_face.create_mtcnn(sess, None)

    img = cv2.imread(imagePath)
    bounding_boxes, _ = detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor)
    nrof_faces = bounding_boxes.shape[0]  # 人臉數目
    print('找到人臉數目爲：{}'.format(nrof_faces))
    print(bounding_boxes)

    crop_faces = []
    for face_position in bounding_boxes:
        face_position = face_position.astype(int)
        print(face_position[0:4])
        for i in range(len(face_position)):
            if face_position[i]<0:
                face_position[i]=0


        # cv2.rectangle(img, (face_position[0], face_position[1]), (face_position[2], face_position[3]), (0, 255, 0), 2)
        # crop = img[face_position[1]:face_position[3],
        #        face_position[0]:face_position[2], ]
        crop=img
        # crop = cv2.resize(crop, (256,256), interpolation=cv2.INTER_CUBIC)
        print(crop.shape)
        crop_faces.append(crop)
        # cv2.imwrite(dstPath+filename ,crop)
        # plt.imshow(crop)
        # plt.show()
        # cv2.imshow("image",crop)
        # cv2.waitKey()
        return(crop,face_position)
    return(img,[0,0,0,0]) #如果沒有人臉，將直接跳出for循環

（註釋掉的部分都是之前修改的痕跡，本來想在文章中刪除，但是怕一不小心刪錯，所以可以自行刪除）

face.py可以檢測粗人臉的框圖 crop，並返回人臉圖像crop和人臉在圖像中的位置face_position，但是這樣的人臉每一張都大小不一，所以我們將能夠框住人臉的224x224的圖裁切下來

facemask裁切人臉周圍的224x224的圖像

因爲pwc需要兩張圖片同樣大小，所以我們統一截取爲224x224

這裏我的存放位置在

srcPath="G:/datasets/myceleb/preal"
dstPath="G:/datasets/maskceleb2/preal_mask/"

"""
@Time    : 2020/3/6 3:30
@Author  : cs
@content:
"""
from image_deal import detectFace
from face import getFace
import cv2 as cv
import numpy as np
import os
from image_deal import delete
import matplotlib.pyplot as plt
def facemask(srcPath,dstPath):
    count=0
    for filename in os.listdir(srcPath):
        count+=1
        print(count)
        # 如果不存在目的目錄則創建一個，保持層級結構
        if not os.path.exists(dstPath):
            os.makedirs(dstPath)
        srcFile=os.path.join(srcPath,filename)
        dstFile=os.path.join(dstPath,filename)
        img,faceRect=getFace(srcFile) #這裏可能爲空
        x1, y1, x2, y2 = faceRect[0], faceRect[1], faceRect[2], faceRect[3]
        h,w,c=img.shape
        print(faceRect)
        if x1+y1+x2+y2>0:
            mask=np.zeros([img.shape[0],img.shape[1]],dtype=np.uint8)

            left=(x1+x2)//2-112
            right=(x1+x2)//2+112
            top=(y1+y2)//2-112
            bottom=(y1+y2)//2+112
            if left<=0:
                left=0
            if right>=w:
                right=w
            if top<=0:
                top=0
            if bottom>=h:
                bottom=h
            # mask[top:bottom,left:right]=255
            image=img[top:bottom,left:right]
            # image=cv.add(img,np.zeros(np.shape(img),dtype=np.uint8),mask=mask)
            cv.imwrite(dstFile,image)
            # cv.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2, 8, 0)
            # cv.imshow("image",img)
            # cv.waitKey()
            print(dstFile+" mask succeeded")
        else:
            print("no face")
            continue

srcPath="G:/datasets/myceleb/preal"
dstPath="G:/datasets/maskceleb2/preal_mask/"
delete(dstPath)
facemask(srcPath,dstPath)

這樣就可以裁出相同大小的圖片了,這裏有一個delete()函數，來自於image_deal，關於圖片的常用處理，我都放在了image_deal中

"""
@Time    : 2020/3/6 3:20
@Author  : cs
@content: 
"""
import numpy as np
import cv2 as cv
import os

# 檢測圖片中的臉部並返回座標
def detectFace(srcFile):
    face_cascade = cv.CascadeClassifier("myxml/haarcascade_frontalface_default.xml")
    eye_cascade = cv.CascadeClassifier("myxml/haarcascade_eye_tree_eyeglasses.xml")

    img = cv.imread(srcFile)
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(gray, 1.1, 5, cv.CASCADE_SCALE_IMAGE, (50, 50), (100, 100))

    if len(faces) > 0:
        for faceRect in faces:
            x, y, w, h = faceRect
            # cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2, 8, 0)

            roi_gray = gray[y:y + h, x:x + w]
            roi_color = img[y:y + h, x:x + w]
            # cv2.imshow("img", img)
            # cv2.waitKey(0)
            # 只討論一張臉的情況
            return [img, [x, y, x + w, y + h]]
    return [img,[0,0,0,0]]   #這裏必須返回值，不然可能沒有返回值

# 圖片壓縮批處理
def compressImage(srcPath, dstPath):
    count = 0
    # 在生成圖片前一定要清空dstPath
    for filename in os.listdir(dstPath):
        if filename:
            os.remove(dstPath + filename)
    for filename in os.listdir(srcPath):
        # 如果不存在目的目錄則創建一個，保持層級結構
        if not os.path.exists(dstPath):
            os.makedirs(dstPath)

            # 拼接完整的文件或文件夾路徑
        srcFile = os.path.join(srcPath, filename)
        dstFile = os.path.join(dstPath, filename)
        print(srcFile)
        print(dstFile)

        # 如果是文件就處理
        if os.path.isfile(srcFile):
            # 打開原圖片縮小後保存，可以用if srcFile.endswith(".jpg")或者split，splitext等函數等針對特定文件壓縮
            sImg = cv.imread(srcFile)
            w, h, c = sImg.shape
            print(w, h)

            """
            保證是同樣大小的圖片，不然報錯
            """
            dImg = cv.resize(sImg, (w//2, h//2), interpolation=cv.INTER_AREA)  # 設置壓縮尺寸和選項，注意尺寸要用括號
            #             dImg.save(dstFile) #也可以用srcFile原路徑保存,或者更改後綴保存，save這個函數後面可以加壓縮編碼選項JPEG之類的
            #             print("壓縮後大小",w//4,h//4)
            cv.imwrite(dstFile, dImg, [int(cv.IMWRITE_JPEG_QUALITY), 5])
            print(dstFile + " compressed succeeded")

        # 如果是文件夾就遞歸
        if os.path.isdir(srcFile):
            compressImage(srcFile, dstFile, [int(cv.IMWRITE_JPEG_QUALITY), 5])


# compressImage(pfake,plittlefake)
def video2image(videoPath, svPath,name):

    # delete(svPath)
    cap = cv.VideoCapture(videoPath)
    numFrame = 0
    test=0
    while True:
        if cap.grab():
            flag, frame = cap.retrieve()
            if not flag:
                continue
            else:
                # cv2.imshow('video', frame)
                numFrame += 1

                newPath = svPath + name+str(numFrame) + ".png"
                cv.imencode('.png', frame)[1].tofile(newPath)
        if cv.waitKey(10) == 27:
            break
        if numFrame==test:
            break
        test=numFrame
        print("已在"+svPath+"生成 %s 張圖片"%numFrame)

def delete(imagePath):
    count=0
    for root,dirs,files in os.walk(imagePath):
        # os.remove(imagePath+"/"+files)
        for file in files:
            os.remove(root+"/"+file)
            count+=1
    print("已刪除 %s 張圖片" % count)

#讓文件名的數字位數相同
def regularName(imagePath):
    for file in os.listdir(imagePath):
        names=file.split("_")
        if len(names[0])==5:
            n=names[0][-1]
            f=names[0][0:4]
            names[0]=f+'00'+n
        elif len(names[0]) == 6:
            n = names[0][4:6]
            f = names[0][0:4]
            names[0] = f + "0" + n
        numbers=names[1].split(".")
        if len(numbers[0])==1:
            names[1]="00"+names[1]
        elif len(numbers[0])==2:
            names[1]="0"+names[1]
        newfile=names[0]+"_"+names[1]
        os.rename(imagePath+file,imagePath+newfile)
        print("已將"+imagePath+file+"改名爲"+newfile)
#if __name__ == '__main__':
    #regularName("G:/datasets/maskceleb/pfake_mask/")

可以看到，這裏麪包含了視頻轉圖片，刪除原有圖片，圖片批量重命名（可以不使用），還有一個opencv自帶的檢測人臉機制，但是比起facenet差遠了，所以不用

可以得到這些圖像

PWC光流處理

好了，進入光流處理環節，具體操作方法可參考

【Tensorflow】如何使用PWC-Net網絡輸出運動中的光流圖像

由於整個代碼是在jupyter中完成，所以顯得比較凌亂，函數和實現代碼混在了一起，這裏要注意pwcnet.ckpt-595000一個光流模型，得預先下載好

fake_mask就是上一步中裁剪出來的文件夾

pflowfake_mask就是已經生成好光流的文件夾

具體代碼如下

"""
pwcnet_predict_from_img_pairs.ipynb

Run inference on a list of images pairs.

Written by Phil Ferriere

Licensed under the MIT License (see LICENSE for details)
"""
from __future__ import absolute_import, division, print_function
from copy import deepcopy
from skimage.io import imread
from model_pwcnet import ModelPWCNet, _DEFAULT_PWCNET_TEST_OPTIONS
from visualize import display_img_pairs_w_flows,archive_img_pairs_w_flow_pyrs,plot_img_pairs_w_flows
from PIL import Image
import os
import cv2 as cv
from optflow import flow_to_img

# TODO: Set device to use for inference
# Here, we're using a GPU (use '/device:CPU:0' to run inference on the CPU)
gpu_devices = ['/device:GPU:0']  
controller = '/device:GPU:0'

# TODO: Set the path to the trained model (make sure you've downloaded it first from http://bit.ly/tfoptflow)
ckpt_path = './models/pwcnet-lg-6-2-multisteps-chairsthingsmix/pwcnet.ckpt-595000'

# Configure the model for inference, starting with the default options
nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS)
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = 1
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller

# We're running the PWC-Net-large model in quarter-resolution mode
# That is, with a 6 level pyramid, and upsampling of level 2 by 4 in each dimension as the final flow prediction
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2

# The size of the images in this dataset are not multiples of 64, while the model generates flows padded to multiples
# of 64. Hence, we need to crop the predicted flows to their original size
nn_opts['adapt_info'] = (1, 436, 1024, 2)

def delete(imagePath):
    count=0
    for root,dirs,files in os.walk(imagePath):
        # os.remove(imagePath+"/"+files)
        for file in files:
            os.remove(root+"/"+file)
            count+=1
    print("已刪除 %s 張圖片" % count)
    
# Instantiate the model in inference mode and display the model configuration
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()



# img_pairs = []
count=0
def getImagePairs(startNumber,times):
    startNumber*=times
    img_pairs = []
    files=os.listdir(srcPath)
    for i in range(startNumber,startNumber+times):
        name1=files[i].split("_")
        name2=files[i+1].split("_")
        if name1[0]!=name2[0]: #保證是同一個人，不然報錯
            continue
        else:
            print(name1,name2)
            image_path1 = srcPath+files[i]
            image_path2 = srcPath+files[i+1]
            image1, image2 = imread(image_path1), imread(image_path2)
            img_pairs.append((image1, image2))
        print(len(img_pairs))
    return img_pairs



def GeneratePre(img_pairs):# Generate the predictions and display them
    pred_labels = nn.predict_from_img_pairs(img_pairs, batch_size=1, verbose=False)
    display_img_pairs_w_flows(img_pairs, pred_labels)
    return pred_labels

# Build a list of image pairs to process 
# srcPath=plittle_real_mask  #光流結對照片的數據集
real_mask="G:/datasets/maskceleb/preal_mask/"
fake_mask="G:/datasets/maskceleb/pfake_mask/"
pflowfake_mask="G:/datasets/maskceleb/pflowfake_mask/"
pflowreal_mask="G:/datasets/maskceleb/pflowreal_mask/"
srcPath=plittlefake_mask  #光流結對照片的數據集
flow_path=pflowfake_mask
def writeFlow(i,img_pairs,pred_labels,times):
    i*=times
    # flow_path=pflowreal_mask
    for row in range(len(img_pairs)):
        i+=1
        if pred_labels is not None:
            img=flow_to_img(pred_labels[row])
            if not os.path.exists(flow_path):
                    os.makedirs(flow_path)
            sNumber=str(i)
            while len(sNumber)<5:
                sNumber="0"+sNumber
            cv.imwrite(flow_path+sNumber+".png",img,[int(cv.IMWRITE_JPEG_QUALITY),5])
times=10
delete(flow_path)
for i in range(3100):
    img_pairs=getImagePairs(i,times)
    pred_labels=GeneratePre(img_pairs)
    writeFlow(i,img_pairs,pred_labels,times)

可以得到如下結果

拆分成數據集

到了最後一步，我們將其分成train validation test，定義一個split_data.py

"""
@Time    : 2020/3/6 9:34
@Author  : cs
@content: 
"""
import os, shutil
from image_deal import delete
# 含有光流圖的總目錄
original = 'G:/datasets/maskceleb/'
# 存放數據集的目錄
base_dir = 'G:/datasets/flowset/'
# os.mkdir(base_dir)

# 切割數據集
# fnames = ['cat.{}.jpg'.format(i) for i in range(1000)]
original_real=original+"pflowreal_mask/"
original_fake=original+"pflowfake_mask/"

print(original_real)
# 建立訓練集、驗證集、測試集目錄
train_dir = os.path.join(base_dir, 'train/')
if not os.path.exists(train_dir):
    os.mkdir(train_dir)
validation_dir = os.path.join(base_dir, 'validation/')
if not os.path.exists(validation_dir):
    os.mkdir(validation_dir)
test_dir = os.path.join(base_dir, 'test/')
if not os.path.exists(test_dir):
    os.mkdir(test_dir)

# 將real,fake照片按照訓練、驗證、測試分類
train_real_dir = os.path.join(train_dir, 'real/')
# if not os.path.exists(train_real_dir):
#     os.mkdir(train_real_dir)

train_fake_dir = os.path.join(train_dir, 'fake/')
# os.mkdir(train_fake_dir)

validation_real_dir = os.path.join(validation_dir, 'real/')
# os.mkdir(validation_real_dir)

validation_fake_dir = os.path.join(validation_dir, 'fake/')
# os.mkdir(validation_fake_dir)

test_real_dir = os.path.join(test_dir, 'real/')
# os.mkdir(test_real_dir)

test_fake_dir = os.path.join(test_dir, 'fake/')
# os.mkdir(test_fake_dir)


trainNumber=30000
valNumber=500
testNumber=500
numbers=[0,trainNumber,trainNumber+valNumber,trainNumber+valNumber+testNumber]
flowRealPaths=[original_real,train_real_dir,validation_real_dir,test_real_dir]
flowFakePaths=[original_fake,train_fake_dir,validation_fake_dir,test_fake_dir]

def cleanPath(imgaePath):
    delete(imgaePath)
def cleanAllPath(flowRealPaths,flowFakePaths):
    for i in range(1,len(flowFakePaths)):
        delete(flowFakePaths[i])
        delete(flowRealPaths[i])
def getDatasets(flowPath_list):
    fnames = os.listdir(flowPath_list[0])
    for i in range(3):
        count=1
        for fname in fnames[numbers[i]:numbers[i+1]]:
            src = os.path.join(flowPath_list[0], fname)
            dat = os.path.join(flowPath_list[i+1], fname)
            shutil.copyfile(src, dat)
            print("已從"+flowPath_list[0]+"生成"+str(count)+"張 圖片到"+flowPath_list[i+1])
            count+=1
if __name__ == '__main__':
    cleanAllPath(flowRealPaths,flowFakePaths)
    getDatasets(flowRealPaths)
    getDatasets(flowFakePaths)

將會生成這樣的文件夾，每個文件夾裏面都有fake和real數據集，train裏面是30000，validation和test分別是500

模型訓練網絡

爲了保證效果，嘗試了不同的網絡，核心是keras+tensorflow

純cnn

# 優化 數據增強
from keras import layers
from keras import models
import matplotlib.pyplot as plt
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
import tensorflow as tf
import keras
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
keras.backend.tensorflow_backend.set_session(tf.Session(config=config))

train_dir = 'G:/datasets/flowset/train/'
validation_dir = 'G:/datasets/flowset/validation/'


model = models.Sequential()
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(224, 224, 3)))
model.add(layers.MaxPool2D((2, 2)))

model.add(layers.Conv2D(64, (3, 3), activation='relu'))
model.add(layers.MaxPool2D((2, 2)))

model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPool2D((2, 2)))

model.add(layers.Conv2D(128, (3, 3), activation='relu'))
model.add(layers.MaxPool2D((2, 2)))

model.add(layers.Flatten())
model.add(layers.Dropout(0.5))
model.add(layers.Dense(512, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc'])



# 調整像素值
train_datagen = ImageDataGenerator(
    rescale=1./255,
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(
    directory=train_dir,
    target_size=(224, 224),
    batch_size=32,
    class_mode='binary')

validation_generator = test_datagen.flow_from_directory(
    directory=validation_dir,
    target_size=(224, 224),
    batch_size=32,
    class_mode='binary')

history = model.fit_generator(
    train_generator,
    steps_per_epoch=100,
    epochs=100,
    validation_data=validation_generator,
    validation_steps=50)

model.save('fake_and_real_small_2.h5')

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

這裏只是簡單使用了4層神經網絡和relu激活函數，最後會生成兩張圖

可以看到，效果不是很理想

VGG+CNN

這裏我們使用VGG網絡進行預特徵處理
網絡架構

"""
@Time    : 2020/3/13 11:50
@Author  : cs
@content: 
"""
from keras.applications import VGG16
import os
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras import models
from keras import layers
from keras import optimizers
import matplotlib.pyplot as plt


conv_base = VGG16(weights='imagenet',
                  include_top=False,
                  input_shape=(224, 224, 3))

base_dir = 'G:/datasets/flowset/'
train_dir = os.path.join(base_dir, 'train')
validation_dir = os.path.join(base_dir, 'validation')
test_dir = os.path.join(base_dir, 'test')

datagen = ImageDataGenerator(rescale=1./255)
batch_size = 20
train_number=30000
validation_number=500
test_number=500

def extarct_features(directory, sample_count):
    features = np.zeros(shape=(sample_count, 4, 4, 512))
    labels = np.zeros(shape=(sample_count))
    generator = datagen.flow_from_directory(
        directory,
        target_size=(224, 224),
        batch_size=batch_size,
        class_mode='binary')

    i = 0
    for inputs_batch, labels_batch in generator:
        features_batch = conv_base.predict(inputs_batch)
        features[i * batch_size : (i + 1) * batch_size] = features_batch
        labels[i * batch_size : (i + 1) * batch_size] = labels_batch
        i += 1
        if i * batch_size >= sample_count:
            break

    return features, labels


train_features, train_labels = extarct_features(train_dir, 2000)
validation_features, validation_labels = extarct_features(validation_dir, 150)
test_features, test_labels = extarct_features(test_dir, 100)

train_features = np.reshape(train_features, (2000, 4 * 4 * 512))
validation_features = np.reshape(validation_features, (1000, 4 * 4 * 512))
test_features = np.reshape(test_features, (1000, 4 * 4 * 512))

model = models.Sequential()
model.add(layers.Dense(256, activation='relu', input_dim=4 * 4 * 512))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(optimizer=optimizers.RMSprop(lr=2e-5),
              loss='binary_crossentropy',
              metrics=['acc'])

history = model.fit(train_features, train_labels,
                    epochs=30,
                    batch_size=20,
                    validation_data=(validation_features, validation_labels))


acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

可以看到，效果有所好轉，也更加規律

總結

光流法能夠檢測出運動中的圖像變化，對於人臉視頻，有其一定的效果