https://blog.csdn.net/qq_39033834
畢業論文已經寫完，經過大量實驗，本系統的性能得到了顯著的提高，本文旨在展示我的系統，並附上代碼。
畢業論文 和 答辯PPT 可以在我的 Git 上下載哦。
目錄
1 系統效果展示
爲了不浪費大家時間，本小節將會展示我的系統效果，如果大家覺得做的還可以再往後看；如果大家覺得做的一般，還望您能不吝賜教，感謝您的觀看。本系統能完成繪畫、娛樂等圖像界面控制功能。
 |
 |
| 圖1-1 繪畫 | 圖1-2 玩水果忍者 |
 |
 |
| 圖1-3 打開我的畢業論文 | 圖1-4 打開菜單 |
2 系統環境
2.1 硬件環境
2.2 軟件環境
3 系統整體展示
4 程序
4.1 模板
4.2 代碼
/*----------------------------------【程序說明】---------------------------------------|
| 程序名稱：基於視覺的手勢識別交互系統 |
| 程序功能：以手勢代替鼠標進行人機交互 |
| 程序時間：2019.1.14 |
| 程序作者：黃俊 |
|-------------------------------------------------------------------------------------*/
/*----------------------------------【頭文件、命名空間包含部分】-----------------------|
| 描述：包含程序所使用的頭文件和命名空間 |
|-------------------------------------------------------------------------------------*/
#include <stdio.h>
#include <tchar.h>
#include <iostream>
#include <vector>
#include <string>
#include <list>
#include <map>
#include <stack>
#include <opencv2/core/core.hpp>
#include <opencv2/features2d/features2d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/imgproc/types_c.h>
#include <Windows.h>
#include<time.h>
using namespace std;
using namespace cv;
/*-----------------------------------【被調函數聲明部分】-------------------------------|
| 描述：被調函數聲明 |
|--------------------------------------------------------------------------------------*/
// 顯示信息
void ShowHelpText()
{
//輸出歡迎信息和OpenCV版本
cout << "\n 《基於視覺的手勢識別交互系統設計》";
cout << "\nDesign of vision-based gesture recognition interactive system";
cout << "\n作者：黃俊 功能：以手勢代替鼠標進行人機交互";
cout << "\n---------------------------------------------------";
cout << "\n手勢1：鼠標移動";
cout << "\n手勢2：單擊鼠標左鍵 手勢3：單擊鼠標右鍵";
cout << "\n手勢4：按下鼠標左鍵 手勢5：鬆開鼠標左鍵";
cout << "\n---------------------------------------------------\n";
}
// 八鄰接種子算法，並返回每塊區域的邊緣框
void Seed_Filling(const cv::Mat& binImg, cv::Mat& labelImg, int& labelNum, int(&ymin)[20], int(&ymax)[20], int(&xmin)[20], int(&xmax)[20]) //種子填充法
{
if (binImg.empty() ||
binImg.type() != CV_8UC1)// 如果圖像是空或者格式不正確就返回
{
return;
}
labelImg.release();
binImg.convertTo(labelImg, CV_32SC1);// 矩陣數據類型轉換
int label = 0;
int rows = binImg.rows;
int cols = binImg.cols;
for (int i = 1; i < rows - 1; i++)
{
int* data = labelImg.ptr<int>(i);
for (int j = 1; j < cols - 1; j++)
{
if (data[j] == 0)
{
std::stack<std::pair<int, int>> neighborPixels;// std::pair主要的作用是將兩個數據組合成一個數據，兩個數據可以是同一類型或者不同類型。
// pair是一個模板結構體，其主要的兩個成員變量是first和second，這兩個變量可以直接使用。
neighborPixels.push(std::pair<int, int>(j, i));// 向棧頂插入元素像素位置: <j,i>
ymin[label] = i;
ymax[label] = i;
xmin[label] = j;
xmax[label] = j;
while (!neighborPixels.empty())
{
std::pair<int, int> curPixel = neighborPixels.top();// 如果與上一行中一個團有重合區域，則將上一行的那個團的標號賦給它
int curX = curPixel.first;
int curY = curPixel.second;
labelImg.at<int>(curY, curX) = 255;
neighborPixels.pop(); // 出棧
if ((curX > 0) && (curY > 0) && (curX < (cols - 1)) && (curY < (rows - 1)))
{
if (labelImg.at<int>(curY - 1, curX) == 0) //上
{
neighborPixels.push(std::pair<int, int>(curX, curY - 1));
//ymin[label] = curY - 1;
}
if (labelImg.at<int>(curY + 1, curX) == 0) //下
{
neighborPixels.push(std::pair<int, int>(curX, curY + 1));
if ((curY + 1) > ymax[label])
ymax[label] = curY + 1;
}
if (labelImg.at<int>(curY, curX - 1) == 0) //左
{
neighborPixels.push(std::pair<int, int>(curX - 1, curY));
if ((curX - 1) < xmin[label])
xmin[label] = curX - 1;
}
if (labelImg.at<int>(curY, curX + 1) == 0) //右
{
neighborPixels.push(std::pair<int, int>(curX + 1, curY));
if ((curX + 1) > xmax[label])
xmax[label] = curX + 1;
}
if (labelImg.at<int>(curY - 1, curX - 1) == 0) //左上
{
neighborPixels.push(std::pair<int, int>(curX - 1, curY - 1));
//ymin[label] = curY - 1;
if ((curX - 1) < xmin[label])
xmin[label] = curX - 1;
}
if (labelImg.at<int>(curY + 1, curX + 1) == 0) //右下
{
neighborPixels.push(std::pair<int, int>(curX + 1, curY + 1));
if ((curY + 1) > ymax[label])
ymax[label] = curY + 1;
if ((curX + 1) > xmax[label])
xmax[label] = curX + 1;
}
if (labelImg.at<int>(curY + 1, curX - 1) == 0) //左下
{
neighborPixels.push(std::pair<int, int>(curX - 1, curY + 1));
if ((curY + 1) > ymax[label])
ymax[label] = curY + 1;
if ((curX - 1) < xmin[label])
xmin[label] = curX - 1;
}
if (labelImg.at<int>(curY - 1, curX + 1) == 0) //右上
{
neighborPixels.push(std::pair<int, int>(curX + 1, curY - 1));
//ymin[label] = curY - 1;
if ((curX + 1) > xmax[label])
xmax[label] = curX + 1;
}
}
}
++label; // 沒有重複的團，開始新的標籤
}
}
}
labelNum = label;
}
// 鏡像
void mirrorY(Mat src, Mat &dst)
{
int row = src.rows;
int col = src.cols;
dst = src.clone();
for (int i = 0; i < col; i++) {
src.col(col - 1 - i).copyTo(dst.col(i));
}
}
// 分水嶺算法
class WatershedSegmenter {
private:
cv::Mat markers;
public:
void setMarkers(const cv::Mat& markerImage) {
// 轉換爲整數圖像
markerImage.convertTo(markers, CV_32S);
}
cv::Mat process(const cv::Mat &image) {
// 適用分水嶺
cv::watershed(image, markers);
return markers;
}
// 以圖像的形式返回結果
cv::Mat getSegmentation() {
cv::Mat tmp;
// 標籤高於255的所有段
// 將被賦值爲255
markers.convertTo(tmp, CV_8U);
return tmp;
}
// 以圖像的形式返回分水嶺
cv::Mat getWatersheds() {
cv::Mat tmp;
markers.convertTo(tmp, CV_8U, 255, 255);
return tmp;
}
};
/*--------------------------------------【main()函數】-----------------------------------|
| 描述：控制檯應用程序的入口函數，我們的程序從這裏開始執行 |
|---------------------------------------------------------------------------------------*/
int main()
{
// 顯示幫助信息
ShowHelpText();
// 設置視頻讀入，括號裏面的數字是攝像頭的選擇，一般自帶的是0
cv::VideoCapture cap(0);
if (!cap.isOpened())
{
return -1;
}
clock_t start, finish;
double totaltime;
string str1, str2, str3;
Mat frame;
Mat binImage, tmp, tmp1;
Mat Y, Cr, Cb, H;
vector<Mat> channels, channels1;
//模板圖片
Mat mu[5];
mu[0] = imread("m1.png", CV_8UC1); // 手勢1
mu[1] = imread("m2.png", CV_8UC1); // 手勢2
mu[2] = imread("m3.png", CV_8UC1); // 手勢3
mu[3] = imread("m4.png", CV_8UC1); // 手勢4
mu[4] = imread("m5.png", CV_8UC1); // 手勢5
float simi[5]; // 模板匹配
float flag; // 標誌
int flagx = 0, flagx1 = 0, flagx2 = 0; // 中間變量
float k = 1.5; // 鼠標靈敏因子
int curpointx = 0, curpointy = 0, prepointx = 367, prepointy = 80; // 鼠標初始位置
int xg_num = 0;
bool stop = false;
while (!stop)
{
cap >> frame;
//imshow("The original image", frame); // 測試顯示【原始圖像】
//start = clock();
// 鏡像
mirrorY(frame, frame);
// 顏色空間變換（RGB to GRAY）
cvtColor(frame, binImage, CV_BGR2GRAY);
// 顏色空間變換（RGB to YCrCb）
frame.copyTo(tmp);
cvtColor(tmp, tmp, CV_BGR2YCrCb);
// 顏色空間變換（RGB to HSV）
frame.copyTo(tmp1);
cvtColor(tmp1, tmp1, CV_BGR2HSV);
// 通道分離
split(tmp, channels);
split(tmp1, channels1);
Cr = channels.at(1); // 分離出【色調Cr】
Cb = channels.at(2); // 分離出【飽和度Cb】
H = channels1.at(0); // 分離出【H】
// 膚色檢測，輸出二值圖像
for (int j = 1; j < Cr.rows - 1; j++) // 遍歷圖像像素點
{
uchar* currentCr = Cr.ptr< uchar>(j);
uchar* currentCb = Cb.ptr< uchar>(j);
uchar* currentH = H.ptr< uchar>(j);
uchar* current = binImage.ptr< uchar>(j);
for (int i = 1; i < Cb.cols - 1; i++)
{
if ((currentCr[i] >= 135) && (currentCr[i] <= 170) && (currentCb[i] >= 94) && (currentCb[i] <= 125) && (currentH[i] >= 1) && (currentH[i] <= 23))
current[i] = 255;
else
current[i] = 0;
}
}
// 灰度形態學處理
erode(binImage, binImage, Mat());
dilate(binImage, binImage, Mat());
// 基於標記的分水嶺算法
cv::Mat fg;
cv::erode(binImage, fg, cv::Mat(), cv::Point(-1, -1), 6); // 六次遞歸腐蝕
// 識別沒有對象的圖像像素
cv::Mat bg;
cv::dilate(binImage, bg, cv::Mat(), cv::Point(-1, -1), 6); // 六次遞歸膨脹
cv::threshold(bg, bg, 1, 128, cv::THRESH_BINARY_INV); // 二進制閾值函數圖像分割cv::THRESH_BINARY_INV 超過閾值 則 值變爲 0,其他爲 128 黑白二值反轉(反轉二值閾值化)
// 顯示標記圖像
cv::Mat markers(binImage.size(), CV_8U, cv::Scalar(0));
markers = fg + bg;
// 創建分水嶺分割對象
WatershedSegmenter segmenter;
segmenter.setMarkers(markers);
segmenter.process(frame);// 應用分水嶺算法
Mat waterShed;
waterShed = segmenter.getSegmentation();
// 8向種子算法，給邊框做標記
Mat labelImg;
int label, ymin[20], ymax[20], xmin[20], xmax[20];
Seed_Filling(waterShed, labelImg, label, ymin, ymax, xmin, xmax);
// 統計一下區域中的膚色區域比例
float fuseratio[20];
for (int k = 0; k < label; k++)
{
fuseratio[k] = 1;
if (((xmax[k] - xmin[k] + 1) > 50) && ((xmax[k] - xmin[k] + 1) < 300) && ((ymax[k] - ymin[k] + 1) > 150) && ((ymax[k] - ymin[k] + 1) < 450))
{
int fusepoint = 0;
for (int j = ymin[k]; j < ymax[k]; j++)
{
uchar* current = waterShed.ptr< uchar>(j);
for (int i = xmin[k]; i < xmax[k]; i++)
{
if (current[i] == 255)
fusepoint++;
}
}
fuseratio[k] = float(fusepoint) / ((xmax[k] - xmin[k] + 1) * (ymax[k] - ymin[k] + 1));
}
}
Size dsize = Size(108, 128);
// 給符合閾值條件的位置畫框
for (int i = 0; i < label; i++)
{
if ((fuseratio[i] < 0.58))
{
// 尺度調整
Mat rROI = Mat(dsize, CV_8UC1);
resize(waterShed(Rect(xmin[i], ymin[i], xmax[i] - xmin[i], ymax[i] - ymin[i])), rROI, dsize);
imshow("手勢區域", rROI);
// 模板匹配
Mat result;
for (int mp = 0; mp < 5; mp++)
{
matchTemplate(rROI, mu[mp], result, CV_TM_SQDIFF_NORMED);
simi[mp] = result.ptr<float>(0)[0];
}
// 尋找最佳匹配
flagx2 = flagx1;
flagx1 = flagx;
flagx = 0;
flag = simi[0];
for (int j = 1; j < 5; j++)
{
if (simi[j] < flag)
{
flagx = j;
flag = simi[j];
}
}
cv::Point end = cv::Point(xmin[i], ymin[i] - 15); // 加標籤【位置】
str1 = "(" + to_string(xmin[i]) + "," + to_string(ymin[i]) + ") " + " (" + to_string(xmax[i]) + "," + to_string(ymax[i]) + ")";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.5, cv::Scalar(255, 0, 0), 2);
putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.5, cv::Scalar(255, 0, 0), 2);
end = cv::Point(xmin[i] - 80, ymin[i] + 20); // 加標籤【手部標記】
str1 = "Hand" + to_string(flagx + 1);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 20); // 加標籤【膚色比例】
str1 = "Skin area ratio: " + to_string(fuseratio[i]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 40); // 加標籤【長寬】
str1 = "Height: " + to_string(ymax[i] - ymin[i]) + " Width: " + to_string(xmax[i] - xmin[i]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 60); // 加標籤【數字 匹配度】
str1 = "Suitability_1: " + to_string(simi[0]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 80); // 加標籤【數字 匹配度】
str1 = "Suitability_2: " + to_string(simi[1]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 100); // 加標籤【數字 匹配度】
str1 = "Suitability_3: " + to_string(simi[2]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 120); // 加標籤【數字 匹配度】
str1 = "Suitability_4: " + to_string(simi[3]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
end = cv::Point(20, 140); // 加標籤【數字 匹配度】
str1 = "Suitability_5: " + to_string(simi[4]);
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
//putText(frame, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
if (flagx == 0)
{
curpointx = xmin[i];
curpointy = ymin[i];
int dx = curpointx - prepointx;
int dy = curpointy - prepointy;
prepointx = curpointx;
prepointy = curpointy;
mouse_event(MOUSEEVENTF_MOVE, k * dx, k * dy, 0, 0); // 鼠標移動
end = cv::Point(20, 400); // 加標籤【數字 匹配度】
str1 = "MOUSEEVENTF_MOVE";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
}
if (flagx == 1 && flagx1 != 1 && flagx2 != 1)
{
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); // 鼠標左鍵按下
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); // 鼠標左鍵鬆開
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); // 鼠標左鍵按下
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); // 鼠標左鍵鬆開
end = cv::Point(20, 440); // 加標籤【數字 匹配度】
str1 = "MOUSEEVENTF_LEFT_CLICK";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
}
if (flagx == 2 && flagx1 != 2 && flagx2 != 2)
{
mouse_event(MOUSEEVENTF_RIGHTDOWN, 0, 0, 0, 0); // 鼠標右鍵按下
mouse_event(MOUSEEVENTF_RIGHTUP, 0, 0, 0, 0); // 鼠標右鍵鬆開
end = cv::Point(20, 440); // 加標籤【數字 匹配度】
str1 = "MOUSEEVENTF_RIGHT_CLICK";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
}
if (flagx == 3)
{
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); // 鼠標左鍵按下
end = cv::Point(20, 440); // 加標籤【數字 匹配度】
str1 = "MOUSEEVENTF_LEFT_DOWN";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
}
if (flagx == 4)
{
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); // 鼠標左鍵鬆開
end = cv::Point(20, 440); // 加標籤【數字 匹配度】
str1 = "MOUSEEVENTF_LEFT_UP";
putText(waterShed, str1, end, cv::FONT_HERSHEY_DUPLEX, 0.7, cv::Scalar(255, 0, 0), 2);
}
rectangle(waterShed, Point(xmin[i], ymin[i]), Point(xmax[i], ymax[i]), Scalar::all(255), 3, 8, 0); // 加框
rectangle(frame, Point(xmin[i], ymin[i]), Point(xmax[i], ymax[i]), cv::Scalar(0, 255, 0), 3, 8, 0); // 加框
}
}
imshow("信息顯示", waterShed);
imshow("功能顯示", frame);
// 時間顯示
//finish = clock();
//totaltime = (double)(finish - start) / CLOCKS_PER_SEC;
//cout << "\n此程序的運行時間爲" << totaltime << "秒！" << endl;
if (waitKey(1) >= 0)
stop = true;
}
return 0;
}