faster r-cnn ckpt模型轉pd模型及其調用C++版本

《faster r-cnn ckpt模型轉pd模型及其調用-python版本》介紹瞭如何將faster r-cnn訓練出來的ckpt模型轉換爲pd模型，以及如何用tensroflow調用pd模型進行目標識別。其中關鍵的兩處是如何確定輸入輸出tensor和使用源碼的demo.py進行模型轉換。下面記錄一下如何使用C++調用pd模型進行目標識別。

準備階段

1、C++版本的tensorflow的編譯和使用
參考《20190415-將基於Keras訓練的Unet模型通過TensorFlow的C++接口部署》、《用keras訓練模型並用Tensorflow的C++API調用模型》
這兩篇文章已經把如何使用C++調用pd模型做預測介紹的很清楚了，我的一些代碼也是參考以上兩位作者的代碼修改的。
2、opencv mat類型與tensorflow::tensor的轉換，tensor與eigen::matrix的轉換,這一部分主要是數據類型的轉換，opencv用來讀取圖片，獲取到的是mat類型的數據，需要轉換成tensor類型，模型預測得到的結果需要經過後處理（檢測框修正和裁剪越界檢測框），可以參考之前的文章《gdal opencv tensorflow eigen數據轉換(C++)》

3、Eigen的使用（主要用於後處理）
Eigen是C++的矩陣運算庫，用來後處理的矩陣運算。

1、ckpt模型轉pb模型

具體操作可以參看之前的python版本的文章。

2、數據預處理

我做的是遙感圖像識別，所以需要分塊讀取，這裏每塊大小爲1000*1000,使用gdal分塊讀取的詳細用法見之前的文章

	//分開讀取遙感影像
	int block_height=1000
	int block_width=1000
	int block_nums = block_height*block_width;
	float *origin_data = new float[block_nums]; //這裏用float型數據，因爲會對原始數據做一個歸一化，需要用到小數。
	
	/*分塊讀取數據的僞代碼，一次讀取1000*1000的數據存放在origin_data中
	input_band->RasterIO(GF_Read, i*slide_size, j*slide_size, block_width, block_height,
				origin_data, block_width, block_height, GDT_Float32, 0, 0);
	*/
	//將數據轉換爲opencv的mat格式存儲
	cv::Mat block_data_cv = cv::Mat::zeros(block_size, block_size,CV_32F);
	block_data_cv = cv::Mat(block_size, block_size, CV_32F, origin_data);
	
	//重採樣,faster_rcnn模型對數據做的預處理是先做歸一化，然後重採樣到1000或者600大小。
	cv::Mat resize_img;
	cv::Mat sub_mean;
	int tensor_size = 1000         //與我的輸入數據大小一致。
	sub_mean = img - 115.3335;     //歸一化，減去訓練數據的均值
	cv::resize(sub_mean, resize_img, cv::Size(tensor_size, tensor_size));                    //重採樣
	
	std::vector<cv::Mat> channels;
	channels.push_back(resize_img);
	channels.push_back(resize_img);
	channels.push_back(resize_img);

	//單通道變三通道
	cv::Mat img_3channel;
	cv::merge(channels, img_3channel);
	

3、準備輸入和輸出tensor，運行模型

創建變量

	string inp_tensor_name_0;     // 輸入節點名字(Placeholder)
	string inp_tensor_name_1;     // 輸入節點名字(Placeholder_1)
	string out_tensor_name_bias;  // 輸出節點名字(vgg_16_3/cls_score/BiasAdd)
	string out_tensor_name_score; // 輸出節點名字(vgg_16_3/cls_prob)
	string out_tensor_name_bbox_pred;  // 輸出節點名字(add)
	string out_tensor_name_rois;  // 輸出節點名字(vgg_16_1/rois/concat)

	Session* session;  // 定義session
	GraphDef graphdef;  // 定義graph

	int tensor_size = 1000;// 定義圖中的tensor尺寸
	int img_size = 1000;  // 輸入圖片尺寸
	float im_scales = float(tensor_size) / img_size;  //圖片縮放比

	float CONF_THRESH = 0.8;
	float NMS_THRESH = 0.3;

	// 定義輸入輸出張量
	Tensor inp_Tensor_0 = Tensor(DT_FLOAT, TensorShape({ 1, tensor_size, tensor_size, 3 }));  // 輸入張量
	Tensor inp_Tensor_1 = Tensor(DT_FLOAT, TensorShape({ 3 }));  //輸入張量

	std::vector<tensorflow::Tensor> out_Tensor; //輸出張量

變量初始化

	//// 配置模型的輸入輸出節點名字 vgg16
	//inp_tensor_name_0 = "Placeholder:0";
	//inp_tensor_name_1 = "Placeholder_1:0";
	//out_tensor_name_bias = "vgg_16_3/cls_score/BiasAdd:0";
	//out_tensor_name_score = "vgg_16_3/cls_prob:0";
	//out_tensor_name_bbox_pred = "add:0";
	//out_tensor_name_rois = "vgg_16_1/rois/concat:0";

	// 配置模型的輸入輸出節點名字 res101
	inp_tensor_name_0 = "Placeholder:0";
	inp_tensor_name_1 = "Placeholder_1:0";
	out_tensor_name_bias = "resnet_v1_101_5/cls_score/BiasAdd:0";
	out_tensor_name_score = "resnet_v1_101_5/cls_prob:0";
	out_tensor_name_bbox_pred = "add:0";
	out_tensor_name_rois = "resnet_v1_101_3/rois/concat:0";

	// 加載模型到計算圖
	Status status_load = ReadBinaryProto(Env::Default(), model_path, &graphdef);
	if (!status_load.ok()) {
		std::cout << "ERROR: Loading model failed..." << std::endl;
	}

	// 創建會話
	NewSession(SessionOptions(), &session);
	Status status_create = session->Create(graphdef);
	if (!status_create.ok()) {
		std::cout << "ERROR: Creating graph in session failed.." << status_create.ToString() << std::endl;
	}
	else {
		std::cout << "----------- Successfully created session and load graph -------------" << std::endl;
	}

填充數據、運行圖

	//輸入tensort inp_Tensor_0填充數據
	float *tensor_data_ptr = inp_Tensor_0.flat<float>().data();
	cv::Mat fake_mat(tensor_size, tensor_size, CV_32FC(3), tensor_data_ptr);
	img_3channel.convertTo(fake_mat, CV_32FC(3));

	//輸入tensort inp_Tensor_1填充數據
	auto tmap_info = inp_Tensor_1.tensor<float, 1>();

	tmap_info(0) = tensor_size;
	tmap_info(1) = tensor_size;
	tmap_info(2) = im_scales;

	std::vector<std::pair<string, Tensor>> inputs = {
		{ inp_tensor_name_0,inp_Tensor_0 },
		{ inp_tensor_name_1,inp_Tensor_1 }
	};

	std::vector<string> output_tensor_names = { out_tensor_name_bias,out_tensor_name_score,
		out_tensor_name_bbox_pred,out_tensor_name_rois };

	// 輸入張量 -> 輸出張量
	Status status_run = session->Run({ inputs }, { output_tensor_names }, {}, &out_Tensor);
	if (!status_run.ok()) {
		std::cout << "ERROR: RUN failed..." << std::endl;
		std::cout << status_run.ToString() << "\n";
	}
	//邊框修正
	Eigen::MatrixXf pred_boxes(300, 8);
	bbox_transform_matrix(out_Tensor[3], out_Tensor[2], pred_boxes);

	//裁剪超過邊界的邊框
	Eigen::MatrixXf bboxes(300, 8);
	clip_bbox(pred_boxes, bboxes);

	//非極大抑制NMS
	std::vector<int> keep;
	keep = cpu_nms(bboxes, out_Tensor[1], NMS_THRESH);
	int keep_size = keep.size();
	auto keep_matrix = Eigen::Map<Eigen::Matrix<int, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(keep.data(), 1, keep_size);

初步識別結果就保存在out_Tensor中。後面需要做一些預處理，包括檢測框修改正、越界檢測框裁剪，非極大抑制去除重複的檢測框,滿足得分閾值的檢查框輸出。

4、後處理

4.1 、檢測框修正

void bbox_transform_matrix(tensorflow::Tensor &rois, tensorflow::Tensor &delta_bbox, Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> &pred_boxes)
{
	//Tensorflow::Tensor轉換爲Eigen::Matrix
	auto m_rois = Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(rois.flat<float>().data(), 300, 5);
	auto m_delta_bbox = Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(delta_bbox.flat<float>().data(), 300, 8);

	Eigen::VectorXf rois_x1 = m_rois.col(1)/im_scales;
	Eigen::VectorXf rois_y1 = m_rois.col(2)/im_scales;
	Eigen::VectorXf rois_x2 = m_rois.col(3)/im_scales;
	Eigen::VectorXf rois_y2 = m_rois.col(4)/im_scales;

	Eigen::VectorXf offset(300);
	offset.setConstant(1.0);

	//(x1,y1,x2,y2)轉換成(center_x,center_y,heights,widths)
	Eigen::VectorXf	widths = rois_x2 - rois_x1 + offset;
	Eigen::VectorXf	heights = rois_y2 - rois_y1 + offset;
	Eigen::VectorXf	ctr_x = rois_x1 + 0.5*widths;
	Eigen::VectorXf	ctr_y = rois_y1 + 0.5*heights;

	//中心點x座標偏移
	Eigen::MatrixXf dx(300, 2);
	dx.col(0) = m_delta_bbox.col(0);
	dx.col(1) = m_delta_bbox.col(4);
	//中心點y座標偏移
	Eigen::MatrixXf dy(300, 2);
	dy.col(0) = m_delta_bbox.col(1);
	dy.col(1) = m_delta_bbox.col(5);
	//bounding box寬縮放係數
	Eigen::MatrixXf dw(300, 2);
	dw.col(0) = m_delta_bbox.col(2);
	dw.col(1) = m_delta_bbox.col(6);
	//bounding box高縮放係數
	Eigen::MatrixXf dh(300, 2);
	dh.col(0) = m_delta_bbox.col(3);
	dh.col(1) = m_delta_bbox.col(7);

	//計算修改後的bounding box
	Eigen::MatrixXf pred_ctr_x(300, 2);
	pred_ctr_x.col(0) = dx.col(0).cwiseProduct(widths) + ctr_x;
	pred_ctr_x.col(1) = dx.col(1).cwiseProduct(widths) + ctr_x;

	Eigen::MatrixXf pred_ctr_y(300, 2);
	pred_ctr_y.col(0) = dy.col(0).cwiseProduct(heights) + ctr_y;
	pred_ctr_y.col(1) = dy.col(1).cwiseProduct(heights) + ctr_y;

	Eigen::MatrixXf pred_w(300, 2);
	pred_w.col(0) = Eigen::VectorXf(dw.col(0).array().exp()).cwiseProduct(widths);
	pred_w.col(1) = Eigen::VectorXf(dw.col(1).array().exp()).cwiseProduct(widths);

	Eigen::MatrixXf pred_h(300, 2);
	pred_h.col(0) = Eigen::VectorXf(dh.col(0).array().exp()).cwiseProduct(heights);
	pred_h.col(1) = Eigen::VectorXf(dh.col(1).array().exp()).cwiseProduct(heights);

	//修正邊框(center_x,center_y,h,w)轉(x1,y1,x2,y2)
	pred_boxes.col(0) = pred_ctr_x.col(0) - 0.5*pred_w.col(0);
	pred_boxes.col(4) = pred_ctr_x.col(1) - 0.5*pred_w.col(1);

	pred_boxes.col(1) = pred_ctr_y.col(0) - 0.5*pred_h.col(0);
	pred_boxes.col(5) = pred_ctr_y.col(1) - 0.5*pred_h.col(1);

	pred_boxes.col(2) = pred_ctr_x.col(0) + 0.5*pred_w.col(0);
	pred_boxes.col(6) = pred_ctr_x.col(1) + 0.5*pred_w.col(1);

	pred_boxes.col(3) = pred_ctr_y.col(0) + 0.5*pred_h.col(0);
	pred_boxes.col(7) = pred_ctr_y.col(1) + 0.5*pred_h.col(1);

}

4.2 、越界檢測框裁剪

void clip_bbox(Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>&pred_bboxes, Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>&bboxes)
{
	//x1>=0
	bboxes.col(0) = pred_bboxes.col(0).cwiseMax(0);
	bboxes.col(4) = pred_bboxes.col(4).cwiseMax(0);

	//y1>=0
	bboxes.col(1) = pred_bboxes.col(1).cwiseMax(0);
	bboxes.col(5) = pred_bboxes.col(5).cwiseMax(0);

	//x2<img_cols
	bboxes.col(2) = pred_bboxes.col(2).cwiseMin(tensor_size -1);
	bboxes.col(6) = pred_bboxes.col(6).cwiseMin(tensor_size -1);

	//y2<img_rows
	bboxes.col(3) = pred_bboxes.col(3).cwiseMin(tensor_size - 1);
	bboxes.col(7) = pred_bboxes.col(7).cwiseMin(tensor_size - 1);
	
}

4.3、NMS（非極大抑制）實現

std::vector<int> cpu_nms(Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>&bboxes, tensorflow::Tensor &scores, float nms_thresh)
{

	auto m_scores = Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(scores.flat<float>().data(), 300, 2);

	Eigen::VectorXf class_vec = m_scores.col(1);

	//按照得分排序
	Eigen::VectorXi score_order(300);
	argsort(class_vec, score_order);

	Eigen::VectorXf x1 = bboxes.col(4);
	Eigen::VectorXf y1 = bboxes.col(5);
	Eigen::VectorXf x2 = bboxes.col(6);
	Eigen::VectorXf y2 = bboxes.col(7);

	Eigen::VectorXf offset(300);
	offset.setConstant(1.0);

	std::vector<int> keep;

	//計算bounding box面積
	Eigen::VectorXf areas = (x2 - x1 + offset).cwiseProduct(y2 - y1 + offset);

	while (score_order.size()>0)
	{
		//std::cout << "order:" << score_order.transpose() << std::endl;
		int i = score_order[0];
		keep.push_back(i);
		//std::cout <<"keep:"<< i << std::endl;
		int order_size = score_order.size();
		Eigen::VectorXf xx1 = x1(score_order.segment(1, order_size - 1)).cwiseMax(x1[i]);
		Eigen::VectorXf yy1 = y1(score_order.segment(1, order_size - 1)).cwiseMax(y1[i]);
		Eigen::VectorXf xx2 = x2(score_order.segment(1, order_size - 1)).cwiseMin(x2[i]);
		Eigen::VectorXf yy2 = y2(score_order.segment(1, order_size - 1)).cwiseMin(y2[i]);

		Eigen::VectorXf w = (xx2 - xx1 + Eigen::VectorXf::Ones(order_size - 1)).cwiseMax(0);
		Eigen::VectorXf h = (yy2 - yy1 + Eigen::VectorXf::Ones(order_size - 1)).cwiseMax(0);
		Eigen::VectorXf inter = w.cwiseProduct(h);

		Eigen::VectorXf area_score_max(order_size - 1);
		area_score_max.setConstant(areas[i]);

		Eigen::VectorXf ovr = inter.cwiseQuotient(area_score_max + areas(score_order.segment(1, order_size - 1)) - inter);
		
		Eigen::VectorXi index_cond = (ovr.array() < nms_thresh).cast<int>();
		//std::cout << "index_cond:" << index_cond.transpose() << std::endl;
		int cond_sum = index_cond.sum();

		Eigen::VectorXi inds(cond_sum);
		inds.setZero();
		select_where(index_cond, inds);
		//std::cout << "inds:" << inds.transpose() << std::endl;
		Eigen::VectorXi offset2(cond_sum);
		offset2.setConstant(1);

		Eigen::VectorXi selcet_elem = score_order(inds+offset2);
		score_order = selcet_elem;
		//std::cout << "selcet_elem:" << selcet_elem.transpose() << std::endl;
	}
	return keep;
}

4.4、滿足閾值的檢測框輸出

	//選出得分超過閾值的邊框
	Eigen::VectorXi keep_vec = keep_matrix.row(0);
	Eigen::MatrixXf dets(keep_size, 4);
	dets.col(0) = bboxes.col(4)(keep_vec);
	dets.col(1) = bboxes.col(5)(keep_vec);
	dets.col(2) = bboxes.col(6)(keep_vec);
	dets.col(3) = bboxes.col(7)(keep_vec);
	auto m_scores = Eigen::Map<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>(out_Tensor[1].flat<float>().data(), 300, 2);

	Eigen::VectorXf dets_score = m_scores.col(1)(keep_vec);

	Eigen::VectorXi score_cond = (dets_score.array() >= CONF_THRESH).cast<int>();
	//std::cout << "index_cond:" << index_cond.transpose() << std::endl;
	int cond_sum = score_cond.sum();

	Eigen::VectorXi inds(cond_sum);
	inds.setZero();
	select_where(score_cond, inds);

	Eigen::MatrixXf res_dets(cond_sum, 4);

	for (int id = 0; id < inds.size(); id++)
	{
		res_dets.row(id) = dets.row(inds[id]);
	}

5、檢測框保存成shapfile文件

最後一步就是將滿足條件的檢測框保存成shapefile矢量文件，需要用到原始影像的轉移矩陣參數和輸入數據在影像中的位置來轉換。

聲明

目前項目還在進行中，還不太方便放上全部的代碼，如果有問題可以在評論區裏提出，看到後會及時回覆。