目标检测算法中ROI提取方法比较+源码分析

本文主要介绍ROI提取结构在目标检测框架中的作用，并结合源码，理解它的实现方式。包含的算法有：ROI-pooling，ROI-align，Deformable-psroi-pooling。
目前，主流的目标检测算法大致分为2种，one-stage和two-stage方法。

• one-stage：典型代表为SSD，相当于two-stage中的rpn结构，先通过基本的特征提取网络如resnet或vggnet得到特征图，再通过5层的卷积得到目标位置和类别。这种方法计算速度快，但是精度较two-stage方法差一些
• two-stage:典型代表为Faster-rcnn。其结构分为RPN(Region Proposal Network)和RCNN(Region Convolution Neural Network)两个部分。RPN的特征通过ROI-Pooling层传递到RCNN中。

本文介绍的方法，仅出现在two-stage的方法中。顾名思义，该层的作用就是将RPN中提取的位置，截取特征图中特征用于进一步的分类和定位。

---

ROI-Pooling

Roi-pooling是Faster-rcnn原版使用的特征提取方式，论文在此。这里用动图来说明roi-pooling的过程(动图来源)

从上图看出，ROI-Pooling层的输入有两个：RPN层得到的位置和特征提取网络得到的特征。参数有pooling结果的宽高。
好了，下面结合图片来理解ROI-pooling代码，代码来源是pytorch-fasterrcnn项目，由于是cuda代码，所以如果你看了另外支持自定义operation的框架（如caffe，mxnet等）的roipooling实现方式，就会发现它们是完全一致的。

代码的注释中，增加了上图实例中各个变量的实际值，以便于读者理解。

_global__ void ROIPoolForward(const int nthreads, const float* bottom_data,
    const float spatial_scale, const int height, const int width,
    const int channels, const int pooled_height, const int pooled_width,
    const float* bottom_rois, float* top_data, int* argmax_data)
{
    CUDA_KERNEL_LOOP(index, nthreads)
    {
    	//index是gpu并行时块的计数
        int pw = index % pooled_width;//pooled_width=2，用户设置的参数，控制pooling输出大小
        int ph = (index / pooled_width) % pooled_height;//pooled_height=2，用户设置的参数，控制pooling输出大小
        int c  = (index / pooled_width / pooled_height) % channels;
        int n  = index / pooled_width / pooled_height / channels;

        // bottom_rois += n * 5;
        int roi_batch_ind = bottom_rois[n * 5 + 0];
        int roi_start_w = round(bottom_rois[n * 5 + 1] * spatial_scale);//rsw=0,左上角点横座标，来自RPN
        int roi_start_h = round(bottom_rois[n * 5 + 2] * spatial_scale);//rsh=3,左上角点纵座标，来自RPN
        int roi_end_w = round(bottom_rois[n * 5 + 3] * spatial_scale);//rew=7,右下角点横座标，来自RPN
        int roi_end_h = round(bottom_rois[n * 5 + 4] * spatial_scale);//reh=8,右下角点纵座标，来自RPN

        // Force malformed ROIs to be 1x1
        int roi_width = fmaxf(roi_end_w - roi_start_w + 1, 1);
        int roi_height = fmaxf(roi_end_h - roi_start_h + 1, 1);
        float bin_size_h = (float)(roi_height) / (float)(pooled_height);
        float bin_size_w = (float)(roi_width) / (float)(pooled_width);

        int hstart = (int)(floor((float)(ph) * bin_size_h));
        int wstart = (int)(floor((float)(pw) * bin_size_w));
        int hend = (int)(ceil((float)(ph + 1) * bin_size_h));
        int wend = (int)(ceil((float)(pw + 1) * bin_size_w));

        // Add roi offsets and clip to input boundaries
        hstart = fminf(fmaxf(hstart + roi_start_h, 0), height);
        hend = fminf(fmaxf(hend + roi_start_h, 0), height);
        wstart = fminf(fmaxf(wstart + roi_start_w, 0), width);
        wend = fminf(fmaxf(wend + roi_start_w, 0), width);
        bool is_empty = (hend <= hstart) || (wend <= wstart);//当roi_width<pooled_width或roi_height<pooled_height时触发，此时bin_size<1

        // Define an empty pooling region to be zero
        float maxval = is_empty ? 0 : -FLT_MAX;
        // If nothing is pooled, argmax = -1 causes nothing to be backprop'd
        int maxidx = -1;
        
        int bottom_data_batch_offset = roi_batch_ind * channels * height * width;
        int bottom_data_offset = bottom_data_batch_offset + c * height * width;
        
		//max-pooling操作，不同的index对应的hstart,wstart,hend,wend不同
        for (int h = hstart; h < hend; ++h) {
            for (int w = wstart; w < wend; ++w) {
                int bottom_index = h * width + w;
                if (bottom_data[bottom_data_offset + bottom_index] > maxval) {
                    maxval = bottom_data[bottom_data_offset + bottom_index];
                    maxidx = bottom_data_offset + bottom_index;
                }
            }
        }
        top_data[index] = maxval;
        if (argmax_data != NULL)
            argmax_data[index] = maxidx;
    }
}

---

ROI-align

从上面ROI-pooling的实现过程不难看出，由于取整的影响，各个index方块中对应的宽高是不同的，有些是2，有些是3。而ROI-align做了一个小改动，使h, w可以是小数，并通过双线性内插取得各个像素值，消除了取整带来的误差。源码位置。
此时，特征图上的pooling方框变成了下面这样

    __global__ void ROIAlignForward(const int nthreads, const float* bottom_data, const float spatial_scale, const int height, const int width,
                                    const int channels, const int aligned_height, const int aligned_width, const float* bottom_rois, float* top_data) {
        CUDA_1D_KERNEL_LOOP(index, nthreads) {

            int pw = index % aligned_width;
            int ph = (index / aligned_width) % aligned_height;
            int c  = (index / aligned_width / aligned_height) % channels;
            int n  = index / aligned_width / aligned_height / channels;

            // bottom_rois += n * 5;
            float roi_batch_ind = bottom_rois[n * 5 + 0];
            float roi_start_w = bottom_rois[n * 5 + 1] * spatial_scale;
            float roi_start_h = bottom_rois[n * 5 + 2] * spatial_scale;
            float roi_end_w = bottom_rois[n * 5 + 3] * spatial_scale;
            float roi_end_h = bottom_rois[n * 5 + 4] * spatial_scale;

            // Force malformed ROIs to be 1x1
            float roi_width = fmaxf(roi_end_w - roi_start_w + 1., 0.);
            float roi_height = fmaxf(roi_end_h - roi_start_h + 1., 0.);
            float bin_size_h = roi_height / (aligned_height - 1.);
            float bin_size_w = roi_width / (aligned_width - 1.);
			
			//注意，此处的h,w变成了float型，避免取整带来的误差
            float h = (float)(ph) * bin_size_h + roi_start_h;
            float w = (float)(pw) * bin_size_w + roi_start_w;
			
			//保留了整数的hstart,hstart，便于下面计算取整带来的位置偏移到底是多少
            int hstart = fminf(floor(h), height - 2);
            int wstart = fminf(floor(w), width - 2);

            int img_start = roi_batch_ind * channels * height * width;

            // bilinear interpolation
            if (h < 0 || h >= height || w < 0 || w >= width) {
                top_data[index] = 0.;
            } else {
            	//计算位置偏移，h是float型，hstart是int型
                float h_ratio = h - (float)(hstart);
                float w_ratio = w - (float)(wstart);
                int upleft = img_start + (c * height + hstart) * width + wstart;
                int upright = upleft + 1;
                int downleft = upleft + width;
                int downright = downleft + 1;
				
				//双线性内插
                top_data[index] = bottom_data[upleft] * (1. - h_ratio) * (1. - w_ratio)
                    + bottom_data[upright] * (1. - h_ratio) * w_ratio
                    + bottom_data[downleft] * h_ratio * (1. - w_ratio)
                    + bottom_data[downright] * h_ratio * w_ratio;
            }
        }
    }

有些细心的同学可能发现了上面的代码没有进行pooling操作，只有内插。那是因为caffe实现roi-align时，在后面又接了一个avg_pooling和max_pooling，以实现不同的roipooling操作，并且避免了重复开发。
代码位置

---

Deformable Roi pooling

可变形roi提取方法来源于论文Deformable Convolutional Networks,文章介绍了形变卷积的方法（增加offset）和所带来的好处。其中，Deformable Roi pooling就是一种由此衍生而来的思路。代码来源
由于要实现可变形，所以代码中加入了offset变量。

    template <typename DType>
    __global__ void DeformablePSROIPoolForwardKernel(
      const int count,
      const DType* bottom_data,
      const DType spatial_scale,
      const int channels,
      const int height, const int width,
      const int pooled_height, const int pooled_width,
      const DType* bottom_rois, const DType* bottom_trans,
      const bool no_trans,
      const DType trans_std,
      const int sample_per_part,
      const int output_dim,
      const int group_size,
      const int part_size,
      const int num_classes,
      const int channels_each_class,
      DType* top_data,
      DType* top_count) {
      CUDA_KERNEL_LOOP(index, count) {
      	//常规套路
        // The output is in order (n, ctop, ph, pw)
        int pw = index % pooled_width;
        int ph = (index / pooled_width) % pooled_height;
        int ctop = (index / pooled_width / pooled_height) % output_dim;
        int n = index / pooled_width / pooled_height / output_dim;

        // [start, end) interval for spatial sampling
        const DType* offset_bottom_rois = bottom_rois + n * 5;
        int roi_batch_ind = offset_bottom_rois[0];
        DType roi_start_w = static_cast<DType>(round(offset_bottom_rois[1])) * spatial_scale - 0.5;
        DType roi_start_h = static_cast<DType>(round(offset_bottom_rois[2])) * spatial_scale - 0.5;
        DType roi_end_w = static_cast<DType>(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5;
        DType roi_end_h = static_cast<DType>(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5;

        // Force too small ROIs to be 1x1
        DType roi_width = max(roi_end_w - roi_start_w, 0.1); //avoid 0
        DType roi_height = max(roi_end_h - roi_start_h, 0.1);

        // Compute w and h at bottom
        DType bin_size_h = roi_height / static_cast<DType>(pooled_height);
        DType bin_size_w = roi_width / static_cast<DType>(pooled_width);
		
		//一个采样点采样几次
        DType sub_bin_size_h = bin_size_h / static_cast<DType>(sample_per_part);
        DType sub_bin_size_w = bin_size_w / static_cast<DType>(sample_per_part);

        int part_h = floor(static_cast<DType>(ph) / pooled_height*part_size);
        int part_w = floor(static_cast<DType>(pw) / pooled_width*part_size);
        int class_id = ctop / channels_each_class;
        
        //传递bottom_trans,也就是采样点位移
        DType trans_x = no_trans ? static_cast<DType>(0) :
          bottom_trans[(((n * num_classes + class_id) * 2) * part_size + part_h)*part_size + part_w] * trans_std;
        DType trans_y = no_trans ? static_cast<DType>(0) :
          bottom_trans[(((n * num_classes + class_id) * 2 + 1) * part_size + part_h)*part_size + part_w] * trans_std;
        
        DType wstart = static_cast<DType>(pw)* bin_size_w
          + roi_start_w;
        wstart += trans_x * roi_width;
        DType hstart = static_cast<DType>(ph) * bin_size_h
          + roi_start_h;
        hstart += trans_y * roi_height;
        
        DType sum = 0;
        int count = 0;
        int gw = floor(static_cast<DType>(pw) * group_size / pooled_width);
        int gh = floor(static_cast<DType>(ph)* group_size / pooled_height);
        gw = min(max(gw, 0), group_size - 1);
        gh = min(max(gh, 0), group_size - 1);
		
		//内插采样（比roi-align更精确）
        const DType* offset_bottom_data = bottom_data + (roi_batch_ind * channels) * height * width;
        for (int ih = 0; ih < sample_per_part; ih++) {
          for (int iw = 0; iw < sample_per_part; iw++) {
            DType w = wstart + iw*sub_bin_size_w;
            DType h = hstart + ih*sub_bin_size_h;
            // bilinear interpolation
            if (w<-0.5 || w>width - 0.5 || h<-0.5 || h>height - 0.5) {
              continue;
            }
            w = min(max(w, 0.), width - 1.);
            h = min(max(h, 0.), height - 1.);
            int c = (ctop*group_size + gh)*group_size + gw;
            DType val = bilinear_interp(offset_bottom_data + c*height*width, w, h, width, height);
            sum += val;
            count++;
          }
        }
        top_data[index] = count == 0 ? static_cast<DType>(0) : sum / count;
        top_count[index] = count;
      }
    }

---

本文描述了各个ROI特征提取技术的流程，每一个新的技术都是对上一代方法缺点进行了改进。
最后，祝您身体健康，再见！

---

https://blog.csdn.net/jiongnima/article/details/80016683
https://towardsdatascience.com/review-dcn-deformable-convolutional-networks-2nd-runner-up-in-2017-coco-detection-object-14e488efce44