【caffe源碼研究】第三章：源碼篇(9) ：DataLayer

先從最基礎的Data層講起。
看看datalayer相關的類的繼承關係

首先定義了一個

template <typename Dtype>
class Batch {
 public:
  Blob<Dtype> data_, label_;
};

base_data_layer.cpp

#include <boost/thread.hpp>
#include <vector>

#include "caffe/blob.hpp"
#include "caffe/data_transformer.hpp"
#include "caffe/internal_thread.hpp"
#include "caffe/layer.hpp"
#include "caffe/layers/base_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/blocking_queue.hpp"

namespace caffe {
//調用模板類layer的構造函數，將param傳遞給模板類layer的構造函數，同時用param.transform_param()初始化BaseDataLayer的transform_param_成員  
template <typename Dtype>
BaseDataLayer<Dtype>::BaseDataLayer(const LayerParameter& param)
    : Layer<Dtype>(param),
      transform_param_(param.transform_param()) {
}

template <typename Dtype>
void BaseDataLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  if (top.size() == 1) {
    output_labels_ = false;
  } else {
    output_labels_ = true;
  }
  data_transformer_.reset(
      new DataTransformer<Dtype>(transform_param_, this->phase_));
  data_transformer_->InitRand();
  // The subclasses should setup the size of bottom and top
  DataLayerSetUp(bottom, top);
}

template <typename Dtype>
BasePrefetchingDataLayer<Dtype>::BasePrefetchingDataLayer(
    const LayerParameter& param)
    : BaseDataLayer<Dtype>(param),
      prefetch_free_(), prefetch_full_() {
  for (int i = 0; i < PREFETCH_COUNT; ++i) {
    prefetch_free_.push(&prefetch_[i]);//壓入batch的地址
  }
}


template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  //1. 調用父類BaseDataLayer構造方法，因爲BasePrefetchingDataLayer沒有覆蓋BaseDataLayer的DataLayerSetUp方法，所有他仍然調用父類的DataLayerSetUp方法
  BaseDataLayer<Dtype>::LayerSetUp(bottom, top);//迴避虛函數的機制，通過作用域符來限定調用的虛函數版本。由於C++的多態性，BaseDataLayer<Dtype>::LayerSetUp(bottom, top)會調用Datalayer的ataLayerSetUp方法，該方法會Reshape prefetch_的batch的data_ blob、label_ blob
  // Before starting the prefetch thread（預取線程）, we make cpu_data and gpu_data
  // calls so that the prefetch thread does not accidentally make simultaneous
  // cudaMalloc calls when the main thread is running. In some GPUs this
  // seems to cause failures if we do not so.
  for (int i = 0; i < PREFETCH_COUNT; ++i) {
    //2. 訪問預取數據空間，這裏是爲了提前分配預取數據的存儲空間
    prefetch_[i].data_.mutable_cpu_data();//由於C++的多態性，BaseDataLayer<Dtype>::LayerSetUp(bottom, top)會調用Datalayer的ataLayerSetUp方法，該方法會Reshape prefetch_的batch的data_ blob、label_ blob
    if (this->output_labels_) {
      prefetch_[i].label_.mutable_cpu_data();
    }
  }
#ifndef CPU_ONLY
  if (Caffe::mode() == Caffe::GPU) {
    for (int i = 0; i < PREFETCH_COUNT; ++i) {
      prefetch_[i].data_.mutable_gpu_data();
      if (this->output_labels_) {
        prefetch_[i].label_.mutable_gpu_data();
      }
    }
  }
#endif
  //3. 創建用於預取數據的線程
  DLOG(INFO) << "Initializing prefetch";
  this->data_transformer_->InitRand();//調用 DataTransformer類的InitRand方法生成一個隨機數生成器。注意，DataTransformer類中有一個成員是shared_ptr<Caffe::RNG> rng_
  StartInternalThread();//會調用InternalThreadEntry方法,因爲Datalayer類沒有覆蓋InternalThreadEntry方法，所以如果真實的對象類型是Data_layer的話，也只會調用其基類的方法，即BasePrefetchingDataLayer<Dtype>::InternalThreadEntry()。
  DLOG(INFO) << "Prefetch initialized.";
}


template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::InternalThreadEntry() {
#ifndef CPU_ONLY
  cudaStream_t stream;
  if (Caffe::mode() == Caffe::GPU) {
    CUDA_CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
  }
#endif


  try {
    while (!must_stop()) {
      Batch<Dtype>* batch = prefetch_free_.pop();//batch是指針
      //load_batch(Batch<Dtype>* batch)方法Reshape了其中的data_ Blob，爲其重新分配所需的內存。做到這一點已經足夠，因爲prefetch_free_中存儲的也只是指針
      load_batch(batch);//實際上會調用DataLayer的load_batch方法，因爲它是個純虛函數
#ifndef CPU_ONLY
      if (Caffe::mode() == Caffe::GPU) {
        batch->data_.data().get()->async_gpu_push(stream);
        CUDA_CHECK(cudaStreamSynchronize(stream));
      }
#endif
      prefetch_full_.push(batch);//batch在經過load_batch(batch)後發生了變化
    }
  } catch (boost::thread_interrupted&) {
    // Interrupted exception is expected on shutdown
  }
#ifndef CPU_ONLY
  if (Caffe::mode() == Caffe::GPU) {
    CUDA_CHECK(cudaStreamDestroy(stream));
  }
#endif
}


//數據層作爲網絡的最底層，其forward功能只需要將設置top[0] top[1]的數據，即拷貝。
template <typename Dtype>
void BasePrefetchingDataLayer<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  Batch<Dtype>* batch = prefetch_full_.pop("Data layer prefetch queue empty");
  // Reshape to loaded data.
  top[0]->ReshapeLike(batch->data_);
  // Copy the data
  caffe_copy(batch->data_.count(), batch->data_.cpu_data(),
             top[0]->mutable_cpu_data());//將數據從batch拷貝到top[0]
  DLOG(INFO) << "Prefetch copied";
  if (this->output_labels_) {
    // Reshape to loaded labels.
    top[1]->ReshapeLike(batch->label_);
    // Copy the labels.
    caffe_copy(batch->label_.count(), batch->label_.cpu_data(),
        top[1]->mutable_cpu_data());//將數據從batch拷貝到top[1]
  }


  prefetch_free_.push(batch);
}
// 如果沒有GPU的話則在BasePrefetchingDataLayer類中生成一個Forward函數 
// 該函數並不前傳，而是直接報錯
#ifdef CPU_ONLY
STUB_GPU_FORWARD(BasePrefetchingDataLayer, Forward);
#endif

INSTANTIATE_CLASS(BaseDataLayer);
INSTANTIATE_CLASS(BasePrefetchingDataLayer);

//最後這兩個再後面的幾篇博客中有介紹到，可以看看定義<code class="prettyprint">INSTANTIATE_CLASS(</code>BaseDataLayer)被用來實例化BaseDataLayer的
<code class="prettyprint"></code>//類模板，<code class="prettyprint">REGISTER_LAYER_CLASS(</code>BaseData)被用來向layer_factory註冊BaseDataLayer的構造方法，方便直接通過層的
<code class="prettyprint"></code>//名稱（BaseData）直接獲取層的對象。Caffe中內置的層在實現的碼的最後都會加上這兩個宏。
 /*
<pre><code class="language-" data-lang="">// ------ in common.hpp ------
// Instantiate a class with float and double specifications.
#define INSTANTIATE_CLASS(classname) \
  char gInstantiationGuard##classname; \
  template class classname<float>; \
  template class classname<double>
// ------ in common.hpp ------

// ------ in layer_factory.hpp ------
#define REGISTER_LAYER_CREATOR(type, creator)                                  \
  static LayerRegisterer<float> g_creator_f_##type(#type, creator<float>);     \
  static LayerRegisterer<double> g_creator_d_##type(#type, creator<double>)    \

#define REGISTER_LAYER_CLASS(type)                                             \
  template <typename Dtype>                                                    \
  shared_ptr<Layer<Dtype> > Creator_##type##Layer(const LayerParameter& param) \
  {                                                                            \
    return shared_ptr<Layer<Dtype> >(new type##Layer<Dtype>(param));           \
  }                                                                            \
  REGISTER_LAYER_CREATOR(type, Creator_##type##Layer)
// ------ in layer_factory.hpp ------
} // namespace caffe

image_data_layer

#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>

#include <fstream>  // NOLINT(readability/streams)
#include <iostream>  // NOLINT(readability/streams)
#include <string>
#include <utility>
#include <vector>

#include "caffe/data_layers.hpp"
#include "caffe/layer.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/util/rng.hpp"

namespace caffe {

template <typename Dtype>
ImageDataLayer<Dtype>::~ImageDataLayer<Dtype>() {
  this->StopInternalThread();
}

template <typename Dtype>
void ImageDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  // 根據參數文件設置參數
  // 圖像的高度、寬度、是否彩色圖像、圖像目錄
  const int new_height = this->layer_param_.image_data_param().new_height();
  const int new_width  = this->layer_param_.image_data_param().new_width();
  const bool is_color  = this->layer_param_.image_data_param().is_color();
  string root_folder = this->layer_param_.image_data_param().root_folder();

  // 當前只支持讀取高度和寬度同樣大小的圖像
  CHECK((new_height == 0 && new_width == 0) ||
      (new_height > 0 && new_width > 0)) << "Current implementation requires "
      "new_height and new_width to be set at the same time.";

  // Read the file with filenames and labels
  // 讀取存放圖像文件名和類標的列表文件
  const string& source = this->layer_param_.image_data_param().source();
  LOG(INFO) << "Opening file " << source;
  std::ifstream infile(source.c_str());
  string filename;
  int label;
  // lines_存放文件名和類標的pair
  while (infile >> filename >> label) {
    lines_.push_back(std::make_pair(filename, label));
  }

  // 是否需要打亂文件的順序
  if (this->layer_param_.image_data_param().shuffle()) {
    // randomly shuffle data
    LOG(INFO) << "Shuffling data";
    const unsigned int prefetch_rng_seed = caffe_rng_rand();
    prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed));
    ShuffleImages();
  }
  LOG(INFO) << "A total of " << lines_.size() << " images.";

  // 隨機跳過的圖像，調過的圖像個數在[0, rand_skip-1]之間
  lines_id_ = 0;
  // Check if we would need to randomly skip a few data points
  // 如果參數中的rand_skip大於1，則隨機跳過[0,rand_skip-1]個圖片
  // 
  if (this->layer_param_.image_data_param().rand_skip()) {
    unsigned int skip = caffe_rng_rand() %
        this->layer_param_.image_data_param().rand_skip();
    LOG(INFO) << "Skipping first " << skip << " data points.";
    CHECK_GT(lines_.size(), skip) << "Not enough points to skip";
    lines_id_ = skip;
  }
  // Read an image, and use it to initialize the top blob.
  // 讀取文件名到Mat
  cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
                                    new_height, new_width, is_color);
  CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
  // Use data_transformer to infer the expected blob shape from a cv_image.
  // 對數據的形狀進行推斷
  vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
  // 設置transformed_data_的形狀
  this->transformed_data_.Reshape(top_shape);
  // Reshape prefetch_data and top[0] according to the batch_size.
  // 設置batch_size
  const int batch_size = this->layer_param_.image_data_param().batch_size();
  CHECK_GT(batch_size, 0) << "Positive batch size required";
  top_shape[0] = batch_size;
  // 設置預取數組中數據的形狀
  for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
    this->prefetch_[i].data_.Reshape(top_shape);
  }
  // 設置輸出的數據的形狀
  top[0]->Reshape(top_shape);

  LOG(INFO) << "output data size: " << top[0]->num() << ","
      << top[0]->channels() << "," << top[0]->height() << ","
      << top[0]->width();
  // label
  // 設置輸出的類標的形狀
  vector<int> label_shape(1, batch_size);
  top[1]->Reshape(label_shape);
  // 設置預取數組中類標的形狀
  for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
    this->prefetch_[i].label_.Reshape(label_shape);
  }
}

// 產生打亂圖像順序的數組
template <typename Dtype>
void ImageDataLayer<Dtype>::ShuffleImages() {
  caffe::rng_t* prefetch_rng =
      static_cast<caffe::rng_t*>(prefetch_rng_->generator());
  shuffle(lines_.begin(), lines_.end(), prefetch_rng);
}

// This function is called on prefetch thread
// 該函數會被內部的線程調用
template <typename Dtype>
void ImageDataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
  CPUTimer batch_timer;
  batch_timer.Start();
  double read_time = 0;
  double trans_time = 0;
  CPUTimer timer;
  CHECK(batch->data_.count());
  CHECK(this->transformed_data_.count());
  // 獲取層參數，具體參見層參數的定義的解釋
  ImageDataParameter image_data_param = this->layer_param_.image_data_param();
  const int batch_size = image_data_param.batch_size();
  const int new_height = image_data_param.new_height();
  const int new_width = image_data_param.new_width();
  const bool is_color = image_data_param.is_color();
  string root_folder = image_data_param.root_folder();

  // Reshape according to the first image of each batch
  // on single input batches allows for inputs of varying dimension.
  // 讀取跳過之後的第一幅圖像，然後根據該圖像設置相撞
  cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
      new_height, new_width, is_color);
  CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
  // Use data_transformer to infer the expected blob shape from a cv_img.
  // 推斷圖像形狀
  vector<int> top_shape = this->data_transformer_->InferBlobShape(cv_img);
  // 設置transformed_data_形狀
  this->transformed_data_.Reshape(top_shape);
  // Reshape batch according to the batch_size.
  // 設置batch_size
  top_shape[0] = batch_size;
  batch->data_.Reshape(top_shape);

  Dtype* prefetch_data = batch->data_.mutable_cpu_data();
  Dtype* prefetch_label = batch->label_.mutable_cpu_data();

  // datum scales
  // 讀取一批圖像，並進行預處理
  const int lines_size = lines_.size();
  for (int item_id = 0; item_id < batch_size; ++item_id) {
    // get a blob
    timer.Start();
    CHECK_GT(lines_size, lines_id_);
    cv::Mat cv_img = ReadImageToCVMat(root_folder + lines_[lines_id_].first,
        new_height, new_width, is_color);
    CHECK(cv_img.data) << "Could not load " << lines_[lines_id_].first;
    read_time += timer.MicroSeconds();
    timer.Start();
    // Apply transformations (mirror, crop...) to the image
    // 進行預處理

    // 根據圖像的批次獲得圖像數據的偏移量
    int offset = batch->data_.offset(item_id);
    // 設置圖像數據的指針到transformed_data_
    this->transformed_data_.set_cpu_data(prefetch_data + offset);
    // 進行預處理
    this->data_transformer_->Transform(cv_img, &(this->transformed_data_));
    trans_time += timer.MicroSeconds();//統計預處理時間

    // 複製類標到prefetch_label
    prefetch_label[item_id] = lines_[lines_id_].second;
    // go to the next iter
    lines_id_++;
    // 是否是圖像目錄中的最後一個圖像
    if (lines_id_ >= lines_size) {
      // We have reached the end. Restart from the first.
      DLOG(INFO) << "Restarting data prefetching from start.";
      lines_id_ = 0;
      // 打亂圖像索引的順序
      if (this->layer_param_.image_data_param().shuffle()) {
        ShuffleImages();
      }
    }
  }
  batch_timer.Stop();
  DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
  DLOG(INFO) << "     Read time: " << read_time / 1000 << " ms.";
  // 預處理時間
  DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}

INSTANTIATE_CLASS(ImageDataLayer);
REGISTER_LAYER_CLASS(ImageData);

}  // namespace caffe
#endif  // USE_OPENCV

data_layer

#ifdef USE_OPENCV
#include <opencv2/core/core.hpp>
#endif  // USE_OPENCV
#include <stdint.h>

#include <string>
#include <vector>

#include "caffe/common.hpp"
#include "caffe/data_layers.hpp"
#include "caffe/layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/io.hpp"

namespace caffe {

// 初始化DataReader，層參數
template <typename Dtype>
DataLayer<Dtype>::DataLayer(const LayerParameter& param)
  : BasePrefetchingDataLayer<Dtype>(param),
    reader_(param) {
}

// 析構函數停止內部線程
template <typename Dtype>
DataLayer<Dtype>::~DataLayer() {
  this->StopInternalThread();
}

// 數據層的初始化
template <typename Dtype>
void DataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
      const vector<Blob<Dtype>*>& top) {
  // 從層參數中讀取batch_size
  const int batch_size = this->layer_param_.data_param().batch_size();
  // Read a data point, and use it to initialize the top blob.
  // 從reader_中獲取一個數據
  Datum& datum = *(reader_.full().peek());

  // Use data_transformer to infer the expected blob shape from datum.
  // 用數據來推斷blob的形狀存放到top_shape
  vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
  this->transformed_data_.Reshape(top_shape);
  // Reshape top[0] and prefetch_data according to the batch_size.
  // 既然獲取了數據的形狀(channel,height,width)，那麼這裏再設置一下batch_size
  // top_shape[0]=batch_size
  // top_shape[1]=channel
  // top_shape[2]=height
  // top_shape[3]=width
  top_shape[0] = batch_size;
  // 根據形狀設置top[0]的形狀
  top[0]->Reshape(top_shape);

  // 設置預取數據的形狀
  for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
    this->prefetch_[i].data_.Reshape(top_shape);
  }
  LOG(INFO) << "output data size: " << top[0]->num() << ","
      << top[0]->channels() << "," << top[0]->height() << ","
      << top[0]->width();
  // label
  // 如果輸出類標的話則把top[1]的形狀也弄一下
  if (this->output_labels_) {
    vector<int> label_shape(1, batch_size);
    top[1]->Reshape(label_shape);
    for (int i = 0; i < this->PREFETCH_COUNT; ++i) {
      this->prefetch_[i].label_.Reshape(label_shape);
    }
  }
}

// This function is called on prefetch thread
// 這個函數是在自己定義的線程執行函數內部執行的
template<typename Dtype>
void DataLayer<Dtype>::load_batch(Batch<Dtype>* batch) {
  CPUTimer batch_timer;
  batch_timer.Start();
  double read_time = 0;
  double trans_time = 0;
  CPUTimer timer;
  CHECK(batch->data_.count());
  CHECK(this->transformed_data_.count());

  // Reshape according to the first datum of each batch
  // on single input batches allows for inputs of varying dimension.
  // 意思是像以下這種做法這樣的話，每個batch的數據的維度可以不一樣
  // 從參數文件獲取batch_size
  const int batch_size = this->layer_param_.data_param().batch_size();
  // 獲取第一個數據
  Datum& datum = *(reader_.full().peek());
  // Use data_transformer to infer the expected blob shape from datum.
  // 使用第一個數據推斷blob的形狀
  vector<int> top_shape = this->data_transformer_->InferBlobShape(datum);
  this->transformed_data_.Reshape(top_shape);
  // Reshape batch according to the batch_size.
  top_shape[0] = batch_size;
  batch->data_.Reshape(top_shape);

  // top_data存數據
  Dtype* top_data = batch->data_.mutable_cpu_data();
  Dtype* top_label = NULL;  // suppress warnings about uninitialized variables

  // top_label存類標
  if (this->output_labels_) {
    top_label = batch->label_.mutable_cpu_data();
  }

  // 對這批數據進行處理
  for (int item_id = 0; item_id < batch_size; ++item_id) {
    timer.Start();
    // get a datum
    Datum& datum = *(reader_.full().pop("Waiting for data"));
    read_time += timer.MicroSeconds();
    timer.Start();
    // Apply data transformations (mirror, scale, crop...)
    // 對於給定批的數據獲取offset，這裏調用的是給定batchid，然後獲取offset
    int offset = batch->data_.offset(item_id);
    this->transformed_data_.set_cpu_data(top_data + offset);
    this->data_transformer_->Transform(datum, &(this->transformed_data_));
    // Copy label.
    // 複製類標
    if (this->output_labels_) {
      top_label[item_id] = datum.label();
    }
    // 數據傳輸時間
    trans_time += timer.MicroSeconds();

    // 將數據指針壓到free隊列
    reader_.free().push(const_cast<Datum*>(&datum));
  }
  timer.Stop();
  batch_timer.Stop();
  DLOG(INFO) << "Prefetch batch: " << batch_timer.MilliSeconds() << " ms.";
  DLOG(INFO) << "     Read time: " << read_time / 1000 << " ms.";
  DLOG(INFO) << "Transform time: " << trans_time / 1000 << " ms.";
}

INSTANTIATE_CLASS(DataLayer);
REGISTER_LAYER_CLASS(Data);

}  // namespace caffe