Ubuntu18.04.3 安裝 Nvidia DeepStream 並在 P4 顯卡上解碼取幀存圖

環境安裝

參照 官方文檔 中的 “dGPU Setup” 部分即可。

我的環境：

①Ubuntu 18.04.3；
②Gstreamer 1.14.5；
③NVIDIA driver 418.87.00；
④CUDA 10.1；
⑤TensorRT 5.1.5；
⑥DeepStream SDK 4.0.1；
⑦P4 顯卡。
本文涉及取幀存圖，用的是OpenCV，故也安裝了OpenCV，版本3.4.4，安裝參照 這裏。

deepstream-app 命令

這裏只進行簡單的測試。
測試文件：deepstream_sdk_v4.0.1_x86_64/samples/configs/deepstream-app/source4_1080p_dec_infer-resnet_tracker_sgie_tiled_display_int8.txt。
修改 [sink0]，將 enable 改爲 0：

修改 [sink1]，將 enable 改爲 1：

執行命令：

deepstream-app -c source4_1080p_dec_infer-resnet_tracker_sgie_tiled_display_int8.txt

我這會報 “Internal data stream error.” 錯誤，因爲主要不是使用 deepstream-app 命令，故不深究。
PS1：在 DeepStream2 中跑不會有問題。
PS2：把 [sink1] 的 type 改成 1，表示使用 Fakesink，是可以避免這個錯誤的。

解碼取幀存圖

使用到例子：deepstream_sdk_v4.0.1_x86_64/sources/apps/sample_apps/deepstream-test1/ 。

make
./deepstream-test1-app ../../../../samples/streams/sample_720p.h264

我這會報 “ERROR from element primary-nvinference-engine: Internal data stream error.” 錯誤。
在 deepstream_test1_app.c 文件中找到：

sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");

將其改爲：

sink = gst_element_factory_make ("fakesink", "nvvideo-renderer");

表示使用 Fakesink，這對我的需求無影響（可以看看 這裏，有我的回覆，我的用戶名爲 “alpaserss”）。再次：

make
./deepstream-test1-app ../../../../samples/streams/sample_720p.h264

我這成功了，然後就是要取幀存圖了，先直接參照官方論壇，看看 這裏。
當然是會報錯的，這是我在上面的回覆：

暫時還沒人回覆我，所以要自己探索咯。
試了很多方法，最後解決了，基本思想就是：刪掉 nvosd，在 nvvidconv 的 src 上或在 sink 的 sink 上通過 gst_pad_add_probe 函數添加 probe，然後再 probe 回調中處理幀。
好吧，上面那句話我自己都看不懂，有點表達不清，所以直接上碼，看代碼大家就懂了。

#include <gst/gst.h>
#include <glib.h>
#include <stdio.h>
#include <opencv2/core.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/imgproc/types_c.h>
#include <vector>
#include "gstnvdsmeta.h"
#include "nvbufsurface.h"
#include "nvbufsurftransform.h"
using namespace std;
using namespace cv;

#define MAX_DISPLAY_LEN 64

#define PGIE_CLASS_ID_VEHICLE 0
#define PGIE_CLASS_ID_PERSON 2

#define MUXER_OUTPUT_WIDTH 1920
#define MUXER_OUTPUT_HEIGHT 1080

#define MUXER_BATCH_TIMEOUT_USEC 4000000

gint frame_number = 0;
gchar pgie_classes_str[4][32] = { "Vehicle", "TwoWheeler", "Person",
  "Roadsign"
};


static void createAlphaMat(Mat &mat)
{
 CV_Assert(mat.channels() == 4);
 for (int i = 0; i < mat.rows; ++i) {
 for (int j = 0; j < mat.cols; ++j) {
 Vec4b& bgra = mat.at<Vec4b>(i, j);
            bgra[0] = UCHAR_MAX; // Blue
            bgra[1] = saturate_cast<uchar>((float (mat.cols - j)) / ((float)mat.cols) * UCHAR_MAX); // Green
            bgra[2] = saturate_cast<uchar>((float (mat.rows - i)) / ((float)mat.rows) * UCHAR_MAX); // Red
            bgra[3] = saturate_cast<uchar>(0.5 * (bgra[1] + bgra[2])); // Alpha
        }
    }
}

static GstPadProbeReturn osd_sink_pad_buffer_probe (GstPad * pad, GstPadProbeInfo * info, gpointer u_data){
    if(++frame_number > 1) return GST_PAD_PROBE_OK;

    GstBuffer *buf = (GstBuffer *) info->data;
    GstMapInfo in_map_info;
    NvBufSurface *surface = NULL;
    NvDsBatchMeta *batch_meta = NULL;
    NvDsMetaList *l_frame = NULL;
    NvDsFrameMeta *frame_meta = NULL;

    memset (&in_map_info, 0, sizeof (in_map_info));

    if (gst_buffer_map (buf, &in_map_info, GST_MAP_READWRITE)){
      surface = (NvBufSurface *) in_map_info.data;

      g_print("%d \n" , NvBufSurfaceMap(surface, -1, -1, NVBUF_MAP_READ_WRITE));
      NvBufSurfaceSyncForCpu(surface, -1, -1);

      batch_meta = gst_buffer_get_nvds_batch_meta(buf);

      for (l_frame = batch_meta->frame_meta_list; l_frame != NULL; l_frame = l_frame->next){
        frame_meta = (NvDsFrameMeta *)(l_frame->data);
        gint frame_width = (gint)surface->surfaceList[frame_meta->batch_id].width;
        gint frame_height = (gint)surface->surfaceList[frame_meta->batch_id].height;
        void *frame_data = surface->surfaceList[frame_meta->batch_id].mappedAddr.addr[0];
        size_t frame_step = surface->surfaceList[frame_meta->batch_id].pitch;

        cv::Mat frame = cv::Mat(frame_height*1.5, frame_width, CV_8UC1, frame_data, frame_step);
        //cv::Mat dst;
        //cv::resize(frame, dst, cv::Size(frame_width, frame_height));
        //cv::Mat out_mat = cv::Mat (cv::Size(frame_width, frame_height), CV_8UC3);
        //cv::cvtColor(frame, out_mat, CV_RGBA2RGB);

        g_print("%d\n",frame.channels());
        g_print("%d\n",frame.rows);
        g_print("%d\n",frame.cols);

        cv::Mat out_mat = cv::Mat (cv::Size(frame_width, frame_height), CV_8UC3);
        cv::cvtColor(frame, out_mat, CV_YUV2RGB_NV21);
        /*createAlphaMat(frame);
        vector<int> compression_params;
        compression_params.push_back(CV_IMWRITE_PNG_COMPRESSION); 
        compression_params.push_back(9);*/
        cv::imwrite("test.jpg", out_mat);
      }

      NvBufSurfaceUnMap(surface, -1, -1);
    }

    gst_buffer_unmap (buf, &in_map_info);

    return GST_PAD_PROBE_OK;
}

static gboolean
bus_call (GstBus * bus, GstMessage * msg, gpointer data)
{
  GMainLoop *loop = (GMainLoop *) data;
  switch (GST_MESSAGE_TYPE (msg)) {
    case GST_MESSAGE_EOS:
      g_print ("End of stream\n");
      g_main_loop_quit (loop);
      break;
    case GST_MESSAGE_ERROR:{
      gchar *debug;
      GError *error;
      gst_message_parse_error (msg, &error, &debug);
      g_printerr ("ERROR from element %s: %s\n",
          GST_OBJECT_NAME (msg->src), error->message);
      if (debug)
        g_printerr ("Error details: %s\n", debug);
      g_free (debug);
      g_error_free (error);
      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }
  return TRUE;
}

int
main (int argc, char *argv[])
{
  GMainLoop *loop = NULL;
  GstElement *pipeline = NULL, *source = NULL, *h264parser = NULL,
      *decoder = NULL, *streammux = NULL, *sink = NULL, *pgie = NULL, *nvvidconv = NULL,
      *nvosd = NULL;
#ifdef PLATFORM_TEGRA
  GstElement *transform = NULL;
#endif
  GstBus *bus = NULL;
  guint bus_watch_id;
  GstPad *osd_sink_pad = NULL;

  /* Check input arguments */
  if (argc != 2) {
    g_printerr ("Usage: %s <H264 filename>\n", argv[0]);
    return -1;
  }

  /* Standard GStreamer initialization */
  gst_init (&argc, &argv);
  loop = g_main_loop_new (NULL, FALSE);

  /* Create gstreamer elements */
  /* Create Pipeline element that will form a connection of other elements */
  pipeline = gst_pipeline_new ("dstest1-pipeline");

  /* Source element for reading from the file */
  source = gst_element_factory_make ("filesrc", "file-source");

  /* Since the data format in the input file is elementary h264 stream,
   * we need a h264parser */
  h264parser = gst_element_factory_make ("h264parse", "h264-parser");

  /* Use nvdec_h264 for hardware accelerated decode on GPU */
  decoder = gst_element_factory_make ("nvv4l2decoder", "nvv4l2-decoder");

  /* Create nvstreammux instance to form batches from one or more sources. */
  streammux = gst_element_factory_make ("nvstreammux", "stream-muxer");

  if (!pipeline || !streammux) {
    g_printerr ("One element could not be created. Exiting.\n");
    return -1;
  }

  /* Use nvinfer to run inferencing on decoder's output,
   * behaviour of inferencing is set through config file */
  pgie = gst_element_factory_make ("nvinfer", "primary-nvinference-engine");

  /* Use convertor to convert from NV12 to RGBA as required by nvosd */
  nvvidconv = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter");
  GstElement *nvvidconv1 = gst_element_factory_make ("nvvideoconvert", "nvvideo-converter1");

  /* Create OSD to draw on the converted RGBA buffer */
  nvosd = gst_element_factory_make ("nvdsosd", "nv-onscreendisplay");

  /* Finally render the osd output */
#ifdef PLATFORM_TEGRA
  transform = gst_element_factory_make ("nvegltransform", "nvegl-transform");
#endif
  //sink = gst_element_factory_make ("nveglglessink", "nvvideo-renderer");
  sink = gst_element_factory_make ("fakesink", "nvvideo-renderer");

  if (!source || !h264parser || !decoder || !pgie
      || !nvvidconv || !nvosd || !sink) {
    g_printerr ("One element could not be created. Exiting.\n");
    return -1;
  }

#ifdef PLATFORM_TEGRA
  if(!transform) {
    g_printerr ("One tegra element could not be created. Exiting.\n");
    return -1;
  }
#endif

  /* we set the input filename to the source element */
  g_object_set (G_OBJECT (source), "location", argv[1], NULL);

  g_object_set (G_OBJECT (streammux), "width", MUXER_OUTPUT_WIDTH, "height",
      MUXER_OUTPUT_HEIGHT, "batch-size", 1,
      "batched-push-timeout", MUXER_BATCH_TIMEOUT_USEC, NULL);

  /* Set all the necessary properties of the nvinfer element,
   * the necessary ones are : */
  g_object_set (G_OBJECT (pgie),
      "config-file-path", "dstest1_pgie_config.txt", NULL);

  /* we add a message handler */
  bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
  bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
  gst_object_unref (bus);

  /* Set up the pipeline */
  /* we add all elements into the pipeline */
#ifdef PLATFORM_TEGRA
  gst_bin_add_many (GST_BIN (pipeline),
      source, h264parser, decoder, streammux, pgie,
      nvvidconv, nvvidconv1, nvosd, transform, sink, NULL);
#else
  gst_bin_add_many (GST_BIN (pipeline),
      source, h264parser, decoder, streammux, pgie,
      nvvidconv, nvvidconv1, nvosd, sink, NULL);
#endif

  GstPad *sinkpad, *srcpad;
  gchar pad_name_sink[16] = "sink_0";
  gchar pad_name_src[16] = "src";

  sinkpad = gst_element_get_request_pad (streammux, pad_name_sink);
  if (!sinkpad) {
    g_printerr ("Streammux request sink pad failed. Exiting.\n");
    return -1;
  }

  srcpad = gst_element_get_static_pad (decoder, pad_name_src);
  if (!srcpad) {
    g_printerr ("Decoder request src pad failed. Exiting.\n");
    return -1;
  }

  if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) {
      g_printerr ("Failed to link decoder to stream muxer. Exiting.\n");
      return -1;
  }

  gst_object_unref (sinkpad);
  gst_object_unref (srcpad);

  /* we link the elements together */
  /* file-source -> h264-parser -> nvh264-decoder ->
   * nvinfer -> nvvidconv -> nvosd -> video-renderer */

  if (!gst_element_link_many (source, h264parser, decoder, NULL)) {
    g_printerr ("Elements could not be linked: 1. Exiting.\n");
    return -1;
  }

#ifdef PLATFORM_TEGRA
  if (!gst_element_link_many (streammux, pgie,
      nvvidconv, transform, sink, NULL)) {
    g_printerr ("Elements could not be linked: 2. Exiting.\n");
    return -1;
  }
#else
  if (!gst_element_link_many (streammux, pgie,
      nvvidconv, sink, NULL)) {
    g_printerr ("Elements could not be linked: 2. Exiting.\n");
    return -1;
  }
#endif

  /* Lets add probe to get informed of the meta data generated, we add probe to
   * the sink pad of the osd element, since by that time, the buffer would have
   * had got all the metadata. */
  osd_sink_pad = gst_element_get_static_pad (sink, "sink");
  if (!osd_sink_pad)
    g_print ("Unable to get sink pad\n");
  else
    gst_pad_add_probe (osd_sink_pad, GST_PAD_PROBE_TYPE_BUFFER,
        osd_sink_pad_buffer_probe, NULL, NULL);

  /* Set the pipeline to "playing" state */
  g_print ("Now playing: %s\n", argv[1]);
  gst_element_set_state (pipeline, GST_STATE_PLAYING);

  /* Wait till pipeline encounters an error or EOS */
  g_print ("Running...\n");
  g_main_loop_run (loop);

  /* Out of the main loop, clean up nicely */
  g_print ("Returned, stopping playback\n");
  gst_element_set_state (pipeline, GST_STATE_NULL);
  g_print ("Deleting pipeline\n");
  gst_object_unref (GST_OBJECT (pipeline));
  g_source_remove (bus_watch_id);
  g_main_loop_unref (loop);
  return 0;
}

以上代碼的編譯腳本：

make clean
g++ -c -o deepstream_test1_app.o -I../../../includes -I/usr/local/cuda-10.1/targets/x86_64-linux/include/ `pkg-config --cflags gstreamer-1.0` deepstream_test1_app.c
g++ -o deepstream-test1-app deepstream_test1_app.o `pkg-config --libs gstreamer-1.0` -L/opt/nvidia/deepstream/deepstream-4.0/lib/ -lnvdsgst_meta -lnvds_meta -Wl,-rpath,/opt/nvidia/deepstream/deepstream-4.0/lib/ -lnvbufsurface -lopencv_imgcodecs -lopencv_core -lopencv_imgproc -lcuda -lcudart -L/usr/local/cuda-10.1/targets/x86_64-linux/lib/ -lnvbufsurftransform

PS：以上用到 OpenCV 將 NV12 轉成 RGB。
上面的代碼可能有我調試的痕跡，我也就不費時去清理了，大家將就着看看。且以上代碼還有很大優化空間，這裏作爲小筆記就不一一贅述了。
以上不是出現了那麼多莫名的錯誤嗎，我沒怎麼搞懂，如果有高手明白，勞煩留個言指明，萬分感謝。
同樣，如果讀者有哪裏不明白的，儘管留言，我會盡可能答覆。