基於live555的rtsp播放器：數據接收（拉流）

live555的使用都是從研究源碼中的testRTSPClient例子開始的，這個例子包含了RTSP消息交互和數據接收。

一.RTSP消息交互

一次基本的RTSP操作過程如下:C表示RTSP客戶端,S表示RTSP服務端
1.第一步：查詢服務器端可用方法
C->S:OPTIONrequest          //詢問S有哪些方法可用
S->C:OPTIONresponse       //S迴應信息的public頭字段中包括提供的所有可用方法

2.第二步：得到媒體描述信息
C->S:DESCRIBE request    //要求得到S提供的媒體描述信息
S->C:DESCRIBE response  //迴應媒體描述信息，一般是sdp信息

3.第三步：建立RTSP會話
C->S:SETUPrequest            //通過Transport頭字段列出可接受的傳輸選項，請求S建立會話
S->C:SETUPresponse          //建立會話，通過Transport頭字段返回選擇的具體轉輸選項，並返回建立的Session ID;

4.第四步：請求開始傳送數據
C->S:PLAY request               //C請求S開始發送數據
S->C:PLAYresponse              //S迴應該請求的信息

5.第五步：數據傳送播放中
S->C:發送流媒體數據             //通過RTP協議傳送數據

6.  第六步：關閉會話，退出
C->S:TEARDOWN request    //C請求關閉會話
S->C:TEARDOWN response //S迴應該請求
上述的過程只是標準的、友好的rtsp流程，但實際的需求中並不一定按此過程。
其中第三和第四步是必需的！第一步，只要服務器客戶端約定好，有哪些方法可用，則option請求可以不要。第二步，如果我們有其他途徑得到媒體初始化描述信息（比如http請求等等），則我們也不需要通過rtsp中的describe請求來完成。

關於RTSP消息格式和SDP協議格式詳見：https://blog.csdn.net/caoshangpa/article/details/53191630

testRTSPClient例子的不足之處在於所有變量、函數及其回調都寫在了一個文件中，不方便管理。而且收到數據後，只是簡單的打印了數據流的類型、字節數和時間戳，並沒有進行下一步的處理。
關於testRTSPClient中變量、函數及其回調的拆分，github上有個demo可以參考：https://github.com/drriguz/Kamera
這個demo中實現了h264視頻流的解碼和顯示，例子中最可取的做法是將數據接收放到了一個線程中，也就是將env->taskScheduler().doEventLoop(&this->eventLoopWatchVariable);這一句直接放到了線程中。線程的結束通過eventLoopWatchVariable變量來控制。這樣的話一個線程就是一個session，拉取多路流只需要創建多個session即可。

二.數據接收

繼承父類MediaSink即可實現數據的接收。在子類的構造函數中，可以先根據MediaSubsession參數獲取一些有用的信息。

1.視頻信息（h264和h265（HEVC））

    if(strcmp(m_subsession.mediumName(), "video") == 0)
    {
        if(strcmp(m_subsession.codecName(), "H264") == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_H264;

            unsigned int extraSize = 0;
            uint8_t *extra = h264_parse_config(m_subsession.fmtp_spropparametersets(),extraSize);
            if(extra&&extraSize>8)
            {
                format->extraSize = extraSize;
                format->extra = new uint8_t[extraSize];
                memcpy(format->extra, extra, extraSize);
                delete [] extra;
            }

            delete format;
        }
        else if(strcmp(m_subsession.codecName(), "H265") == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_H265;

            unsigned int extraSizeVPS = 0, extraSizeSPS = 0, extraSizePPS = 0, extraSizeTotal = 0;
            uint8_t *extraVPS = h264_parse_config(m_subsession.fmtp_spropvps(),extraSizeVPS);
            uint8_t *extraSPS = h264_parse_config(m_subsession.fmtp_spropsps(),extraSizeSPS);
            uint8_t *extraPPS = h264_parse_config(m_subsession.fmtp_sproppps(),extraSizePPS);
            extraSizeTotal = extraSizeVPS + extraSizeSPS + extraSizePPS;
            if(extraVPS&&extraSPS&&extraPPS&&extraSizeTotal>12)
            {
                format->extraSize = extraSizeTotal;
                format->extra = new uint8_t[extraSizeTotal];
                memcpy(format->extra, extraVPS, extraSizeVPS);
                memcpy(format->extra+extraSizeVPS, extraSPS, extraSizeSPS);
                memcpy(format->extra+extraSizeVPS+extraSizeSPS, extraPPS, extraSizePPS);
                delete [] extraVPS;
                delete [] extraSPS;
                delete [] extraPPS;
            }

            delete format;
        }
    }

對於h264格式，fmtp_spropparametersets包含了從SDP中獲取的SPS和PPS信息的base64編碼，SPS和PPS中間用逗號隔開。解析後的數據extra在初始化視頻解碼器的時候需要用到。
一個典型的h264視頻流的SDP信息如下圖：

紅圈處sprop-parameter-sets等號後面就是fmtp_spropparametersets函數獲取到的值。
需要特別注意一點的是SPS和PPS信息並不是在SDP中強制提供的，即sprop-parameter-sets等號後面可以是空的。因此爲了保險起見，在視頻流中總首次接收到SPS和PPS信息時，需要再解碼之前，再次傳給解碼器。
對於h265格式，VPS和SPS和PPS是分三次獲取的，解析後要組裝到一起，同理，組裝後的數據extra在初始化視頻解碼器的時候需要用到。h265也要注意上面提到的問題。

2.音頻信息（MPEG4-GENERIC（AAC）、PCMA（G711a）、PCMU（G711u）和G726）

之所以要處理這麼多音頻編碼類型，是因爲這些類型幾乎所有的安防攝像機都支持。

    if(strcmp(m_subsession.mediumName(), "audio") == 0)
    {
        if(strcmp(m_subsession.codecName(), "MPEG4-GENERIC") == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_AAC;

            unsigned int extraSize = 0;
            uint8_t *extra = parseGeneralConfigStr(m_subsession.fmtp_config(),extraSize);
            if(extra)
            {
                format->extraSize = extraSize;
                format->extra = new uint8_t[extraSize];
                memcpy(format->extra, extra, extraSize);
                delete [] extra;
            }

            delete format;
        }
        else if(strcmp(m_subsession.codecName(), "PCMA") == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_PCM_ALAW;
            format->samplerate = m_subsession.rtpTimestampFrequency();
            format->channels = m_subsession.numChannels();
            format->bitspersample = 8;

            delete format;
        }
        else if(strcmp(m_subsession.codecName(), "PCMU") == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_PCM_MULAW;
            format->samplerate = m_subsession.rtpTimestampFrequency();
            format->channels = m_subsession.numChannels();
            format->bitspersample = 8;

            delete format;
        }
        else if(strncmp(m_subsession.codecName(), "G726", 4) == 0)
        {
            Format *format = new Format();

            format->codecID = AV_CODEC_ID_ADPCM_G726;
            format->samplerate = 8000;
            format->channels = 1;
            if(strcmp(m_subsession.codecName()+5, "40") == 0)
            {
                format->bitrate = 40000;
            }
            else if(strcmp(m_subsession.codecName()+5, "32") == 0)
            {
                format->bitrate = 32000;
            }
            else if(strcmp(m_subsession.codecName()+5, "24") == 0)
            {
                format->bitrate = 24000;
            }
            else if(strcmp(m_subsession.codecName()+5, "16") == 0)
            {
                format->bitrate = 16000;
            }

            delete format;
        }
    }

fmtp_config包含了採樣率和通道等信息，由下圖紅圈處的rtpmap參數獲取。這個信息SDP是必須提供的，因爲在接收數據時無法再獲取到這些信息了，它們將用於初始化音頻解碼器。

3.流數據處理
通過循環調用FrameSouce類的getNextFrame(m_receiveBuffer, RECEIVE_BUFFER_SIZE,afterGettingFrame, this, onSourceClosure, this);函數來獲取數據，參數m_receiveBuffer是接收buffer，afterGettingFrame是回調函數。
在回調函數中afterGettingFrame(unsigned frameSize, unsigned numTruncatedBytes, struct timeval presentationTime, unsigned /*durationInMicroseconds*/)可以對接收到的音視頻流數據進行處理。
此時獲取到的是完整的幀數據，參數frameSize是數據大小，參數presentationTime的類型timeval是live555內置的時間結構，轉換成pts的方法如下：

int64_t pts = (int64_t)presentationTime.tv_sec * INT64_C(1000000) + (int64_t)presentationTime.tv_usec;

這個時間是絕對時間，單位是微秒。
在處理數據之前，先看看live555官方的FAQ：http://www.live555.com/liveMedia/faq.html#testRTSPClient-how-to-decode-data
Question:I have successfully used the "testRTSPClient" demo application to receive a RTSP/RTP stream. Using this application code as a model, how can I decode the received video (and/or audio) data?
The "testRTSPClient" demo application receives each (video and/or audio) frame into a memory buffer, but does not do anything with the frame data. You can, however, use this code as a model for a 'media player' application that decodes and renders these frames. Note, in particular, the "DummySink" class that the "testRTSPClient" demo application uses - and the (non-static) "DummySink::afterGettingFrame()" function. When this function is called, a complete 'frame' (for H.264 or H.265, this will be a "NAL unit") will have already been delivered into "fReceiveBuffer". Note that our "DummySink" implementation doesn't actually do anything with this data; that's why it's called a 'dummy' sink.
If you want to decode (or otherwise process) these frames, you would replace "DummySink" with your own "MediaSink" subclass. Its "afterGettingFrame()" function would pass the data (at "fReceiveBuffer", of length "frameSize") to a decoder. (A decoder would also use the "presentationTime" timestamp to properly time the rendering of each frame, and to synchronize audio and video.)
If you are receiving H.264 video data, there is one more thing that you have to do before you start feeding frames to your decoder. H.264 streams have out-of-band configuration information ("SPS" and "PPS" NAL units) that you may need to feed to the decoder to initialize it. To get this information, call "MediaSubsession::fmtp_spropparametersets()" (on the video 'subsession' object). This will give you a (ASCII) character string. You can then pass this to "parseSPropParameterSets()" (defined in the file "include/H264VideoRTPSource.hh"), to generate binary NAL units for your decoder.
(If you are receiving H.265 video, then you do the same thing, except that you have three separate configuration strings, that you get by calling "MediaSubsession::fmtp_spropvps()", "MediaSubsession::fmtp_spropsps()", and "MediaSubsession::fmtp_sproppps()". For each of these three strings, in turn, pass them to "parseSPropParameterSets()", then feed the resulting binary NAL unit to your decoder.)
大致意思對h264或h265, 接收到的buffer是一個 "NAL unit"，需要注意的是這個NAL unit不帶四字節起始碼0x00000001，但是ffmpeg解碼的時候需要在接收buffer前添加起始碼，否則ffmpeg就解碼錯誤no frame。

Block *block=new Block();
block->esType=ES_VIDEO;
block->codecType=CODEC_H264;
block->frameType=frameType;
block->pts=pts;
block->dts=pts;

uint8_t *receiveBufferAV = new uint8_t[frameSize + 4];
receiveBufferAV[0] = 0;
receiveBufferAV[1] = 0;
receiveBufferAV[2] = 0;
receiveBufferAV[3] = 1;
memcpy(receiveBufferAV + 4, m_receiveBuffer, frameSize);
block->buffer=receiveBufferAV;
block->bufferSize=frameSize + 4;

每個NAL unit的第一個字節是NAL頭，通過NAL頭可以解析出NAL unit的類型，對於h264：uint8_t nalType=(m_receiveBuffer[0] & 0x1f)。nalType=5表示當前NAL unit是IDR圖像，nalType=1表示當前NAL unit是非IDR圖像，nalType=7或8表示當前NAL unit是SPS或者PPS。

關於h264的格式分析詳見：https://blog.csdn.net/caoshangpa/article/details/53019793?utm_source=blogxgwz3
SPS信息和PPS信息可用於初始化視頻解碼器，如前文所述，如果從SDP中未獲取到，從這裏也可以獲取。
SPS中包含了視頻的寬高和幀率等信息，如何解碼SPS獲取分辨率和幀率可參考：https://blog.csdn.net/caoshangpa/article/details/53083410?utm_source=blogxgwz6
需要注意的是幀率並不是SPS中必須包含的，因爲有些流的幀率是可變的，解出來幀率值可能爲0。
這裏IDR圖像一定是I幀，但是I幀不一定是IDR圖像，關於I幀、P幀和B幀將在下篇文章中總結一下。
爲了防止解碼第一幀時出現馬賽克，接收時需要判斷接收到幀是否是IDR圖像（注意不是判斷I幀），如果是IDR圖像，則開始解碼。我的做法是SPS和PPS都接收到且傳給瞭解碼器，再判斷當前幀是否是IDR圖像，如果是則開始解碼當前幀和之後收到的IDR圖像和非IDR圖像，也就是說SPS和PPS只用處理一次。大概如下：
0x00, 0x00, 0x00, 0x01, pps, 0x00, 0x00, 0x00, 0x01, sps, 0x00, 0x00, 0x00, 0x01, IDR frame...........
對於h265：uint8_t nalType=((m_receiveBuffer[0] & 0x7e)>>1)，具體類型判斷如下。

uint8_t nalType=((m_receiveBuffer[0] & 0x7e)>>1);

if(nalType == 32)//VPS
{   
       
}
else if(nalType == 33)//SPS
{  
           
}
else if(nalType == 34)//PPS
{
                
}

if(nalType==19 || nalType==20)//key frame
{
     m_hasKeyFrame=true;
}

if(((nalType>=1 && nalType<=9) || (nalType>=16 && nalType<=18) || nalType==21) && m_hasKeyFrame)//I/P/B
{
               
}

音頻數據的處理就簡單很多，不需要添加起始碼和判斷幀類型。

    if(strcmp(m_subsession.mediumName(), "audio") == 0)
    {
        if(strcmp(m_subsession.codecName(), "MPEG4-GENERIC") == 0
                || strcmp(m_subsession.codecName(), "PCMA") == 0
                || strcmp(m_subsession.codecName(), "PCMU") == 0
                || strncmp(m_subsession.codecName(), "G726", 4) == 0)
        {
            Block *block=new Block();
            if(strcmp(m_subsession.codecName(), "MPEG4-GENERIC") == 0)
            {
                block->codecType=CODEC_AAC;
            }
            else if(strcmp(m_subsession.codecName(), "PCMA") == 0)
            {
                block->codecType=CODEC_G711A;
            }
            else if(strcmp(m_subsession.codecName(), "PCMU") == 0)
            {
                block->codecType=CODEC_G711U;
            }
            else if(strncmp(m_subsession.codecName(), "G726" , 4) == 0)
            {
                block->codecType=CODEC_G726;
            }
            block->esType=ES_AUDIO;
            block->pts=pts;
            block->dts=pts;

            uint8_t *receiveBufferAV = new uint8_t[frameSize];
            memcpy(receiveBufferAV, m_receiveBuffer, frameSize);
            block->buffer=receiveBufferAV;
            block->bufferSize=frameSize;
        }
    }

原創不易，轉載請標明出處：https://blog.csdn.net/caoshangpa/article/details/112063679