音視頻學習（十、再探rtmp拉流）

昨天把推流寫了，不知道看的懂的有多少，確實沒有看到代碼直接看文字有點難，這不是自己寫的代碼，所以還是隻要是分析爲主，不能全部粘貼出來。

10.1 再探rtmp拉流

相比推流還有視頻採集模塊，拉流就簡單了一點，只有一個Rtmpplayer模塊和vidoeDecode模塊，Rtmpplayer只要負責拉流工作，vidoeDecode負責解碼工作。

10.2 Rtmpplayer分析

Rtmpplayer內部啓動了一個單獨的線程，這個單獨的線程專門拉去視頻流數據，也是判斷Rtmp包中是否有數據，如果有數據就取出來分析分析。

void* RtmpPlayer::readPacketThread()
{
    RTMPPacket packet = {0};

    while(!_exit_thread)
    {
        //短線重連
        if(!isConnect())
        {
            printf("短線重連 re connect");
            if(!connect(_url))      //重連失敗
            {
                printf("短線重連 reConnect fail %s",_url.c_str());
                msleep(10);
                continue;
            }
        }


        RTMP_ReadPacket(_rtmp, &packet);		//rtmp讀取包數據

        if(RTMPPacket_IsReady(&packet))    //判斷是不是整個包都組好了
        {
            uint8_t nalu_header[4] = { 0x00, 0x00, 0x00, 0x01 };

            if(!packet.m_nBodySize)
                continue;

            if(packet.m_packetType == RTMP_PACKET_TYPE_VIDEO)
            {
                //這個就是視頻數據，視頻數據中，包括兩個部分，一個是AVC sequence header，一個是nalu
                //這兩個分支下面都做了不同的處理

                // 解析完數據再發送給解碼器
                // 判斷起始字節, 檢測是不是spec config, 還原出sps pps等
                // 重組幀
                bool keyframe = 0x17 == packet.m_body[0] ? true : false;
                // AVC NALU : 0x01/ AVC sequence header : 0x00
                bool sequence = 0x00 == packet.m_body[1];        

                //SPS/PPS sequence
                if(sequence)
                {
                   
                }
                // Nalu frames
                else
                {
                    
                }
            }
            else if(packet.m_packetType == RTMP_PACKET_TYPE_AUDIO)
            {
                printf("rtmp audio\n");
                //目前不處理音頻，以後處理
            }
            else if(packet.m_packetType == RTMP_PACKET_TYPE_INFO)               
            //一啓動就能收到服務器發送過來的兩條info信息
            {
                printf("rtmp info\n");
                //這個就是視頻的info信息，也就是FLV中的metadata數據，metadata只要包含視頻中配置信息

                //解析信息tag
                parse_script_tag(packet);
                if(video_width > 0 && video_height>0)
                {
                    FLVMetadataMsg *metadata = new FLVMetadataMsg();
                    metadata->width = video_width;//720;
                    metadata->height = video_height;//480;
                    video_callable_object_(RTMP_BODY_METADATA, metadata, false);
                }
            }
            else
            {
                printf("rtmp else\n");
                RTMP_ClientPacket(_rtmp, &packet);
            }
        }

        RTMPPacket_Free(&packet);

        memset(&packet,0,sizeof(RTMPPacket));
    }
    printf("thread exit\n");
    return NULL;

}

10.2.1 metadata數據解析

//解析腳本文件
void RtmpPlayer::parse_script_tag(RTMPPacket &packet)
{
    AMFObject obj;
    AVal val;
    AMFObjectProperty * property;
    AMFObject subObject;
	//使用rtmp庫中的AMF模塊進行metadata解析
    if(AMF_Decode(&obj, packet.m_body, packet.m_nBodySize, FALSE) < 0 )
    {
        printf("error AMF Decode\n");
    }

    AMF_Dump(&obj);
    printf(" amf obj %d\n",obj.o_num);
	//爲什麼有兩個for循環，就是因爲metadata數據是多個類的方式封裝的
    for(int n=0; n<obj.o_num; n++)			
    {
        property = AMF_GetProp(&obj, NULL, n);  //通過父類中提取出子對象的信息
        if(property != NULL)
        {
           if(property->p_type == AMF_OBJECT)	//如果子對象也是一個類，就要再次提取
           {
                AMFProp_GetObject(property, &subObject);      //獲取子類對象
                for(int m = 0; m < subObject.o_num; m++)
                {
                    property = AMF_GetProp(&subObject, NULL, m);	//這裏纔是再次提取
                    printf("val = %s\n",property->p_name.av_val);

                    if(property != NULL)
                    {
                        if (property->p_type == AMF_OBJECT)		//應該沒有對象了
                        {

                        }
                        else if ( property->p_type == AMF_BOOLEAN )  //獲取bool值
                        {
                            int bval = AMFProp_GetBoolean(property);
                            if(strncasecmp("stereo", property->p_name.av_val, property->p_name.av_len) == 0)
                            {
                                audio_channel = bval > 0 ? 2 : 1;
                                printf("parse channel %d\n", audio_channel);
                            }
                        }
                        else if (property->p_type == AMF_NUMBER)	//這裏是提取值，這個比較多
                       {
                       	//包括視頻的寬 高  碼流  幀率等
                           double dVal = AMFProp_GetNumber(property);
                           if (strncasecmp("width", property->p_name.av_val, property->p_name.av_len) == 0)
                           {
                               video_width = (int)dVal;
                               printf("parse widht %d\n",video_width);
                           }
                           else if (strcasecmp("height", property->p_name.av_val) == 0)
                           {
                               video_height = (int)dVal;
                               printf("parse Height %d\n",video_height);
                           }
                           else if (strcasecmp("framerate", property->p_name.av_val) == 0)
                           {
                               video_frame_rate = (int)dVal;
                               printf("parse frame_rate %d\n",video_frame_rate);
                           }
                           else if (strcasecmp("videocodecid", property->p_name.av_val) == 0)
                           {
                               video_codec_id = (int)dVal;
                               printf("parse video_codec_id %d\n",video_codec_id);
                           }
                           else if (strcasecmp("audiosamplerate", property->p_name.av_val) == 0)
                           {
                               audio_sample_rate = (int)dVal;
                               printf("parse audiosamplerate %d\n",audio_sample_rate);
                           }
                           else if (strcasecmp("audiodatarate", property->p_name.av_val) == 0)
                           {
                               audio_bit_rate = (int)dVal;
                               printf("parse audiodatarate %d\n",audio_bit_rate);
                           }
                           else if (strcasecmp("audiosamplesize", property->p_name.av_val) == 0)
                           {
                               audio_sample_size = (int)dVal;
                               printf("parse audiosamplesize %d\n",audio_sample_size);
                           }
                           else if (strcasecmp("audiocodecid", property->p_name.av_val) == 0)
                           {
                               audio_codec_id = (int)dVal;
                               printf("parse audiocodecid %d\n",audio_codec_id);
                           }
                           else if (strcasecmp("filesize", property->p_name.av_val) == 0)
                           {
                               file_size = (int)dVal;
                               printf("parse filesize %d\n",file_size);
                           }
                       }
                       else if (property->p_type == AMF_STRING)
                       {
                           AMFProp_GetString(property, &val);
                       }
                    }
                }
           }
           else
           {
               AMFProp_GetString(property, &val);

               printf("val = %s\n",val.av_val);
           }

        }
    }
}

這樣就能完全解析出來，要想深入研究，可以去分析源碼，我這裏就先不分析了，先用起來先

10.2.2 視頻數據解析

//printf("rtmp video\n");

// 解析完數據再發送給解碼器
// 判斷起始字節, 檢測是不是spec config, 還原出sps pps等
// 重組幀
bool keyframe = 0x17 == packet.m_body[0] ? true : false;
bool sequence = 0x00 == packet.m_body[1];         // AVC NALU : 0x01/ AVC sequence header : 0x00

//SPS/PPS sequence   昨天分析可以看出，第二個字節爲0代表是sps/pps
if(sequence)
{
    uint32_t offset = 10;       //偏移量=10 sps  
    //可以看到昨天推流的數據，10剛好就是 sps nums   body[i++] = 0xE1;           //&0x1f
    uint32_t sps_num = packet.m_body[offset++] & 0x1f;    //需要 & 0x1f 才能算出真正的長度
    if(sps_num > 0)
    {
        _sps_vector.clear();    // 先清空原來的緩存
    }
    for (int i = 0; i < sps_num; i++)				//保存sps數據
    {
        uint8_t len0 = packet.m_body[offset];
        uint8_t len1 = packet.m_body[offset + 1];
        uint32_t sps_len = ((len0 << 8) | len1);
        offset += 2;
        // Write sps data
        std::string sps;
        sps.append(nalu_header, nalu_header + 4); // 存儲 start code
        sps.append(packet.m_body + offset, packet.m_body + offset + sps_len);
        _sps_vector.push_back(sps);
        offset += sps_len;                  //sps data
    }

	//後面就是保存pps的數據了
    uint32_t pps_num = packet.m_body[offset++] & 0x1f;
    if(pps_num > 0)
    {
        _pps_vector.clear();    // 先清空原來的緩存
    }
    for (int i = 0; i < pps_num; i++)
    {
        uint8_t len0 = packet.m_body[offset];
        uint8_t len1 = packet.m_body[offset + 1];
        uint32_t pps_len = ((len0 << 8) | len1);
        offset += 2;
        // Write pps data
        std::string pps;
        pps.append(nalu_header, nalu_header + 4); // 存儲 start code
        pps.append(packet.m_body + offset, packet.m_body + offset + pps_len);
        _pps_vector.push_back(pps);
        offset += pps_len;
    }
    VideoSequenceHeaderMsg * vid_config_msg = new VideoSequenceHeaderMsg(
                                (uint8_t *)_sps_vector[0].c_str(),
                                _sps_vector[0].size(),
                                (uint8_t *)_pps_vector[0].c_str(),
                                _pps_vector[0].size()
                                );
    //調用回調
    video_callable_object_(RTMP_BODY_VID_CONFIG, vid_config_msg, false);
}
// Nalu frames
else
{
    //nalu 數據就比較簡單了，直接跳到長度位置，提取4個字節的數據代表nalu數據長度
    uint32_t offset = 5;
    uint8_t len0 = packet.m_body[offset];
    uint8_t len1 = packet.m_body[offset + 1];
    uint8_t len2 = packet.m_body[offset + 2];
    uint8_t len3 = packet.m_body[offset + 3];
    uint32_t data_len = ((len0 << 24) | (len1 << 16) | (len2 << 8) | len3);
    offset += 4;

    NaluStruct * nalu = new NaluStruct(data_len + 4);
    memcpy(nalu->data, nalu_header, 4);
    memcpy(nalu->data + 4, packet.m_body + offset, data_len);
    if(_video_pre_pts == -1){
        _video_pre_pts= packet.m_nTimeStamp;
        if(!packet.m_hasAbsTimestamp) {
            printf("no init video pts\n");
        }
    }
    else {
        if(packet.m_hasAbsTimestamp)
            _video_pre_pts= packet.m_nTimeStamp;
        else
            _video_pre_pts += packet.m_nTimeStamp;
    }
    nalu->pts = _video_pre_pts;
    video_callable_object_(RTMP_BODY_VID_RAW, nalu, false);
    offset += data_len;
}

不難發現我們採集到數據之後，都會調用一個video_callable函數，這是在初始化的時候綁定的，接下分析一下video_callable函數。

10.2.3 視頻回調函數分析

這次回調函數比較簡單，回調函數收到Rtmpplayer收到的數據，直接post到隊列中。

10.3 videoDecodeLoop分析

videoDecodeLoop模塊是視頻解碼模塊，在初始化中，初始化了一個h264decoder模塊，還有初始胡一個Looper循環隊列，這個循環隊列內部帶線程，線程只要作用就是檢測隊列中是否有數據，如果有數據，提取數據，傳給回調函數處理，也就是videoDecodeLoop：：handle函數。

10.3.1 videoDecodeLoop：：handle分析

這個函數是取到隊列中的數據，然後調用的：

void videoDecodeLoop::handle(int what, MsgBaseObj *data)
{
    if(what == RTMP_BODY_METADATA)				//接收到metadata數據，進行播放器的初始化
    {
        if(!_video_out_sdl)
        {
            _video_out_sdl = new VideoOutSDL();
            if(!_video_out_sdl)
            {
                printf("new VideoOutSDL() failed\n");
                return;
            }
            Properties vid_out_properties;
            FLVMetadataMsg *metadata = (FLVMetadataMsg *)data;
            vid_out_properties.SetProperty("video_width", metadata->width);
            vid_out_properties.SetProperty("video_height",  metadata->height);
            vid_out_properties.SetProperty("win_x", 1000);
            vid_out_properties.SetProperty("win_title", "pull video display");
            delete metadata;
            if(_video_out_sdl->init(vid_out_properties) != 0)
            {
                printf("video_out_sdl Init failed\n");
                return;
            }
        }
    }
    else if(what == RTMP_BODY_VID_CONFIG)			//把接收到的sps/pps送到解碼器中解碼
    {
        VideoSequenceHeaderMsg *vid_config = (VideoSequenceHeaderMsg *)data;

        // 把sps送給解碼器
        _yuv_buf_size = YUV_BUF_MAX_SIZE;
        _h264decoder->decode(vid_config->_sps, vid_config->_sps_size,
                                        _yuv_buf, _yuv_buf_size);
        // 把pps送給解碼器
        _yuv_buf_size = YUV_BUF_MAX_SIZE;
        _h264decoder->decode(vid_config->_pps, vid_config->_pps_size,
                                        _yuv_buf, _yuv_buf_size);
        delete vid_config;
    }
    else							//最後就是發送nalu數據到解壓器中解碼
    {
        NaluStruct *nalu = (NaluStruct *)data;
        _yuv_buf_size = YUV_BUF_MAX_SIZE;
        if(_h264decoder->decode(nalu->data, nalu->size,
                                        _yuv_buf, _yuv_buf_size) == 0)
        {
            callable_object_(_yuv_buf, _yuv_buf_size);
        }
        delete nalu;     // 要手動釋放資源
    }
}

這個都比較簡單，接下來重點的還是h264解碼器。

10.3.2 h264decoder分析

h264decode解碼初始化跟demo的解壓代碼差不多，所以這裏最主要還是要看解碼函數：

int h264decoder::decode(uint8_t *in, int32_t in_len, uint8_t *out, int32_t &out_len)
{
   int got_picture=0;

   AVPacket pkt;
   av_init_packet(&pkt);
   pkt.data = in;
   pkt.size = in_len;
   
   int readed = avcodec_decode_video2(_ctx, _frame, &got_picture, &pkt);

   //Si hay picture_
   if (got_picture && readed>0)
   {
       if(_ctx->width==0 || _ctx->height==0)
       {
           printf("-Wrong dimmensions [%d,%d]\n", _ctx->width, _ctx->height);
           return 0;
       }
       int width = _frame->width;       //720
       int height = _frame->height;     //480

       out_len = _frame->width * _frame->height * 1.5;    //圖片的長度，這是h264輸入，然後YUV數據出來
       /*memcpy(out, _frame->data[0], _frame->width * _frame->height);        //Y
       memcpy(out + _frame->width * _frame->height, _frame->data[1],        //U
               (_frame->width * _frame->height) /4);
       memcpy(out + (_frame->width * _ctx->height) + (_frame->width * _ctx->height) /4,
              _frame->data[2],
               (_frame->width * _frame->height) /4);   */                   //V

	//解碼出來的數據是對齊的，所以需要這樣子複製，如果是上面那樣複製不行的
       for(int j=0; j<height; j++)
       {
           memcpy(out + j*width, _frame->data[0] + j * _frame->linesize[0], width);    
           //768   64字節對齊
       }
       out += width * height;
       for(int j=0; j<height/2; j++)
       {
           memcpy(out + j*width/2, _frame->data[1] + j * _frame->linesize[1], width/2);
       }
       out += width * height/2/2;
       for(int j=0; j<height/2; j++)
       {
           memcpy(out + j*width/2, _frame->data[2] + j * _frame->linesize[2], width/2);
       }

		//下面是保存成YUV格式的，保存成YUV格式也是對齊保存
       static FILE *dump_yuv = NULL;
       if(!dump_yuv)
       {
           dump_yuv = fopen("h264_dump_320x240.yuv", "wb");
           if(!dump_yuv)
           {
               printf("fopen h264_dump.yuv failed");
           }
       }
       if(dump_yuv)
       {//ffplay -ar 48000 -ac 2 -f s16le -i aac_dump.pcm
           //AVFrame存儲YUV420P對齊分析
           //https://blog.csdn.net/dancing_night/article/details/80830920?depth_1-utm_source=distribute.pc_relevant.none-task&utm_source=distribute.pc_relevant.none-task
           for(int j=0; j<height; j++)
               fwrite(_frame->data[0] + j * _frame->linesize[0], 1, width, dump_yuv);
           for(int j=0; j<height/2; j++)
               fwrite(_frame->data[1] + j * _frame->linesize[1], 1, width/2, dump_yuv);
           for(int j=0; j<height/2; j++)
               fwrite(_frame->data[2] + j * _frame->linesize[2], 1, width/2, dump_yuv);

           fflush(dump_yuv);
       }
       return 0;
   }
   out_len = 0;
   return 0;
}

優化過後的推流拉流就先講到這裏，下一節是要音頻數據添加上來，現在爲了簡單處理都只是使用了視頻數據。