live555 二次開發 接收RTP PS流(支持udp，tcp主動，被動) 轉發rtsp裸流，支持H264，H265

經過一個多月的努力，終於完成rtp PS流轉rtsp裸流的開發。大部分時間都花在了理解live555的框架，幾乎翻遍了關於live555博客，加上自己的調試跟蹤，算是有了些淺薄理解，在這裏分享給大家，關於live555框架理解（後續有時間補上），網上有很多優秀的博客，大家可以查查，在這裏只分享rtp PS流轉rtsp裸流大概流程。

1.首先需要擴展FramedFilter類專門用來解封裝PS流，即輸入一幀PS rtp數據，輸出一幀裸流數據，h264或者h265

class MP2PVideoToRawLiveSource: public  FramedFilter{

public:
  static MP2PVideoToRawLiveSource* createNew(UsageEnvironment& env, FramedSource* inputSource);
  //static MP2PVideoToRawLiveSource* createNew(UsageEnvironment& env, FramedSource* inputSource, MP2PAudioToRawLiveSource *audioSource);
  FramedSource *getInputSource(){return fInputSource;}
protected:
  virtual ~MP2PVideoToRawLiveSource();

public:
  MP2PVideoToRawLiveSource(UsageEnvironment& env, FramedSource* inputSource);
  //MP2PVideoToRawLiveSource(UsageEnvironment& env, FramedSource* inputSource, MP2PAudioToRawLiveSource *audioSource);
  bool isH264(){
        if(fVideoCodec == H264_CODEC){
            return true;
        }
        else{
            return false;
        }
  }
  bool isH265(){
        if(fVideoCodec == H265_CODEC){
            return true;
        }
        else{
            return false;
        }

  }


  
public:
    bool fRtp_receiving;
    bool fVideoCodecChange;

protected:
  // redefined virtual functions:
virtual void doGetNextFrame();

  static void afterGettingFrame(void* clientData, unsigned frameSize,
                                unsigned numTruncatedBytes,
                                struct timeval presentationTime,
                                unsigned durationInMicroseconds);
  void afterGettingFrame1(unsigned frameSize,
					  	unsigned numTruncatedBytes,
						struct timeval presentationTime,
						unsigned durationInMicroseconds);
protected:
  VIDEO_CODEC fVideoCodec;
  
private:
  //FramedSource* fInputSource;
  unsigned int fIframegop;
  unsigned char *fMP2P_frame_data;
  unsigned int fMP2P_frame_data_size;
  unsigned char *fMp2p_next_start_slice;
  unsigned int fMp2p_next_start_size;
  
  unsigned fPacketSize;
  unsigned fNumBitsSeenSoFar; // used by the getNextFrameBit*() routines
  unsigned char* fRaw_buf;
  RawFrameCache  fRawFrameCache[RAW_CACHE_NUM];
  int fRawFrameCount;
  int64_t fLastPesTime;
  unsigned int fTimestamp;
  struct timeval fMP2PPreTime;
  struct timeval fMP2PPreVideoTime;
  

private:
  void pushRawCache(unsigned char *data, unsigned int size, unsigned int numTruncatedBytes, struct timeval presentationTime,
  unsigned int durationInMicroseconds);

  void createRawCache();
  RawFrameCache * popRawCache();
  void destroyRawCache();
};

這個類可以看成一個管道，進去一個source出去一個source， 其中doGetNextFrame是繼承來的虛函數，live555框架會去調用這個接口，並且把afterGettingFrame函數註冊進去。

void MP2PVideoToRawLiveSource::doGetNextFrame() {

if(fInputSource != NULL) {

 RawFrameCache* frame_cache = popRawCache();
 if(frame_cache){
    memcpy(fTo, frame_cache->frame_data, frame_cache->frame_size);
    delete[] frame_cache->frame_data;
    frame_cache->frame_data = NULL;
    fFrameSize = frame_cache->frame_size;
    fNumTruncatedBytes = frame_cache->numTruncatedBytes;
    fPresentationTime = frame_cache->presentationTime;
    fDurationInMicroseconds = frame_cache->durationInMicroseconds;  
    fMP2PPreVideoTime = fPresentationTime;
    //printf("######### video time %u\n", fPresentationTime.tv_sec);
    afterGetting(this);
    return;
    
 }

  fInputSource->getNextFrame(fTo, fMaxSize,
                             MP2PVideoToRawLiveSource::afterGettingFrame, this,
                             FramedSource::handleClosure, this);
}else{
  nextTask() = envir().taskScheduler().scheduleDelayedTask(0,
                                 (TaskFunc*)FramedSource::handleClosure, this);    
    }
}

等到下一次有幀數據的時候會調用afterGettingFrame函數，然後會調用afterGettingFrame1函數，在這裏MP2PVideoToRawLiveSource::afterGettingFrame1就是解PS流封裝的核心函數。

void MP2PVideoToRawLiveSource
 ::afterGettingFrame1(unsigned frameSize, unsigned numTruncatedBytes,
                      struct timeval presentationTime,
                      unsigned durationInMicroseconds) {

   //printf("########## fTo 4byte   %x  %x   %x  %x, frameSize %d, fMaxSize %u\n ", fTo[0], fTo[1], fTo[2], fTo[3], frameSize, fMaxSize);
     
 // 00 00 01 e0  + byte1 + byte2 + byte3 + byte4 + byte5(pes頭後剩下的長度 len)  + len  + raw
     if(frameSize > 4){
        fRtp_receiving = true;
     }
     else{
        fFrameSize = 0;
        afterGetting(this);
        return;
    }


     
     int fpos = 0;
     int first_pes_find = 0;
     int new_ps_find = 0;
     int raw_pos = 0;
     int raw_buf_pos = 0;
 
 
     fFrameSize = frameSize;
     fNumTruncatedBytes = numTruncatedBytes;
     fPresentationTime = presentationTime;
     fDurationInMicroseconds = durationInMicroseconds;    

     fRawFrameCount = 0;


     
#if 0
      //test write file 
      static FILE *mpg_fp = NULL;
      if(mpg_fp == NULL){
          mpg_fp = fopen("test_108_hevc.mpg","w+");
      }
  
      if(mpg_fp != NULL){
         
          //printf("######### frameSize %d\n",frameSize);
          fwrite(fTo, frameSize, 1, mpg_fp);
      }
#endif  

    if(fMP2P_frame_data_size > 0 && fTo[0] == 0x00 && fTo[1] == 0x00 && fTo[2] == 0x01 && (fTo[3] == 0xba || fTo[3] == 0xc0)){


        memcpy(fMp2p_next_start_slice, fTo, frameSize);
        fMp2p_next_start_size = frameSize;

        memcpy(fTo, fMP2P_frame_data, fMP2P_frame_data_size);
        frameSize = fMP2P_frame_data_size;
        fFrameSize = frameSize;
        presentationTime = fMP2PPreTime;
        
            //fMP2PPreTime = fPresentationTime;
        if(fTo[3] == 0xc0){
            fMP2PPreTime = fPresentationTime;
        }
            

        memcpy(fMP2P_frame_data, fMp2p_next_start_slice, fMp2p_next_start_size);
        fMP2P_frame_data_size = fMp2p_next_start_size;
        
        
    }
    else{
        
        memcpy(fMP2P_frame_data + fMP2P_frame_data_size, fTo, frameSize);
        fMP2P_frame_data_size += frameSize;
        fFrameSize = 0;
        fMP2PPreTime = presentationTime;
        afterGetting(this);
        return;
    }
    
    
     if(frameSize < 4  || !(fTo[fpos] == 0x00 && fTo[fpos + 1] == 0x00 &&  fTo[fpos + 2] == 0x01)){

             printf("##########  invaild  data  fTo 4byte   %x  %x   %x  %x, frameSize %d, fMaxSize %u\n ", fTo[0], fTo[1], fTo[2], fTo[3], frameSize, fMaxSize); 
         
             fFrameSize = 0;
             afterGetting(this);
             return;
     }


     unsigned char *fRaw_buf = new unsigned char[frameSize];
     memset(fRaw_buf, 0, frameSize);



    #if 0
    fTimestamp  += 3600;
    unsigned int tv_sec = fTimestamp/(double)90000;
    //unsigned int tv_usec = tv_sec*1000000; 
    fMP2PPreTime.tv_sec = tv_sec;
    fMP2PPreTime.tv_usec = (unsigned)((fTimestamp/(double)90000 - tv_sec)*1000000);
    
    presentationTime = fMP2PPreTime;
    //printf("######### presentationTime time %u\n", presentationTime.tv_sec);
    #endif   


     
     
     //去掉PS，PES頭信息，fRaw_buf含有起始碼00 00 00 01的裸流
     unsigned char *pes_header = NULL;
     unsigned char *pts_pos = NULL;
     int64_t pts = 0;
     unsigned char PTS_DTS_flags;
     unsigned int private_data_size = 0;
     while(fpos  < frameSize){

            #if 0  //parse  pes  pts and dts
            if(fTo[fpos] == 0x00 && fTo[fpos + 1] == 0x00 &&  fTo[fpos + 2] == 0x01){
                    if(fTo[fpos + 3] == 0xe0 || fTo[fpos + 3] == 0xc0){                    
                     pes_header = fTo + fpos + 3;
                     //PTS_DTS_flags 
                     PTS_DTS_flags = pes_header[4]>>6;
                     //pes頭部有PTS
                     if(PTS_DTS_flags == 0x2 || PTS_DTS_flags == 0x3){
                        
                    //PTS佔用5個字節， 在這5個字節中取33位
                    /*
                        5             4               3               2               1 
                        7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 7 6 5 4 3 2 1 0 
                       |0 0 1 1|PTS1 |1|       PTS2                  |1|       PTS3                   |
                               |32-30| |       29-15                 | |       14-0                   |
                       PTS[32…30]：佔位3bit；  PTS[29…15]：佔位15bit； PTS[14…0]：佔位15bit；
                       PTS = （PTS1 & 0x0e) << 29 + (PTS2 & 0xfffe) << 14 + (PTS3 & 0xfffe ) >> 1;
                    */
                       pts_pos = pes_header + 6;
                       pts = (int64_t)(*pts_pos & 0x0e) << 29 | ((*(unsigned short*)(pts_pos + 1) >> 1) << 15) |
                       (*(unsigned short*)(pts_pos + 3) >> 1);

                       if(fTo[fpos + 3] == 0xe0){
                            unsigned int tv_sec = pts/(double)90000;
                            unsigned int tv_usec = tv_sec*1000000; 
                            presentationTime.tv_sec += tv_sec;
                            presentationTime.tv_usec += (unsigned)((pts/(double)90000 - tv_sec)*1000000);
                            fMP2PPreTime = presentationTime;
                            printf("########## PTS exit in PES heade  pts  %u, sec pts %f\n",pts, pts/(double)90000);
                       }
                       else{
                            presentationTime = fMP2PPreTime;
                       }
                        //printf("########## PTS exit in PES heade  pts  %u, sec pts %f\n",pts, pts/(double)90000);
                            
                     }
                     else
                        printf("#### no PTS PTS_DTS_flags %u\n", PTS_DTS_flags);
                       
                    }
           }
            #endif
     
             //視頻
             if(fTo[fpos] == 0x00 && fTo[fpos + 1] == 0x00 &&  fTo[fpos + 2] == 0x01 &&  fTo[fpos + 3] == 0xe0){
                 if(fpos + 3 + 5 < frameSize){
                     if(raw_pos > 0){
                         memcpy(fRaw_buf + raw_buf_pos, fTo + raw_pos, fpos - raw_pos);                         
                         raw_buf_pos  = raw_buf_pos + fpos - raw_pos;
                     }
                    
                     
                     //拿到ES流裸流的偏移
                     raw_pos = fpos + 3 + 5 + 1 + (int)fTo[fpos + 3 + 5];
                     fpos = raw_pos;
                 }
                 else{
                     break;
                 }
             }
             //音頻
             else if(fTo[fpos] == 0x00 && fTo[fpos + 1] == 0x00 &&  fTo[fpos + 2] == 0x01 &&  fTo[fpos + 3] == 0xc0){
                //放入音頻緩衝
                //printf("########### find audio data\n");
                 int audio_buf_pos = fpos + 3 + 5 + 1 + (int)fTo[fpos + 3 + 5];
                 int audio_buf_size = frameSize - audio_buf_pos;
                 //presentationTime = fMP2PPreVideoTime;
                 presentationTime = fMP2PPreTime;
                 //printf("######### audio time %u\n", presentationTime.tv_sec);
                 audioFrameAfterGetting(fTo + audio_buf_pos, audio_buf_size, numTruncatedBytes, presentationTime, durationInMicroseconds);
                 fFrameSize = 0;
                 afterGetting(this);
                 delete fRaw_buf;
                 return;
             }

             //海康PS流私有數據
             else if(fTo[fpos] == 0x00 && fTo[fpos + 1] == 0x00 &&  fTo[fpos + 2] == 0x01 &&  fTo[fpos + 3] == 0xbd){
                //printf("############## PS stream  private data will lost it\n");
                private_data_size = frameSize - fpos;
                break;
             }
             else{
                 fpos++;
             }
         }
 
     

     
     memcpy(fRaw_buf + raw_buf_pos, fTo + raw_pos, frameSize - raw_pos - private_data_size);
     raw_buf_pos = raw_buf_pos + frameSize - raw_pos;
 
     int raw_buf_size = raw_buf_pos; 


     
     //去掉裸流起始碼 00 00 00 01 sps pps vps 分包發送
     int start_nalu_pos = 0;
     int no_start_raw_buf_pos = 0;
     int copy_size = 0;
     int no_start_raw_len = 0;
     for(int i = 0; i + 4 < raw_buf_size;){
         if(fRaw_buf[i] == 0x00 && fRaw_buf[i+1] == 0x00 && fRaw_buf[i+2] == 0x00 && fRaw_buf[i+3] == 0x01){
            
             if(fRaw_buf[i + 4] == 0x67 || fRaw_buf[i + 4] == 0x68 || fRaw_buf[i + 4] == 0x06 || fRaw_buf[i + 4] == 0x65 ){
                 if(fVideoCodec != NONE_CODEC && fVideoCodec != H264_CODEC){
                    printf("######## VideoCodec change  to H264\n");
                        fVideoCodecChange = true;
                        fFrameSize = 0;
                        doStopGettingFrames();
                        return;
                 }
                 if(fVideoCodec == NONE_CODEC)
                    fVideoCodec = H264_CODEC;
                 
             }

             else if(fRaw_buf[i + 4] == 0x40 || fRaw_buf[i + 4] == 0x26 ||fRaw_buf[i + 4] == 0x42 || fRaw_buf[i + 4] == 0x44 || fRaw_buf[i + 4] == 0x4e){
                if(fVideoCodec != NONE_CODEC && fVideoCodec != H265_CODEC){
                        printf("######## VideoCodec change  to H265\n");
                        fVideoCodecChange = true;
                        fFrameSize = 0;
                        doStopGettingFrames();
                        return;
                }
                
                if(fVideoCodec == NONE_CODEC){
                    fVideoCodec = H265_CODEC;
                }
             }
             
             if(start_nalu_pos > 0){
                copy_size = i - start_nalu_pos;
                
                pushRawCache(fRaw_buf + start_nalu_pos, copy_size, numTruncatedBytes,  presentationTime,durationInMicroseconds);
                fFrameSize = (unsigned int)copy_size; 
                no_start_raw_buf_pos += copy_size;
             }
             start_nalu_pos = i + 4;
             i += 4;
         }
         else
            i++;
         
     }

     copy_size = raw_buf_size - start_nalu_pos;
     if(copy_size > 0){         
         pushRawCache(fRaw_buf + start_nalu_pos, copy_size, numTruncatedBytes,  presentationTime,durationInMicroseconds);
     }
     
     fFrameSize = 0;
     RawFrameCache* frame_cache = popRawCache();
     if(frame_cache){
        memcpy(fTo, frame_cache->frame_data, frame_cache->frame_size);
        delete[] frame_cache->frame_data;
        frame_cache->frame_data = NULL;
        fFrameSize = frame_cache->frame_size;
        fNumTruncatedBytes = frame_cache->numTruncatedBytes;
        fPresentationTime = frame_cache->presentationTime;
        fDurationInMicroseconds = frame_cache->durationInMicroseconds;
        //printf("######### video time %u\n", fPresentationTime.tv_sec);
        fMP2PPreVideoTime = fPresentationTime;
     }

    delete fRaw_buf;
    afterGetting(this);
 }

首先需要說明下，live555在正常收rtp流的時候，當調用這個接口的時候fTo裏面放的是一幀數據，但是對於接收rtp PS流，這裏每次回調只有一個rtp包數據。我們也可以通過修改參數，讓fTo每次回調有一幀數據，但是當數據源的rtp包沒有mark標誌的時候這裏將一直無法被調用，因爲live底層認爲只有遇到mark標誌纔是新的一幀的開始。所以這裏還是得一個個rtp包處理，開發時我遇到過有些攝像機得出PS流就沒有mark標誌。所以在這裏說明下這個接口主要工作：
1.將收到的rtp數據保存在緩衝中，暫時不傳遞給下一個souce，等到發現有PS頭的時候再將完整一幀的數據回調給下一個source或sink。
2.去掉PS頭，PES頭，解析PES包裏面的PTS，在這裏我用的是rtp包的時間戳，PES包的時間戳轉發出去的流感覺不是很流暢。
3.將流數據的nalu起始碼去掉，將完整的裸流數據回調給H264VideoStreamDiscreteFramer處理即可

還有一個地方也需要注意，對於裸流的sps,pps,vps等視頻參數信息，有時候是放到一個PS包裏面的，但是這個接口每次又只能回調一幀數據，也就是說對於sps,pps每次只能回調一個。所以這裏我實現了一個數組這些數據，等到下一次回調的時候，再把緩衝的sps,pps數據傳給後面的source。上面的doGetNextFrame函數很好說明了這個機制，每次判斷緩衝是否有數據，如果有則將緩衝數據回調下去。