最近做項目用到Android native層的MediaCodec的接口對H264進行解碼，通過在解碼前和解碼後加打印日誌，發現解碼耗時200多ms，和IOS的解碼耗時10ms相比實在是延時好大。後來研究了兩週也沒能解決延時問題，很悲慘……不過還是將這過程中分析的思路記錄下來，說不定以後萬一靈感來了就解決了呢。
起初在https://software.intel.com/en-us/forums/android-applications-on-intel-architecture/topic/536355和https://software.intel.com/en-us/android/articles/android-hardware-codec-mediacodec這兩個網址中找到問題原因和一種解決方法。
如果編碼器類型是H.264，MediaCodec可能會有幾幀的延遲（對於IA平臺，它是7幀，而另一些則是3-5幀）。如果幀速率爲30 FPS，則7幀將具有200毫秒的延遲。什麼導致了硬件解碼器的延遲？原因可能是主或高配置文件有一個B幀，並且如果當B幀引用後續緩衝區時解碼器沒有緩衝區，則該播放會短暫停止。英特爾已經考慮了硬件解碼器的這種情況，併爲主要或高配置文件提供了7個緩衝區，但是由於基線沒有B幀，因此將其減少到基線的零緩衝區。其他供應商可能爲所有配置文件使用4個緩衝區。
因此，在基於IA的Android平臺上，解決方案是將編碼器更改爲基線配置文件。如果編碼器配置文件無法更改，則可行的解決方法是更改解碼器端的配置數據。通常，配置文件位於第五個字節中，因此將基準配置文件更改爲66會在IA平臺上產生最低延遲。也就是說要想低延時，必須：
- 不要設置MediaFormat.KEY_MAX_WIDTH和MediaFormat.KEY_MAX_HEIGHT。
- 將第一個SPS中的配置文件切換到基線，然後在第一個PPS後以正確的配置文件重新提交。
先附上解碼部分代碼：
int OpenMetaCUDAVideoDecoderContext::OnVideoH264(uint8_t * frame, uint32_t frameSize){
DP_MSG("onVideoData=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
frameSize,
frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7]
);
int pos = 0;
if( (frame[4] & 0X1F ) == 0X09 ){
// this is an iframe
frame += 6;
frameSize -= 6;
}
if( (frame[4] & 0X1F ) == 0X07 ){
// this is an iframe
iFrameFound = true;
}
else if(!iFrameFound){
// iframe not found, do nothing
return -1;
}
DP_MSG("video264::::%d",frameSize)
uint8_t *data = NULL;
uint8_t *pps = NULL;
uint8_t *sps = NULL;
// I know what my H.264 data source's NALUs look like so I know start code index is always 0.
// if you don't know where it starts, you can use a for loop similar to how i find the 2nd and 3rd start codes
int currentIndex = 0;
long blockLength = 0;
int nalu_type = (frame[currentIndex + 4] & 0x1F);
// if we havent already set up our format description with our SPS PPS parameters, we
// can't process any frames except type 7 that has our parameters
if (nalu_type != 7 && formatDesc == NULL)
{
return -1;
}
if (nalu_type == 7)
{
currentIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
size_t spsSize = currentIndex;
// find what the second NALU type is
nalu_type = (frame[currentIndex + 4] & 0x1F);
DP_MSG("onVideoDataSPs=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
frameSize,
frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7]
);
// type 8 is the PPS parameter NALU
if(nalu_type == 8){
int nextIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
size_t ppsSize = nextIndex - currentIndex;
DP_MSG("onVideoDataPPs=%6d, %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X %.2X\n",
frameSize,
frame[0],frame[1],frame[2],frame[3],frame[4],frame[5],frame[6],frame[7],
frame[spsSize+1],frame[spsSize+2],frame[spsSize+3],frame[spsSize+4]
);
if(decompressionSession == NULL) {
formatDesc = AMediaFormat_new();
AMediaFormat_setString(formatDesc, "mime", "video/avc");
AMediaFormat_setInt32(formatDesc, AMEDIAFORMAT_KEY_WIDTH, 1920); // 視頻寬度
AMediaFormat_setInt32(formatDesc, AMEDIAFORMAT_KEY_HEIGHT, 1088); // 視頻高度
sps = (uint8_t *)malloc(spsSize);
pps = (uint8_t *)malloc(ppsSize);
memcpy (sps, &frame[0], spsSize);
memcpy (pps, &frame[spsSize], ppsSize);
AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps
DP_MSG("createDecompSession ========= ")
this->createDecompSession();
if(sps != NULL)
{
free(sps);
sps = NULL;
}
if(pps != NULL)
{
free(pps);
pps = NULL;
}
}
// now lets handle the IDR frame that (should) come after the parameter sets
// I say "should" because that's how I expect my H264 stream to work, YMMV
currentIndex = nextIndex;
nalu_type = (frame[currentIndex + 4] & 0x1F);
while(currentIndex != -1 && nalu_type != 5 && nalu_type != 1){
currentIndex = H264_findStartCode(frame,frameSize,currentIndex+4);
nalu_type = (frame[currentIndex + 4] & 0x1F);
}
}
}
int32_t afout = 0;
nalu_type = 5; // temp
// type 5 is an IDR frame NALU. The SPS and PPS NALUs should always be followed by an IDR (or IFrame) NALU, as far as I know
if(nalu_type == 5)
{
DP_MSG("jniPlayTs size=%d and %d \n",AMediaFormat_getInt32(formatDesc,AMEDIAFORMAT_KEY_COLOR_FORMAT,&afout),afout);
//DP_MSG("theNativeWindow format %p and %d \n",theNativeWindow,ANativeWindow_getFormat(theNativeWindow));
// find the offset, or where the SPS and PPS NALUs end and the IDR frame NALU begins
int offset = currentIndex;//
blockLength = frameSize - offset;
data = &frame[offset];
DP_MSG("nalu_type ===5 ")
//DP_MSG("nalu_type ===5 %s /n ",AMediaFormat_toString(formatDesc))
if(decompressionSession == nullptr)
{
DP_MSG("decompressionSession is null")
}
DP_MSG("OpenMetaCUDAVideoDecoder Start decoding: frameIdentifier=%lld",frameIdentifier);
avx_info("OpenMetaCUDAVideoDecoder Start decoding: ","frameIdentifier=%lld",frameIdentifier);
static int64_t llPts = 0;
ssize_t index = -1;
index = AMediaCodec_dequeueInputBuffer(decompressionSession, 10000);
if(index>=0)
{
size_t bufsize;
auto buf = AMediaCodec_getInputBuffer(decompressionSession, index, &bufsize);
if(buf!= nullptr )
{
DP_MSG("buf!= nullptr5======ww")
memcpy(buf,data,blockLength);
AMediaCodec_queueInputBuffer(decompressionSession, index, 0, blockLength, (uint64_t)frameIdentifier, 0);
llPts += 3000;
}
}
AMediaCodecBufferInfo info;
auto status = AMediaCodec_dequeueOutputBuffer(decompressionSession, &info, 0);
if (status >= 0) {
if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
DP_MSG("output EOS");
}
size_t outsize = 0;
u_int8_t * outData = AMediaCodec_getOutputBuffer(decompressionSession,status,&outsize);
auto formatType = AMediaCodec_getOutputFormat(decompressionSession);
DP_MSG("format formatType to: %s", AMediaFormat_toString(formatType));
int colorFormat = 0;
int width = 0;
int height = 0;
AMediaFormat_getInt32(formatType,"color-format",&colorFormat);
AMediaFormat_getInt32(formatType,"width",&width);
AMediaFormat_getInt32(formatType,"height",&height);
DP_MSG("format color-format : %d width ：%d height :%d", colorFormat,width,height);
AMediaFormat_delete(formatType);
DP_MSG("OpenMetaCUDAVideoDecoder End decoding: frameIdentifier=%lld,%d,%d,%p",info.presentationTimeUs,info.size,outsize,outData)
if(this->kController){
//OpenMetaPixelBuffer _kPixelBuffer(pixelBuffer,sizeof(pixelBuffer));
OpenMetaPixelBuffer _kPixelBuffer(outData,sizeof(outData));
_kPixelBuffer.kLine[0] = outData;
_kPixelBuffer.kLine[1] = outData + width * height;
_kPixelBuffer.kSizeX = width;
_kPixelBuffer.kSizeY = height;
_kPixelBuffer.kDataType = 1;
_kPixelBuffer.kTimeStamp = info.presentationTimeUs;
if(this!=NULL && this->kController!=NULL)
{
this->kController->OnVideoImage(&_kPixelBuffer);
}
}
avx_info("OpenMetaCUDAVideoDecoder End decoding:","%lld,%d,%d,%p",info.presentationTimeUs,info.size,outsize,outData);
AMediaCodec_releaseOutputBuffer(decompressionSession, status, info.size != 0);
} else if (status == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) {
DP_MSG("output buffers changed");
} else if (status == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
auto format = AMediaCodec_getOutputFormat(decompressionSession);
DP_MSG("format changed to: %s", AMediaFormat_toString(format));
AMediaFormat_delete(format);
} else if (status == AMEDIACODEC_INFO_TRY_AGAIN_LATER) {
DP_MSG("no output buffer right now");
} else {
DP_MSG("unexpected info code: %zd", status);
}
}
return 0;
}
針對查到的解決辦法，對以上代碼進行了修改，分別是：
//將第一個sps的改爲基線配置
sps = (uint8_t *)malloc(spsSize);
sps[0+5] = 66;
memcpy (sps, &frame[0], spsSize);
pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);
AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps
//拷貝兩份sps，將第一個sps的改爲基線配置
sps = (uint8_t *)malloc(spsSize * 2);
memcpy (sps, &frame[0], spsSize);
sps[0+5] = 66;
memcpy (sps + spsSize, &frame[0], spsSize);
pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);
AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize * 2); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps
//拷貝兩份sps，將第二份的第一個sps的改爲基線配置
sps = (uint8_t *)malloc(spsSize * 2);
memcpy (sps, &frame[0], spsSize);
memcpy (sps + spsSize, &frame[0], spsSize);
sps[spsSize+5] = 66;
pps = (uint8_t *)malloc(ppsSize);
memcpy (pps, &frame[spsSize], ppsSize);
AMediaFormat_setBuffer(formatDesc, "csd-0", sps, spsSize * 2); // sps
AMediaFormat_setBuffer(formatDesc, "csd-1", pps, ppsSize); // pps
通過查看打印的開始解碼與結束解碼的時間發現，幾種方法的時間差距並不是很大，依舊是200-250ms左右。
---------------------
作者：珠雨妮兒
來源：CSDN
原文：https://blog.csdn.net/zhuyunier/article/details/79730872
版權聲明：本文爲博主原創文章，轉載請附上博文鏈接！