隨着VR類、遊戲類場景的快速發展,開發者對Unity3d低延遲的直播需求量越來越大,前兩年,大牛直播SDK發佈了Windows平臺、Android平臺和iOS平臺的Unity3d RTMP和RTSP的播放,好多公司用起來體驗都非常好,以下介紹大概實現流程。
本文以Android平臺爲例,我們的實現:基於大牛直播SDK現有非常成熟的native RTMP和RTSP播放模塊,回調解碼後的原始數據,傳遞給Unity3d,實現相應的繪製即可,對應demo,可以參考 Github。
具體步驟如下:
1. Native RTSP或RTSP直播播放SDK回調RGB/YUV420/NV12等其中的一種未壓縮的圖像格式;
2. Unity3D創建相應的RGB/YUV420等Shader;
3.Unity3D從各個平臺獲取圖像數據來填充紋理即可。
以Android平臺爲例,我們在原有接口基礎上,做了橋接接口,供Unity平臺使用:
/// <summary>
/// Init
/// </summary>
public int NT_U3D_Init()
{
return DANIULIVE_RETURN_OK;
}
/// <summary>
/// 開始
/// 返回播放句柄
/// </summary>
public long NT_U3D_Open()
{
if ( 0 != player_obj_.Call<int>("Init", java_obj_cur_activity_) )
{
return 0;
}
return player_obj_.Call<long>("Open");
}
/// <summary>
/// Register Game Object,用於消息傳遞
/// </summary>
public int NT_U3D_Set_Game_Object(long handle, string gameObjectName)
{
return player_obj_.Call<int>("SetGameObject", handle, gameObjectName);
}
/// <summary>
/// 設置H.264解碼方式 false 軟件解碼 true 硬件解碼 默認爲false
/// </summary>
/// <param name="isHwDecoder"></param>
public int NT_U3D_SetVideoDecoderMode(long handle, int isHwDecoder)
{
return player_obj_.Call<int>("SetPlayerVideoHWDecoder", handle, isHwDecoder);
}
/// <summary>
/// 設置H.265 解碼方式 false 軟件解碼 true 硬件解碼 默認爲false
/// </summary>
/// <param name="isHevcHwDecoder"></param>
public int NT_U3D_SetVideoHevcDecoderMode(long handle, int isHevcHwDecoder)
{
return player_obj_.Call<int>("SetPlayerVideoHevcHWDecoder", handle, isHevcHwDecoder);
}
/// <summary>
/// 設置音頻輸出模式: if 0: 自動選擇; if with 1: audiotrack模式
/// </summary>
/// <param name="use_audiotrack"></param>
public int NT_U3D_SetAudioOutputType(long handle, int use_audiotrack)
{
return player_obj_.Call<int>("SetAudioOutputType", handle, use_audiotrack);
}
/// <summary>
/// 設置播放端緩存大小, 默認200毫秒
/// </summary>
/// <param name="buffer"></param>
public int NT_U3D_SetBuffer(long handle, int buffer)
{
return player_obj_.Call<int>("SetBuffer", handle, buffer);
}
/// <summary>
/// 接口可實時調用:設置是否實時靜音,1:靜音; 0: 取消靜音
/// </summary>
/// <param name="is_mute"></param>
public int NT_U3D_SetMute(long handle, int is_mute)
{
return player_obj_.Call<int>("SetMute", handle, is_mute);
}
/// <summary>
/// 設置RTSP TCP模式, 1: TCP; 0: UDP
/// </summary>
/// <param name="is_using_tcp"></param>
public int NT_U3D_SetRTSPTcpMode(long handle, int is_using_tcp)
{
return player_obj_.Call<int>("SetRTSPTcpMode", handle, is_using_tcp);
}
/// <summary>
/// 設置RTSP超時時間, timeout單位爲秒,必須大於0
/// </summary>
/// <param name="timeout"></param>
public int NT_U3D_SetRTSPTimeout(long handle, int timeout)
{
return player_obj_.Call<int>("SetRTSPTimeout", handle, timeout);
}
/// <summary>
/// 設置RTSP TCP/UDP自動切換
/// NOTE: 對於RTSP來說,有些可能支持rtp over udp方式,有些可能支持使用rtp over tcp方式.
/// 爲了方便使用,有些場景下可以開啓自動嘗試切換開關, 打開後如果udp無法播放,sdk會自動嘗試tcp, 如果tcp方式播放不了,sdk會自動嘗試udp.
/// </summary>
/// <param name="timeout"></param>
/// timeout:如果設置1的話, sdk將在tcp和udp之間嘗試切換播放,如果設置爲0,則不嘗試切換.
public int NT_U3D_SetRTSPAutoSwitchTcpUdp(long handle, int is_auto_switch_tcp_udp)
{
return player_obj_.Call<int>("SetRTSPAutoSwitchTcpUdp", handle, is_auto_switch_tcp_udp);
}
/// <summary>
/// 設置快速啓動該模式,
/// </summary>
/// <param name="is_fast_startup"></param>
public int NT_U3D_SetFastStartup(long handle, int is_fast_startup)
{
return player_obj_.Call<int>("SetFastStartup", handle, is_fast_startup);
}
/// <summary>
/// 設置超低延遲模式 false不開啓 true開啓 默認false
/// </summary>
/// <param name="mode"></param>
public int NT_U3D_SetPlayerLowLatencyMode(long handle, int mode)
{
return player_obj_.Call<int>("SetPlayerLowLatencyMode", handle, mode);
}
/// <summary>
/// 設置視頻垂直反轉
/// is_flip: 0: 不反轉, 1: 反轉
/// </summary>
/// <param name="is_flip"></param>
public int NT_U3D_SetFlipVertical(long handle, int is_flip)
{
return player_obj_.Call<int>("SetFlipVertical", handle, is_flip);
}
/// <summary>
/// 設置視頻水平反轉
/// is_flip: 0: 不反轉, 1: 反轉
/// </summary>
/// <param name="is_flip"></param>
public int NT_U3D_SetFlipHorizontal(long handle, int is_flip)
{
return player_obj_.Call<int>("SetFlipHorizontal", handle, is_flip);
}
/// <summary>
/// 設置順時針旋轉, 注意除了0度之外, 其他角度都會額外消耗性能
/// degress: 當前支持 0度,90度, 180度, 270度 旋轉
/// </summary>
/// <param name="degress"></param>
public int NT_U3D_SetRotation(long handle, int degress)
{
return player_obj_.Call<int>("SetRotation", handle, degress);
}
/// <summary>
/// 設置是否回調下載速度
/// is_report: if 1: 上報下載速度, 0: 不上報.
/// report_interval: 上報間隔,以秒爲單位,>0.
/// </summary>
/// <param name="is_report"></param>
/// <param name="report_interval"></param>
public int NT_U3D_SetReportDownloadSpeed(long handle, int is_report, int report_interval)
{
return player_obj_.Call<int>("SetReportDownloadSpeed", handle, is_report, report_interval);
}
/// <summary>
/// 設置是否需要在播放或錄像過程中快照
/// </summary>
/// <param name="is_save_image"></param>
public int NT_U3D_SetSaveImageFlag(long handle, int is_save_image)
{
return player_obj_.Call<int>("SetSaveImageFlag", handle, is_save_image);
}
/// <summary>
/// 播放或錄像過程中快照
/// </summary>
/// <param name="imageName"></param>
public int NT_U3D_SaveCurImage(long handle, string imageName)
{
return player_obj_.Call<int>("SaveCurImage", handle, imageName);
}
/// <summary>
/// 播放或錄像過程中,快速切換url
/// </summary>
/// <param name="uri"></param>
public int NT_U3D_SwitchPlaybackUrl(long handle, string uri)
{
return player_obj_.Call<int>("SwitchPlaybackUrl", handle, uri);
}
/// <summary>
/// 創建錄像存儲路徑
/// </summary>
/// <param name="path"></param>
public int NT_U3D_CreateFileDirectory(string path)
{
return player_obj_.Call<int>("CreateFileDirectory", path);
}
/// <summary>
/// 設置錄像存儲路徑
/// </summary>
/// <param name="path"></param>
public int NT_U3D_SetRecorderDirectory(long handle, string path)
{
return player_obj_.Call<int>("SetRecorderDirectory", handle, path);
}
/// <summary>
/// 設置單個錄像文件大小
/// </summary>
/// <param name="size"></param>
public int NT_U3D_SetRecorderFileMaxSize(long handle, int size)
{
return player_obj_.Call<int>("SetRecorderFileMaxSize", handle, size);
}
/// <summary>
/// 設置錄像時音頻轉AAC編碼的開關
/// aac比較通用,sdk增加其他音頻編碼(比如speex, pcmu, pcma等)轉aac的功能.
/// 注意: 轉碼會增加性能消耗
/// </summary>
/// <param name="is_transcode"></param>
/// is_transcode:設置爲1的話,如果音頻編碼不是aac,則轉成aac,如果是aac,則不做轉換. 設置爲0的話,則不做任何轉換. 默認是0.
public int NT_U3D_SetRecorderAudioTranscodeAAC(long handle, int is_transcode)
{
return player_obj_.Call<int>("SetRecorderAudioTranscodeAAC", handle, is_transcode);
}
/// <summary>
/// 設置播放路徑
/// </summary>
public int NT_U3D_SetUrl(long handle, string url)
{
return player_obj_.Call<int>("SetUrl", handle, url);
}
/// <summary>
/// 開始播放
/// </summary>
public int NT_U3D_StartPlay(long handle)
{
return player_obj_.Call<int>("StartPlay", handle);
}
/// <summary>
/// 獲取YUV數據
/// </summary>
public AndroidJavaObject NT_U3D_GetVideoFrame(long handle)
{
return player_obj_.Call<AndroidJavaObject>("GetVideoFrame", handle);
}
/// <summary>
/// 停止播放
/// </summary>
public int NT_U3D_StopPlay(long handle)
{
return player_obj_.Call<int>("StopPlay", handle);
}
/// <summary>
/// 開始錄像
/// </summary>
public int NT_U3D_StartRecorder(long handle)
{
return player_obj_.Call<int>("StartRecorder", handle);
}
/// <summary>
/// 停止錄像
/// </summary>
public int NT_U3D_StopRecorder(long handle)
{
return player_obj_.Call<int>("StopRecorder", handle);
}
/// <summary>
/// 關閉播放
/// </summary>
public int NT_U3D_Close(long handle)
{
return player_obj_.Call<int>("Close", handle);
}
/// <summary>
/// UnInit Player
/// </summary>
public int NT_U3D_UnInit()
{
return DANIULIVE_RETURN_OK;
}
Event傳遞,我們在unity3d裏面解析相應的字串即可:
/// <summary>
/// android 傳遞過來 code
/// </summary>
/// <param name="code"></param>
public void onNTSmartEvent(string param)
{
if (!param.Contains(","))
{
Debug.Log("[onNTSmartEvent] android傳遞參數錯誤");
return;
}
string[] strs = param.Split(',');
string player_handle =strs[0];
string code = strs[1];
string param1 = strs[2];
string param2 = strs[3];
string param3 = strs[4];
string param4 = strs[5];
Debug.Log("[onNTSmartEvent] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16));
switch (Convert.ToInt32(code))
{
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
Debug.Log("開始。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
Debug.Log("連接中。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
Debug.Log("連接失敗。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
Debug.Log("連接成功。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
Debug.Log("連接斷開。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
Debug.Log("停止播放。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
Debug.Log("分辨率信息: width: " + Convert.ToInt32(param1) + ", height: " + Convert.ToInt32(param2));
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
Debug.Log("收不到媒體數據,可能是url錯誤。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
Debug.Log("切換播放URL。。");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
Debug.Log("快照: " + param1 + " 路徑:" + param3);
if (Convert.ToInt32(param1) == 0)
{
Debug.Log("截取快照成功。.");
}
else
{
Debug.Log("截取快照失敗。.");
}
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
Debug.Log("[record]開始一個新的錄像文件 : " + param3);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
Debug.Log("[record]已生成一個錄像文件 : " + param3);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
Debug.Log("Start_Buffering");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
Debug.Log("Buffering: " + Convert.ToInt32(param1));
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
Debug.Log("Stop_Buffering");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
Debug.Log("download_speed:" + param1 + "Byte/s" + ", "
+ (Convert.ToInt32(param1) * 8 / 1000) + "kbps" + ", " + (Convert.ToInt32(param1) / 1024)
+ "KB/s");
break;
}
}
開始播放:
public void Play()
{
if (is_running)
{
Debug.Log("已經在播放。。");
return;
}
//獲取輸入框的url
string url = input_url_.text.Trim();
if (!url.StartsWith("rtmp://") && !url.StartsWith("rtsp://"))
{
videoUrl = "rtmp://202.69.69.180:443/webcast/bshdlive-pc";
}
else
{
videoUrl = url;
}
OpenPlayer();
if ( player_handle_ == 0 )
return;
NT_U3D_Set_Game_Object(player_handle_, game_object_);
NT_U3D_SetUrl(player_handle_, videoUrl);
/* ++ 播放前參數配置可加在此處 ++ */
int is_using_tcp = 0; //TCP/UDP模式設置
NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp);
int is_report = 0;
int report_interval = 1;
NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval); //下載速度回調
NT_U3D_SetBuffer(player_handle_, play_buffer_time_); //設置buffer time
NT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_ ? 1 : 0); //設置是否啓用低延遲模式
NT_U3D_SetMute(player_handle_, is_mute_ ? 1 : 0); //是否啓動播放的時候靜音
NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //設置H.264軟硬解模式
NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); //設置H.265軟硬解模式
int is_fast_startup = 1;
NT_U3D_SetFastStartup(player_handle_, is_fast_startup); //設置快速啓動模式
int rtsp_timeout = 10;
NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout); //設置RTSP超時時間
int is_auto_switch_tcp_udp = 1;
NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp); //設置TCP/UDP模式自動切換
/* -- 播放前參數配置可加在此處 -- */
int flag = NT_U3D_StartPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
is_need_get_frame_ = true;
Debug.Log("播放成功");
}
else
{
is_need_get_frame_ = false;
Debug.LogError("播放失敗");
}
is_running = true;
}
開始播放後,native模塊,回調yuv/rgb數據,unity3d模塊,做相應的繪製處理即可。
private void Update()
{
if (!is_need_get_frame_)
return;
if (player_handle_ == 0)
return;
AndroidJavaObject u3d_video_frame_obj = NT_U3D_GetVideoFrame(player_handle_);
if (u3d_video_frame_obj == null)
return;
VideoFrame converted_video_frame = ConvertToVideoFrame(u3d_video_frame_obj);
if (converted_video_frame == null)
return;
if ( !is_need_init_texture_)
{
if (converted_video_frame.width_ != video_width_
|| converted_video_frame.height_ != video_height_
|| converted_video_frame.y_stride_ != y_row_bytes_
|| converted_video_frame.u_stride_ != u_row_bytes_
|| converted_video_frame.v_stride_ != v_row_bytes_)
{
is_need_init_texture_ = true;
}
}
if (is_need_init_texture_)
{
if (InitYUVTexture(converted_video_frame))
{
is_need_init_texture_ = false;
}
}
UpdateYUVTexture(converted_video_frame);
converted_video_frame = null;
}
關閉播放:
private void ClosePlayer()
{
is_need_get_frame_ = false;
is_need_init_texture_ = false;
int flag = NT_U3D_StopPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("停止成功");
}
else
{
Debug.LogError("停止失敗");
}
flag = NT_U3D_Close(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("關閉成功");
}
else
{
Debug.LogError("關閉失敗");
}
player_handle_ = 0;
NT_U3D_UnInit();
is_running = false;
}
以上是大概的實現流程,Unity環境下,實際測試時延,和 native直播模塊相差無幾,都是毫秒級延遲。Windows平臺和iOS平臺也是如此。