With the rapid development of VR and game scenes, developers have an increasing demand for Unity3d low-delay live streaming. In the past two years, Niu Live SDK released the playback of Unity3d RTMP and RTSP on Windows platform, Android platform and iOS platform, and many companies have very good experience when using it. The following is an overview of the implementation process.
In this paper, the Android platform is taken as an example. Our implementation is based on the existing mature native RTMP and RTSP playback modules of Daniu Live SDK, and the original data after the mediation code is passed to Unity3d to realize the corresponding rendering. For the corresponding demo, please refer to Github.
The specific steps are as follows:
1. Native RTSP or RTSP live playback SDK callback RGB/YUV420/NV12 one of the uncompressed image formats;
2. Unity3D creates the corresponding RGB/YUV420 Shader;
3.Unity3D can obtain image data from various platforms to fill the texture.
Taking Android platform as an example, we have made a bridge interface for Unity platform on the basis of the original interface:
/// <summary>
/// Init
/// </summary>
public int NT_U3D_Init()
{
return DANIULIVE_RETURN_OK;
}
/// <summary>
///start
///Returns the play handle
/// </summary>
public long NT_U3D_Open()
{
if ( 0! = player_obj_.Call<int> ("Init", java_obj_cur_activity_) )
{
return 0;
}
return player_obj_.Call<long> ("Open");
}
/// <summary>
///Register Game Object for messaging
/// </summary>
public int NT_U3D_Set_Game_Object(long handle, string gameObjectName)
{
return player_obj_.Call<int> ("SetGameObject", handle, gameObjectName);
}
/// <summary>
///Set the h.264 decoding mode. False Software decoding true Hardware decoding The default value is False
/// </summary>
/// <param name="isHwDecoder"></param>
public int NT_U3D_SetVideoDecoderMode(long handle, int isHwDecoder)
{
return player_obj_.Call<int> ("SetPlayerVideoHWDecoder", handle, isHwDecoder);
}
/// <summary>
///H.265 Decoding mode False Software decoding true Hardware decoding The default value is False
/// </summary>
/// <param name="isHevcHwDecoder"></param>
public int NT_U3D_SetVideoHevcDecoderMode(long handle, int isHevcHwDecoder)
{
return player_obj_.Call<int> ("SetPlayerVideoHevcHWDecoder", handle, isHevcHwDecoder);
}
/// <summary>
///Set the audio output mode: if 0: Automatic selection; If with 1: indicates the AudioTrack mode
/// </summary>
/// <param name="use_audiotrack"></param>
public int NT_U3D_SetAudioOutputType(long handle, int use_audiotrack)
{
return player_obj_.Call<int> ("SetAudioOutputType", handle, use_audiotrack);
}
/// <summary>
///Set the size of the playback side cache. The default size is 200 milliseconds
/// </summary>
/// <param name="buffer"></param>
public int NT_U3D_SetBuffer(long handle, int buffer)
{
return player_obj_.Call<int> ("SetBuffer", handle, buffer);
}
/// <summary>
///The interface can be called in real time: set real-time mute or not: 1: mute. 0: Unmute
/// </summary>
/// <param name="is_mute"></param>
public int NT_U3D_SetMute(long handle, int is_mute)
{
return player_obj_.Call<int> ("SetMute", handle, is_mute);
}
/// <summary>
///Set the RTSP TCP mode to 1: TCP. 0: UDP
/// </summary>
/// <param name="is_using_tcp"></param>
public int NT_U3D_SetRTSPTcpMode(long handle, int is_using_tcp)
{
return player_obj_.Call<int> ("SetRTSPTcpMode", handle, is_using_tcp);
}
/// <summary>
///Set the RTSP timeout period. The unit of timeout is second and must be greater than 0
/// </summary>
/// <param name="timeout"></param>
public int NT_U3D_SetRTSPTimeout(long handle, int timeout)
{
return player_obj_.Call<int> ("SetRTSPTimeout", handle, timeout);
}
/// <summary>
///The RTSP TCP/UDP switchover is set
/// NOTE:For RTSP, some may support RTP over UDP, while others may support RTP over TCP.
///For convenience, you can enable the automatic attempt switch in some scenarios. If UDP cannot be played, THE SDK will automatically try TCP. If TCP cannot be played, the SDK will automatically try UDP.
/// </summary>
/// <param name="timeout"></param>
///Timeout: If set to 1, SDK will try to switch between TCP and UDP. If set to 0, SDK will not try to switch.
public int NT_U3D_SetRTSPAutoSwitchTcpUdp(long handle, int is_auto_switch_tcp_udp)
{
return player_obj_.Call<int> ("SetRTSPAutoSwitchTcpUdp", handle, is_auto_switch_tcp_udp);
}
/// <summary>
///Set the quick start mode,
/// </summary>
/// <param name="is_fast_startup"></param>
public int NT_U3D_SetFastStartup(long handle, int is_fast_startup)
{
return player_obj_.Call<int> ("SetFastStartup", handle, is_fast_startup);
}
/// <summary>
///Set the ultra-low latency mode false Disable true Enable the default false
/// </summary>
/// <param name="mode"></param>
public int NT_U3D_SetPlayerLowLatencyMode(long handle, int mode)
{
return player_obj_.Call<int> ("SetPlayerLowLatencyMode", handle, mode);
}
/// <summary>
///Set the video to vertical inversion
///Is_flip: 0: do not flip, 1: flip
/// </summary>
/// <param name="is_flip"></param>
public int NT_U3D_SetFlipVertical(long handle, int is_flip)
{
return player_obj_.Call<int> ("SetFlipVertical", handle, is_flip);
}
/// <summary>
///Set the video to horizontal inversion
///Is_flip: 0: do not flip, 1: flip
/// </summary>
/// <param name="is_flip"></param>
public int NT_U3D_SetFlipHorizontal(long handle, int is_flip)
{
return player_obj_.Call<int> ("SetFlipHorizontal", handle, is_flip);
}
/// <summary>
///Set the rotation clockwise, and note that any Angle other than 0 degrees will cost extra performance
///Degress: The rotation at 0, 90, 180, or 270 degrees is supported
/// </summary>
/// <param name="degress"></param>
public int NT_U3D_SetRotation(long handle, int degress)
{
return player_obj_.Call<int> ("SetRotation", handle, degress);
}
/// <summary>
///Set whether to call back the download speed
///Is_report: if 1: The download speed is reported. 0: the download speed is not reported.
///Report_interval: indicates the reporting interval in seconds. The value is greater than 0.
/// </summary>
/// <param name="is_report"></param>
/// <param name="report_interval"></param>
public int NT_U3D_SetReportDownloadSpeed(long handle, int is_report, int report_interval)
{
return player_obj_.Call<int> ("SetReportDownloadSpeed", handle, is_report, report_interval);
}
/// <summary>
///Set whether to take a snapshot during playback or recording
/// </summary>
/// <param name="is_save_image"></param>
public int NT_U3D_SetSaveImageFlag(long handle, int is_save_image)
{
return player_obj_.Call<int> ("SetSaveImageFlag", handle, is_save_image);
}
/// <summary>
///A snapshot is taken during playback or recording
/// </summary>
/// <param name="imageName"></param>
public int NT_U3D_SaveCurImage(long handle, string imageName)
{
return player_obj_.Call<int> ("SaveCurImage", handle, imageName);
}
/// <summary>
///Quickly switch urls during playback or video recording
/// </summary>
/// <param name="uri"></param>
public int NT_U3D_SwitchPlaybackUrl(long handle, string uri)
{
return player_obj_.Call<int> ("SwitchPlaybackUrl", handle, uri);
}
/// <summary>
///Example Create a video storage path
/// </summary>
/// <param name="path"></param>
public int NT_U3D_CreateFileDirectory(string path)
{
return player_obj_.Call<int> ("CreateFileDirectory", path);
}
/// <summary>
///Set the video storage path
/// </summary>
/// <param name="path"></param>
public int NT_U3D_SetRecorderDirectory(long handle, string path)
{
return player_obj_.Call<int> ("SetRecorderDirectory", handle, path);
}
/// <summary>
///Set the size of a single video file
/// </summary>
/// <param name="size"></param>
public int NT_U3D_SetRecorderFileMaxSize(long handle, int size)
{
return player_obj_.Call<int> ("SetRecorderFileMaxSize", handle, size);
}
/// <summary>
///Set the switch of audio to AAC coding when recording
///Aac is more general, SDK adds other audio encoding (such as Speex, PCMU, PCMA, etc.) to AAC function.
///Note: Transcoding increases performance costs
/// </summary>
/// <param name="is_transcode"></param>
///Is_transcode: If set to 1, if the audio code is not AAC, it will be converted to AAC, if it is aAC, it will not be converted. If set to 0, no conversion is done. The default is 0.
public int NT_U3D_SetRecorderAudioTranscodeAAC(long handle, int is_transcode)
{
return player_obj_.Call<int> ("SetRecorderAudioTranscodeAAC", handle, is_transcode);
}
/// <summary>
///Setting the play path
/// </summary>
public int NT_U3D_SetUrl(long handle, string url)
{
return player_obj_.Call<int> ("SetUrl", handle, url);
}
/// <summary>
///Start playing
/// </summary>
public int NT_U3D_StartPlay(long handle)
{
return player_obj_.Call<int> ("StartPlay", handle);
}
/// <summary>
///Get YUV data
/// </summary>
public AndroidJavaObject NT_U3D_GetVideoFrame(long handle)
{
return player_obj_.Call<AndroidJavaObject>("GetVideoFrame", handle);
}
/// <summary>
///Stop playing
/// </summary>
public int NT_U3D_StopPlay(long handle)
{
return player_obj_.Call<int> ("StopPlay", handle);
}
/// <summary>
///Start the video
/// </summary>
public int NT_U3D_StartRecorder(long handle)
{
return player_obj_.Call<int> ("StartRecorder", handle);
}
/// <summary>
///Stop the video
/// </summary>
public int NT_U3D_StopRecorder(long handle)
{
return player_obj_.Call<int> ("StopRecorder", handle);
}
/// <summary>
///Close play
/// </summary>
public int NT_U3D_Close(long handle)
{
return player_obj_.Call<int> ("Close", handle);
}
/// <summary>
/// UnInit Player
/// </summary>
public int NT_U3D_UnInit()
{
return DANIULIVE_RETURN_OK;
}
Copy the codeWe can parse the corresponding string in unity3D:
/// <summary>
///Android passes the code
/// </summary>
/// <param name="code"></param>
public void onNTSmartEvent(string param)
{
if(! param.Contains(","))
{
Debug.Log([onNTSmartEvent] Android passing parameter error");
return;
}
string[] strs = param.Split(', ');
string player_handle =strs[0];
string code = strs[1];
string param1 = strs[2];
string param2 = strs[3];
string param3 = strs[4];
string param4 = strs[5];
Debug.Log("[onNTSmartEvent] code: 0x" + Convert.ToString(Convert.ToInt32(code), 16));
switch (Convert.ToInt32(code))
{
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
Debug.Log("Here we go.);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
Debug.Log("Connecting.");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
Debug.Log("Connection failed.");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
Debug.Log("Connection successful.");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
Debug.Log("Connection is down.");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
Debug.Log("Stop playing.");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
Debug.Log("Width:" + Convert.ToInt32(param1) + ", height: " + Convert.ToInt32(param2));
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
Debug.Log("Cannot receive media data, may be url error...");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
Debug.Log("Switch play URL...");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
Debug.Log("Snapshot:" + param1 + "Path:" + param3);
if (Convert.ToInt32(param1) == 0)
{
Debug.Log("Snapshot capture succeeded. .");
}
else
{
Debug.Log("Failed to intercept a snapshot. .");
}
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
Debug.Log("[record] Starts a new video file:" + param3);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
Debug.Log([record] has generated a video file: + param3);
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
Debug.Log("Start_Buffering");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
Debug.Log("Buffering: " + Convert.ToInt32(param1));
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
Debug.Log("Stop_Buffering");
break;
case EVENTID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
Debug.Log("download_speed:" + param1 + "Byte/s" + ","
+ (Convert.ToInt32(param1) * 8 / 1000) + "kbps" + "," + (Convert.ToInt32(param1) / 1024)
+ "KB/s");
break; }}Copy the codeStart playing:
public void Play()
{
if (is_running)
{
Debug.Log("It's already on...");
return;
}
// Get the URL of the input box
string url = input_url_.text.Trim();
if(! url.StartsWith("rtmp://") && !url.StartsWith("rtsp://"))
{
videoUrl = "RTMP: / / 202.69.69.180:443 / webcasts/bshdlive - PC";
}
else
{
videoUrl = url;
}
OpenPlayer();
if ( player_handle_ == 0 )
return;
NT_U3D_Set_Game_Object(player_handle_, game_object_);
NT_U3D_SetUrl(player_handle_, videoUrl);
/* ++ Parameter Settings can be added here ++ */
int is_using_tcp = 0; // Set the TCP/UDP mode
NT_U3D_SetRTSPTcpMode(player_handle_, is_using_tcp);
int is_report = 0;
int report_interval = 1;
NT_U3D_SetReportDownloadSpeed(player_handle_, is_report, report_interval); // Download speed callback
NT_U3D_SetBuffer(player_handle_, play_buffer_time_); // Set the buffer time
NT_U3D_SetPlayerLowLatencyMode(player_handle_, is_low_latency_ ? 1 : 0); // Set whether to enable the low latency mode
NT_U3D_SetMute(player_handle_, is_mute_ ? 1 : 0); // Whether to mute playback
NT_U3D_SetVideoDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); // Set the h.264 soft/hard mode
NT_U3D_SetVideoHevcDecoderMode(player_handle_, is_hw_decode_ ? 1 : 0); // Set the h.265 soft/hard mode
int is_fast_startup = 1;
NT_U3D_SetFastStartup(player_handle_, is_fast_startup); // Set the quick start mode
int rtsp_timeout = 10;
NT_U3D_SetRTSPTimeout(player_handle_, rtsp_timeout); // Set RTSP timeout
int is_auto_switch_tcp_udp = 1;
NT_U3D_SetRTSPAutoSwitchTcpUdp(player_handle_, is_auto_switch_tcp_udp); // Set the TCP/UDP mode to automatically switch
/* -- Parameter Settings can be added here -- */
int flag = NT_U3D_StartPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
is_need_get_frame_ = true;
Debug.Log("Play successful");
}
else
{
is_need_get_frame_ = false;
Debug.LogError("Play failed");
}
is_running = true;
}
Copy the codeAfter playing, native module will call back YUV/RGB data, unity3D module, and do corresponding drawing processing.
private void Update()
{
if(! is_need_get_frame_)return;
if (player_handle_ == 0)
return;
AndroidJavaObject u3d_video_frame_obj = NT_U3D_GetVideoFrame(player_handle_);
if (u3d_video_frame_obj == null)
return;
VideoFrame converted_video_frame = ConvertToVideoFrame(u3d_video_frame_obj);
if (converted_video_frame == null)
return;
if ( !is_need_init_texture_)
{
if(converted_video_frame.width_ ! = video_width_ || converted_video_frame.height_ ! = video_height_ || converted_video_frame.y_stride_ ! = y_row_bytes_ || converted_video_frame.u_stride_ ! = u_row_bytes_ || converted_video_frame.v_stride_ ! = v_row_bytes_) { is_need_init_texture_ =true; }}if (is_need_init_texture_)
{
if (InitYUVTexture(converted_video_frame))
{
is_need_init_texture_ = false;
}
}
UpdateYUVTexture(converted_video_frame);
converted_video_frame = null;
}
Copy the codeClose play:
private void ClosePlayer()
{
is_need_get_frame_ = false;
is_need_init_texture_ = false;
int flag = NT_U3D_StopPlay(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("Stop success");
}
else
{
Debug.LogError("Stop failing");
}
flag = NT_U3D_Close(player_handle_);
if (flag == DANIULIVE_RETURN_OK)
{
Debug.Log("Closed successfully");
}
else
{
Debug.LogError("Shutdown failed");
}
player_handle_ = 0;
NT_U3D_UnInit();
is_running = false;
}
Copy the codeThe above is the general implementation process. In Unity environment, the actual test delay is similar to that of native live broadcast module, both of which are millisecond delay. The same is true for Windows and iOS.