Technical background

As the application scope of Unity3D becomes wider and wider, more and more industries begin to develop products based on Unity3D, such as virtual simulation education, aviation industry, interior design, urban planning, industrial simulation and other fields in traditional industries.

Based on this, a lot of developers are suffering under the environment of Unity, there is no low latency of push-pull flow solution, a few years ago, we launched a cross-platform low latency in Unity environment RTMP | RTSP live player, very good to solve a lot of delay demanding usage scenario.

As time goes by, more and more developers contact us, hoping that we can launch the RTMP push module in Unity environment, get real-time data of Unity, and realize data transmission push with lower delay and higher efficiency. Based on this, we released the RTMP push module in Unity environment.

This article takes Windows platform as an example, the data source is Unity window, camera or the whole screen, coding transmission module, or call Daniel live SDK (official) native interface, simple interface preview:

​

The technical implementation

1. Basic initialization

private bool InitSDK() { if (! Is_pusher_sdk_init_) {// set the log path (make sure the directory exists) String log_path = "D:\ pulisherlog"; NTSmartLog.NT_SL_SetPath(log_path); UInt32 isInited = NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero); if (isInited ! = 0) {debug.log (" call to NT_PB_Init failed.." ); return false; } is_pusher_sdk_init_ = true; } return true; }Copy the code

2. Call the Open() interface to get the push instance

public bool OpenPublisherHandle(uint video_option, uint audio_option) { if (publisher_handle_ ! = IntPtr.Zero) { return true; } publisher_handle_count_ = 0; if (NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_Open(out publisher_handle_, video_option, audio_option, 0, IntPtr.Zero)) { return false; } if (publisher_handle_ ! = IntPtr.Zero) { pb_event_call_back_ = new NT_PB_SDKEventCallBack(PbEventCallBack); NTSmartPublisherSDK.NT_PB_SetEventCallBack(publisher_handle_, IntPtr.Zero, pb_event_call_back_); return true; } else { return false; }}Copy the code

3. Initialize parameter Settings

One thing to note here is that if you want to capture the Unity window, you need to set the layer mode, first fill a layer with RGBA black background, then add a layer for overlaying external data.

private void SetCommonOptionToPublisherSDK() { if (! IsPublisherHandleAvailable()) { Debug.Log("SetCommonOptionToPublisherSDK, publisher handle with null.." ); return; } NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0, 0, IntPtr.Zero); If (video_option = = NTSmartPublisherDefine. NT_PB_E_VIDEO_OPTION. NT_PB_E_VIDEO_OPTION_LAYER) {/ / 0 layer filling RGBA rectangle, the purpose is to ensure the frame rate, Int red = 0; int green = 0; int blue = 0; int alpha = 255; NT_PB_RGBARectangleLayerConfig rgba_layer_c0 = new NT_PB_RGBARectangleLayerConfig(); rgba_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE; rgba_layer_c0.base_.index_ = 0; rgba_layer_c0.base_.enable_ = 1; rgba_layer_c0.base_.region_.x_ = 0; rgba_layer_c0.base_.region_.y_ = 0; rgba_layer_c0.base_.region_.width_ = video_width_; rgba_layer_c0.base_.region_.height_ = video_height_; rgba_layer_c0.base_.offset_ = Marshal.OffsetOf(rgba_layer_c0.GetType(), "base_").ToInt32(); rgba_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(rgba_layer_c0); rgba_layer_c0.red_ = System.BitConverter.GetBytes(red)[0]; rgba_layer_c0.green_ = System.BitConverter.GetBytes(green)[0]; rgba_layer_c0.blue_ = System.BitConverter.GetBytes(blue)[0]; rgba_layer_c0.alpha_ = System.BitConverter.GetBytes(alpha)[0]; IntPtr rgba_conf = Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0)); Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true); UInt32 rgba_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0, rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE, 0, IntPtr.Zero); Marshal.FreeHGlobal(rgba_conf); NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig(); external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; external_layer_c1.base_.index_ = 1; external_layer_c1.base_.enable_ = 1; external_layer_c1.base_.region_.x_ = 0; external_layer_c1.base_.region_.y_ = 0; external_layer_c1.base_.region_.width_ = video_width_; external_layer_c1.base_.region_.height_ = video_height_; external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32(); external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1); IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1)); Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true); UInt32 external_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0, external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME, 0, IntPtr.Zero); Marshal.FreeHGlobal(external_layer_conf); } else if (video_option == NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA) { CameraInfo camera = cameras_[cur_sel_camera_index_]; NT_PB_VideoCaptureCapability cap = camera.capabilities_[cur_sel_camera_resolutions_index_]; SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), (UInt32)cap.width_, (UInt32)cap.height_); } SetFrameRate((UInt32)CalBitRate(edit_key_frame_, video_width_, video_height_)); Int32 type = 0; Int32 encoder_id = 1; UInt32 codec_id = (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264; Int32 param1 = 0; SetVideoEncoder(type, encoder_id, codec_id, param1); SetVideoQualityV2(CalVideoQuality(video_width_, video_height_, is_h264_encoder)); SetVideoMaxBitRate((CalMaxKBitRate(edit_key_frame_, video_width_, video_height_, false))); SetVideoKeyFrameInterval((edit_key_frame_)); if (is_h264_encoder) { SetVideoEncoderProfile(1); } SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder)); // Audio related Settings SetAuidoInputDeviceId(0); SetPublisherAudioCodecType(1); SetPublisherMute(is_mute); SetEchoCancellation(0, 0); SetNoiseSuppression(0); SetAGC(0); SetVAD(0); SetInputAudioVolume(Convert.ToSingle(edit_audio_input_volume_)); }Copy the code

4. Data collection

Camera and screen data collection, or call the native SDK interface, this article will not repeat, if you need to collect Unity form data, you can refer to the following code:

if ( texture_ == null || video_width_ ! = Screen.width || video_height_ ! = Screen.height) { Debug.Log("OnPostRender screen changed++ scr_width: " + Screen.width + " scr_height: " + Screen.height); if (screen_image_ ! = IntPtr.Zero) { Marshal.FreeHGlobal(screen_image_); screen_image_ = IntPtr.Zero; } if (texture_ ! = null) { UnityEngine.Object.Destroy(texture_); texture_ = null; } video_width_ = Screen.width; video_height_ = Screen.height; texture_ = new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false); screen_image_ = Marshal.AllocHGlobal(video_width_ * 4 * video_height_); Debug.Log("OnPostRender screen changed--"); return; } texture_.ReadPixels(new Rect(0, 0, video_width_, video_height_), 0, 0, false); texture_.Apply();Copy the code

From inside the texture, get the raw data by calling GetRawTextureData().

5. Data connection

Once the data is obtained, it is passed to the SDK layer by calling the OnPostRGBAData() interface.

6. Preview local data

public bool StartPreview() { if(CheckPublisherHandleAvailable() == false) return false; video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack); NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_); if (NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } return false; } publisher_handle_count_++; is_previewing_ = true; return true; }Copy the code

After you set Preview, handle preview’s data callbacks

/ / the preview data callback public void SDKVideoPreviewImageCallBack (IntPtr handle, IntPtr user_data, IntPtr image) { NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image)); NT_VideoFrame pVideoFrame = new NT_VideoFrame(); pVideoFrame.width_ = pb_image.width_; pVideoFrame.height_ = pb_image.height_; pVideoFrame.stride_ = pb_image.stride_[0]; Int32 argb_size = pb_image.stride_[0] * pb_image.height_; pVideoFrame.plane_data_ = new byte[argb_size]; if (argb_size > 0) { Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size); } { cur_image_ = pVideoFrame; }}Copy the code

7. Related Event callback processing

private void PbEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5, [MarshalAs(UnmanagedType.LPStr)] String param6, IntPtr param7) { String event_log = ""; The switch (event_id) {case (uint) NTSmartPublisherDefine. NT_PB_E_EVENT_ID. NT_PB_E_EVENT_ID_CONNECTING: event_log = "connection"; if (! String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; Case (uint) NTSmartPublisherDefine. NT_PB_E_EVENT_ID. NT_PB_E_EVENT_ID_CONNECTION_FAILED: event_log = "connection"; if (! String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; Case (uint) NTSmartPublisherDefine. NT_PB_E_EVENT_ID. NT_PB_E_EVENT_ID_CONNECTED: event_log = "connected"; if (! String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; Case (uint) NTSmartPublisherDefine. NT_PB_E_EVENT_ID. NT_PB_E_EVENT_ID_DISCONNECTED: event_log = "disconnected"; if (! String.IsNullOrEmpty(param5)) { event_log = event_log + " url:" + param5; } break; default: break; } if(OnLogEventMsg ! = null) OnLogEventMsg.Invoke(event_id, event_log); }Copy the code

8. Start and stop the push

public bool StartPublisher(String url) { if (CheckPublisherHandleAvailable() == false) return false; if (publisher_handle_ == IntPtr.Zero) { return false; } if (! String.IsNullOrEmpty(url)) { NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero); } if (NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; return false; } publisher_handle_count_++; is_publishing_ = true; return true; } public void StopPublisher() { if (is_publishing_ == false) return; publisher_handle_count_--; NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_); if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; }Copy the code

9. Close the instance

public void Close() { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; }}Copy the code

conclusion

After testing, the Unity environment, through efficient data collection, coding and push, combined with SmartPlayer player, the overall delay can be controlled in milliseconds, can be applied to most Unity environment on the delay and stability requirements of the scene.