At present, most one-to-one interactions in the market are based on WebRTC. The disadvantages are as follows:
- Server deployment is very complex, which is not conducive to private deployment. It cannot be used in some high privacy scenarios, such as public security and municipal systems.
- Based on UDP, it is difficult to ensure the transmission quality. UDP is an unreliable transmission protocol. In a complex public network environment, various burst traffic, occasional transmission errors, network jitter, and timeout will cause packet loss exceptions, which will affect the quality of audio and video communication to a certain extent.
- It is difficult to deal with complex Internet environment, such as cross-region and cross-carrier scenarios, low bandwidth and high packet loss scenarios.
- The whole framework system is not flexible enough, the code complexity is high, the jargon is good: from demo to practical, there are still 10,000 WebRTC.
RTMP one-to-one interaction technology features:
- Based on the existing RTMP push-pull flow system, the product has high stability and low overall delay.
- Add noise suppression, echo cancellation, automatic gain control and other features to ensure the call effect;
- The use of generic RTMP and RTSP servers such as Nginx and SRS is more conducive to private deployment.
- Extended SEI message sending mechanism supporting H.264;
- Support H.265 coding and H.264 variable bit rate setting;
- Support H.265 decoding, live player support functions, one-to-one interaction modules can be selected support;
- It is suitable for emergency command, education and training.
Without further ado, on the encapsulation code:
Based on the github.com/daniulive/S… Pull stream side package code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
using System.Windows.Forms;
using System.Drawing;
using System.Drawing.Drawing2D;
using NT;
namespace SmartEchoCancellationDemo
{
public delegate void DelGetPlayerEventMsg(String msg);
public delegate void DelGetVideoSize(String size);
class nt_player_wrapper : IDisposable{[DllImport("kernel32", EntryPoint = "CopyMemory")]
static extern void CopyMemory(IntPtr Destination, IntPtr Source, uint Length);
private bool disposed_ = false;
private IntPtr player_handle_ = IntPtr.Zero;
private System.Windows.Forms.Control render_wnd_ = null;
private System.Windows.Forms.PaintEventHandler render_wnd_paint_event_ = null;
private bool is_playing_ = false;
private bool is_mute_ = false;
private int play_buffer_ = 100;
private NT_SP_VideoFrame cur_video_frame_ = new NT_SP_VideoFrame();
private WeakReference sync_invoke_ = null;
// Resolution information callback
delegate void ResolutionNotifyCallback(Int32 width, Int32 height);
ResolutionNotifyCallback resolution_notify_callback_;
SP_SDKVideoSizeCallBack video_size_call_back_;
// Video data callback
SP_SDKVideoFrameCallBack video_frame_call_back_;
delegate void VideoFrameCallBack(UInt32 status, NT_SP_VideoFrame frame);
VideoFrameCallBack set_video_frame_call_back_;
//event Indicates the event callback
// Pull stream events
SP_SDKEventCallBack pull_event_call_back_;
delegate void SetPullEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4,
[MarshalAs(UnmanagedType.LPStr)] String param5,
IntPtr param6);
SetPullEventCallBack set_pull_event_call_back_;
private UInt32 connection_status_ = 0;
private UInt32 buffer_status_ = 0;
private Int32 buffer_percent_ = 0;
private Int32 download_speed_ = - 1;
public event DelGetPlayerEventMsg EventGetPlayerEventMsg;
public event DelGetVideoSize EventGetVideoSize;
public nt_player_wrapper(System.Windows.Forms.Control render_wnd, System.ComponentModel.ISynchronizeInvoke sync_invoke)
{
render_wnd_ = render_wnd;
sync_invoke_ = new WeakReference(sync_invoke);
set_pull_event_call_back_ = new SetPullEventCallBack(PullEventCallBack);
if(render_wnd_ ! =null)
{
render_wnd_paint_event_ = new System.Windows.Forms.PaintEventHandler(this.OnRenderWindowPaint); render_wnd_.Paint += render_wnd_paint_event_; }}public void Dispose()
{
Dispose(true);
// This object will be cleaned up by the Dispose method.
// Therefore, you should call GC.SupressFinalize to
// take this object off the finalization queue
// and prevent finalization code for this object
// from executing a second time.
// GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!this.disposed_)
{
if (disposing)
{
}
if (IsPlaying())
{
StopPlay(false);
}
if(render_wnd_ ! =null&& render_wnd_paint_event_ ! =null)
{
render_wnd_.Paint -= render_wnd_paint_event_;
}
render_wnd_paint_event_ = null;
if(cur_video_frame_.plane0_ ! = IntPtr.Zero) { Marshal.FreeHGlobal(cur_video_frame_.plane0_); cur_video_frame_.plane0_ = IntPtr.Zero; }// Note disposing has been done.
disposed_ = true;
}
}
~nt_player_wrapper()
{
Dispose(false);
}
public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
if (frame == IntPtr.Zero)
{
return;
}
// To process RGB data directly, please refer to the following process
NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
if(video_frame.format_ ! = (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32)return;
NT_SP_VideoFrame pVideoFrame = new NT_SP_VideoFrame();
pVideoFrame.format_ = video_frame.format_;
pVideoFrame.width_ = video_frame.width_;
pVideoFrame.height_ = video_frame.height_;
pVideoFrame.timestamp_ = video_frame.timestamp_;
pVideoFrame.stride0_ = video_frame.stride0_;
pVideoFrame.stride1_ = video_frame.stride1_;
pVideoFrame.stride2_ = video_frame.stride2_;
pVideoFrame.stride3_ = video_frame.stride3_;
if(sync_invoke_ ! =null)
{
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if(sync_invoke_target ! =null)
{
Int32 argb_size = video_frame.stride0_ * video_frame.height_;
pVideoFrame.plane0_ = Marshal.AllocHGlobal(argb_size);
CopyMemory(pVideoFrame.plane0_, video_frame.plane0_, (UInt32)argb_size);
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.BeginInvoke(set_video_frame_call_back_, new object[] { status, pVideoFrame });
}
else{ set_video_frame_call_back_(status, pVideoFrame); }}}}public void SDKVideoFrameCallBack(UInt32 status, NT_SP_VideoFrame frame)
{
if(cur_video_frame_.plane0_ ! = IntPtr.Zero) { Marshal.FreeHGlobal(cur_video_frame_.plane0_); cur_video_frame_.plane0_ = IntPtr.Zero; } cur_video_frame_ = frame;if(render_wnd_ ! =null) { render_wnd_.Invalidate(); }}public void SDKPullEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4,
[MarshalAs(UnmanagedType.LPStr)] String param5,
IntPtr param6)
{
if(sync_invoke_ ! =null)
{
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if(sync_invoke_target ! =null )
{
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.BeginInvoke(set_pull_event_call_back_, new object[] { event_id, param1, param2, param3, param4, param5, param6 });
}
else{ set_pull_event_call_back_(event_id, param1, param2, param3, param4, param5, param6); }}}}private void PullEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, [MarshalAs(UnmanagedType.LPStr)] String param4,
[MarshalAs(UnmanagedType.LPStr)] String param5,
IntPtr param6)
{
if(! is_playing_) {return;
}
String show_str = "";
if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_PLAYBACK_REACH_EOS == event_id)
{
StopPlay();
return;
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_RTSP_STATUS_CODE == event_id)
{
int status_code = (int)param1;
show_str = "RTSP incorrect status code received: " + status_code.ToString() + ", please make sure username/password is correct";
}
if((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTING == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTION_FAILED == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTED == event_id || (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DISCONNECTED == event_id) { connection_status_ = event_id; }if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_START_BUFFERING == event_id
|| (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == event_id
|| (UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_STOP_BUFFERING == event_id)
{
buffer_status_ = event_id;
if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == event_id)
{
buffer_percent_ = (Int32)param1;
}
}
if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DOWNLOAD_SPEED == event_id)
{
download_speed_ = (Int32)param1;
}
if(connection_status_ ! =0)
{
show_str += "Connection status:";
if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTING == connection_status_)
{
show_str += "Connected";
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTION_FAILED == connection_status_)
{
show_str += "Connection failed";
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_CONNECTED == connection_status_)
{
show_str += "Connection successful";
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_DISCONNECTED == connection_status_)
{
show_str += "Disconnect"; }}if(download_speed_ ! =- 1)
{
String ss = "Download speed:" + (download_speed_ * 8 / 1000).ToString() + "kbps " + (download_speed_ / 1024).ToString() + "KB/s";
show_str += ss;
}
if(buffer_status_ ! =0)
{
show_str += "Buffer state:";
if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_START_BUFFERING == buffer_status_)
{
show_str += "Start buffering";
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_BUFFERING == buffer_status_)
{
String ss = "In buffer" + buffer_percent_.ToString() + "%";
show_str += ss;
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_STOP_BUFFERING == buffer_status_)
{
show_str += "End buffer"; }}if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_NEED_KEY == event_id)
{
show_str = "RTMP encrypted stream, please set the Key required for playback..";
}
else if ((UInt32)NTSmartPlayerDefine.NT_SP_E_EVENT_ID.NT_SP_E_EVENT_ID_KEY_ERROR == event_id)
{
show_str = "RTMP encrypted stream, Key error, please reset..";
}
EventGetPlayerEventMsg(show_str);
}
public void SetMute(bool is_mute)
{
is_mute_ = is_mute;
if ( !is_playing_ )
return;
NTSmartPlayerSDK.NT_SP_SetMute(player_handle_, is_mute ? 1 : 0);
}
public void SetBuffer(int buffer_time)
{
if (buffer_time >= 0) { play_buffer_ = buffer_time; }}public bool IsPlaying()
{
return is_playing_;
}
public bool OpenPullHandle(String url, bool is_rtsp_tcp_mode, bool is_mute)
{
if( player_handle_ ! = IntPtr.Zero )return true;
if ( String.IsNullOrEmpty(url) )
return false;
IntPtr pull_handle = IntPtr.Zero;
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPlayerSDK.NT_SP_Open(out pull_handle, IntPtr.Zero, 0, IntPtr.Zero))
{
return false;
}
if (pull_handle == IntPtr.Zero)
{
return false;
}
pull_event_call_back_ = new SP_SDKEventCallBack(SDKPullEventCallBack);
NTSmartPlayerSDK.NT_SP_SetEventCallBack(pull_handle, IntPtr.Zero, pull_event_call_back_);
resolution_notify_callback_ = new ResolutionNotifyCallback(PlaybackWindowResized);
set_video_frame_call_back_ = new VideoFrameCallBack(SDKVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetBuffer(pull_handle, play_buffer_);
NTSmartPlayerSDK.NT_SP_SetFastStartup(pull_handle, 1);
NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(pull_handle, 1);
NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(pull_handle, is_rtsp_tcp_mode ? 1 : 0);
NTSmartPlayerSDK.NT_SP_SetMute(pull_handle, is_mute_ ? 1 : 0);
/ / RTSP timeout Settings
Int32 rtsp_timeout = 10;
NTSmartPlayerSDK.NT_SP_SetRtspTimeout(pull_handle, rtsp_timeout);
//RTSP TCP/UDP automatic switchover Settings
Int32 is_auto_switch_tcp_udp = 1;
NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(pull_handle, is_auto_switch_tcp_udp);
NTSmartPlayerSDK.NT_SP_SetMute(pull_handle, is_mute ? 1 : 0);
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPlayerSDK.NT_SP_SetURL(pull_handle, url)) { NTSmartPlayerSDK.NT_SP_Close(pull_handle); pull_handle = IntPtr.Zero;return false;
}
player_handle_ = pull_handle;
return true;
}
private void PlaybackWindowResized(Int32 width, Int32 height)
{
String resolution = width + "*" + height;
EventGetVideoSize(resolution);
}
public void SP_SDKVideoSizeHandle(IntPtr handle, IntPtr userData, Int32 width, Int32 height)
{
if (null == sync_invoke_)
return;
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if(sync_invoke_target ! =null)
{
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.BeginInvoke(resolution_notify_callback_, new object[] { width, height });
}
else{ resolution_notify_callback_(width, height); }}}public bool StartPlay(String url, bool is_rtsp_tcp_mode, bool is_mute)
{
if ( is_playing_ )
return false;
if ( !OpenPullHandle(url, is_rtsp_tcp_mode, is_mute) )
return false;
NTSmartPlayerSDK.NT_SP_SetMute(player_handle_, is_mute ? 1 : 0);
//video resolution callback
video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);
bool is_support_d3d_render = false;
Int32 in_support_d3d_render = 0;
if (NT.NTBaseCodeDefine.NT_ERC_OK == NTSmartPlayerSDK.NT_SP_IsSupportD3DRender(player_handle_, render_wnd_.Handle, ref in_support_d3d_render))
{
if (1 == in_support_d3d_render)
{
is_support_d3d_render = true; }}// is_support_d3d_render = false;
if (is_support_d3d_render)
{
// If you support D3D rendering, use D3D rendering
NTSmartPlayerSDK.NT_SP_SetRenderWindow(player_handle_, render_wnd_.Handle);
NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player_handle_, 1);
}
else
{
// If D3D is not supported, let the player spit out the data and draw with GDI. This demo is only used to show one-to-one interaction. For details, please refer to the demo of the player
//video frame callback (YUV/RGB)
// Format see NT_SP_E_VIDEO_FRAME_FORMAT. If you need to call YUV, set it to NT_SP_E_VIDEO_FRAME_FROMAT_I420
video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FORMAT_RGB32, IntPtr.Zero, video_frame_call_back_);
}
uint ret = NTSmartPlayerSDK.NT_SP_StartPlay(player_handle_);
if( NTBaseCodeDefine.NT_ERC_OK ! = ret ) { NTSmartPlayerSDK.NT_SP_Close(player_handle_); player_handle_ = IntPtr.Zero;return false;
}
is_playing_ = true;
return true;
}
public void StopPlay(bool is_update_ui =true)
{
if ( !is_playing_ )
return;
NTSmartPlayerSDK.NT_SP_StopPlay(player_handle_);
NTSmartPlayerSDK.NT_SP_Close(player_handle_);
player_handle_ = IntPtr.Zero;
is_playing_ = false;
if(is_update_ui && render_wnd_ ! =null) { render_wnd_.Invalidate(); }}private void GetRenderRect(int limtWidth, int limtHeight, int image_w, int image_h, ref int left_offset, ref int top_offset, ref int dw, ref int dh)
{
if (limtWidth < 1 || limtHeight < 1)
{
left_offset = 0;
top_offset = 0;
dw = limtWidth;
dh = limtHeight;
return;
}
if (image_w < 1 || image_h < 1)
{
left_offset = 0;
top_offset = 0;
dw = limtWidth;
dh = limtHeight;
return;
}
/ / in proportion
double limit_ratio = limtWidth * 1.0 / limtHeight;
double video_ratio = image_w * 1.0 / image_h;
if (video_ratio > limit_ratio)
{
dw = limtWidth;
dh = (int)(dw * image_h * 1.0 / image_w);
if (dh > limtHeight)
dh = limtHeight;
}
else
{
dh = limtHeight;
dw = (int)(dh * image_w * 1.0 / image_h);
if (dw > limtWidth)
dw = limtWidth;
}
left_offset = limtWidth / 2 - dw / 2;
if (left_offset < 0)
left_offset = 0;
top_offset = limtHeight / 2 - dh / 2;
if (top_offset < 0)
top_offset = 0;
}
private void OnRenderWindowPaint(object sender, PaintEventArgs e)
{
if (render_wnd_.Width < 1 || render_wnd_.Height < 1)
return;
Graphics g = e.Graphics;
Brush brush = new SolidBrush(Color.Black);
g.FillRectangle(brush, 0.0, render_wnd_.Width, render_wnd_.Height);
if( IsPlaying() && cur_video_frame_.plane0_ ! = IntPtr.Zero) { g.SmoothingMode = SmoothingMode.HighSpeed;int image_width = cur_video_frame_.width_;
int image_height = cur_video_frame_.height_;
Bitmap bitmap = new Bitmap(image_width, image_height, cur_video_frame_.stride0_,
System.Drawing.Imaging.PixelFormat.Format32bppRgb, cur_video_frame_.plane0_);
int d_w = 0, d_h = 0;
int left_offset = 0;
int top_offset = 0;
GetRenderRect(render_wnd_.Width, render_wnd_.Height, image_width, image_height, ref left_offset, ref top_offset, ref d_w, ref d_h);
g.DrawImage(bitmap, left_offset, top_offset, d_w, d_h); // Draw an image in memory on the canvas of the form}}}}Copy the codeBased on the github.com/daniulive/S… The code encapsulated in the push stream side:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
using System.Windows.Forms;
using System.Drawing;
using System.Drawing.Drawing2D;
using NT;
namespace SmartEchoCancellationDemo
{
public delegate void DelGetPublisherEventMsg(String msg); // Push the Event message
public struct NT_VideoFrame
{
public Int32 width_; / / image width
public Int32 height_; / / image
public IntPtr plane_;
public Int32 stride_;
}
public struct CameraInfo
{
public String name_;
public String id_;
public List<NT_PB_VideoCaptureCapability> capabilities_;
};
class nt_publisher_wrapper : IDisposable{[DllImport("kernel32", EntryPoint = "CopyMemory")]
static extern void CopyMemory(IntPtr Destination, IntPtr Source, uint Length);
private bool disposed_ = false;
private IntPtr publisher_handle_ = IntPtr.Zero;
private System.Windows.Forms.Control render_wnd_ = null;
private System.Windows.Forms.PaintEventHandler render_wnd_paint_event_ = null;
private int publisher_handle_count_;
private bool is_publishing_ = false;
private bool is_previewing_ = false;
private WeakReference sync_invoke_ = null;
//event Indicates the event callback
NT_PB_SDKEventCallBack pb_event_call_back_;
delegate void PbSetEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5,
[MarshalAs(UnmanagedType.LPStr)] String param6,
IntPtr param7);
PbSetEventCallBack pb_set_event_call_back_;
// Preview the data callback
NT_PB_SDKVideoPreviewImageCallBack video_preview_image_callback_;
delegate void SetVideoPreviewImageCallBack(NT_VideoFrame frame);
SetVideoPreviewImageCallBack set_video_preview_image_callback_;
private NT_VideoFrame cur_image_ = new NT_VideoFrame();
public event DelGetPublisherEventMsg EventGetPublisherEventMsg;
public nt_publisher_wrapper(System.Windows.Forms.Control render_wnd, System.ComponentModel.ISynchronizeInvoke sync_invoke)
{
render_wnd_ = render_wnd;
sync_invoke_ = new WeakReference(sync_invoke); ;
pb_set_event_call_back_ = new PbSetEventCallBack(PbEventCallBack);
if(render_wnd_ ! =null)
{
render_wnd_paint_event_ = new System.Windows.Forms.PaintEventHandler(this.OnRenderWindowPaint); render_wnd_.Paint += render_wnd_paint_event_; }}public void Dispose()
{
Dispose(true);
// This object will be cleaned up by the Dispose method.
// Therefore, you should call GC.SupressFinalize to
// take this object off the finalization queue
// and prevent finalization code for this object
// from executing a second time.
// GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
// Check to see if Dispose has already been called.
if (!this.disposed_)
{
if (disposing)
{
}
if(render_wnd_ ! =null&& render_wnd_paint_event_ ! =null)
{
render_wnd_.Paint -= render_wnd_paint_event_;
}
render_wnd_paint_event_ = null;
if(cur_image_.plane_ ! = IntPtr.Zero) { Marshal.FreeHGlobal(cur_image_.plane_); cur_image_.plane_ = IntPtr.Zero; }// Note disposing has been done.
disposed_ = true;
}
}
~nt_publisher_wrapper()
{
Dispose(false);
}
private void PbEventCallBack(UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5,
[MarshalAs(UnmanagedType.LPStr)] String param6,
IntPtr param7)
{
String event_log = "";
switch (event_id)
{
case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTING:
event_log = "Connected";
if(! String.IsNullOrEmpty(param5)) { event_log = event_log +" url:" + param5;
}
break;
case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTION_FAILED:
event_log = "Connection failed";
if(! String.IsNullOrEmpty(param5)) { event_log = event_log +" url:" + param5;
}
break;
case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_CONNECTED:
event_log = "Connected";
if(! String.IsNullOrEmpty(param5)) { event_log = event_log +" url:" + param5;
}
break;
case (uint)NTSmartPublisherDefine.NT_PB_E_EVENT_ID.NT_PB_E_EVENT_ID_DISCONNECTED:
event_log = "Disconnect";
if(! String.IsNullOrEmpty(param5)) { event_log = event_log +" url:" + param5;
}
break;
default:
break;
}
EventGetPublisherEventMsg(event_log);
}
public int CalBitRate(int frame_rate, int w, int h)
{
int kbit_rate = 2000;
int area = w * h;
if (area <= (320 * 300))
{
kbit_rate = 280;
}
else if (area <= (360 * 320))
{
kbit_rate = 360;
}
else if (area <= (640 * 480))
{
kbit_rate = 580;
}
else if (area <= (800 * 600))
{
kbit_rate = 620;
}
else if (area <= (900 * 700))
{
kbit_rate = 820;
}
else if (area <= (1280 * 720))
{
kbit_rate = 1600;
}
else if (area <= (1366 * 768))
{
kbit_rate = 2000;
}
else if (area <= (1600 * 900))
{
kbit_rate = 2300;
}
else if (area <= (1600 * 1050))
{
kbit_rate = 2500;
}
else
{
kbit_rate = 2800;
}
kbit_rate = kbit_rate * frame_rate / 25;
if (kbit_rate < 80)
kbit_rate = 80;
return kbit_rate;
}
public int CalMaxKBitRate(int frame_rate, int w, int h, bool is_var_bitrate)
{
int max_kbit_rate = 2000;
int area = w * h;
if (area <= (320 * 300))
{
max_kbit_rate = is_var_bitrate ? 320 : 600;
}
else if (area <= (360 * 320))
{
max_kbit_rate = is_var_bitrate ? 400 : 800;
}
else if (area <= (640 * 360))
{
max_kbit_rate = is_var_bitrate ? 600 : 1000;
}
else if (area <= (640 * 480))
{
max_kbit_rate = is_var_bitrate ? 680 : 1300;
}
else if (area <= (800 * 600))
{
max_kbit_rate = is_var_bitrate ? 700 : 1500;
}
else if (area <= (900 * 700))
{
max_kbit_rate = is_var_bitrate ? 920 : 2200;
}
else if (area <= (1280 * 720))
{
max_kbit_rate = is_var_bitrate ? 1600 : 3000;
}
else if (area <= (1366 * 768))
{
max_kbit_rate = is_var_bitrate ? 1700 : 3300;
}
else if (area <= (1600 * 900))
{
max_kbit_rate = is_var_bitrate ? 2400 : 3400;
}
else if (area <= (1600 * 1050))
{
max_kbit_rate = is_var_bitrate ? 2600 : 3600;
}
else if (area <= (1920 * 1080))
{
max_kbit_rate = is_var_bitrate ? 2900 : 3800;
}
else
{
max_kbit_rate = is_var_bitrate ? 3500 : 5500;
}
max_kbit_rate = max_kbit_rate * frame_rate / 25;
if (area <= (320 * 240))
{
if (max_kbit_rate < 150)
max_kbit_rate = 150;
}
else if (area <= (640 * 480))
{
if (max_kbit_rate < 300)
max_kbit_rate = 300;
}
else if (area <= (1280 * 720))
{
if (max_kbit_rate < 600)
max_kbit_rate = 600;
}
else if (area <= (1920 * 1080))
{
if (max_kbit_rate < 960)
max_kbit_rate = 960;
}
else
{
if (max_kbit_rate < 1500)
max_kbit_rate = 1500;
}
return max_kbit_rate;
}
public int CalVideoQuality(int w, int h, bool is_h264)
{
int area = w * h;
int quality = is_h264 ? 23 : 28;
if (area <= (320 * 240))
{
quality = is_h264 ? 23 : 27;
}
else if (area <= (640 * 360))
{
quality = is_h264 ? 25 : 28;
}
else if (area <= (640 * 480))
{
quality = is_h264 ? 25 : 28;
}
else if (area <= (960 * 600))
{
quality = is_h264 ? 26 : 28;
}
else if (area <= (1280 * 720))
{
quality = is_h264 ? 27 : 29;
}
else if (area <= (1600 * 900))
{
quality = is_h264 ? 28 : 30;
}
else if (area <= (1920 * 1080))
{
quality = is_h264 ? 29 : 31;
}
else
{
quality = is_h264 ? 30 : 32;
}
return quality;
}
public int CalVideoEncoderSpeed(int w, int h, bool is_h264)
{
if (is_h264)
return 3;
int area = w * h;
if (area <= (960 * 600))
{
return 3;
}
else if (area <= (1280 * 720))
{
return 2;
}
else
{
return 1; }}public int GetAudioInputDeviceNumber()
{
int auido_devices = 0;
NTSmartPublisherSDK.NT_PB_GetAuidoInputDeviceNumber(ref auido_devices);
return auido_devices;
}
public List<String> GetAudioInputDeviceName(int auido_devices)
{
List<String> audio_device_name = new List<string> ();if (auido_devices > 0)
{
for (int i = 0; i < auido_devices; ++i)
{
byte[] deviceNameBuffer = new byte[512];
string name = "";
if (NTBaseCodeDefine.NT_ERC_OK == NTSmartPublisherSDK.NT_PB_GetAuidoInputDeviceName((uint)i, deviceNameBuffer, 512))
{
int count = 0;
for (int j = 0; j < deviceNameBuffer.Length; ++j)
{
if(deviceNameBuffer[j] ! =0)
{
count++;
}
else
{
break; }}if (count > 0)
{
name = Encoding.UTF8.GetString(deviceNameBuffer, 0, count); }}var audio_name = "";
if (name.Length == 0)
{
audio_name = "Audio acquisition equipment -";
}
else
{
audio_name = name + "-";
}
audio_name = audio_name + (i + 1); audio_device_name.Add(name); }}return audio_device_name;
}
public bool IsCanCaptureSpeaker()
{
int is_capture_speader = 0;
if (NTBaseCodeDefine.NT_ERC_OK == NTSmartPublisherSDK.NT_PB_IsCanCaptureSpeaker(ref is_capture_speader))
{
if (1 == is_capture_speader)
{
return true; }}return false;
}
public bool OpenPublisherHandle(uint video_option, uint audio_option)
{
if(publisher_handle_ ! = IntPtr.Zero) {return true;
}
publisher_handle_count_ = 0;
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_Open(out publisher_handle_,
video_option, audio_option, 0, IntPtr.Zero))
{
return false;
}
if(publisher_handle_ ! = IntPtr.Zero) { pb_event_call_back_ =new NT_PB_SDKEventCallBack(PbSDKEventCallBack);
NTSmartPublisherSDK.NT_PB_SetEventCallBack(publisher_handle_, IntPtr.Zero, pb_event_call_back_);
set_video_preview_image_callback_ = new SetVideoPreviewImageCallBack(VideoPreviewImageCallBack);
return true;
}
else
{
return false; }}public void PbSDKEventCallBack(IntPtr handle, IntPtr user_data, UInt32 event_id, Int64 param1, Int64 param2, UInt64 param3, UInt64 param4, [MarshalAs(UnmanagedType.LPStr)] String param5,
[MarshalAs(UnmanagedType.LPStr)] String param6,
IntPtr param7)
{
if(sync_invoke_ ! =null)
{
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if(sync_invoke_target ! =null)
{
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.BeginInvoke(pb_set_event_call_back_, new object[] { event_id, param1, param2, param3, param4, param5, param6, param7 });
}
else{ pb_set_event_call_back_(event_id, param1, param2, param3, param4, param5, param6, param7); }}}}// Preview the data callback
public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
{
NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));
NT_VideoFrame pVideoFrame = new NT_VideoFrame();
pVideoFrame.width_ = pb_image.width_;
pVideoFrame.height_ = pb_image.height_;
pVideoFrame.stride_ = pb_image.stride_[0];
Int32 argb_size = pb_image.stride_[0] * pb_image.height_;
pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);
CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);
if(sync_invoke_ ! =null)
{
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if(sync_invoke_target ! =null)
{
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
}
else{ set_video_preview_image_callback_(pVideoFrame); }}}}public void VideoPreviewImageCallBack(NT_VideoFrame frame)
{
if(cur_image_.plane_ ! = IntPtr.Zero) { Marshal.FreeHGlobal(cur_image_.plane_); cur_image_.plane_ = IntPtr.Zero; } cur_image_ = frame;if( render_wnd_ ! =null) { render_wnd_.Invalidate(); }}public List<CameraInfo> GetCameraInfos()
{
List<CameraInfo> cameras = new List<CameraInfo>();
int device_number = 0;
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceNumber(ref device_number))
{
return cameras;
}
if (device_number < 1)
{
return cameras;
}
for (int i = 0; i < device_number; ++i)
{
CameraInfo info = new CameraInfo();
info.capabilities_ = new List<NT_PB_VideoCaptureCapability>();
StringBuilder name = new StringBuilder(256);
StringBuilder id = new StringBuilder(1024);
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceInfo(i, name,256,
id, 1024))
{
continue;
}
info.name_ = name.ToString();
info.id_ = id.ToString();
int capability_number = 0;
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceCapabilityNumber( id.ToString(),ref capability_number))
{
continue;
}
bool is_failed = false;
for (int j = 0; j < capability_number; ++j)
{
NT_PB_VideoCaptureCapability capability = new NT_PB_VideoCaptureCapability();
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_GetVideoCaptureDeviceCapability( id.ToString(), j,ref capability))
{
is_failed = true;
break;
}
info.capabilities_.Add(capability);
}
if (!is_failed)
{
cameras.Add(info);
}
}
return cameras;
}
public bool StartPreview()
{
video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);
if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_,0, IntPtr.Zero))
{
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
return false;
}
publisher_handle_count_++;
is_previewing_ = true;
return true;
}
public void StopPreview()
{
is_previewing_ = false;
publisher_handle_count_--;
NTSmartPublisherSDK.NT_PB_StopPreview(publisher_handle_);
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
if(render_wnd_ ! =null) { render_wnd_.Invalidate(); }}public bool StartPublisher(String url)
{
if (publisher_handle_ == IntPtr.Zero)
{
return false;
}
if(! String.IsNullOrEmpty(url)) { NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero); }if(NTBaseCodeDefine.NT_ERC_OK ! = NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)) {if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
is_publishing_ = false;
return false;
}
publisher_handle_count_++;
is_publishing_ = true;
return true;
}
public void StopPublisher()
{
publisher_handle_count_--;
NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_);
if (0 == publisher_handle_count_)
{
NTSmartPublisherSDK.NT_PB_Close(publisher_handle_);
publisher_handle_ = IntPtr.Zero;
}
is_publishing_ = false;
}
public void Close()
{
if (0== publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; }}public bool IsPreviewing()
{
return is_previewing_;
}
public bool IsPublishing()
{
return is_publishing_;
}
public bool IsPublisherHandleAvailable()
{
returnpublisher_handle_ ! = IntPtr.Zero ?true : false;
}
public int GetPublisherHandleCount()
{
return publisher_handle_count_;
}
public void SetVideoCaptureDeviceBaseParameter(String camera_id, UInt32 width, UInt32 height)
{
NTSmartPublisherSDK.NT_PB_SetVideoCaptureDeviceBaseParameter(publisher_handle_, camera_id, width, height);
}
public void SetFrameRate(UInt32 frame_rate)
{
NTSmartPublisherSDK.NT_PB_SetFrameRate(publisher_handle_, frame_rate);
}
public void SetVideoEncoderType(Int32 encode_type)
{
NTSmartPublisherSDK.NT_PB_SetVideoEncoderType(publisher_handle_, encode_type);
}
public void SetVideoQualityV2(Int32 quality)
{
NTSmartPublisherSDK.NT_PB_SetVideoQualityV2(publisher_handle_, quality);
}
public void SetVideoMaxBitRate(Int32 kbit_rate)
{
NTSmartPublisherSDK.NT_PB_SetVideoMaxBitRate(publisher_handle_, kbit_rate);
}
public void SetVideoKeyFrameInterval(Int32 interval)
{
NTSmartPublisherSDK.NT_PB_SetVideoKeyFrameInterval(publisher_handle_, interval);
}
public void SetVideoEncoderProfile(Int32 profile)
{
NTSmartPublisherSDK.NT_PB_SetVideoEncoderProfile(publisher_handle_, profile);
}
public void SetVideoEncoderSpeed(Int32 speed)
{
NTSmartPublisherSDK.NT_PB_SetVideoEncoderSpeed(publisher_handle_, speed);
}
public void SetAuidoInputDeviceId(UInt32 device_id)
{
NTSmartPublisherSDK.NT_PB_SetAuidoInputDeviceId(publisher_handle_, device_id);
}
public void SetPublisherAudioCodecType(Int32 type)
{
NTSmartPublisherSDK.NT_PB_SetPublisherAudioCodecType(publisher_handle_, type);
}
public void SetPublisherMute(bool is_mute)
{
NTSmartPublisherSDK.NT_PB_SetMute(publisher_handle_, is_mute ? 1 : 0);
}
public void SetEchoCancellation(Int32 isCancel, Int32 delay)
{
NTSmartPublisherSDK.NT_PB_SetEchoCancellation(publisher_handle_, isCancel, delay);
}
public void SetNoiseSuppression(Int32 isNS)
{
NTSmartPublisherSDK.NT_PB_SetNoiseSuppression(publisher_handle_, isNS);
}
public void SetAGC(Int32 isAGC)
{
NTSmartPublisherSDK.NT_PB_SetAGC(publisher_handle_, isAGC);
}
public void SetVAD(Int32 isVAD)
{
NTSmartPublisherSDK.NT_PB_SetVAD(publisher_handle_, isVAD);
}
private void GetRenderRect(int limtWidth, int limtHeight, int image_w, int image_h, ref int left_offset, ref int top_offset, ref int dw, ref int dh)
{
if (limtWidth < 1 || limtHeight < 1)
{
left_offset = 0;
top_offset = 0;
dw = limtWidth;
dh = limtHeight;
return;
}
if (image_w < 1 || image_h < 1)
{
left_offset = 0;
top_offset = 0;
dw = limtWidth;
dh = limtHeight;
return;
}
/ / in proportion
double limit_ratio = limtWidth * 1.0 / limtHeight;
double video_ratio = image_w * 1.0 / image_h;
if (video_ratio > limit_ratio)
{
dw = limtWidth;
dh = (int)(dw * image_h * 1.0 / image_w);
if (dh > limtHeight)
dh = limtHeight;
}
else
{
dh = limtHeight;
dw = (int)(dh * image_w * 1.0 / image_h);
if (dw > limtWidth)
dw = limtWidth;
}
left_offset = limtWidth / 2 - dw / 2;
if (left_offset < 0)
left_offset = 0;
top_offset = limtHeight / 2 - dh / 2;
if (top_offset < 0)
top_offset = 0;
}
private void OnRenderWindowPaint(object sender, PaintEventArgs e)
{
if (render_wnd_.Width < 1 || render_wnd_.Height < 1)
return;
Graphics g = e.Graphics;
Brush brush = new SolidBrush(Color.Black);
g.FillRectangle(brush, 0.0, render_wnd_.Width, render_wnd_.Height);
if(is_previewing_ && cur_image_.plane_ ! = IntPtr.Zero) { g.SmoothingMode = SmoothingMode.HighSpeed;int image_width = cur_image_.width_;
int image_height = cur_image_.height_;
Bitmap bitmap = new Bitmap(image_width, image_height, cur_image_.stride_,
System.Drawing.Imaging.PixelFormat.Format32bppRgb, cur_image_.plane_);
int d_w = 0, d_h = 0;
int left_offset = 0;
int top_offset = 0;
GetRenderRect(render_wnd_.Width, render_wnd_.Height, image_width, image_height, ref left_offset, ref top_offset, ref d_w, ref d_h);
g.DrawImage(bitmap, left_offset, top_offset, d_w, d_h); // Draw an image in memory on the canvas of the form}}}}Copy the codeSince RTMP is buffered at 0, the latency is in the 200-400 ms range, which is sufficient for normal scenarios where latency is not very critical.