技术背景
随着物联网等行业的崛起，越来越多的传统行业如虚拟仿真、航天工业、工业仿真、城市规划等，对Linux下的生态构建，有了更大的期望，Linux平台下，可选的直播推拉流解决方案相对Windows和移动端，非常少，基于Unity的Linux推送方案，更是几无参考。本文以Unity3d环境下Linux平台推送Unity窗体和Unity采集的音频，然后编码推送到RTMP服务器为例，大概说下实现过程。
技术实现
本文以采集Unity窗体数据为例，如果需要对接摄像头和屏幕亦可。简单来说，高效率的获取到原始的窗体Texture，拿到对应的RGB数据，传给现有的RTMP推送模块，音频的话，只要获取到AudioClip数据，实时读取然后传PCM数据。如需2路PCM数据混音，可以采集两路AudioClip数据，分别调用原生的Linux推送RTMP模块的PCM数据投递接口，编码打包推送即可，无图无真相：
下图以Linux平台下，unity3d环境采集窗体（为方便测试，加了个实时时间显示）和unity声音为例，整体延迟可达毫秒级。
音频的话，可以循环采集播放、也可以播放完毕后，加实时静音。
AudioClip数据采集，如需audiosource数据播放完毕后，填充静音帧，可调用FillPcmMuteData()接口。
/// <summary> /// 获取AudioClip数据 /// </summary> private void PostUnityAudioClipData() { //重新初始化 TimerManager.UnRegister(this); audio_clip_info_ = new AudioClipInfo(); audio_clip_info_.audio_source_ = GameObject.Find("Canvas/Panel/AudioSource").GetComponent<AudioSource>(); if (audio_clip_info_.audio_source_ == null) { Debug.LogError("audio source is null.."); return; } if (audio_clip_info_.audio_source_.clip == null) { Debug.LogError("audio clip is null.."); return; } audio_clip_info_.audio_clip_ = audio_clip_info_.audio_source_.clip; audio_clip_info_.audio_clip_offset_ = 0; audio_clip_info_.audio_clip_length_ = audio_clip_info_.audio_clip_.samples * audio_clip_info_.audio_clip_.channels; pcm_mute_data_ = FillPcmMuteData(audio_clip_info_.audio_clip_); TimerManager.Register(this, 0.019999f, (PostPCMData), false); }
采集后的数据，定时调用原生接口，发送到原生接口：
var sample_length = sizeof(float) * pcm_sample.Length; pcm_data.data_ = Marshal.AllocHGlobal(sample_length); Marshal.Copy(pcm_sample, 0, pcm_data.data_, pcm_sample.Length); pcm_data.size_ = (uint)sample_length; publisher_wrapper_.OnPostAudioPCMFloatData(pcm_data.data_, pcm_data.size_, pcm_time_stamp_, pcm_data.sample_rate_, pcm_data.channels_, pcm_data.per_channel_sample_number_); Marshal.FreeHGlobal(pcm_data.data_); pcm_data.data_ = IntPtr.Zero; pcm_data = null; pcm_time_stamp_ += 10; //时间戳自增10毫秒
视频窗体数据采集：
if (video_push_type_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER) return; if (texture_ == null || video_width_ != Screen.width || video_height_ != Screen.height) { Debug.Log("OnPostRender screen changed++ scr_width: " + Screen.width + " scr_height: " + Screen.height); if (screen_image_ != IntPtr.Zero) { Marshal.FreeHGlobal(screen_image_); screen_image_ = IntPtr.Zero; } if (texture_ != null) { UnityEngine.Object.Destroy(texture_); texture_ = null; } video_width_ = Screen.width; video_height_ = Screen.height; texture_ = new Texture2D(video_width_, video_height_, TextureFormat.BGRA32, false); screen_image_ = Marshal.AllocHGlobal(video_width_ * 4 * video_height_); Debug.Log("OnPostRender screen changed--"); return; }
从Texture拿到数据，然后调用OnPostRGBAData()数据传递到原生接口即可。
原生接口模块初始化：
/* * nt_publisher_wrapper.cs * Github：https://github.com/daniulive/SmarterStreaming * * InitSDK主要完成原生模块的初始化工作 */ private bool InitSDK() { if (!is_pusher_sdk_init_) { // 设置日志路径(请确保目录存在) Debug.LogError("NT_SL_SetPath++"); String log_path = "./"; NTSmartLog.NT_SL_SetPath(log_path); UInt32 isInited = NTSmartPublisherSDK.NT_PB_Init(0, IntPtr.Zero); if (isInited != 0) { Debug.Log("调用NT_PB_Init失败.."); return false; } is_pusher_sdk_init_ = true; } return true; }
RTMP推送模块基础参数配置：
private void SetCommonOptionToPublisherSDK() { if (!IsPublisherHandleAvailable()) { Debug.Log("SetCommonOptionToPublisherSDK, publisher handle with null.."); return; } NTSmartPublisherSDK.NT_PB_ClearLayersConfig(publisher_handle_, 0, 0, IntPtr.Zero); if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER) { // 第0层填充RGBA矩形, 目的是保证帧率, 颜色就填充全黑 int red = 0; int green = 0; int blue = 0; int alpha = 255; NT_PB_RGBARectangleLayerConfig rgba_layer_c0 = new NT_PB_RGBARectangleLayerConfig(); rgba_layer_c0.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE; rgba_layer_c0.base_.index_ = 0; rgba_layer_c0.base_.enable_ = 1; rgba_layer_c0.base_.region_.x_ = 0; rgba_layer_c0.base_.region_.y_ = 0; rgba_layer_c0.base_.region_.width_ = video_width_; rgba_layer_c0.base_.region_.height_ = video_height_; rgba_layer_c0.base_.offset_ = Marshal.OffsetOf(rgba_layer_c0.GetType(), "base_").ToInt32(); rgba_layer_c0.base_.cb_size_ = (uint)Marshal.SizeOf(rgba_layer_c0); rgba_layer_c0.red_ = System.BitConverter.GetBytes(red)[0]; rgba_layer_c0.green_ = System.BitConverter.GetBytes(green)[0]; rgba_layer_c0.blue_ = System.BitConverter.GetBytes(blue)[0]; rgba_layer_c0.alpha_ = System.BitConverter.GetBytes(alpha)[0]; IntPtr rgba_conf = Marshal.AllocHGlobal(Marshal.SizeOf(rgba_layer_c0)); Marshal.StructureToPtr(rgba_layer_c0, rgba_conf, true); UInt32 rgba_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0, rgba_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE, 0, IntPtr.Zero); Marshal.FreeHGlobal(rgba_conf); NT_PB_ExternalVideoFrameLayerConfig external_layer_c1 = new NT_PB_ExternalVideoFrameLayerConfig(); external_layer_c1.base_.type_ = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME; external_layer_c1.base_.index_ = 1; external_layer_c1.base_.enable_ = 1; external_layer_c1.base_.region_.x_ = 0; external_layer_c1.base_.region_.y_ = 0; external_layer_c1.base_.region_.width_ = video_width_; external_layer_c1.base_.region_.height_ = video_height_; external_layer_c1.base_.offset_ = Marshal.OffsetOf(external_layer_c1.GetType(), "base_").ToInt32(); external_layer_c1.base_.cb_size_ = (uint)Marshal.SizeOf(external_layer_c1); IntPtr external_layer_conf = Marshal.AllocHGlobal(Marshal.SizeOf(external_layer_c1)); Marshal.StructureToPtr(external_layer_c1, external_layer_conf, true); UInt32 external_r = NTSmartPublisherSDK.NT_PB_AddLayerConfig(publisher_handle_, 0, external_layer_conf, (int)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME, 0, IntPtr.Zero); Marshal.FreeHGlobal(external_layer_conf); } else if (video_option_ == (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA) { CameraInfo camera = cameras_[cur_sel_camera_index_]; NT_PB_VideoCaptureCapability cap = camera.capabilities_[cur_sel_camera_resolutions_index_]; SetVideoCaptureDeviceBaseParameter(camera.id_.ToString(), cap.width_, cap.height_); } SetFrameRate((uint)video_fps_); Int32 type = 0; //软编码 Int32 encoder_id = 1; UInt32 codec_id = (UInt32)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264; Int32 param1 = 0; SetVideoEncoder(type, encoder_id, codec_id, param1); SetVideoQuality(CalVideoQuality(video_width_, video_height_, is_h264_encoder_)); SetVideoBitRate(CalBitRate(video_fps_, video_width_, video_height_)); SetVideoMaxBitRate((CalMaxKBitRate(video_fps_, video_width_, video_height_, false))); SetVideoKeyFrameInterval((key_frame_interval_)); if (is_h264_encoder_) { SetVideoEncoderProfile(1); } SetVideoEncoderSpeed(CalVideoEncoderSpeed(video_width_, video_height_, is_h264_encoder_)); // 音频相关设置 SetAuidoInputDeviceId(0); SetPublisherAudioCodecType(1); SetPublisherMute(is_mute_); SetEchoCancellation(0, 0); SetNoiseSuppression(0); SetAGC(0); SetVAD(0); SetInputAudioVolume(Convert.ToSingle(audio_input_volume_)); }
开始和停止预览：
public bool StartPreview() { if(CheckPublisherHandleAvailable() == false) return false; video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack); NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(publisher_handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_); if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(publisher_handle_, 0, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } return false; } publisher_handle_count_++; is_previewing_ = true; return true; } public void StopPreview() { if (is_previewing_ == false) return; is_previewing_ = false; publisher_handle_count_--; NTSmartPublisherSDK.NT_PB_StopPreview(publisher_handle_); if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } }
预览数据回调：
//预览数据回调 public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image) { NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image)); NT_VideoFrame pVideoFrame = new NT_VideoFrame(); pVideoFrame.width_ = pb_image.width_; pVideoFrame.height_ = pb_image.height_; pVideoFrame.stride_ = pb_image.stride_[0]; Int32 argb_size = pb_image.stride_[0] * pb_image.height_; pVideoFrame.plane_data_ = new byte[argb_size]; if (argb_size > 0) { Marshal.Copy(pb_image.plane_[0],pVideoFrame.plane_data_,0, argb_size); } { cur_image_ = pVideoFrame; } }
开始推送和停止推送：
public bool StartPublisher(String url) { if (CheckPublisherHandleAvailable() == false) return false; if (publisher_handle_ == IntPtr.Zero) { return false; } if (!String.IsNullOrEmpty(url)) { NTSmartPublisherSDK.NT_PB_SetURL(publisher_handle_, url, IntPtr.Zero); } if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(publisher_handle_, IntPtr.Zero)) { if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; return false; } publisher_handle_count_++; is_publishing_ = true; return true; } public void StopPublisher() { if (is_publishing_ == false) return; publisher_handle_count_--; NTSmartPublisherSDK.NT_PB_StopPublisher(publisher_handle_); if (0 == publisher_handle_count_) { NTSmartPublisherSDK.NT_PB_Close(publisher_handle_); publisher_handle_ = IntPtr.Zero; } is_publishing_ = false; }
对应Unity层调用：
public void btn_publish_Click() { if (publisher_wrapper_.IsPublishing()) { return; } String url = rtmp_pusher_url.text; if (url.Length < 8) { publisher_wrapper_.Close(); Debug.Log("请输入RTMP推送地址"); return; } publisher_wrapper_.SetVideoPushType(video_push_type_); publisher_wrapper_.SetAudioPushType(audio_push_type_); if (!publisher_wrapper_.StartPublisher(url)) { Debug.LogError("调用StartPublisher失败.."); return; } startPublishBtn.interactable = false; stopPublishBtn.interactable = !startPublishBtn.interactable; videoOptionSel.interactable = false; audioOptionSel.interactable = false; if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA || audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER) { PostUnityAudioClipData(); } } public void btn_stop_publish_Click() { if (!publisher_wrapper_.IsPublishing()) { return; } StopAudioSource(); if (texture_ != null) { UnityEngine.Object.Destroy(texture_); texture_ = null; } if (screen_image_ != IntPtr.Zero) { Marshal.FreeHGlobal(screen_image_); screen_image_ = IntPtr.Zero; } publisher_wrapper_.StopPublisher(); videoOptionSel.interactable = true; audioOptionSel.interactable = true; startPublishBtn.interactable = true; stopPublishBtn.interactable = !startPublishBtn.interactable; }
总结
以上是Unity环境下，以采集Unity窗体和unity声音为例，对接Linux平台RTMP推送，整体延迟也可以达到毫秒级，基本满足常用场景了，感兴趣的开发者可酌情参考。
