This article will show how FFmpeg plays RTSP/Webcam/File streams. The process is as follows:

RTSP/Webcam/File > FFmpeg open and decode to BGR/YUV > OpenCV/OpenGL display
Copy the code
  • Code: github.com/ikuokuo/rts… , sub-module RtSP-local-player

FFmpeg to prepare

git clone https://github.com/ikuokuo/rtsp-wasm-player.git
cd rtsp-wasm-player
export MY_ROOT=`pwd`

# ffmpeg: https://ffmpeg.org/
git clone1 - b - the depth n4.4 https://git.ffmpeg.org/ffmpeg.git$MY_ROOT/3rdparty/source/ffmpeg
cd $MY_ROOT/3rdparty/source/ffmpeg
./configure --prefix=$MY_ROOT/3rdparty/ ffmPEG-4.4 \ --enable-gpl --enable-version3 \ --disable-programs --disable-doc --disable-everything \ --enable-decoder=h264 --enable-parser=h264 \ --enable-decoder=hevc --enable-parser=hevc \ --enable-hwaccel=h264_nvdec --enable-hwaccel=hevc_nvdec \ --enable-demuxer=rtsp \ --enable-demuxer=rawvideo --enable-decoder=rawvideo --enable-indev=v4l2 \ --enable-protocol=file make -j 'nproc' make install ln -s ffmPEG-4.4$MY_ROOT/3rdparty/ffmpeg
Copy the code

./configure manually selects: decode H264, HEVC, unpack RTSP, RAWVideo, and protocol file to support RTSP/Webcam/ file streams.

Among them, Webcam is based on Linux, so v4L2 is used. Dshow is available for Windows and AVFoundation is available for macOS. See Capture/Webcam for details.

You can make your own choices here, or you can compile all of them.

FFmpeg pull flow

Pull flow process, mainly involved modules:

  • Avdevice: IO device support (secondary, for Webcam)
  • Avformat: Open stream, unpack, take small package (main)
  • Avcodec: Packet receiving, decoding, frame taking (main)
  • Swscale: image scaling, transcoding (minor)

Unpack it. Grab the bag

For the complete code, see stream.cc.

Open the input stream:

// IO device register for Webcam
avdevice_register_all(a);// Network initializes for RTSP
avformat_network_init(a);// Open the input stream
format_ctx_ = avformat_alloc_context(a);avformat_open_input(&format_ctx_, "rtsp://".nullptr.nullptr);
Copy the code

Find the video stream:

avformat_find_stream_info(format_ctx_, nullptr);

video_stream_ = nullptr;

for (unsigned int i = 0; i < format_ctx_->nb_streams; i++) {
  auto codec_type = format_ctx_->streams[i]->codecpar->codec_type;
  if (codec_type == AVMEDIA_TYPE_VIDEO) {
    video_stream_ = format_ctx_->streams[i];
    break;
  } else if (codec_type == AVMEDIA_TYPE_AUDIO) {
    // ignore}}Copy the code

Cycle bag:

if (packet_ == nullptr) {
  packet_ = av_packet_alloc(a); }av_read_frame(format_ctx_, packet_);
if (packet_->stream_index == video_stream_->GetIndex()) {
  // If it is a video stream, process its decoding, capture frames, etc
}
av_packet_unref(packet_);
Copy the code

Decode, take the frame

For the complete code, see stream_video.cc.

Decoding initialization:

if (codec_ctx_ == nullptr) {
  AVCodec *codec_ = avcodec_find_decoder(video_stream_->codecpar->codec_id);

  codec_ctx_ = avcodec_alloc_context3(codec_);

  avcodec_parameters_to_context(codec_ctx_, stream_->codecpar);
  avcodec_open2(codec_ctx_, codec_, nullptr);

  frame_ = av_frame_alloc(a);/ / frame
}
Copy the code

Decode the packet, return frame:

int ret = avcodec_send_packet(codec_ctx_, packet);
if(ret ! =0&& ret ! =AVERROR(EAGAIN) && ret ! = AVERROR_EOF) {throw StreamError(ret);
}

ret = avcodec_receive_frame(codec_ctx_, frame_);
if(ret ! =0&& ret ! =AVERROR(EAGAIN) && ret ! = AVERROR_EOF) {throw StreamError(ret);
}

// frame_ is ok here
Copy the code

Note the special return codes: EAGAIN indicates to continue receiving the package, EOF indicates to end, and other special codes.

Zooming, transcoding

/ / initialization
if (sws_ctx_ == nullptr) {
  // Set the target size and encoding
  auto pix_fmt = options_.sws_dst_pix_fmt;
  int width = options_.sws_dst_width;
  int height = options_.sws_dst_height;
  int align = 1;
  int flags = SWS_BICUBIC;

  sws_frame_ = av_frame_alloc(a);int bytes_n = av_image_get_buffer_size(pix_fmt, width, height, align);
  uint8_t *buffer = static_cast<uint8_t* > (av_malloc(bytes_n * sizeof(uint8_t)));
  av_image_fill_arrays(sws_frame_->data, sws_frame_->linesize, buffer,
    pix_fmt, width, height, align);

  sws_frame_->width = width;
  sws_frame_->height = height;

  / / instantiate
  sws_ctx_ = sws_getContext(
      codec_ctx_->width, codec_ctx_->height, codec_ctx_->pix_fmt,
      width, height, pix_fmt, flags, nullptr.nullptr.nullptr);
  if (sws_ctx_ == nullptr) throw StreamError("Get sws context fail");
}

// Zoom or transcode
sws_scale(sws_ctx_, frame_->data, frame_->linesize, 0, codec_ctx_->height,
  sws_frame_->data, sws_frame_->linesize);

// sws_frame_ as the result frame
Copy the code

OpenCV display

For the complete code, see main_UI_with_opencv.cc.

Transcoding to BGR24 for display:

cv::namedWindow("ui");

try {
  Stream stream;
  stream.Open(options);

  while (1) {
    auto frame = stream.GetFrameVideo(a);if(frame ! =nullptr) {
      cv::Mat image(frame->height, frame->width, CV_8UC3,
        frame->data[0], frame->linesize[0]);
      cv::imshow(win_name, image);
    }
    char key = static_cast<char>(cv::waitKey(10));
    if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q
      break;
    }
  }

  stream.Close(a); }catch (const StreamError &err) {
  LOG(ERROR) << err.what(a); } cv::destroyAllWindows(a);Copy the code

OpenGL display

For the complete code, see glfw_frame.h, main_UI_with_opengl.cc.

Transcoding to YUYV420P for display:

void OnDraw(a) override {
  if(frame_ ! =nullptr) {
    auto width = frame_->width;
    auto height = frame_->height;
    auto data = frame_->data[0];

    auto len_y = width * height;
    auto len_u = (width >> 1) * (height >> 1);

    // Yuyv420p can directly address three planes of data, assign values into the texture
    texture_y_->Fill(width, height, data);
    texture_u_->Fill(width >> 1, height >> 1, data + len_y);
    texture_v_->Fill(width >> 1, height >> 1, data + len_y + len_u);
  }

  glBindVertexArray(vao_);
  glDrawArrays(GL_TRIANGLE_STRIP, 0.4);
}
Copy the code

Fragment shader, convert directly to RGB:

#version 330 core
in vec2 vTexCoord;

uniform sampler2D yTex;
uniform sampler2D uTex;
uniform sampler2D vTex;

// yuv420p to rgb888 matrix
const mat4 YUV2RGB = mat4(
  1.1643828125.0.1.59602734375.- 87078515625..1.1643828125.- 39176171875..- 81296875..52959375..1.1643828125.2.017234375.0.1.081390625.0.0.0.1
);

void main(a) {
  gl_FragColor = vec4(
    texture(yTex, vTexCoord).x,
    texture(uTex, vTexCoord).x,
    texture(vTex, vTexCoord).x,
    1
  ) * YUV2RGB;
}
Copy the code

conclusion

To compile and run the code in this article, follow the README.

GoCoding personal practice experience sharing, please pay attention to the public account!