This article was first published on the wechat official account Byteflow

FFmpeg development series serial:

FFmpeg Development (01) : FFmpeg compilation and integration

FFmpeg development (02) : FFmpeg + ANativeWindow video decoding playback

FFmpeg development (03) : FFmpeg + OpenSLES audio decoding playback

FFmpeg development (04) : FFmpeg + OpenGLES audio visual playback

FFmpeg development (05) : FFmpeg + OpenGLES video decoding playback and video filter

FFmpeg development (06) : FFmpeg player to achieve audio and video synchronization in three ways

FFmpeg development (07) : FFmpeg + OpenGLES implementation of 3D panorama player

In the previous article, we have implemented a multimedia player using FFmpeg + OpenGLES + OpenSLES. This article will optimize the player for video rendering.

Video Rendering optimization

In the previous article, we converted the decoded video frames into RGBA format through SWScale library, and then sent them to OpenGL for rendering. The usual format of video frames is YUV420P/YUV420SP, so swScale is required for format conversion in most cases.

When the video size is relatively large, there will be a performance bottleneck if swscale is used for format conversion. Therefore, this paper puts YUV to RGBA format conversion into SHAder and uses GPU to achieve format conversion to improve rendering efficiency.

In this paper, video rendering optimization is essentially to improve the OpenGLRender video renderer, so that it supports YUV420P, NV21 and NV12 these commonly used formats of image rendering.

We know in the previous article to master YUV image processing, YUV420P format image has 3 planes in memory, YUV420SP (NV21, NV12) format image has 2 planes in memory, and RGBA format image only one plane.

Therefore, OpenGLRender video renderer to compatible with YUV420P, YUV420SP and RGBA format, need to create 3 textures to store the data to be rendered, rendering YUV420P format image need to use 3 textures, YUV420SP images need only 2 textures to render, while RGBA images need only 1 texture to render.

Determine the format of the decoded video frame. AVFrame is the decoded video frame.

void VideoDecoder::OnFrameAvailable(AVFrame *frame) {
    LOGCATE("VideoDecoder::OnFrameAvailable frame=%p", frame);
    if(m_VideoRender ! =nullptr&& frame ! =nullptr) {
        NativeImage image;
 		//YUV420P
		if(GetCodecContext()->pix_fmt == AV_PIX_FMT_YUV420P || GetCodecContext()->pix_fmt == AV_PIX_FMT_YUVJ420P) { image.format =  IMAGE_FORMAT_I420; image.width = frame->width; image.height = frame->height; image.pLineSize[0] = frame->linesize[0];
            image.pLineSize[1] = frame->linesize[1];
            image.pLineSize[2] = frame->linesize[2];
            image.ppPlane[0] = frame->data[0];
            image.ppPlane[1] = frame->data[1];
            image.ppPlane[2] = frame->data[2];
            if(frame->data[0] && frame->data[1] && !frame->data[2] && frame->linesize[0] == frame->linesize[1] && frame->linesize[2] = =0) {
                // On some Android device, output of H264 mediacodec decoder is NV12 compatible with some devices may appear format mismatch problemimage.format = IMAGE_FORMAT_NV12; }}else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_NV12) { //NV12
            image.format = IMAGE_FORMAT_NV12;
            image.width = frame->width;
            image.height = frame->height;
            image.pLineSize[0] = frame->linesize[0];
            image.pLineSize[1] = frame->linesize[1];
            image.ppPlane[0] = frame->data[0];
            image.ppPlane[1] = frame->data[1];
        } else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_NV21) { //NV21
            image.format = IMAGE_FORMAT_NV21;
            image.width = frame->width;
            image.height = frame->height;
            image.pLineSize[0] = frame->linesize[0];
            image.pLineSize[1] = frame->linesize[1];
            image.ppPlane[0] = frame->data[0];
            image.ppPlane[1] = frame->data[1];
        } else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_RGBA) { //RGBA
            image.format = IMAGE_FORMAT_RGBA;
            image.width = frame->width;
            image.height = frame->height;
            image.pLineSize[0] = frame->linesize[0];
            image.ppPlane[0] = frame->data[0];
        } else {  // Other formats are converted from SWscale to RGBA
            sws_scale(m_SwsContext, frame->data, frame->linesize, 0,
                      m_VideoHeight, m_RGBAFrame->data, m_RGBAFrame->linesize);
            image.format = IMAGE_FORMAT_RGBA;
            image.width = m_RenderWidth;
            image.height = m_RenderHeight;
            image.ppPlane[0] = m_RGBAFrame->data[0];
        }
		
		// Pass the image to the renderer for renderingm_VideoRender->RenderVideoFrame(&image); }}Copy the code

Creates 3 textures, but does not specify the format of the image to load.

// TEXTURE_NUM = 3
glGenTextures(TEXTURE_NUM, m_TextureIds);
for (int i = 0; i < TEXTURE_NUM ; ++i) {
    glActiveTexture(GL_TEXTURE0 + i);
    glBindTexture(GL_TEXTURE_2D, m_TextureIds[i]);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glBindTexture(GL_TEXTURE_2D, GL_NONE);
}
Copy the code

Load data in different formats to textures.

switch (m_RenderImage.format)
{   
    // Load RGBA data
    case IMAGE_FORMAT_RGBA:
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, m_RenderImage.width, m_RenderImage.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, m_RenderImage.ppPlane[0]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);
        break;
	// Load YUV420SP data
    case IMAGE_FORMAT_NV21:
    case IMAGE_FORMAT_NV12:
        //upload Y plane data
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width,
                     m_RenderImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
                     m_RenderImage.ppPlane[0]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);

        //update UV plane data
        glActiveTexture(GL_TEXTURE1);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[1]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, m_RenderImage.width >> 1,
                     m_RenderImage.height >> 1.0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
                     m_RenderImage.ppPlane[1]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);
        break;
	// Load data of type YUV420P
    case IMAGE_FORMAT_I420:
        //upload Y plane data
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width,
                     m_RenderImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
                     m_RenderImage.ppPlane[0]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);

        //update U plane data
        glActiveTexture(GL_TEXTURE1);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[1]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width >> 1,
                     m_RenderImage.height >> 1.0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
                     m_RenderImage.ppPlane[1]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);

        //update V plane data
        glActiveTexture(GL_TEXTURE2);
        glBindTexture(GL_TEXTURE_2D, m_TextureIds[2]);
        glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width >> 1,
                     m_RenderImage.height >> 1.0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
                     m_RenderImage.ppPlane[2]);
        glBindTexture(GL_TEXTURE_2D, GL_NONE);
        break;
    default:
        break;
}
Copy the code

Vertex shaders and fragment shaders. Importantly, fragment shaders need different sampling strategies for different image formats.

// Vertex shader
#version 300 es
layout(location = 0) in vec4 a_Position;
layout(location = 1) in vec2 a_texCoord;
out vec2 v_texCoord;
void main(a)
{
    gl_Position = a_Position;
    v_texCoord = a_texCoord;
}

// Fragment shader
#version 300 es
precision highp float;
in vec2 v_texCoord;
layout(location = 0) out vec4 outColor;
uniform sampler2D s_texture0;
uniform sampler2D s_texture1;
uniform sampler2D s_texture2;
uniform int u_ImgType;// 1:RGBA, 2:NV21, 3:NV12, 4:I420

void main(a)
{

    if(u_ImgType == 1) //RGBA
    {
        outColor = texture(s_texture0, v_texCoord);
    }
    else if(u_ImgType == 2) //NV21
    {
        vec3 yuv;
        yuv.x = texture(s_texture0, v_texCoord).r;
        yuv.y = texture(s_texture1, v_texCoord).a - 0.5;
        yuv.z = texture(s_texture1, v_texCoord).r - 0.5;
        highp vec3 rgb = mat3(1.0.1.0.1.0.0.0.0.344.1.770.1.403.0.714.0.0) * yuv;
        outColor = vec4(rgb, 1.0);

    }
    else if(u_ImgType == 3) //NV12
    {
        vec3 yuv;
        yuv.x = texture(s_texture0, v_texCoord).r;
        yuv.y = texture(s_texture1, v_texCoord).r - 0.5;
        yuv.z = texture(s_texture1, v_texCoord).a - 0.5;
        highp vec3 rgb = mat3(1.0.1.0.1.0.0.0.0.344.1.770.1.403.0.714.0.0) * yuv;
        outColor = vec4(rgb, 1.0);
    }
    else if(u_ImgType == 4) //I420
    {
        vec3 yuv;
        yuv.x = texture(s_texture0, v_texCoord).r;
        yuv.y = texture(s_texture1, v_texCoord).r - 0.5;
        yuv.z = texture(s_texture2, v_texCoord).r - 0.5;
        highp vec3 rgb = mat3(1.0.1.0.1.0.0.0.0.344.1.770.1.403.0.714.0.0) * yuv;
        outColor = vec4(rgb, 1.0);
    }
    else
    {
        outColor = vec4(1.0); }}Copy the code

The fragment shader u_ImgType variable is used to set the format type of the image to be rendered, thus adopting different sampling conversion strategies.

It should be noted that the default value of UV component of YUV format image is 127, the default value of Y component is 0, and the value range of 8 bits is 0 ~ 255. Since the texture sampling value needs to be normalized in shader, the sampling value of UV component needs to be subtracted 0.5 respectively. Make sure YUV to RGB conversion is correct.

The source code

LearnFFmpeg source code

Technical communication

Technical exchange/get source code can add my wechat: byte-flow