FFmpeg+SDL2 simple audio and video synchronization player

List of SDL2 articles

Introduction to SDL2

SDL2 event processing

SDL2 texture rendering

SDL2 audio playback

FFmpeg+SDL2 video streaming

FFmpeg+SDL2 audio stream playback

FFmpeg audio and video synchronization

After the previous series of SDL2 learning, finally to achieve a complete simple player.

Threading model

This is the implementation of a simple player thread model, and through this diagram, combined with what we learned in the previous blog, we can basically understand the overall flow of the player. The specific code is also based on this diagram.

Important structure

VideoState

The most important structure in the whole player, demultiplexing, decoding, audio and video synchronization, render related parameters are in this structure, it runs through the whole playback process.

typedef struct VideoState {
    
    char filename[1024]; // File name
    AVFormatContext *pFormatCtx; / / context
    int videoStream, audioStream; // Stream index


    //// Synchronization correlation
    double audio_clock;
    double frame_timer;
    double frame_last_pts;
    double frame_last_delay;

    double video_clock; 
    double video_current_pts; 
    int64_t video_current_pts_time;  

    // Audio correlation
    AVStream *audio_st; / / audio stream
    AVCodecContext *audio_ctx; // Audio decoding context
    PacketQueue audioq; // Audio queue
    uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2]; // Audio cache
    unsigned int audio_buf_size;
    unsigned int audio_buf_index;
    AVFrame audio_frame; / / audio frame
    AVPacket audio_pkt; / / audio packets
    uint8_t *audio_pkt_data;
    int audio_pkt_size;
    struct SwrContext *audio_swr_ctx; // Audio resampling


    //video
    AVStream *video_st; / / video
    AVCodecContext *video_ctx; // Video stream decoding context
    PacketQueue videoq; // Video stream queue


    VideoPicture pictq[VIDEO_PICTURE_QUEUE_SIZE]; // Decoded video frame array
    int pictq_size, pictq_rindex, pictq_windex;
    SDL_mutex *pictq_mutex;
    SDL_cond *pictq_cond;

    SDL_Thread *parse_tid; // Demultiplexing threads
    SDL_Thread *video_tid;// Video decoder thread

    int quit; // Exits the flag bit
} VideoState;
Copy the code

PacketQueue

//// Save queue of audio and video packet after demultiplexing
typedef struct PacketQueue {
    AVPacketList *first_pkt, *last_pkt;
    int nb_packets;
    int size;
    SDL_mutex *mutex;
    SDL_cond *cond;
} PacketQueue;

Copy the code

VideoPicture

//// Decoded video frame
typedef struct VideoPicture {
    AVFrame *frame;
    int width, height;
    double pts; // The time when the video frame should play after audio and video synchronization
} VideoPicture;
Copy the code

Specific code

Main

1. Initialize the
2. Create a timer to schedule the refresh of video frames
3. Create a demultiplexing thread
4. Wait for events

int WinMain(int argc, char *argv[]) {
    char *file = "C:\\Users\\lenovo\\Desktop\\IMG_5950.mp4";
    SDL_Event event;
    VideoState *is;
    is = av_mallocz(sizeof(VideoState));

    if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
        fprintf(stderr."Could not initialize SDL - %s\n", SDL_GetError());
        exit(1);
    }
    // Create SDL Window
    win = SDL_CreateWindow("Media Player".100.100.640.480,
                           SDL_WINDOW_RESIZABLE);
    if(! win) {fprintf(stderr."SDL_CreateWindow error, exit!", SDL_GetError());
        exit(1);
    }

    renderer = SDL_CreateRenderer(win, - 1.0);
    text_mutex = SDL_CreateMutex();
    
    strlcpy(is->filename, file, sizeof(is->filename));
    is->pictq_mutex = SDL_CreateMutex();
    is->pictq_cond = SDL_CreateCond();

    // Timer refresh, mainly used to control the video refresh
    schedule_refresh(is, 40);

    // Create a demultiplexing thread
    is->parse_tid = SDL_CreateThread(demux_thread, "demux_thread", is);
    if(! is->parse_tid) { av_free(is);return - 1;
    }


    for (;;) {
        // Wait for SDL events, otherwise block
        SDL_WaitEvent(&event);
        switch (event.type) {
            case FF_QUIT_EVENT:
            case SDL_QUIT: / / exit
                is->quit = 1;
                goto Destroy;
            case SDL_KEYDOWN:/ / ESC to exit
                if (event.key.keysym.sym == SDLK_ESCAPE) {
                    is->quit = 1;
                    goto Destroy;
                }
                break;
            case FF_REFRESH_EVENT: // Timer refresh event
                video_refresh_timer(event.user.data1);
                break;
            default:
                break; }}/ / exit
    Destroy:
    SDL_Quit();
    return 0;

}
Copy the code

The solution to reuse

1. Open the file
2. Find audio and video streams
3. Open audio and video streams, create video decoding threads, and prepare to decode
4. Read the packet, put the audio and video packet into the queue respectively, and wait for the decoding thread to take out

int demux_thread(void *arg) {
    
    if ((err_code = avformat_open_input(&pFormatCtx, is->filename, NULL.NULL))"0) {
        av_strerror(err_code, errors, 1024);
        return - 1;
    }

    // Find the first video stream
    for (i = 0; i < pFormatCtx->nb_streams; i++) {
        if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO &&
            video_index < 0) {
            video_index = i;
        }
        if (pFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO &&
            audio_index < 0) { audio_index = i; }}// Open audio stream, create decoder, configure parameters
    if (audio_index >= 0) {
        stream_component_open(is, audio_index);
    }
    // Open video stream, create decoder, create decoder thread
    if (video_index >= 0) {
        stream_component_open(is, video_index);
        // video_tid = SDL_CreateThread(decode_video_thread, "decode_video_thread", is);
    }

    for (;;) {
        if (av_read_frame(is->pFormatCtx, packet) < 0) {
            if (is->pFormatCtx->pb->error == 0) {
                SDL_Delay(100); /* no error; wait for user input */
                continue;
            } else {
                break; }}// Put the packet into the queue
        if (packet->stream_index == is->videoStream) {
            packet_queue_put(&is->videoq, packet);
        } else if (packet->stream_index == is->audioStream) {
            packet_queue_put(&is->audioq, packet);
        } else{ av_packet_unref(packet); }}return 0;
}
Copy the code

Video decoding

1. Fetch the video packet from the queue
2. Decoding, synchronization
3. Frame is stored in the array and waits for the video to render

//// Video decoding
int decode_video_thread(void *arg) {
    VideoState *is = (VideoState *) arg;
    AVPacket pkt1, *packet = &pkt1;
    AVFrame *pFrame;
    double pts;

    pFrame = av_frame_alloc();

    for (;;) {
        // Retrieve the packet from the video queue
        if (packet_queue_get(&is->videoq, packet, 1) < 0) {
            break;
        }
        
        / / decoding
        avcodec_send_packet(is->video_ctx, packet);
        while (avcodec_receive_frame(is->video_ctx, pFrame) == 0) {
            if((pts = pFrame->best_effort_timestamp) ! = AV_NOPTS_VALUE) { }else {
                pts = 0;
            }
            pts *= av_q2d(is->video_st->time_base);

            / / synchronize
            pts = synchronize_video(is, pFrame, pts);
            if (queue_picture(is, pFrame, pts) < 0) {
                break;
            }
            av_packet_unref(packet);
        }
    }
    av_frame_free(&pFrame);
    return 0;
}
Copy the code

Audio decoding

//// Audio device callback
void audio_callback(void *userdata, Uint8 *stream, int len) {

    VideoState *is = (VideoState *) userdata;
    int len1, audio_size;
    double pts;

    SDL_memset(stream, 0, len);

    while (len > 0) {
        if (is->audio_buf_index >= is->audio_buf_size) {
            // Audio decoding
            audio_size = audio_decode_frame(is, is->audio_buf, sizeof(is->audio_buf), &pts);
            if (audio_size < 0) {
                // Audio decoding error, play mute
                is->audio_buf_size = 1024 * 2 * 2;
                memset(is->audio_buf, 0, is->audio_buf_size);
            } else {
                is->audio_buf_size = audio_size;
            }
            is->audio_buf_index = 0;
        }
        len1 = is->audio_buf_size - is->audio_buf_index;
        if (len1 > len)
            len1 = len;
        // Remix playback
        SDL_MixAudio(stream, (uint8_t*) is->audio_buf + is->audio_buf_index, len1, SDL_MIX_MAXVOLUME); len -= len1; stream += len1; is->audio_buf_index += len1; }}Copy the code

Video refresh play

//// Refresh the video and predict the playing time of the next frame and set a new timer
void video_refresh_timer(void *userdata) {

    VideoState *is = (VideoState *) userdata;
    VideoPicture *vp;
    double actual_delay, delay, sync_threshold, ref_clock, diff;

    if (is->video_st) {
        if (is->pictq_size == 0) {
            schedule_refresh(is, 1);
        } else {
            // Get a video frame from the array
            vp = &is->pictq[is->pictq_rindex];

            is->video_current_pts = vp->pts;
            is->video_current_pts_time = av_gettime();
            // Subtract the current Frame time from the last Frame time to get the time difference between the two frames
            delay = vp->pts - is->frame_last_pts;
            if (delay <= 0 || delay >= 1.0) {
                // Delay less than 0 or more than 1 second (too long) is an error, set the delay time to the last delay time
                delay = is->frame_last_delay;
            }
            // Save the delay and PTS for next use
            is->frame_last_delay = delay;
            is->frame_last_pts = vp->pts;

            // Get audio Audio_Clock
            ref_clock = get_audio_clock(is);
            // Get the difference between the current PTS and Audio_Clock
            diff = vp->pts - ref_clock;

            // AV_SYNC_THRESHOLD Minimum refresh time
            sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay : AV_SYNC_THRESHOLD;
            // Diff is lower than the non-synchronization threshold and can be synchronized
            if (fabs(diff) < AV_NOSYNC_THRESHOLD) {
                if (diff <= -sync_threshold) {
                    // The video should be played as soon as possible before the audio time
                    delay = 0;
                } else if (diff >= sync_threshold) {
                     // After the audio time, the video should be delayed
                    delay = 2 * delay;
                }
            }
            is->frame_timer += delay;
            // Finally the actual delay time
            actual_delay = is->frame_timer - (av_gettime() / 1000000.0);
            if (actual_delay < 0.010) {
                // If the delay time is too small, set the minimum value
                actual_delay = 0.010;
            }
            // Reset the timer according to the delay time to refresh the video
            schedule_refresh(is, (int) (actual_delay * 1000 + 0.5));

            // Video frame display
            video_display(is);

            // Update the subscript of the video frame array
            if (++is->pictq_rindex == VIDEO_PICTURE_QUEUE_SIZE) {
                is->pictq_rindex = 0;
            }
            SDL_LockMutex(is->pictq_mutex);
            // Video frame array minus oneis->pictq_size--; SDL_CondSignal(is->pictq_cond); SDL_UnlockMutex(is->pictq_mutex); }}else {
        schedule_refresh(is, 100); }}Copy the code

This is the general process, which will be much more complicated than the previous Demo, but all the knowledge is covered in the previous blog, and I can’t explain anything in the blog. It is better to run it directly by myself, and then look at the code. It will clarify the process, and the whole player code will not be difficult to understand. Github-SimplePlay is attached here

Learning audio and video recommendations:

The first of course recommended Thor Lei Xiaohua, China FFmpeg first person, systematically sorted out the FFmpeg related knowledge points, the entry will see, but early death, memory of thor. Lei Xiaohua’s blog

The second recommended god Li Chao moOCs network video, speaking very practical, audio and video xiaobai is worth a look.