阅读LXH《FFMPEG+SDL的视频播放器》总结

一、原文地址

　　https://blog.csdn.net/leixiaohua1020/article/details/46889389

　　在此向雷霄骅致敬！！！

 

二、视频播放器实现思路

　　1）视频播放器大致可分为，视频文件IO模块，解复用模块，音视频解码模块，视频渲染模块，音频播放模块

　　2）ffmpeg中的代码可以实现上面所有的内容，但是为了手工实现一个播放器，上面的项目中只用ffmpeg来读取视频文件和解码视频文件

　　3）SDL是一个主要用于游戏领域的跨平台音视频渲染库，上面的项目中使用SDL渲染解码之后的YUV图像

 

三、项目代码分析

　　文件目录：

　　

 

　　核心代码：

　　

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
typedef struct VideoState {
  AVFormatContext *pFormatCtx;
  int             videoStream, audioStream;
  AVStream        *audio_st;
  PacketQueue     audioq;
  uint8_t         audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2];
  unsigned int    audio_buf_size;
  unsigned int    audio_buf_index;
  AVFrame         audio_frame;
  AVPacket        audio_pkt;
  uint8_t         *audio_pkt_data;
  int             audio_pkt_size;
  AVStream        *video_st;
  PacketQueue     videoq;
 
  VideoPicture    pictq[VIDEO_PICTURE_QUEUE_SIZE];
  int             pictq_size, pictq_rindex, pictq_windex;
  //picture原子保护
  SDL_mutex       *pictq_mutex;
  SDL_cond        *pictq_cond;
 
  SDL_Thread      *parse_tid;
  SDL_Thread      *video_tid;
 
  char            filename[1024];
  int             quit;
 
  AVIOContext     *io_context;
  struct SwsContext *sws_ctx;
 
  double video_clock;
  double audio_clock;
  double frame_last_delay;
  double frame_last_pts;
  double frame_timer;
} VideoState;

　　上面的结构体包含了播放器中共有的核心对象，整个程序中都会使用到，是整个播放器的上下文

　　可以看到里面包含两个队列，一个是视频队列，一个音频队列；看一下队列的实现：

?
1
2
3
4
5
6
7
8
9
typedef struct PacketQueue {
    AVPacketList *first_pkt, *last_pkt;
    int nb_packets;
    int size;
    //保持队列的原子性
    SDL_mutex *mutex;
    //put get 操作通信
    SDL_cond *cond;
} PacketQueue;

　　队列的定义中包含一个临界区锁和一个信号量，这个保证了队列的存取多线程的安全性，实现如下：

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
int packet_queue_put(PacketQueue *q, AVPacket *pkt) {
 
    AVPacketList *pkt1;
    if(av_dup_packet(pkt) < 0) {
        return -1;
    }
    pkt1 = (AVPacketList*)av_malloc(sizeof(AVPacketList));
    if (!pkt1)
        return -1;
    pkt1->pkt = *pkt;
    pkt1->next = NULL;
 
    SDL_LockMutex(q->mutex);
 
    if (!q->last_pkt)
        q->first_pkt = pkt1;
    else
        q->last_pkt->next = pkt1;
    q->last_pkt = pkt1;
    q->nb_packets++;
    q->size += pkt1->pkt.size;
    SDL_CondSignal(q->cond);
 
    SDL_UnlockMutex(q->mutex);
    return 0;
}
 
int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block)
{
    AVPacketList *pkt1;
    int ret;
 
    SDL_LockMutex(q->mutex);
 
    while(1){
        if(global_video_state->quit) {
            ret = -1;
            break;
        }
 
        pkt1 = q->first_pkt;
        if (pkt1) {
            q->first_pkt = pkt1->next;
            if (!q->first_pkt)
                q->last_pkt = NULL;
            q->nb_packets--;
            q->size -= pkt1->pkt.size;
            *pkt = pkt1->pkt;
            av_free(pkt1);
            ret = 1;
            break;
        } else if (!block) {
            ret = 0;
            break;
        } else {
            SDL_CondWait(q->cond, q->mutex);
        }
    }
    SDL_UnlockMutex(q->mutex);
    return ret;
}

多线程访问这个队列的时候，可以较好的实现数据之间的同步。

main入口函数分析:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
int main(int argc, char *argv[])
{
    SDL_Event       event;
    VideoState      *is;
 
    is = (VideoState*)av_mallocz(sizeof(VideoState));
    global_video_state = is;
 
    if(argc < 2) {
        fprintf(stderr, "Usage: test <file>\n");
        exit(1);
    }
    // Register all formats and codecs
    av_register_all();
    avformat_network_init();
 
    if(<strong>SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)</strong>) {
        fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
        exit(1);
    }
 
    // Make a screen to put our video
    screen = SDL_SetVideoMode(640, 480, 0, 0);
    if(!screen) {
        fprintf(stderr, "SDL: could not set video mode - exiting\n");
        exit(1);
    }
 
    av_strlcpy(is->filename, argv[1], 1024);
    is->pictq_mutex = SDL_CreateMutex();
    is->pictq_cond = SDL_CreateCond();
 
    schedule_refresh(is, 40);
    is->parse_tid = <strong>SDL_CreateThread(decode_thread, is);</strong>
    if(!is->parse_tid) {
        av_free(is);
        return -1;
    }
    while(1){
        SDL_WaitEvent(&event);
        switch(event.type) {
            case FF_QUIT_EVENT:
            case SDL_QUIT:
                is->quit = 1;
                /*
                 * If the video has finished playing, then both the picture and
                 * audio queues are waiting for more data.  Make them stop
                 * waiting and terminate normally.
                 */
                SDL_CondSignal(is->audioq.cond);
                SDL_CondSignal(is->videoq.cond);
                SDL_Quit();
                return 0;
                break;
            case FF_ALLOC_EVENT:
                alloc_picture(event.user.data1);
                break;
            case FF_REFRESH_EVENT:
                video_refresh_timer(event.user.data1);
                break;
            default:
                break;
        }
    }
    return 0;
}

 

先看上面两行加粗的代码

第一行初始化音频、视频、定时器模块；这里是SDL初始化的代码，以上都是SDL初始化相关内容

第二行是真正的入口，创建一个解码线程，在解码线程中同时创建视频渲染线程

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
int stream_component_open(VideoState *is, int stream_index)
{
    AVFormatContext *pFormatCtx = is->pFormatCtx;
    AVCodecContext *codecCtx = NULL;
    AVCodec *codec = NULL;
    AVDictionary *optionsDict = NULL;
    SDL_AudioSpec wanted_spec, spec;
 
    if(stream_index < 0 || (unsigned int)stream_index >= pFormatCtx->nb_streams) {
        return -1;
    }
 
    // Get a pointer to the codec context for the video stream
    codecCtx = pFormatCtx->streams[stream_index]->codec;
 
    if(codecCtx->codec_type == AVMEDIA_TYPE_AUDIO) {
        // Set audio settings from codec info
        wanted_spec.freq = codecCtx->sample_rate;
        wanted_spec.format = AUDIO_S16SYS;
        wanted_spec.channels = codecCtx->channels;
        wanted_spec.silence = 0;
        wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
        wanted_spec.callback = audio_callback;
        wanted_spec.userdata = is;
 
        if(SDL_OpenAudio(&wanted_spec, &spec) < 0) {
            fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
            return -1;
        }
    }
    codec = avcodec_find_decoder(codecCtx->codec_id);
    if(!codec || (avcodec_open2(codecCtx, codec, &optionsDict) < 0)) {
        fprintf(stderr, "Unsupported codec!\n");
        return -1;
    };
    wanted_spec.callback = audio_callback;
 
 
    switch(codecCtx->codec_type) {
        case AVMEDIA_TYPE_AUDIO:
            is->audioStream = stream_index;
            is->audio_st = pFormatCtx->streams[stream_index];
            is->audio_buf_size = 0;
            is->audio_buf_index = 0;
            memset(&is->audio_pkt, 0, sizeof(is->audio_pkt));
            packet_queue_init(&is->audioq);
            SDL_PauseAudio(0);
            break;
        case AVMEDIA_TYPE_VIDEO:
            is->videoStream = stream_index;
            is->frame_timer = (double)av_gettime() / 1000000.0;
            is->video_st = pFormatCtx->streams[stream_index];
 
            packet_queue_init(&is->videoq);
            is->video_tid = SDL_CreateThread(video_thread, is);
            is->sws_ctx =
                sws_getContext
                (
                 is->video_st->codec->width,
                 is->video_st->codec->height,
                 is->video_st->codec->pix_fmt,
                 is->video_st->codec->width,
                 is->video_st->codec->height,
                 AV_PIX_FMT_YUV420P,
                 SWS_BILINEAR,
                 0,
                 0,
                 0
                );
            /* codecCtx->get_buffer2 = our_get_buffer; */
            break;
        default:
            break;
    }
    return 0;
}

　那现在解码线程+渲染线程都有了；他们之间的信息（task）是如何传递的呢？

　就是刚才全局上下文中的音频和视频队列；

　这个队列实现很精妙，在队列满的时候会导致生产者阻塞，在队列空的时候会导致消费者阻塞，如此一来，在播放器因为网络差的时候得不到视频文件，因此队列为空，后续所有的任务自动“暂停”

　

?
1
2
3
4
5
6
7
8
9
10
while(1){
    if(is->quit) {
        break;
    }
    // seek stuff goes here
    if(is->audioq.size > MAX_AUDIOQ_SIZE ||
            is->videoq.size > MAX_VIDEOQ_SIZE) {
        SDL_Delay(10);
        continue;
    }

　队列满的时候，停止放数据到队列中；

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
int queue_picture(VideoState *is, AVFrame *pFrame,double pts)
{
    VideoPicture *vp;
    AVPicture pict;
 
    /* wait until we have space for a new pic */
    SDL_LockMutex(is->pictq_mutex);
    //最大显示缓存队列已蛮，等待信号
    while(is->pictq_size >= VIDEO_PICTURE_QUEUE_SIZE &&
            !is->quit) {
        SDL_CondWait(is->pictq_cond, is->pictq_mutex);
    }
    SDL_UnlockMutex(is->pictq_mutex);
 
    if(is->quit)
        return -1;

　　

同时在放数据的业务层，也判断一下自定义的缓冲区大小，如果满了就阻塞，等待队列释放空间，这样能精确控制内存

更精妙的是，在解码一帧图像的时候，这里利用信号和锁+SDL事件队列，方便控制了一帧图像的渲染；因为这张渲染的bmp是全局共享的，同样需要保证线程安全

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
if(!vp->bmp ||
        vp->width != is->video_st->codec->width ||
        vp->height != is->video_st->codec->height) {
    SDL_Event event;
 
    vp->allocated = 0;
    /* we have to do it in the main thread */
    event.type = FF_ALLOC_EVENT;
    event.user.data1 = is;
    SDL_PushEvent(&event);
 
    /* wait until we have a picture allocated */
    SDL_LockMutex(is->pictq_mutex);
    while(!vp->allocated && !is->quit) {
        SDL_CondWait(is->pictq_cond, is->pictq_mutex);
    }
    SDL_UnlockMutex(is->pictq_mutex);
    if(is->quit) {
        return -1;
    }
}

　　

借助上面的思路可以完全实现内存的上限控制，限制队列的大小就可以实现，这样在嵌入式设备上面可以精确控制内存的使用。

 

四、音视频同步

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
void video_refresh_timer(void *userdata) {
 
    VideoState *is = (VideoState *)userdata;
    VideoPicture *vp;
    double actual_delay, delay, sync_threshold, ref_clock, diff;
 
    if(is->video_st) {
        if(is->pictq_size == 0) {
            schedule_refresh(is, 1);
        } else {
 
            /* printf("vidoe clock %f  audio clock %f \n",is->video_clock,is->audio_clock); */
            /* printf("audio clock %f",is->audio_clock); */
 
            vp = &is->pictq[is->pictq_rindex];
 
            delay = vp->pts - is->frame_last_pts; /* the pts from last time */
            /* printf("delay %f ",delay); */
            if(delay <= 0 || delay >= 1.0) {
                /* if incorrect delay, use previous one */
                delay = is->frame_last_delay;
            }
            /* save for next time */
            is->frame_last_delay = delay;
            is->frame_last_pts = vp->pts;
 
            /* update delay to sync to audio */
            ref_clock = get_audio_clock(is);
            diff = vp->pts - ref_clock;
            /* printf("diff %f \n",diff); */
 
            /* Skip or repeat the frame. Take delay into account
               FFPlay still doesn't "know if this is the best guess." */
            sync_threshold = (delay > AV_SYNC_THRESHOLD) ? delay : AV_SYNC_THRESHOLD;
            if(fabs(diff) < AV_NOSYNC_THRESHOLD) {
                if(diff <= -sync_threshold) {
                    delay = 0;
                } else if(diff >= sync_threshold) {
                    delay = 2 * delay;
                }
            }
 
            is->frame_timer += delay;
            /* computer the REAL delay */
            actual_delay = is->frame_timer - (av_gettime() / 1000000.0);
            /* printf("diff %f actual delay %f \n",diff,actual_delay); */
            if(actual_delay < 0.010) {
                /* Really it should skip the picture instead */
                actual_delay = 0.010;
            }
            schedule_refresh(is, (int)(actual_delay * 1000 + 0.5));
            /* show the picture! */
            video_display(is);
 
            /* update queue for next picture! */
            if(++is->pictq_rindex == VIDEO_PICTURE_QUEUE_SIZE) {
                is->pictq_rindex = 0;
            }
            SDL_LockMutex(is->pictq_mutex);
            is->pictq_size--;
            SDL_CondSignal(is->pictq_cond);
            SDL_UnlockMutex(is->pictq_mutex);
        }
    } else {
        schedule_refresh(is, 100);
    }
}

 

利用SDL的timer模块，我们每次注册一个timer，渲染完一帧图像之后，根据pts，当前的clock，音频的clock设置下一帧渲染的timer

这样可以实现视频的连续刷新

posted @   兜兜有糖的博客  阅读(655)  评论(0)  编辑  收藏  举报
编辑推荐：
· 10年+ .NET Coder 心语，封装的思维：从隐藏、稳定开始理解其本质意义
· .NET Core 中如何实现缓存的预热？
· 从 HTTP 原因短语缺失研究 HTTP/2 和 HTTP/3 的设计差异
· AI与.NET技术实操系列：向量存储与相似性搜索在 .NET 中的实现
· 基于Microsoft.Extensions.AI核心库实现RAG应用
阅读排行：
· 10年+ .NET Coder 心语 ── 封装的思维：从隐藏、稳定开始理解其本质意义
· 地球OL攻略 —— 某应届生求职总结
· 提示词工程——AI应用必不可少的技术
· Open-Sora 2.0 重磅开源！
· 周边上新：园子的第一款马克杯温暖上架
历史上的今天：
2017-07-06 Postman POST多个文件
点击右上角即可分享
微信分享提示
AI IDE Trae