ffplay.c源码分析【2】
ffplay.c源码分析【1】讲了ffplay基本架构和相关的数据结构等内容，本文主要进行数据读取线程、解码线程的源码进行分析，充分理解ffplay整个处理流程，其整体架构如下图所示。
（1）数据读取线程
（a）准备阶段
avformat_alloc_context 创建上下文
ic->interrupt_callback.callback = decode_interrupt_cb 设置中断回调函数
avformat_open_input 打开媒体文件
avformat_find_stream_info 读取媒体文件的包获取更多的stream信息
av_find_best_stream 查找流，从流中获取相关参数，设置显示窗口的宽高
stream_component_open 打开音频、视频、字幕解码器，并创建解码线程
（b）循环读取数据
static int read_thread(void *arg)
{
VideoState *is = arg;
AVFormatContext *ic = NULL;
int err, i, ret;
int st_index[AVMEDIA_TYPE_NB];
AVPacket pkt1, *pkt = &pkt1;
int64_t stream_start_time;
int pkt_in_play_range = 0;
AVDictionaryEntry *t;
SDL_mutex *wait_mutex = SDL_CreateMutex();
int scan_all_pmts_set = 0;
int64_t pkt_ts;
if (!wait_mutex) {
av_log(NULL, AV_LOG_FATAL, "SDL_CreateMutex(): %s\n", SDL_GetError());
ret = AVERROR(ENOMEM);
goto fail;
}
memset(st_index, -1, sizeof(st_index));
is->eof = 0;
ic = avformat_alloc_context();//创建上下文
if (!ic) {
av_log(NULL, AV_LOG_FATAL, "Could not allocate context.\n");
ret = AVERROR(ENOMEM);
goto fail;
}
ic->interrupt_callback.callback = decode_interrupt_cb;
ic->interrupt_callback.opaque = is;
if (!av_dict_get(format_opts, "scan_all_pmts", NULL, AV_DICT_MATCH_CASE)) {
av_dict_set(&format_opts, "scan_all_pmts", "1", AV_DICT_DONT_OVERWRITE);
scan_all_pmts_set = 1;
}
//打开媒体文件，内部调用probe相关函数根据文件名找到对应的解复用器
err = avformat_open_input(&ic, is->filename, is->iformat, &format_opts);
if (err < 0) {
print_error(is->filename, err);
ret = -1;
goto fail;
}
if (scan_all_pmts_set)
av_dict_set(&format_opts, "scan_all_pmts", NULL, AV_DICT_MATCH_CASE);
if ((t = av_dict_get(format_opts, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
av_log(NULL, AV_LOG_ERROR, "Option %s not found.\n", t->key);
ret = AVERROR_OPTION_NOT_FOUND;
goto fail;
}
is->ic = ic;
if (genpts)
ic->flags |= AVFMT_FLAG_GENPTS;
av_format_inject_global_side_data(ic);
if (find_stream_info) {
AVDictionary **opts = setup_find_stream_info_opts(ic, codec_opts);
int orig_nb_streams = ic->nb_streams;
//通过读取媒体文件的部分数据来分析流信息，获取更多stream信息
err = avformat_find_stream_info(ic, opts);
for (i = 0; i < orig_nb_streams; i++)
av_dict_free(&opts[i]);
av_freep(&opts);
if (err < 0) {
av_log(NULL, AV_LOG_WARNING,
"%s: could not find codec parameters\n", is->filename);
ret = -1;
goto fail;
}
}
if (ic->pb)
ic->pb->eof_reached = 0; // FIXME hack, ffplay maybe should not use avio_feof() to test for the end
if (seek_by_bytes < 0)
seek_by_bytes = !!(ic->iformat->flags & AVFMT_TS_DISCONT) && strcmp("ogg", ic->iformat->name);
is->max_frame_duration = (ic->iformat->flags & AVFMT_TS_DISCONT) ? 10.0 : 3600.0;
if (!window_title && (t = av_dict_get(ic->metadata, "title", NULL, 0)))
window_title = av_asprintf("%s - %s", t->value, input_filename);
/* 检测是否指定播放起始时间，如果指定时间则seek到指定位置，通过ffplay -ss 设置*/
if (start_time != AV_NOPTS_VALUE) {
int64_t timestamp;
timestamp = start_time;
/* add the stream start time */
if (ic->start_time != AV_NOPTS_VALUE)
timestamp += ic->start_time;
//seek的指定位置开始播放
ret = avformat_seek_file(ic, -1, INT64_MIN, timestamp, INT64_MAX, 0);
if (ret < 0) {
av_log(NULL, AV_LOG_WARNING, "%s: could not seek to position %0.3f\n",
is->filename, (double)timestamp / AV_TIME_BASE);
}
}
is->realtime = is_realtime(ic);
if (show_status)
av_dump_format(ic, 0, is->filename, 0);
for (i = 0; i < ic->nb_streams; i++) {
AVStream *st = ic->streams[i];
enum AVMediaType type = st->codecpar->codec_type;
st->discard = AVDISCARD_ALL;
if (type >= 0 && wanted_stream_spec[type] && st_index[type] == -1)
if (avformat_match_stream_specifier(ic, st, wanted_stream_spec[type]) > 0)
st_index[type] = i;
}
for (i = 0; i < AVMEDIA_TYPE_NB; i++) {
if (wanted_stream_spec[i] && st_index[i] == -1) {
av_log(NULL, AV_LOG_ERROR, "Stream specifier %s does not match any %s stream\n", wanted_stream_spec[i], av_get_media_type_string(i));
st_index[i] = INT_MAX;
}
}
//对音频、视频、字幕进行查找流
if (!video_disable)
st_index[AVMEDIA_TYPE_VIDEO] =
av_find_best_stream(ic, AVMEDIA_TYPE_VIDEO,
st_index[AVMEDIA_TYPE_VIDEO], -1, NULL, 0);
if (!audio_disable)
st_index[AVMEDIA_TYPE_AUDIO] =
av_find_best_stream(ic, AVMEDIA_TYPE_AUDIO,
st_index[AVMEDIA_TYPE_AUDIO],
st_index[AVMEDIA_TYPE_VIDEO],
NULL, 0);
if (!video_disable && !subtitle_disable)
st_index[AVMEDIA_TYPE_SUBTITLE] =
av_find_best_stream(ic, AVMEDIA_TYPE_SUBTITLE,
st_index[AVMEDIA_TYPE_SUBTITLE],
(st_index[AVMEDIA_TYPE_AUDIO] >= 0 ?
st_index[AVMEDIA_TYPE_AUDIO] :
st_index[AVMEDIA_TYPE_VIDEO]),
NULL, 0);
is->show_mode = show_mode;
if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {
//从视频流中获取宽高参数，设置窗口的宽高
AVStream *st = ic->streams[st_index[AVMEDIA_TYPE_VIDEO]];
AVCodecParameters *codecpar = st->codecpar;
AVRational sar = av_guess_sample_aspect_ratio(ic, st, NULL);
if (codecpar->width)
set_default_window_size(codecpar->width, codecpar->height, sar);
}
/* 打开音频流、视频流、字幕流，并创建解码线程 */
if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {
stream_component_open(is, st_index[AVMEDIA_TYPE_AUDIO]);
}
ret = -1;
if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {
ret = stream_component_open(is, st_index[AVMEDIA_TYPE_VIDEO]);
}
if (is->show_mode == SHOW_MODE_NONE)
is->show_mode = ret >= 0 ? SHOW_MODE_VIDEO : SHOW_MODE_RDFT; //选择怎么显示，如果视频打开成功，就显示视频，否则显示频谱对应的频谱图
if (st_index[AVMEDIA_TYPE_SUBTITLE] >= 0) {
stream_component_open(is, st_index[AVMEDIA_TYPE_SUBTITLE]);
}
if (is->video_stream < 0 && is->audio_stream < 0) {
av_log(NULL, AV_LOG_FATAL, "Failed to open file '%s' or configure filtergraph\n",
is->filename);
ret = -1;
goto fail;
}
if (infinite_buffer < 0 && is->realtime)
infinite_buffer = 1;
//循环读取数据
for (;;) {
if (is->abort_request)//是否退出
break;
if (is->paused != is->last_paused) {//是否暂停/继续
is->last_paused = is->paused;
if (is->paused)
is->read_pause_return = av_read_pause(ic);//这里的暂停、继续只是对网络流的时候有作用
else
av_read_play(ic);
}
#if CONFIG_RTSP_DEMUXER || CONFIG_MMSH_PROTOCOL
if (is->paused &&
(!strcmp(ic->iformat->name, "rtsp") ||
(ic->pb && !strncmp(input_filename, "mmsh:", 5)))) {
/* wait 10 ms to avoid trying to get another packet */
/* XXX: horrible */
SDL_Delay(10);
continue;
}
#endif
if (is->seek_req) {//是否需要seek
int64_t seek_target = is->seek_pos;
int64_t seek_min = is->seek_rel > 0 ? seek_target - is->seek_rel + 2: INT64_MIN;
int64_t seek_max = is->seek_rel < 0 ? seek_target - is->seek_rel - 2: INT64_MAX;
// FIXME the +-2 is due to rounding being not done in the correct direction in generation
// of the seek_pos/seek_rel variables
//注意：mp4可以seek到指定的时间戳，ts是seek到文件的某个position，而不能直接seek到指定时间点
ret = avformat_seek_file(is->ic, -1, seek_min, seek_target, seek_max, is->seek_flags);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR,
"%s: error while seeking\n", is->ic->url);
} else {
//清空音频、视频、字幕队列节点和数据，同时发送flush_pkt清空解码器缓存
if (is->audio_stream >= 0) {
packet_queue_flush(&is->audioq);//清空packet队列
packet_queue_put(&is->audioq, &flush_pkt);//用来开起新的一个播放序列，解码器读取到一个flush_pkt也清空解码器
}
if (is->subtitle_stream >= 0) {
packet_queue_flush(&is->subtitleq);
packet_queue_put(&is->subtitleq, &flush_pkt);
}
if (is->video_stream >= 0) {
packet_queue_flush(&is->videoq);
packet_queue_put(&is->videoq, &flush_pkt);
}
if (is->seek_flags & AVSEEK_FLAG_BYTE) {
set_clock(&is->extclk, NAN, 0);
} else {
set_clock(&is->extclk, seek_target / (double)AV_TIME_BASE, 0);
}
}
is->seek_req = 0;
is->queue_attachments_req = 1;
is->eof = 0;
if (is->paused)
step_to_next_frame(is);//如果本身是pause状态，则显示一帧继续暂停
}
if (is->queue_attachments_req) {
if (is->video_st && is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC) {
AVPacket copy;
if ((ret = av_packet_ref(©, &is->video_st->attached_pic)) < 0)
goto fail;
packet_queue_put(&is->videoq, ©);
packet_queue_put_nullpacket(&is->videoq, is->video_stream);
}
is->queue_attachments_req = 0;
}
/* PacketQueue队列缓冲区已满 最大值为MAX_QUEUE_SIZE 15M，不需要继续读取数据*/
if (infinite_buffer<1 &&
(is->audioq.size + is->videoq.size + is->subtitleq.size > MAX_QUEUE_SIZE
|| (stream_has_enough_packets(is->audio_st, is->audio_stream, &is->audioq) &&
stream_has_enough_packets(is->video_st, is->video_stream, &is->videoq) &&
stream_has_enough_packets(is->subtitle_st, is->subtitle_stream, &is->subtitleq)))) {
/* wait 10 ms */
SDL_LockMutex(wait_mutex);
SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10);//如果没有唤醒，则超时10ms退出
SDL_UnlockMutex(wait_mutex);
continue;
}
if (!is->paused &&
(!is->audio_st || (is->auddec.finished == is->audioq.serial && frame_queue_nb_remaining(&is->sampq) == 0)) &&
(!is->video_st || (is->viddec.finished == is->videoq.serial && frame_queue_nb_remaining(&is->pictq) == 0))) {
if (loop != 1 && (!loop || --loop)) {
stream_seek(is, start_time != AV_NOPTS_VALUE ? start_time : 0, 0, 0);
} else if (autoexit) {
ret = AVERROR_EOF;
goto fail;
}
}
ret = av_read_frame(ic, pkt);//读取数据包，不会释放其数据，要自己去释放
if (ret < 0) {
if ((ret == AVERROR_EOF || avio_feof(ic->pb)) && !is->eof) {//数据读取完毕，往队列添加nullpacket
if (is->video_stream >= 0)
packet_queue_put_nullpacket(&is->videoq, is->video_stream);
if (is->audio_stream >= 0)
packet_queue_put_nullpacket(&is->audioq, is->audio_stream);
if (is->subtitle_stream >= 0)
packet_queue_put_nullpacket(&is->subtitleq, is->subtitle_stream);
is->eof = 1;
}
if (ic->pb && ic->pb->error)
break;
SDL_LockMutex(wait_mutex);
SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10);
SDL_UnlockMutex(wait_mutex);
continue;
} else {
is->eof = 0;
}
/* 检测是否在播放范围内 */
stream_start_time = ic->streams[pkt->stream_index]->start_time;//获取流的起始时间
pkt_ts = pkt->pts == AV_NOPTS_VALUE ? pkt->dts : pkt->pts;//读取到pkt的时间戳
pkt_in_play_range = duration == AV_NOPTS_VALUE ||
(pkt_ts - (stream_start_time != AV_NOPTS_VALUE ? stream_start_time : 0)) *
av_q2d(ic->streams[pkt->stream_index]->time_base) -
(double)(start_time != AV_NOPTS_VALUE ? start_time : 0) / 1000000
<= ((double)duration / 1000000);
//插入队列
if (pkt->stream_index == is->audio_stream && pkt_in_play_range) {
packet_queue_put(&is->audioq, pkt);
} else if (pkt->stream_index == is->video_stream && pkt_in_play_range
&& !(is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC)) {
packet_queue_put(&is->videoq, pkt);
} else if (pkt->stream_index == is->subtitle_stream && pkt_in_play_range) {
packet_queue_put(&is->subtitleq, pkt);
} else {
av_packet_unref(pkt);//不入队列的pkt，则直接释放
}
}
ret = 0;
fail://退出线程
if (ic && !is->ic)
avformat_close_input(&ic);
if (ret != 0) {
SDL_Event event;
event.type = FF_QUIT_EVENT;
event.user.data1 = is;
SDL_PushEvent(&event);
}
SDL_DestroyMutex(wait_mutex);
return 0;
}
中断回调函数ic->interrupt_callback.callback用于ffmpeg内部在执行耗时操作时，检查上层调用者是否有退出请求，避免用户退出请求没有及时响应。回调函数中返回1则代表ffmpeg结束耗时操作退出当前函数的调用，返回0则代表ffmpeg内部继续执⾏耗时操作，直到完成既定的任务。使用gdb给回调函数decode_interrupt_cb下断点，观察调用栈可知avformat_open_input，avformat_find_stream_info，av_read_frame等函数都会触发。
有两种情况PacketQueue缓冲区满：
audioq、videoq、subtitleq三个PacketQueue之和最大为15M，这是一个经验值；
stream_has_enough_packets，判断音频、视频、字幕流都已有够用的包，需要三个同时满足才成立
static int stream_has_enough_packets(AVStream *st, int stream_id, PacketQueue *queue) {
return stream_id < 0 ||//流没有打开
queue->abort_request ||//有请求退出
(st->disposition & AV_DISPOSITION_ATTACHED_PIC) ||//配置了AV_DISPOSITION_ATTACHED_PIC
queue->nb_packets > MIN_FRAMES && (!queue->duration || av_q2d(st->time_base) * queue->duration > 1.0);//队列包个数大于25，并满足总时长为0或总时长超过1s
}
stream_component_open函数，打开音频、视频、字幕对应的解码器，并创建解码线程。
static int stream_component_open(VideoState *is, int stream_index)
{
AVFormatContext *ic = is->ic;
AVCodecContext *avctx;
AVCodec *codec;
const char *forced_codec_name = NULL;
AVDictionary *opts = NULL;
AVDictionaryEntry *t = NULL;
int sample_rate, nb_channels;
int64_t channel_layout;
int ret = 0;
int stream_lowres = lowres;
if (stream_index < 0 || stream_index >= ic->nb_streams)
return -1;
//分配解码器上下文
avctx = avcodec_alloc_context3(NULL);
if (!avctx)
return AVERROR(ENOMEM);
//将码流中解码参数拷贝到新分配的解码器上下文结构体
ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar);
if (ret < 0)
goto fail;
avctx->pkt_timebase = ic->streams[stream_index]->time_base;
//根据codec_id查找解码器
codec = avcodec_find_decoder(avctx->codec_id);
switch(avctx->codec_type){//获取指定解码器的名字
case AVMEDIA_TYPE_AUDIO : is->last_audio_stream = stream_index; forced_codec_name = audio_codec_name; break;
case AVMEDIA_TYPE_SUBTITLE: is->last_subtitle_stream = stream_index; forced_codec_name = subtitle_codec_name; break;
case AVMEDIA_TYPE_VIDEO : is->last_video_stream = stream_index; forced_codec_name = video_codec_name; break;
}
if (forced_codec_name)
codec = avcodec_find_decoder_by_name(forced_codec_name);
if (!codec) {
if (forced_codec_name) av_log(NULL, AV_LOG_WARNING,
"No codec could be found with name '%s'\n", forced_codec_name);
else av_log(NULL, AV_LOG_WARNING,
"No decoder could be found for codec %s\n", avcodec_get_name(avctx->codec_id));
ret = AVERROR(EINVAL);
goto fail;
}
avctx->codec_id = codec->id;
if (stream_lowres > codec->max_lowres) {
av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n",
codec->max_lowres);
stream_lowres = codec->max_lowres;
}
avctx->lowres = stream_lowres;
if (fast)
avctx->flags2 |= AV_CODEC_FLAG2_FAST;
opts = filter_codec_opts(codec_opts, avctx->codec_id, ic, ic->streams[stream_index], codec);
if (!av_dict_get(opts, "threads", NULL, 0))
av_dict_set(&opts, "threads", "auto", 0);
if (stream_lowres)
av_dict_set_int(&opts, "lowres", stream_lowres, 0);
if (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO)
av_dict_set(&opts, "refcounted_frames", "1", 0);
//打开解码器
if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
goto fail;
}
if ((t = av_dict_get(opts, "", NULL, AV_DICT_IGNORE_SUFFIX))) {
av_log(NULL, AV_LOG_ERROR, "Option %s not found.\n", t->key);
ret = AVERROR_OPTION_NOT_FOUND;
goto fail;
}
is->eof = 0;
ic->streams[stream_index]->discard = AVDISCARD_DEFAULT;
switch (avctx->codec_type) {
case AVMEDIA_TYPE_AUDIO:
#if CONFIG_AVFILTER
{
AVFilterContext *sink;
is->audio_filter_src.freq = avctx->sample_rate;
is->audio_filter_src.channels = avctx->channels;
is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels);
is->audio_filter_src.fmt = avctx->sample_fmt;
if ((ret = configure_audio_filters(is, afilters, 0)) < 0)
goto fail;
sink = is->out_audio_filter;
sample_rate = av_buffersink_get_sample_rate(sink);
nb_channels = av_buffersink_get_channels(sink);
channel_layout = av_buffersink_get_channel_layout(sink);
}
#else
sample_rate = avctx->sample_rate;
nb_channels = avctx->channels;
channel_layout = avctx->channel_layout;
#endif
/* 准备音频输出 */
if ((ret = audio_open(is, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0)
goto fail;
is->audio_hw_buf_size = ret;
is->audio_src = is->audio_tgt;
is->audio_buf_size = 0;
is->audio_buf_index = 0;
/* init averaging filter */
is->audio_diff_avg_coef = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
is->audio_diff_avg_count = 0;
/* since we do not have a precise anough audio FIFO fullness,
we correct audio sync only if larger than this threshold */
is->audio_diff_threshold = (double)(is->audio_hw_buf_size) / is->audio_tgt.bytes_per_sec;
is->audio_stream = stream_index;
is->audio_st = ic->streams[stream_index];
//初始化解码器
decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);
//启动解码器，创建解码线程
if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) {
is->auddec.start_pts = is->audio_st->start_time;
is->auddec.start_pts_tb = is->audio_st->time_base;
}
if ((ret = decoder_start(&is->auddec, audio_thread, "audio_decoder", is)) < 0)
goto out;
SDL_PauseAudioDevice(audio_dev, 0);
break;
case AVMEDIA_TYPE_VIDEO:
is->video_stream = stream_index;//获取video的stream索引
is->video_st = ic->streams[stream_index];//获取video的stream指针
//初始化解码器
decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
//启动解码器，创建解码线程
if ((ret = decoder_start(&is->viddec, video_thread, "video_decoder", is)) < 0)
goto out;
is->queue_attachments_req = 1;
break;
case AVMEDIA_TYPE_SUBTITLE:
is->subtitle_stream = stream_index;
is->subtitle_st = ic->streams[stream_index];
//初始化解码器
decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);
//启动解码器，创建解码线程
if ((ret = decoder_start(&is->subdec, subtitle_thread, "subtitle_decoder", is)) < 0)
goto out;
break;
default:
break;
}
goto out;
fail:
avcodec_free_context(&avctx);
out:
av_dict_free(&opts);
return ret;
}
decoder_init 初始化解码器
d->avctx = avctx; 绑定对应的解码器上下⽂
d->queue = queue; 绑定对应的packet队列
d->empty_queue_cond = empty_queue_cond; 绑定VideoState的continue_read_thread，当解码线程没有packet可读时唤醒read_thread赶紧读取数据
decoder_start启动解码器
packet_queue_start 启⽤对应的packet 队列
SDL_CreateThread 创建对应的解码线程
（2）解码线程
ffplay的解码线程独立于数据读取线程，并且每种类型的流都有各自的解码线程。
（a）video_thread
static int video_thread(void *arg)
{
VideoState *is = arg;
AVFrame *frame = av_frame_alloc();//分配解码帧
double pts;
double duration;//帧持续时间
int ret;
AVRational tb = is->video_st->time_base;//获取stream timebase
AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);//获取帧率
#if CONFIG_AVFILTER
AVFilterGraph *graph = NULL;
AVFilterContext *filt_out = NULL, *filt_in = NULL;
int last_w = 0;
int last_h = 0;
enum AVPixelFormat last_format = -2;
int last_serial = -1;
int last_vfilter_idx = 0;
#endif
if (!frame)
return AVERROR(ENOMEM);
//循环取出解码的帧数据
for (;;) {
ret = get_video_frame(is, frame);//里面包含了从PacketQueue取一个pkt，avcodec_send_packet发送到解码器、avcodec_receive_frame从解码器读取到图像帧的解码过程
if (ret < 0)
goto the_end;//解码结束
if (!ret)
continue;//没有解码得到画面
//过滤器
#if CONFIG_AVFILTER
if ( last_w != frame->width
|| last_h != frame->height
|| last_format != frame->format
|| last_serial != is->viddec.pkt_serial
|| last_vfilter_idx != is->vfilter_idx) {
av_log(NULL, AV_LOG_DEBUG,
"Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n",
last_w, last_h,
(const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none"), last_serial,
frame->width, frame->height,
(const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none"), is->viddec.pkt_serial);
avfilter_graph_free(&graph);
graph = avfilter_graph_alloc();
if (!graph) {
ret = AVERROR(ENOMEM);
goto the_end;
}
graph->nb_threads = filter_nbthreads;
if ((ret = configure_video_filters(graph, is, vfilters_list ? vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) {
SDL_Event event;
event.type = FF_QUIT_EVENT;
event.user.data1 = is;
SDL_PushEvent(&event);
goto the_end;
}
filt_in = is->in_video_filter;
filt_out = is->out_video_filter;
last_w = frame->width;
last_h = frame->height;
last_format = frame->format;
last_serial = is->viddec.pkt_serial;
last_vfilter_idx = is->vfilter_idx;
frame_rate = av_buffersink_get_frame_rate(filt_out);
}
ret = av_buffersrc_add_frame(filt_in, frame);
if (ret < 0)
goto the_end;
while (ret >= 0) {
is->frame_last_returned_time = av_gettime_relative() / 1000000.0;
ret = av_buffersink_get_frame_flags(filt_out, frame, 0);
if (ret < 0) {
if (ret == AVERROR_EOF)
is->viddec.finished = is->viddec.pkt_serial;
ret = 0;
break;
}
is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time;
if (fabs(is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0)
is->frame_last_filter_delay = 0;
tb = av_buffersink_get_time_base(filt_out);
#endif
//计算帧持续时间
duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
//根据AVStream timebase计算出pts值，单位为秒
pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
//将解码后的视频帧插入队列
ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
av_frame_unref(frame);
#if CONFIG_AVFILTER
if (is->videoq.serial != is->viddec.pkt_serial)
break;
}
#endif
if (ret < 0)
goto the_end;
}
the_end:
#if CONFIG_AVFILTER
avfilter_graph_free(&graph);
#endif
av_frame_free(&frame);//释放frame
return 0;
}
上面的流程中主要理解get_video_frame获和queue_picture的实现，而get_video_frame实现真正解码的是decoder_decode_frame。
static int get_video_frame(VideoState *is, AVFrame *frame)
{
int got_picture;
//解码并获取解码后的帧
if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0)
return -1;
//分析得到的帧是否要drop掉，目的是在放入frameQueue前，drop掉过时的帧
if (got_picture) {
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
dpts = av_q2d(is->video_st->time_base) * frame->pts;
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
//framedrop为1则始终判断是否要丢帧，为0则始终不丢帧，为-1则在同步时钟不是按video时钟时，判断是否要丢帧
//如果刚解出来的帧就落后主时钟，就没必要放入队列，进行后面的播放
if (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
if (frame->pts != AV_NOPTS_VALUE) {
double diff = dpts - get_master_clock(is);//计算当前pts和主时钟的差值，!isnan(diff)差值有效，fabs(diff)差值在范围内
if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
diff - is->frame_last_filter_delay < 0 &&
is->viddec.pkt_serial == is->vidclk.serial &&
is->videoq.nb_packets) {
is->frame_drops_early++;
av_frame_unref(frame);
got_picture = 0;
}
}
}
}
return got_picture;
}
decoder_decode_frame的实现
static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub) {
int ret = AVERROR(EAGAIN);
for (;;) {
AVPacket pkt;
//1.流连续情况下获取解码帧
if (d->queue->serial == d->pkt_serial) {
do {
if (d->queue->abort_request)
return -1;//是否请求退出
switch (d->avctx->codec_type) {
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_receive_frame(d->avctx, frame);//从视频解码器中读取一帧
if (ret >= 0) {
if (decoder_reorder_pts == -1) {
frame->pts = frame->best_effort_timestamp;
} else if (!decoder_reorder_pts) {
frame->pts = frame->pkt_dts;
}
}
break;
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);//从音频解码器中读取一帧
if (ret >= 0) {
AVRational tb = (AVRational){1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb);
else if (d->next_pts != AV_NOPTS_VALUE)
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;
}
}
break;
}
if (ret == AVERROR_EOF) {//解码器内没有帧可读了，解码结束
d->finished = d->pkt_serial;
avcodec_flush_buffers(d->avctx);//清解码器缓存，
return 0;
}
if (ret >= 0)
return 1;//读取到解码帧
} while (ret != AVERROR(EAGAIN));
}
//2.获取一个packet，如果播放序列不一致，则过滤掉过时的pkt
do {
if (d->queue->nb_packets == 0)//如果没有数据可读，则唤醒数据读取线程read_thread
SDL_CondSignal(d->empty_queue_cond);
if (d->packet_pending) {//如果之前有缓存的d->pkt，则把之前缓存的d->pkt拷贝到pkt，然后立马发送给编码器
av_packet_move_ref(&pkt, &d->pkt);
d->packet_pending = 0;
} else {//没缓存有，则从PacketQueue中获取
if (packet_queue_get(d->queue, &pkt, 1, &d->pkt_serial) < 0)//从packet队列获取一个packet，并获取serial
return -1;
}
//d->queue就是videoq
//d->pkt_serial是最近一次取packet的序列号
if (d->queue->serial == d->pkt_serial)//序列一致，退出循环
break;
av_packet_unref(&pkt);//释放过滤的pkt
} while (1);
//如果获取到的pkt是plush_pkt
if (pkt.data == flush_pkt.data) {
avcodec_flush_buffers(d->avctx);//清空解码器里面的缓存
d->finished = 0;
d->next_pts = d->start_pts;
d->next_pts_tb = d->start_pts_tb;
} else {
if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
int got_frame = 0;
ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt);
if (ret < 0) {
ret = AVERROR(EAGAIN);
} else {
if (got_frame && !pkt.data) {
d->packet_pending = 1;
av_packet_move_ref(&d->pkt, &pkt);
}
ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF);
}
} else {
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) {//发送一个packet到解码器中
//返回EAGAIN解码器没有收到pkt，说明还要继续调用avcodec_send_packet将frame读取，再avcodec_send_packet发送该pkt
//由于解码器没有收到pkt，但又没办法放回PacketQueue，所以就缓存到自个封装的Decoder的pkt，并将d->packet_pending=1，以备下次继续使用该pkt
av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n");
d->packet_pending = 1;//说明send_packet失败时，需要重新发送
av_packet_move_ref(&d->pkt, &pkt);//拷贝到缓存的d->pkt
}
}
av_packet_unref(&pkt);//发送到解码器后，释放pkt
}
}
}
queue_picture 把解码得到的frame添加到队列中
static int queue_picture(VideoState *is, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial)
{
Frame *vp;
#if defined(DEBUG_SYNC)
printf("frame_type=%c pts=%0.3f\n",
av_get_picture_type_char(src_frame->pict_type), pts);
#endif
if (!(vp = frame_queue_peek_writable(&is->pictq)))//检测队列可写，并从队列尾部申请一可写帧空间，返回可写帧指针
return -1;
//对可写帧进行赋值
vp->sar = src_frame->sample_aspect_ratio;
vp->uploaded = 0;
vp->width = src_frame->width;
vp->height = src_frame->height;
vp->format = src_frame->format;
vp->pts = pts;
vp->duration = duration;
vp->pos = pos;
vp->serial = serial;
set_default_window_size(vp->width, vp->height, vp->sar);
//将src_frame中所有数据拷贝到vp->frame，使用av_frame_move_ref，使用后的src_frame是无效了
av_frame_move_ref(vp->frame, src_frame);
//更新索引位置
frame_queue_push(&is->pictq);
return 0;
}
（b）audio_thread
static int audio_thread(void *arg)
{
VideoState *is = arg;
AVFrame *frame = av_frame_alloc();
Frame *af;
#if CONFIG_AVFILTER
int last_serial = -1;
int64_t dec_channel_layout;
int reconfigure;
#endif
int got_frame = 0;
AVRational tb;
int ret = 0;
if (!frame)
return AVERROR(ENOMEM);
do {
//1.读取解码帧
if ((got_frame = decoder_decode_frame(&is->auddec, frame, NULL)) < 0)
goto the_end;
if (got_frame) {
tb = (AVRational){1, frame->sample_rate};//设置sample_rate为timebase
//过滤器
#if CONFIG_AVFILTER
dec_channel_layout = get_valid_channel_layout(frame->channel_layout, frame->channels);
reconfigure =
cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels,
frame->format, frame->channels) ||
is->audio_filter_src.channel_layout != dec_channel_layout ||
is->audio_filter_src.freq != frame->sample_rate ||
is->auddec.pkt_serial != last_serial;
if (reconfigure) {
char buf1[1024], buf2[1024];
av_get_channel_layout_string(buf1, sizeof(buf1), -1, is->audio_filter_src.channel_layout);
av_get_channel_layout_string(buf2, sizeof(buf2), -1, dec_channel_layout);
av_log(NULL, AV_LOG_DEBUG,
"Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n",
is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial,
frame->sample_rate, frame->channels, av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial);
is->audio_filter_src.fmt = frame->format;
is->audio_filter_src.channels = frame->channels;
is->audio_filter_src.channel_layout = dec_channel_layout;
is->audio_filter_src.freq = frame->sample_rate;
last_serial = is->auddec.pkt_serial;
if ((ret = configure_audio_filters(is, afilters, 1)) < 0)
goto the_end;
}
if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0)
goto the_end;
while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) {
tb = av_buffersink_get_time_base(is->out_audio_filter);
#endif
//2.从队列中获取可写帧指针
if (!(af = frame_queue_peek_writable(&is->sampq)))
goto the_end;
//给可写帧赋值
af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
af->pos = frame->pkt_pos;
af->serial = is->auddec.pkt_serial;
af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate});
//3.将解码帧拷贝到可写帧，放入队列
av_frame_move_ref(af->frame, frame);//拷贝
frame_queue_push(&is->sampq);//更新索引
#if CONFIG_AVFILTER
if (is->audioq.serial != is->auddec.pkt_serial)
break;
}
if (ret == AVERROR_EOF)
is->auddec.finished = is->auddec.pkt_serial;
#endif
}
} while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF);
the_end:
#if CONFIG_AVFILTER
avfilter_graph_free(&is->agraph);
#endif
av_frame_free(&frame);//此时经过av_frame_move_ref的解码帧已无用，释放即可
return ret;
}
音频解码也是调用了decoder_decode_frame，主要看音频处理不同部分，其他从队列获取packet，过滤packet，发送packet到解码器，流程都与视频解码一致。
case AVMEDIA_TYPE_AUDIO:
ret = avcodec_receive_frame(d->avctx, frame);//从解码器中读取一帧
if (ret >= 0) {
AVRational tb = (AVRational){1, frame->sample_rate};
if (frame->pts != AV_NOPTS_VALUE)
//如果frame->pts正常则先将其从pkt_timebase 转成{1, frame->sample_rate}
frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb);
else if (d->next_pts != AV_NOPTS_VALUE)
//如果frame->pts不正常则使⽤上⼀帧更新的next_ pts和next_pts_tb 转成 {1, frame->sample_rate}
frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb);
if (frame->pts != AV_NOPTS_VALUE) {
//根据当前帧的pts和nb_samples预估下⼀帧的pts
d->next_pts = frame->pts + frame->nb_samples;
d->next_pts_tb = tb;//设置timebase
}
}
break;
音频解码线程和视频解码线程还有一点不同，就是解码得到的frame后，并没有做过时帧判断以及drop处理。
