ffplay.c源码分析【2】
ffplay.c源码分析【1】讲了ffplay基本架构和相关的数据结构等内容,本文主要进行数据读取线程、解码线程的源码进行分析,充分理解ffplay整个处理流程,其整体架构如下图所示。
(1)数据读取线程
(a)准备阶段
avformat_alloc_context 创建上下文
ic->interrupt_callback.callback = decode_interrupt_cb 设置中断回调函数
avformat_open_input 打开媒体文件
avformat_find_stream_info 读取媒体文件的包获取更多的stream信息
av_find_best_stream 查找流,从流中获取相关参数,设置显示窗口的宽高
stream_component_open 打开音频、视频、字幕解码器,并创建解码线程
(b)循环读取数据
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 | static int read_thread( void *arg) { VideoState *is = arg; AVFormatContext *ic = NULL; int err, i, ret; int st_index[AVMEDIA_TYPE_NB]; AVPacket pkt1, *pkt = &pkt1; int64_t stream_start_time; int pkt_in_play_range = 0; AVDictionaryEntry *t; SDL_mutex *wait_mutex = SDL_CreateMutex(); int scan_all_pmts_set = 0; int64_t pkt_ts; if (!wait_mutex) { av_log(NULL, AV_LOG_FATAL, "SDL_CreateMutex(): %s\n" , SDL_GetError()); ret = AVERROR(ENOMEM); goto fail; } memset (st_index, -1, sizeof (st_index)); is->eof = 0; ic = avformat_alloc_context(); //创建上下文 if (!ic) { av_log(NULL, AV_LOG_FATAL, "Could not allocate context.\n" ); ret = AVERROR(ENOMEM); goto fail; } ic->interrupt_callback.callback = decode_interrupt_cb; ic->interrupt_callback.opaque = is; if (!av_dict_get(format_opts, "scan_all_pmts" , NULL, AV_DICT_MATCH_CASE)) { av_dict_set(&format_opts, "scan_all_pmts" , "1" , AV_DICT_DONT_OVERWRITE); scan_all_pmts_set = 1; }<br> //打开媒体文件,内部调用probe相关函数根据文件名找到对应的解复用器 err = avformat_open_input(&ic, is->filename, is->iformat, &format_opts); if (err < 0) { print_error(is->filename, err); ret = -1; goto fail; } if (scan_all_pmts_set) av_dict_set(&format_opts, "scan_all_pmts" , NULL, AV_DICT_MATCH_CASE); if ((t = av_dict_get(format_opts, "" , NULL, AV_DICT_IGNORE_SUFFIX))) { av_log(NULL, AV_LOG_ERROR, "Option %s not found.\n" , t->key); ret = AVERROR_OPTION_NOT_FOUND; goto fail; } is->ic = ic; if (genpts) ic->flags |= AVFMT_FLAG_GENPTS; av_format_inject_global_side_data(ic); if (find_stream_info) { AVDictionary **opts = setup_find_stream_info_opts(ic, codec_opts); int orig_nb_streams = ic->nb_streams; //通过读取媒体文件的部分数据来分析流信息,获取更多stream信息 err = avformat_find_stream_info(ic, opts); for (i = 0; i < orig_nb_streams; i++) av_dict_free(&opts[i]); av_freep(&opts); if (err < 0) { av_log(NULL, AV_LOG_WARNING, "%s: could not find codec parameters\n" , is->filename); ret = -1; goto fail; } } if (ic->pb) ic->pb->eof_reached = 0; // FIXME hack, ffplay maybe should not use avio_feof() to test for the end if (seek_by_bytes < 0) seek_by_bytes = !!(ic->iformat->flags & AVFMT_TS_DISCONT) && strcmp ( "ogg" , ic->iformat->name); is->max_frame_duration = (ic->iformat->flags & AVFMT_TS_DISCONT) ? 10.0 : 3600.0; if (!window_title && (t = av_dict_get(ic->metadata, "title" , NULL, 0))) window_title = av_asprintf( "%s - %s" , t->value, input_filename); /* 检测是否指定播放起始时间,如果指定时间则seek到指定位置,通过ffplay -ss 设置*/ if (start_time != AV_NOPTS_VALUE) { int64_t timestamp; timestamp = start_time; /* add the stream start time */ if (ic->start_time != AV_NOPTS_VALUE) timestamp += ic->start_time;<br> //seek的指定位置开始播放 ret = avformat_seek_file(ic, -1, INT64_MIN, timestamp, INT64_MAX, 0); if (ret < 0) { av_log(NULL, AV_LOG_WARNING, "%s: could not seek to position %0.3f\n" , is->filename, ( double )timestamp / AV_TIME_BASE); } } is->realtime = is_realtime(ic); if (show_status) av_dump_format(ic, 0, is->filename, 0); for (i = 0; i < ic->nb_streams; i++) { AVStream *st = ic->streams[i]; enum AVMediaType type = st->codecpar->codec_type; st->discard = AVDISCARD_ALL; if (type >= 0 && wanted_stream_spec[type] && st_index[type] == -1) if (avformat_match_stream_specifier(ic, st, wanted_stream_spec[type]) > 0) st_index[type] = i; } for (i = 0; i < AVMEDIA_TYPE_NB; i++) { if (wanted_stream_spec[i] && st_index[i] == -1) { av_log(NULL, AV_LOG_ERROR, "Stream specifier %s does not match any %s stream\n" , wanted_stream_spec[i], av_get_media_type_string(i)); st_index[i] = INT_MAX; } } //对音频、视频、字幕进行查找流 if (!video_disable) st_index[AVMEDIA_TYPE_VIDEO] = av_find_best_stream(ic, AVMEDIA_TYPE_VIDEO, st_index[AVMEDIA_TYPE_VIDEO], -1, NULL, 0); if (!audio_disable) st_index[AVMEDIA_TYPE_AUDIO] = av_find_best_stream(ic, AVMEDIA_TYPE_AUDIO, st_index[AVMEDIA_TYPE_AUDIO], st_index[AVMEDIA_TYPE_VIDEO], NULL, 0); if (!video_disable && !subtitle_disable) st_index[AVMEDIA_TYPE_SUBTITLE] = av_find_best_stream(ic, AVMEDIA_TYPE_SUBTITLE, st_index[AVMEDIA_TYPE_SUBTITLE], (st_index[AVMEDIA_TYPE_AUDIO] >= 0 ? st_index[AVMEDIA_TYPE_AUDIO] : st_index[AVMEDIA_TYPE_VIDEO]), NULL, 0); is->show_mode = show_mode; if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {<br> //从视频流中获取宽高参数,设置窗口的宽高 AVStream *st = ic->streams[st_index[AVMEDIA_TYPE_VIDEO]]; AVCodecParameters *codecpar = st->codecpar; AVRational sar = av_guess_sample_aspect_ratio(ic, st, NULL);<br> if (codecpar->width) set_default_window_size(codecpar->width, codecpar->height, sar); } /* 打开音频流、视频流、字幕流,并创建解码线程 */ if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) { stream_component_open(is, st_index[AVMEDIA_TYPE_AUDIO]); } ret = -1; if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) { ret = stream_component_open(is, st_index[AVMEDIA_TYPE_VIDEO]); } if (is->show_mode == SHOW_MODE_NONE) is->show_mode = ret >= 0 ? SHOW_MODE_VIDEO : SHOW_MODE_RDFT; //选择怎么显示,如果视频打开成功,就显示视频,否则显示频谱对应的频谱图 if (st_index[AVMEDIA_TYPE_SUBTITLE] >= 0) { stream_component_open(is, st_index[AVMEDIA_TYPE_SUBTITLE]); } if (is->video_stream < 0 && is->audio_stream < 0) { av_log(NULL, AV_LOG_FATAL, "Failed to open file '%s' or configure filtergraph\n" , is->filename); ret = -1; goto fail; } if (infinite_buffer < 0 && is->realtime) infinite_buffer = 1; //循环读取数据 for (;;) { if (is->abort_request) //是否退出 break ; if (is->paused != is->last_paused) { //是否暂停/继续 is->last_paused = is->paused; if (is->paused) is->read_pause_return = av_read_pause(ic); //这里的暂停、继续只是对网络流的时候有作用 else av_read_play(ic); } #if CONFIG_RTSP_DEMUXER || CONFIG_MMSH_PROTOCOL if (is->paused && (! strcmp (ic->iformat->name, "rtsp" ) || (ic->pb && ! strncmp (input_filename, "mmsh:" , 5)))) { /* wait 10 ms to avoid trying to get another packet */ /* XXX: horrible */ SDL_Delay(10); continue ; } #endif if (is->seek_req) { //是否需要seek int64_t seek_target = is->seek_pos; int64_t seek_min = is->seek_rel > 0 ? seek_target - is->seek_rel + 2: INT64_MIN; int64_t seek_max = is->seek_rel < 0 ? seek_target - is->seek_rel - 2: INT64_MAX; // FIXME the +-2 is due to rounding being not done in the correct direction in generation // of the seek_pos/seek_rel variables //注意:mp4可以seek到指定的时间戳,ts是seek到文件的某个position,而不能直接seek到指定时间点 ret = avformat_seek_file(is->ic, -1, seek_min, seek_target, seek_max, is->seek_flags); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "%s: error while seeking\n" , is->ic->url); } else {<br> //清空音频、视频、字幕队列节点和数据,同时发送flush_pkt清空解码器缓存 if (is->audio_stream >= 0) { packet_queue_flush(&is->audioq); //清空packet队列 packet_queue_put(&is->audioq, &flush_pkt); //用来开起新的一个播放序列,解码器读取到一个flush_pkt也清空解码器 } if (is->subtitle_stream >= 0) { packet_queue_flush(&is->subtitleq); packet_queue_put(&is->subtitleq, &flush_pkt); } if (is->video_stream >= 0) { packet_queue_flush(&is->videoq); packet_queue_put(&is->videoq, &flush_pkt); } if (is->seek_flags & AVSEEK_FLAG_BYTE) { set_clock(&is->extclk, NAN, 0); } else { set_clock(&is->extclk, seek_target / ( double )AV_TIME_BASE, 0); } } is->seek_req = 0; is->queue_attachments_req = 1; is->eof = 0; if (is->paused) step_to_next_frame(is); //如果本身是pause状态,则显示一帧继续暂停 } if (is->queue_attachments_req) { if (is->video_st && is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC) { AVPacket copy; if ((ret = av_packet_ref(©, &is->video_st->attached_pic)) < 0) goto fail; packet_queue_put(&is->videoq, ©); packet_queue_put_nullpacket(&is->videoq, is->video_stream); } is->queue_attachments_req = 0; } /* PacketQueue队列缓冲区已满 最大值为MAX_QUEUE_SIZE 15M,不需要继续读取数据*/ if (infinite_buffer<1 && (is->audioq.size + is->videoq.size + is->subtitleq.size > MAX_QUEUE_SIZE || (stream_has_enough_packets(is->audio_st, is->audio_stream, &is->audioq) && stream_has_enough_packets(is->video_st, is->video_stream, &is->videoq) && stream_has_enough_packets(is->subtitle_st, is->subtitle_stream, &is->subtitleq)))) { /* wait 10 ms */ SDL_LockMutex(wait_mutex); SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10); //如果没有唤醒,则超时10ms退出 SDL_UnlockMutex(wait_mutex); continue ; } if (!is->paused && (!is->audio_st || (is->auddec.finished == is->audioq.serial && frame_queue_nb_remaining(&is->sampq) == 0)) && (!is->video_st || (is->viddec.finished == is->videoq.serial && frame_queue_nb_remaining(&is->pictq) == 0))) { if (loop != 1 && (!loop || --loop)) { stream_seek(is, start_time != AV_NOPTS_VALUE ? start_time : 0, 0, 0); } else if (autoexit) { ret = AVERROR_EOF; goto fail; } } ret = av_read_frame(ic, pkt); //读取数据包,不会释放其数据,要自己去释放 if (ret < 0) { if ((ret == AVERROR_EOF || avio_feof(ic->pb)) && !is->eof) { //数据读取完毕,往队列添加nullpacket if (is->video_stream >= 0) packet_queue_put_nullpacket(&is->videoq, is->video_stream); if (is->audio_stream >= 0) packet_queue_put_nullpacket(&is->audioq, is->audio_stream); if (is->subtitle_stream >= 0) packet_queue_put_nullpacket(&is->subtitleq, is->subtitle_stream); is->eof = 1; } if (ic->pb && ic->pb->error) break ; SDL_LockMutex(wait_mutex); SDL_CondWaitTimeout(is->continue_read_thread, wait_mutex, 10); SDL_UnlockMutex(wait_mutex); continue ; } else { is->eof = 0; } /* 检测是否在播放范围内 */ stream_start_time = ic->streams[pkt->stream_index]->start_time; //获取流的起始时间 pkt_ts = pkt->pts == AV_NOPTS_VALUE ? pkt->dts : pkt->pts; //读取到pkt的时间戳 pkt_in_play_range = duration == AV_NOPTS_VALUE || (pkt_ts - (stream_start_time != AV_NOPTS_VALUE ? stream_start_time : 0)) * av_q2d(ic->streams[pkt->stream_index]->time_base) - ( double )(start_time != AV_NOPTS_VALUE ? start_time : 0) / 1000000 <= (( double )duration / 1000000);<br> //插入队列 if (pkt->stream_index == is->audio_stream && pkt_in_play_range) { packet_queue_put(&is->audioq, pkt); } else if (pkt->stream_index == is->video_stream && pkt_in_play_range && !(is->video_st->disposition & AV_DISPOSITION_ATTACHED_PIC)) { packet_queue_put(&is->videoq, pkt); } else if (pkt->stream_index == is->subtitle_stream && pkt_in_play_range) { packet_queue_put(&is->subtitleq, pkt); } else { av_packet_unref(pkt); //不入队列的pkt,则直接释放 } } ret = 0; fail: //退出线程 if (ic && !is->ic) avformat_close_input(&ic); if (ret != 0) { SDL_Event event; event.type = FF_QUIT_EVENT; event.user.data1 = is; SDL_PushEvent(&event); } SDL_DestroyMutex(wait_mutex); return 0; } |
中断回调函数ic->interrupt_callback.callback用于ffmpeg内部在执行耗时操作时,检查上层调用者是否有退出请求,避免用户退出请求没有及时响应。回调函数中返回1则代表ffmpeg结束耗时操作退出当前函数的调用,返回0则代表ffmpeg内部继续执⾏耗时操作,直到完成既定的任务。使用gdb给回调函数decode_interrupt_cb下断点,观察调用栈可知avformat_open_input,avformat_find_stream_info,av_read_frame等函数都会触发。
有两种情况PacketQueue缓冲区满:
audioq、videoq、subtitleq三个PacketQueue之和最大为15M,这是一个经验值;
stream_has_enough_packets,判断音频、视频、字幕流都已有够用的包,需要三个同时满足才成立
1 2 3 4 5 6 | static int stream_has_enough_packets(AVStream *st, int stream_id, PacketQueue *queue) { return stream_id < 0 || //流没有打开 queue->abort_request || //有请求退出 (st->disposition & AV_DISPOSITION_ATTACHED_PIC) || //配置了AV_DISPOSITION_ATTACHED_PIC queue->nb_packets > MIN_FRAMES && (!queue->duration || av_q2d(st->time_base) * queue->duration > 1.0); //队列包个数大于25,并满足总时长为0或总时长超过1s } |
stream_component_open函数,打开音频、视频、字幕对应的解码器,并创建解码线程。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | static int stream_component_open(VideoState *is, int stream_index) { AVFormatContext *ic = is->ic; AVCodecContext *avctx; AVCodec *codec; const char *forced_codec_name = NULL; AVDictionary *opts = NULL; AVDictionaryEntry *t = NULL; int sample_rate, nb_channels; int64_t channel_layout; int ret = 0; int stream_lowres = lowres; if (stream_index < 0 || stream_index >= ic->nb_streams) return -1; //分配解码器上下文 avctx = avcodec_alloc_context3(NULL); if (!avctx) return AVERROR(ENOMEM); //将码流中解码参数拷贝到新分配的解码器上下文结构体 ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar); if (ret < 0) goto fail; avctx->pkt_timebase = ic->streams[stream_index]->time_base; //根据codec_id查找解码器 codec = avcodec_find_decoder(avctx->codec_id); switch (avctx->codec_type){ //获取指定解码器的名字 case AVMEDIA_TYPE_AUDIO : is->last_audio_stream = stream_index; forced_codec_name = audio_codec_name; break ; case AVMEDIA_TYPE_SUBTITLE: is->last_subtitle_stream = stream_index; forced_codec_name = subtitle_codec_name; break ; case AVMEDIA_TYPE_VIDEO : is->last_video_stream = stream_index; forced_codec_name = video_codec_name; break ; } if (forced_codec_name) codec = avcodec_find_decoder_by_name(forced_codec_name); if (!codec) { if (forced_codec_name) av_log(NULL, AV_LOG_WARNING, "No codec could be found with name '%s'\n" , forced_codec_name); else av_log(NULL, AV_LOG_WARNING, "No decoder could be found for codec %s\n" , avcodec_get_name(avctx->codec_id)); ret = AVERROR(EINVAL); goto fail; } avctx->codec_id = codec->id; if (stream_lowres > codec->max_lowres) { av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n" , codec->max_lowres); stream_lowres = codec->max_lowres; } avctx->lowres = stream_lowres; if (fast) avctx->flags2 |= AV_CODEC_FLAG2_FAST; opts = filter_codec_opts(codec_opts, avctx->codec_id, ic, ic->streams[stream_index], codec); if (!av_dict_get(opts, "threads" , NULL, 0)) av_dict_set(&opts, "threads" , "auto" , 0); if (stream_lowres) av_dict_set_int(&opts, "lowres" , stream_lowres, 0); if (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO) av_dict_set(&opts, "refcounted_frames" , "1" , 0);<br> //打开解码器 if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { goto fail; } if ((t = av_dict_get(opts, "" , NULL, AV_DICT_IGNORE_SUFFIX))) { av_log(NULL, AV_LOG_ERROR, "Option %s not found.\n" , t->key); ret = AVERROR_OPTION_NOT_FOUND; goto fail; } is->eof = 0; ic->streams[stream_index]->discard = AVDISCARD_DEFAULT; switch (avctx->codec_type) { case AVMEDIA_TYPE_AUDIO: #if CONFIG_AVFILTER { AVFilterContext *sink; is->audio_filter_src.freq = avctx->sample_rate; is->audio_filter_src.channels = avctx->channels; is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels); is->audio_filter_src.fmt = avctx->sample_fmt; if ((ret = configure_audio_filters(is, afilters, 0)) < 0) goto fail; sink = is->out_audio_filter; sample_rate = av_buffersink_get_sample_rate(sink); nb_channels = av_buffersink_get_channels(sink); channel_layout = av_buffersink_get_channel_layout(sink); } #else sample_rate = avctx->sample_rate; nb_channels = avctx->channels; channel_layout = avctx->channel_layout; #endif /* 准备音频输出 */ if ((ret = audio_open(is, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0) goto fail; is->audio_hw_buf_size = ret; is->audio_src = is->audio_tgt; is->audio_buf_size = 0; is->audio_buf_index = 0; /* init averaging filter */ is->audio_diff_avg_coef = exp ( log (0.01) / AUDIO_DIFF_AVG_NB); is->audio_diff_avg_count = 0; /* since we do not have a precise anough audio FIFO fullness, we correct audio sync only if larger than this threshold */ is->audio_diff_threshold = ( double )(is->audio_hw_buf_size) / is->audio_tgt.bytes_per_sec; is->audio_stream = stream_index; is->audio_st = ic->streams[stream_index]; //初始化解码器 decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);<br> //启动解码器,创建解码线程 if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) { is->auddec.start_pts = is->audio_st->start_time; is->auddec.start_pts_tb = is->audio_st->time_base; } if ((ret = decoder_start(&is->auddec, audio_thread, "audio_decoder" , is)) < 0) goto out; SDL_PauseAudioDevice(audio_dev, 0); break ; case AVMEDIA_TYPE_VIDEO: is->video_stream = stream_index; //获取video的stream索引 is->video_st = ic->streams[stream_index]; //获取video的stream指针 //初始化解码器 decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);<br> //启动解码器,创建解码线程 if ((ret = decoder_start(&is->viddec, video_thread, "video_decoder" , is)) < 0) goto out; is->queue_attachments_req = 1; break ; case AVMEDIA_TYPE_SUBTITLE: is->subtitle_stream = stream_index; is->subtitle_st = ic->streams[stream_index]; //初始化解码器 decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);<br> //启动解码器,创建解码线程 if ((ret = decoder_start(&is->subdec, subtitle_thread, "subtitle_decoder" , is)) < 0) goto out; break ; default : break ; } goto out; fail: avcodec_free_context(&avctx); out: av_dict_free(&opts); return ret; } |
decoder_init 初始化解码器
d->avctx = avctx; 绑定对应的解码器上下⽂
d->queue = queue; 绑定对应的packet队列
d->empty_queue_cond = empty_queue_cond; 绑定VideoState的continue_read_thread,当解码线程没有packet可读时唤醒read_thread赶紧读取数据
decoder_start启动解码器
packet_queue_start 启⽤对应的packet 队列
SDL_CreateThread 创建对应的解码线程
(2)解码线程
ffplay的解码线程独立于数据读取线程,并且每种类型的流都有各自的解码线程。
(a)video_thread
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | static int video_thread( void *arg) { VideoState *is = arg; AVFrame *frame = av_frame_alloc(); //分配解码帧 double pts; double duration; //帧持续时间 int ret; AVRational tb = is->video_st->time_base; //获取stream timebase AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL); //获取帧率 #if CONFIG_AVFILTER AVFilterGraph *graph = NULL; AVFilterContext *filt_out = NULL, *filt_in = NULL; int last_w = 0; int last_h = 0; enum AVPixelFormat last_format = -2; int last_serial = -1; int last_vfilter_idx = 0; #endif if (!frame) return AVERROR(ENOMEM); //循环取出解码的帧数据 for (;;) { ret = get_video_frame(is, frame); //里面包含了从PacketQueue取一个pkt,avcodec_send_packet发送到解码器、avcodec_receive_frame从解码器读取到图像帧的解码过程 if (ret < 0) goto the_end; //解码结束 if (!ret) continue ; //没有解码得到画面 //过滤器 #if CONFIG_AVFILTER if ( last_w != frame->width || last_h != frame->height || last_format != frame->format || last_serial != is->viddec.pkt_serial || last_vfilter_idx != is->vfilter_idx) { av_log(NULL, AV_LOG_DEBUG, "Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n" , last_w, last_h, ( const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none" ), last_serial, frame->width, frame->height, ( const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none" ), is->viddec.pkt_serial); avfilter_graph_free(&graph); graph = avfilter_graph_alloc(); if (!graph) { ret = AVERROR(ENOMEM); goto the_end; } graph->nb_threads = filter_nbthreads; if ((ret = configure_video_filters(graph, is, vfilters_list ? vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) { SDL_Event event; event.type = FF_QUIT_EVENT; event.user.data1 = is; SDL_PushEvent(&event); goto the_end; } filt_in = is->in_video_filter; filt_out = is->out_video_filter; last_w = frame->width; last_h = frame->height; last_format = frame->format; last_serial = is->viddec.pkt_serial; last_vfilter_idx = is->vfilter_idx; frame_rate = av_buffersink_get_frame_rate(filt_out); } ret = av_buffersrc_add_frame(filt_in, frame); if (ret < 0) goto the_end; while (ret >= 0) { is->frame_last_returned_time = av_gettime_relative() / 1000000.0; ret = av_buffersink_get_frame_flags(filt_out, frame, 0); if (ret < 0) { if (ret == AVERROR_EOF) is->viddec.finished = is->viddec.pkt_serial; ret = 0; break ; } is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time; if ( fabs (is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0) is->frame_last_filter_delay = 0; tb = av_buffersink_get_time_base(filt_out); #endif<br> //计算帧持续时间 duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0); //根据AVStream timebase计算出pts值,单位为秒<br> pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);<br> //将解码后的视频帧插入队列 ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial); av_frame_unref(frame); #if CONFIG_AVFILTER if (is->videoq.serial != is->viddec.pkt_serial) break ; } #endif if (ret < 0) goto the_end; } the_end: #if CONFIG_AVFILTER avfilter_graph_free(&graph); #endif av_frame_free(&frame); //释放frame return 0; } |
上面的流程中主要理解get_video_frame获和queue_picture的实现,而get_video_frame实现真正解码的是decoder_decode_frame。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | static int get_video_frame(VideoState *is, AVFrame *frame) { int got_picture; //解码并获取解码后的帧 if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0) return -1; //分析得到的帧是否要drop掉,目的是在放入frameQueue前,drop掉过时的帧 if (got_picture) { double dpts = NAN; if (frame->pts != AV_NOPTS_VALUE) dpts = av_q2d(is->video_st->time_base) * frame->pts; frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame); //framedrop为1则始终判断是否要丢帧,为0则始终不丢帧,为-1则在同步时钟不是按video时钟时,判断是否要丢帧<br> //如果刚解出来的帧就落后主时钟,就没必要放入队列,进行后面的播放 if (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) { if (frame->pts != AV_NOPTS_VALUE) { double diff = dpts - get_master_clock(is); //计算当前pts和主时钟的差值,!isnan(diff)差值有效,fabs(diff)差值在范围内 if (!isnan(diff) && fabs (diff) < AV_NOSYNC_THRESHOLD && diff - is->frame_last_filter_delay < 0 && is->viddec.pkt_serial == is->vidclk.serial && is->videoq.nb_packets) { is->frame_drops_early++; av_frame_unref(frame); got_picture = 0; } } } } return got_picture; } |
decoder_decode_frame的实现
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub) { int ret = AVERROR(EAGAIN); for (;;) { AVPacket pkt; //1.流连续情况下获取解码帧 if (d->queue->serial == d->pkt_serial) { do { if (d->queue->abort_request) return -1; //是否请求退出 switch (d->avctx->codec_type) { case AVMEDIA_TYPE_VIDEO: ret = avcodec_receive_frame(d->avctx, frame); //从视频解码器中读取一帧 if (ret >= 0) { if (decoder_reorder_pts == -1) { frame->pts = frame->best_effort_timestamp; } else if (!decoder_reorder_pts) { frame->pts = frame->pkt_dts; } } break ; case AVMEDIA_TYPE_AUDIO: ret = avcodec_receive_frame(d->avctx, frame); //从音频解码器中读取一帧 if (ret >= 0) { AVRational tb = (AVRational){1, frame->sample_rate}; if (frame->pts != AV_NOPTS_VALUE) frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb); else if (d->next_pts != AV_NOPTS_VALUE) frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb); if (frame->pts != AV_NOPTS_VALUE) { d->next_pts = frame->pts + frame->nb_samples; d->next_pts_tb = tb; } } break ; } if (ret == AVERROR_EOF) { //解码器内没有帧可读了,解码结束 d->finished = d->pkt_serial; avcodec_flush_buffers(d->avctx); //清解码器缓存, return 0; } if (ret >= 0) return 1; //读取到解码帧 } while (ret != AVERROR(EAGAIN)); } //2.获取一个packet,如果播放序列不一致,则过滤掉过时的pkt do { if (d->queue->nb_packets == 0) //如果没有数据可读,则唤醒数据读取线程read_thread SDL_CondSignal(d->empty_queue_cond); if (d->packet_pending) { //如果之前有缓存的d->pkt,则把之前缓存的d->pkt拷贝到pkt,然后立马发送给编码器 av_packet_move_ref(&pkt, &d->pkt); d->packet_pending = 0; } else { //没缓存有,则从PacketQueue中获取 if (packet_queue_get(d->queue, &pkt, 1, &d->pkt_serial) < 0) //从packet队列获取一个packet,并获取serial return -1; }<br> //d->queue就是videoq<br> //d->pkt_serial是最近一次取packet的序列号 if (d->queue->serial == d->pkt_serial) //序列一致,退出循环 break ; av_packet_unref(&pkt); //释放过滤的pkt } while (1); //如果获取到的pkt是plush_pkt if (pkt.data == flush_pkt.data) { avcodec_flush_buffers(d->avctx); //清空解码器里面的缓存 d->finished = 0; d->next_pts = d->start_pts; d->next_pts_tb = d->start_pts_tb; } else { if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) { int got_frame = 0; ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt); if (ret < 0) { ret = AVERROR(EAGAIN); } else { if (got_frame && !pkt.data) { d->packet_pending = 1; av_packet_move_ref(&d->pkt, &pkt); } ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF); } } else { if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) { //发送一个packet到解码器中<br> //返回EAGAIN解码器没有收到pkt,说明还要继续调用avcodec_send_packet将frame读取,再avcodec_send_packet发送该pkt<br> //由于解码器没有收到pkt,但又没办法放回PacketQueue,所以就缓存到自个封装的Decoder的pkt,并将d->packet_pending=1,以备下次继续使用该pkt av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation.\n" ); d->packet_pending = 1; //说明send_packet失败时,需要重新发送 av_packet_move_ref(&d->pkt, &pkt); //拷贝到缓存的d->pkt } } av_packet_unref(&pkt); //发送到解码器后,释放pkt } } } |
queue_picture 把解码得到的frame添加到队列中
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | static int queue_picture(VideoState *is, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial) { Frame *vp; #if defined(DEBUG_SYNC) printf ( "frame_type=%c pts=%0.3f\n" , av_get_picture_type_char(src_frame->pict_type), pts); #endif if (!(vp = frame_queue_peek_writable(&is->pictq))) //检测队列可写,并从队列尾部申请一可写帧空间,返回可写帧指针 return -1; //对可写帧进行赋值 vp->sar = src_frame->sample_aspect_ratio; vp->uploaded = 0; vp->width = src_frame->width; vp->height = src_frame->height; vp->format = src_frame->format; vp->pts = pts; vp->duration = duration; vp->pos = pos; vp->serial = serial; set_default_window_size(vp->width, vp->height, vp->sar); //将src_frame中所有数据拷贝到vp->frame,使用av_frame_move_ref,使用后的src_frame是无效了 av_frame_move_ref(vp->frame, src_frame);<br> //更新索引位置 frame_queue_push(&is->pictq); return 0; } |
(b)audio_thread
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | static int audio_thread( void *arg) { VideoState *is = arg; AVFrame *frame = av_frame_alloc(); Frame *af; #if CONFIG_AVFILTER int last_serial = -1; int64_t dec_channel_layout; int reconfigure; #endif int got_frame = 0; AVRational tb; int ret = 0; if (!frame) return AVERROR(ENOMEM); do {<br> //1.读取解码帧 if ((got_frame = decoder_decode_frame(&is->auddec, frame, NULL)) < 0) goto the_end; if (got_frame) { tb = (AVRational){1, frame->sample_rate}; //设置sample_rate为timebase //过滤器 #if CONFIG_AVFILTER dec_channel_layout = get_valid_channel_layout(frame->channel_layout, frame->channels); reconfigure = cmp_audio_fmts(is->audio_filter_src.fmt, is->audio_filter_src.channels, frame->format, frame->channels) || is->audio_filter_src.channel_layout != dec_channel_layout || is->audio_filter_src.freq != frame->sample_rate || is->auddec.pkt_serial != last_serial; if (reconfigure) { char buf1[1024], buf2[1024]; av_get_channel_layout_string(buf1, sizeof (buf1), -1, is->audio_filter_src.channel_layout); av_get_channel_layout_string(buf2, sizeof (buf2), -1, dec_channel_layout); av_log(NULL, AV_LOG_DEBUG, "Audio frame changed from rate:%d ch:%d fmt:%s layout:%s serial:%d to rate:%d ch:%d fmt:%s layout:%s serial:%d\n" , is->audio_filter_src.freq, is->audio_filter_src.channels, av_get_sample_fmt_name(is->audio_filter_src.fmt), buf1, last_serial, frame->sample_rate, frame->channels, av_get_sample_fmt_name(frame->format), buf2, is->auddec.pkt_serial); is->audio_filter_src.fmt = frame->format; is->audio_filter_src.channels = frame->channels; is->audio_filter_src.channel_layout = dec_channel_layout; is->audio_filter_src.freq = frame->sample_rate; last_serial = is->auddec.pkt_serial; if ((ret = configure_audio_filters(is, afilters, 1)) < 0) goto the_end; } if ((ret = av_buffersrc_add_frame(is->in_audio_filter, frame)) < 0) goto the_end; while ((ret = av_buffersink_get_frame_flags(is->out_audio_filter, frame, 0)) >= 0) { tb = av_buffersink_get_time_base(is->out_audio_filter); #endif<br> //2.从队列中获取可写帧指针 if (!(af = frame_queue_peek_writable(&is->sampq))) goto the_end; //给可写帧赋值 af->pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb); af->pos = frame->pkt_pos; af->serial = is->auddec.pkt_serial; af->duration = av_q2d((AVRational){frame->nb_samples, frame->sample_rate}); //3.将解码帧拷贝到可写帧,放入队列 av_frame_move_ref(af->frame, frame); //拷贝 frame_queue_push(&is->sampq); //更新索引 #if CONFIG_AVFILTER if (is->audioq.serial != is->auddec.pkt_serial) break ; } if (ret == AVERROR_EOF) is->auddec.finished = is->auddec.pkt_serial; #endif } } while (ret >= 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF); the_end: #if CONFIG_AVFILTER avfilter_graph_free(&is->agraph); #endif av_frame_free(&frame); //此时经过av_frame_move_ref的解码帧已无用,释放即可 return ret; } |
音频解码也是调用了decoder_decode_frame,主要看音频处理不同部分,其他从队列获取packet,过滤packet,发送packet到解码器,流程都与视频解码一致。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | case AVMEDIA_TYPE_AUDIO: ret = avcodec_receive_frame(d->avctx, frame); //从解码器中读取一帧 if (ret >= 0) { AVRational tb = (AVRational){1, frame->sample_rate}; if (frame->pts != AV_NOPTS_VALUE)<br> //如果frame->pts正常则先将其从pkt_timebase 转成{1, frame->sample_rate} frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb); else if (d->next_pts != AV_NOPTS_VALUE)<br> //如果frame->pts不正常则使⽤上⼀帧更新的next_ pts和next_pts_tb 转成 {1, frame->sample_rate} frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb); if (frame->pts != AV_NOPTS_VALUE) {<br> //根据当前帧的pts和nb_samples预估下⼀帧的pts d->next_pts = frame->pts + frame->nb_samples; d->next_pts_tb = tb; //设置timebase } } break ; |
音频解码线程和视频解码线程还有一点不同,就是解码得到的frame后,并没有做过时帧判断以及drop处理。
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· 清华大学推出第四讲使用 DeepSeek + DeepResearch 让科研像聊天一样简单!
· 推荐几款开源且免费的 .NET MAUI 组件库
· 实操Deepseek接入个人知识库
· 易语言 —— 开山篇
· Trae初体验