FFmpeg5.1 解码rtsp 并用OpenCV 播放
RTSP 连接过程如下图
看下实际过程中FFmpeg 的日志情况：
[tcp @ 0000014CC3256D40] No default whitelist set
[tcp @ 0000014CC3256D40] Original list of addresses:
[tcp @ 0000014CC3256D40] Address ::1 port 8554
[tcp @ 0000014CC3256D40] Address 127.0.0.1 port 8554
[tcp @ 0000014CC3256D40] Interleaved list of addresses:
[tcp @ 0000014CC3256D40] Address ::1 port 8554
[tcp @ 0000014CC3256D40] Address 127.0.0.1 port 8554
[tcp @ 0000014CC3256D40] Starting connection attempt to ::1 port 8554
[tcp @ 0000014CC3256D40] Successfully connected to ::1 port 8554
===========================================================================================
[rtsp @ 0000014CC325DA80] Sending:
OPTIONS rtsp://:8554/test RTSP/1.0
CSeq: 1
User-Agent: Lavf59.34.101
--
[rtsp @ 0000014CC325DA80] line='RTSP/1.0 200 OK'
[rtsp @ 0000014CC325DA80] line='Server: VLC/3.0.17.4'
[rtsp @ 0000014CC325DA80] line='Content-Length: 0'
[rtsp @ 0000014CC325DA80] line='Cseq: 1'
[rtsp @ 0000014CC325DA80] line='Public: DESCRIBE,SETUP,TEARDOWN,PLAY,PAUSE,GET_PARAMETER'
[rtsp @ 0000014CC325DA80] line=''
===========================================================================================
[rtsp @ 0000014CC325DA80] Sending:
DESCRIBE rtsp://:8554/test RTSP/1.0
Accept: application/sdp
CSeq: 2
User-Agent: Lavf59.34.101
--
[rtsp @ 0000014CC325DA80] line='RTSP/1.0 200 OK'
[rtsp @ 0000014CC325DA80] line='Server: VLC/3.0.17.4'
[rtsp @ 0000014CC325DA80] line='Date: Thu, 17 Nov 2022 06:45:09 GMT'
[rtsp @ 0000014CC325DA80] line='Content-Type: application/sdp'
[rtsp @ 0000014CC325DA80] line='Content-Base: rtsp://[::1]:8554/test'
[rtsp @ 0000014CC325DA80] line='Content-Length: 542'
[rtsp @ 0000014CC325DA80] line='Cache-Control: no-cache'
[rtsp @ 0000014CC325DA80] line='Cseq: 2'
[rtsp @ 0000014CC325DA80] line=''
[rtsp @ 0000014CC325DA80] SDP:
v=0
o=- 16654407034258208241 16654407034258208241 IN IP6 tayu-pc
s=Unnamed
i=N/A
c=IN IP6 ::
t=0 0
a=tool:vlc 3.0.17.4
a=recvonly
a=type:broadcast
a=charset:UTF-8
a=control:rtsp://[::1]:8554/test
m=audio 0 RTP/AVP 14
b=AS:128
b=RR:0
a=rtpmap:14 MPA/90000/2
a=control:rtsp://[::1]:8554/test/trackID=0
m=video 0 RTP/AVP 96
b=RR:0
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1;profile-level-id=640020;sprop-parameter-sets=Z2QAIKzZQFAGWwFqBAQCgAAAAwCAAAAFR4wYyw==,aOvjyyLA;
a=control:rtsp://[::1]:8554/test/trackID=1
===========================================================================================
[rtp @ 0000014CC32558C0] No default whitelist set
[udp @ 0000014CC3262100] No default whitelist set
[udp @ 0000014CC3262100] end receive buffer size reported is 393216
[udp @ 0000014CC3262000] No default whitelist set
[udp @ 0000014CC3262000] end receive buffer size reported is 393216
===========================================================================================
[rtsp @ 0000014CC325DA80] Sending:
SETUP rtsp://[::1]:8554/test/trackID=0 RTSP/1.0
Transport: RTP/AVP/UDP;unicast;client_port=20332-20333
CSeq: 3
User-Agent: Lavf59.34.101
--
[rtsp @ 0000014CC325DA80] line='RTSP/1.0 200 OK'
[rtsp @ 0000014CC325DA80] line='Server: VLC/3.0.17.4'
[rtsp @ 0000014CC325DA80] line='Date: Thu, 17 Nov 2022 06:45:21 GMT'
[rtsp @ 0000014CC325DA80] line='Transport: RTP/AVP/UDP;unicast;client_port=20332-20333;server_port=61687-61688;ssrc=416BF87E;mode=play'
[rtsp @ 0000014CC325DA80] line='Session: a242cd292f307f4b;timeout=60'
[rtsp @ 0000014CC325DA80] line='Content-Length: 0'
[rtsp @ 0000014CC325DA80] line='Cache-Control: no-cache'
[rtsp @ 0000014CC325DA80] line='Cseq: 3'
[rtsp @ 0000014CC325DA80] line=''
===========================================================================================
[rtsp @ 0000014CC325DA80] Sending:
SETUP rtsp://[::1]:8554/test/trackID=1 RTSP/1.0
Transport: RTP/AVP/UDP;unicast;client_port=20334-20335
CSeq: 4
User-Agent: Lavf59.34.101
Session: a242cd292f307f4b
--
[rtsp @ 0000014CC325DA80] line='RTSP/1.0 200 OK'
[rtsp @ 0000014CC325DA80] line='Server: VLC/3.0.17.4'
[rtsp @ 0000014CC325DA80] line='Date: Thu, 17 Nov 2022 06:45:46 GMT'
[rtsp @ 0000014CC325DA80] line='Transport: RTP/AVP/UDP;unicast;client_port=20334-20335;server_port=61688-61689;ssrc=4ACDD3CA;mode=play'
[rtsp @ 0000014CC325DA80] line='Session: a242cd292f307f4b;timeout=60'
[rtsp @ 0000014CC325DA80] line='Content-Length: 0'
[rtsp @ 0000014CC325DA80] line='Cache-Control: no-cache'
[rtsp @ 0000014CC325DA80] line='Cseq: 4'
[rtsp @ 0000014CC325DA80] line=''
===========================================================================================
[rtsp @ 0000014CC325DA80] Sending:
PLAY rtsp://[::1]:8554/test RTSP/1.0
Range: npt=0.000-
CSeq: 5
User-Agent: Lavf59.34.101
Session: a242cd292f307f4b
--
[rtsp @ 0000014CC325DA80] line='RTSP/1.0 200 OK'
[rtsp @ 0000014CC325DA80] line='Server: VLC/3.0.17.4'
[rtsp @ 0000014CC325DA80] line='Date: Thu, 17 Nov 2022 06:45:56 GMT'
[rtsp @ 0000014CC325DA80] line='RTP-Info: url=rtsp://[::1]:8554/test/trackID=0;seq=12541;rtptime=1127806156, url=rtsp://[::1]:8554/test/trackID=1;seq=942;rtptime=1127806156'
[rtsp @ 0000014CC325DA80] line='Range: npt=58.978649-'
[rtsp @ 0000014CC325DA80] line='Session: a242cd292f307f4b;timeout=60'
[rtsp @ 0000014CC325DA80] line='Content-Length: 0'
[rtsp @ 0000014CC325DA80] line='Cache-Control: no-cache'
[rtsp @ 0000014CC325DA80] line='Cseq: 5'
[rtsp @ 0000014CC325DA80] line=''
===========================================================================================
日志到Play 就没再记录时因为后面日志就比较多了，没仔细看。
总体逻辑就是客户端向服务端发送几次请求，并会收到服务端的一些响应。
Sending: OPTIONS Cseq: 1
Receive: 'RTSP/1.0 200 OK Cseq: 1
…
Sending: DESCRIBE Accept: application/sdp Cseq: 2
Receive: 'RTSP/1.0 200 OK Cseq: 2
SDP:
v=0
o=- 16654407034258208241 16654407034258208241 IN IP6 tayu-pc
s=Unnamed
i=N/A
c=IN IP6 ::
t=0 0
a=tool:vlc 3.0.17.4
a=recvonly
a=type:broadcast
a=charset:UTF-8
a=control:rtsp://[::1]:8554/test
m=audio 0 RTP/AVP 14
b=AS:128
b=RR:0
a=rtpmap:14 MPA/90000/2
a=control:rtsp://[::1]:8554/test/trackID=0
m=video 0 RTP/AVP 96
b=RR:0
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1;profile-level-id=640020;sprop-parameter-sets=Z2QAIKzZQFAGWwFqBAQCgAAAAwCAAAAFR4wYyw==,aOvjyyLA;
a=control:rtsp://[::1]:8554/test/trackID=1
…
Sending: SETUP Transport: RTP/AVP/UDP;unicast;client_port=20332-20333 trackID=0 Cseq: 3 //视频轨道
Receive: 'RTSP/1.0 200 OK Cseq: 3
'Transport: RTP/AVP/UDP;unicast;client_port=20332-20333;server_port=61687-61688;ssrc=416BF87E;mode=play
1
…
Sending: SETUP Transport: RTP/AVP/UDP;unicast;client_port=20332-20333 trackID=1 Cseq: 4 //音频轨道
Receive: 'RTSP/1.0 200 OK Cseq: 4
'Transport: RTP/AVP/UDP;unicast;client_port=20332-20333;server_port=61687-61688;ssrc=416BF87E;mode=play
1
…
Sending: PLAY Session: a242cd292f307f4b Cseq: 5
Receive: 'RTSP/1.0 200 OK Session: a242cd292f307f4b;timeout=60 Cseq: 5
…
上面说了这么多消息对应早FFmpeg 中其实就是一个API 做的事情：
API avformat_open_input
当返回值为0时上面的操作就基本都完成了。
需要注意的是 SDP 协议是使用TCP 连接，RTSP 是会建立RTP 连接，FFMpeg 默认使用UDP 建立RTP连接，从上面日志中也可以看出来。
所以如果API 一直失败可以考虑是否不是UDP 的连接，可以使用AVDictionary了修改为TCP 连接。
UDP代码
auto ret = avformat_open_input(&format_ctx, "rtsp://192.168.10.182/video/pri", NULL, nullptr);
1
TCP 代码
AVDictionary* format_opts = NULL;
av_dict_set(&format_opts, "stimeout", std::to_string(2 * 1000000).c_str(), 0); //设置链接超时时间（us）
av_dict_set(&format_opts, "rtsp_transport", "tcp", 0); //设置推流的方式，默认udp。
auto ret = avformat_open_input(&format_ctx, "rtsp://192.168.10.182/video/pri", NULL, &format_opts);
到这里基本RTSP 的连接工作就结束了。。。。
API av_read_frame
while (av_read_frame(format_ctx, &packet) >= 0 && cnt < 1000)
可以循环使用这个API 从RTSP 中读取h264的流，packet 中包换的nal 的buffer。
到这里思路就应该是如何从h264得到yuv或rgb/bgr了。我们使用Opencv 就获取BGR24即可。
之后的代码和解码代码类似，直接放全部代码仅供参考。
#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <fstream>
#include <sstream>
extern "C" {
#include "libavcodec/avcodec.h"
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libswscale/swscale.h"
}
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
using namespace cv;
int main(int argc, char** argv) {
// Open the initial context variables that are needed
SwsContext* img_convert_ctx;
AVFormatContext* format_ctx = avformat_alloc_context();
AVCodecContext* codec_ctx = NULL;
int video_stream_index;
av_log_set_level(AV_LOG_TRACE); //AV_LOG_TRACE //AV_LOG_DEBUG
//open RTSP
AVDictionary* format_opts = NULL;
av_dict_set(&format_opts, "stimeout", std::to_string(2 * 1000000).c_str(), 0); //设置链接超时时间（us）
av_dict_set(&format_opts, "rtsp_transport", "tcp", 0); //设置推流的方式，默认udp。
auto ret = avformat_open_input(&format_ctx, "rtsp://:8554/test", NULL, &format_opts);
ret = avformat_find_stream_info(format_ctx, NULL);
//search video stream
for (int i = 0; i < format_ctx->nb_streams; i++) {
if (format_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
video_stream_index = i;
}
AVPacket packet;
av_init_packet(&packet);
AVStream* stream = NULL;
int cnt = 0;
//start reading packets from stream and write them to file
av_read_play(format_ctx); //play RTSP
// Get the codec
const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec) {
exit(1);
}
std::ofstream myfile;
// Add this to allocate the context by codec
codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_from_context(format_ctx->streams[video_stream_index]->codecpar, codec_ctx);
std::ofstream output_file;
codec_ctx->width = 640;
codec_ctx->height = 480;
codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
if (avcodec_open2(codec_ctx, codec, NULL) < 0)
exit(1);
img_convert_ctx = sws_getContext(codec_ctx->width, codec_ctx->height, codec_ctx->pix_fmt
, codec_ctx->width, codec_ctx->height, AV_PIX_FMT_BGR24,
SWS_BICUBIC, NULL, NULL, NULL);
AVFrame* picture = av_frame_alloc();
picture = av_frame_alloc();
AVFrame* picturergb = av_frame_alloc();
picturergb = av_frame_alloc();
if (!picture) {
fprintf(stderr, "Could not allocate video frame\n");
exit(1);
}
picture->format = codec_ctx->pix_fmt;
picture->width = codec_ctx->width;
picture->height = codec_ctx->height;
picturergb->format = AV_PIX_FMT_BGR24;
picturergb->width = codec_ctx->width;
picturergb->height = codec_ctx->height;
ret = av_frame_get_buffer(picture, 0);
ret = av_frame_get_buffer(picturergb, 0);
AVCodecParserContext* parser;
parser = av_parser_init(codec->id);
if (!parser) {
fprintf(stderr, "parser not found\n");
exit(1);
}
while (av_read_frame(format_ctx, &packet) >= 0 && cnt < 1000)
{
if (stream == NULL)
{
stream = avformat_new_stream(format_ctx, codec);
avcodec_parameters_from_context(stream->codecpar, codec_ctx);
}
ret = avcodec_send_packet(codec_ctx, &packet);
if (ret < 0) {
continue;
}
while (ret >= 0) {
ret = avcodec_receive_frame(codec_ctx, picture);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
else if (ret < 0) {
fprintf(stderr, "Error during decoding\n");
return -1;
}
ret = sws_scale(img_convert_ctx, picture->data, picture->linesize, 0, 480, picturergb->data, picturergb->linesize);
Mat src(cv::Size(640,480), CV_8UC3, picturergb->buf[0]->data);
imshow("test", src);
waitKey(1);
}
}
av_free(picture);
av_read_pause(format_ctx);
return (EXIT_SUCCESS);
}
