ffmpeg NVIDIA编解码二：英伟达硬解码

ffmpeg NVIDIA编解码系列

ffmpeg所有解码流程都是一样的：打开解码器、分配解码器上下文，av_read_frame读取视频帧、avcodec_send_packet送入解码器、avcodec_receive_frame获取解码后的视频帧、关闭解码器。不过硬解码在这个流程基础上增加了几步：获取硬件加速的解码格式、创建设备上下文、把解码后的数据从GPU拷贝达到CPU上。

下面是ffmpeg NVIDIA硬解码流程图，带的为硬解码新增的API：

完整代码：

#include <stdio.h>
#include <libavcodec/avcodec.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libavutil/hwcontext.h>
#include <libavutil/imgutils.h>
#include <libavutil/opt.h>
#include <libavutil/pixdesc.h>
#include <libavutil/pixfmt.h>
#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
/**
 * enum AVHWDeviceType {
 *     AV_HWDEVICE_TYPE_NONE,
 *     AV_HWDEVICE_TYPE_VDPAU,
 *     AV_HWDEVICE_TYPE_CUDA,
 *     AV_HWDEVICE_TYPE_VAAPI,
 *     AV_HWDEVICE_TYPE_DXVA2,
 *     AV_HWDEVICE_TYPE_QSV,
 *     AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
 *     AV_HWDEVICE_TYPE_D3D11VA,
 *     AV_HWDEVICE_TYPE_DRM,
 *     AV_HWDEVICE_TYPE_OPENCL,
 *     AV_HWDEVICE_TYPE_MEDIACODEC,
 *     AV_HWDEVICE_TYPE_VULKAN,
 * }
 * 通过 av_hwdevice_get_type_name 方法可以将这些枚举值转换成对应的字符串，比如 AV_HWDEVICE_TYPE_MEDIACODEC 对应的字符串就是 mediacodec ，其实在源码里面也是有的：
 *
 * static const char *const hw_type_names[] = {
 *     [AV_HWDEVICE_TYPE_CUDA]   = "cuda",
 *     [AV_HWDEVICE_TYPE_DRM]    = "drm",
 *     [AV_HWDEVICE_TYPE_DXVA2]  = "dxva2",
 *     [AV_HWDEVICE_TYPE_D3D11VA] = "d3d11va",
 *     [AV_HWDEVICE_TYPE_OPENCL] = "opencl",
 *     [AV_HWDEVICE_TYPE_QSV]    = "qsv",
 *     [AV_HWDEVICE_TYPE_VAAPI]  = "vaapi",
 *     [AV_HWDEVICE_TYPE_VDPAU]  = "vdpau",
 *     [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
 *     [AV_HWDEVICE_TYPE_MEDIACODEC] = "mediacodec",
 *     [AV_HWDEVICE_TYPE_VULKAN] = "vulkan",
 * }
 * 新版本avcodec_send_packet一次，需要循环调用avcodec_receive_frame多次，返回EAGAIN后，结束当前这次的解码，音频解码也是一样
 * AV_PIX_FMT_QSV 英特尔的qsv
 * AV_PIX_FMT_CUDA 英伟达cuda
 */
AVCodec *decoder = NULL;
AVCodecContext *decoder_ctx = NULL;
enum AVPixelFormat hw_pix_fmt;
AVBufferRef *hw_device_ctx = NULL;
FILE *output_file = NULL;
// H265解码器名称:hevc_cuvid H264解码器名称:h264_cuvid
static int dec_init(AVCodecParameters *codec_params, AVStream *video)
{
    int ret;
    // 列举支持的硬解码
    enum AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
    printf("Available device types:\n");
    while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) {
        const char *type_name = av_hwdevice_get_type_name(type);
        if (type_name) {
            printf("%s\n", type_name);
        } else {
            printf("Unknown device type\n");
        }
    }
    type = av_hwdevice_find_type_by_name("cuda");
    if (type == AV_HWDEVICE_TYPE_NONE) {
        printf("not support nvidia codec\n");
        exit(0);
    }
    decoder = avcodec_find_decoder(codec_params->codec_id); // H264 : avcodec_find_decoder_by_name("h264_cuvid"); H265 : avcodec_find_decoder_by_name("hevc_cuvid");
    // 获取该硬解码器的像素格式。cuda对应的hw_pix_fmt是AV_PIX_FMT_CUDA
    for (int i = 0;; i++) {
        const AVCodecHWConfig *config = avcodec_get_hw_config(decoder, i);
        if (!config) {
            printf("get config error\n");
            exit(0);
        }
        if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
            config->device_type == type) {
            hw_pix_fmt = config->pix_fmt;
            break;
        }
    }
    const char *pixname = av_get_pix_fmt_name(hw_pix_fmt);
    printf("hw_pix_fmt:%s\n", pixname);
    // 初始化解码器
    decoder_ctx = avcodec_alloc_context3(decoder);
    if (decoder_ctx == NULL) {
        printf("avcodec_alloc_context3 error\n");
        exit(0);
    }
    avcodec_parameters_to_context(decoder_ctx, video->codecpar); // 拷贝解码参数
    // 设置硬解码格式
    decoder_ctx->pix_fmt = hw_pix_fmt;

    // 创建一个硬件设备上下文
    int err = 0;
    if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type, NULL, NULL, 0)) < 0) {
        printf("Failed to create specified HW device.\n");
        exit(0);
    }
    decoder_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);

    // 打开解码器
    ret = avcodec_open2(decoder_ctx, decoder, NULL);
    if (ret < 0) {
        printf("avcodec_open2 error\n");
        exit(0);
    }
    printf("dec_init ok\n");
    return 0;
}
static int dec_uninit()
{
    avcodec_close(decoder_ctx);
    avcodec_free_context(&decoder_ctx);
    av_buffer_unref(&hw_device_ctx);
    printf("dec_uninit ok\n");
    return 0;
}
static int decode_write(AVPacket *packet)
{
    AVFrame *frame = NULL, *sw_frame = NULL;
    AVFrame *tmp_frame = NULL;
    uint8_t *buffer = NULL;
    int size;
    int ret = 0;
    ret = avcodec_send_packet(decoder_ctx, packet);
    if (ret < 0) {
        fprintf(stderr, "Error during decoding\n");
        return ret;
    }
    while (1) {
        if (!(frame = av_frame_alloc()) || !(sw_frame = av_frame_alloc())) {
            fprintf(stderr, "Can not alloc frame\n");
            ret = AVERROR(ENOMEM);
            goto fail;
        }
        ret = avcodec_receive_frame(decoder_ctx, frame);
        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
            av_frame_free(&frame);
            av_frame_free(&sw_frame);
            return 0;
        } else if (ret < 0) {
            fprintf(stderr, "Error while decoding\n");
            goto fail;
        }

        if (frame->format == hw_pix_fmt) { // 硬解码，解码后的图像在GPU上
            // 把图像从GPU拷贝到CPU
            // av_hwframe_transfer_data转换后的格式是NV12的格式转换
            if ((ret = av_hwframe_transfer_data(sw_frame, frame, 0)) < 0) {
                fprintf(stderr, "Error transferring the data to system memory\n");
                goto fail;
            }
            tmp_frame = sw_frame;
        } else { // 解码之后的图像在CPU上
            tmp_frame = frame;
        }
        size = av_image_get_buffer_size(tmp_frame->format, tmp_frame->width, tmp_frame->height, 1);
        buffer = av_malloc(size);
        if (!buffer) {
            fprintf(stderr, "Can not alloc buffer\n");
            ret = AVERROR(ENOMEM);
            goto fail;
        }
        ret = av_image_copy_to_buffer(buffer, size,
                                      (const uint8_t *const *)tmp_frame->data,
                                      (const int *)tmp_frame->linesize, tmp_frame->format,
                                      tmp_frame->width, tmp_frame->height, 1);
        if (ret < 0) {
            fprintf(stderr, "Can not copy image to buffer\n");
            goto fail;
        }
        // 写入文件的格式是NV12
        fwrite(buffer, 1, size, output_file);
    fail:
        av_frame_free(&frame);
        av_frame_free(&sw_frame);
        av_freep(&buffer);
        if (ret < 0)
            return ret;
    }
}

int main(int argc, char *argv[])
{
    if (argc < 3) {
        printf("./bin input output\n");
        return -1;
    }
    av_register_all();
    int ret = -1;
    char errors[1024];
    char *in_filename = argv[1];
    char *out_filename = argv[2];
    output_file = fopen(out_filename, "w+");

    int audio_index = -1;
    int video_index = -1;

    AVFormatContext *ifmt_ctx = NULL;
    AVPacket pkt;
    av_init_packet(&pkt);

    if ((ret = avformat_open_input(&ifmt_ctx, in_filename, NULL, NULL)) < 0) {
        av_strerror(ret, errors, 1024);
        av_log(NULL, AV_LOG_DEBUG, "Could not open source file: %s, %d(%s)\n",
               in_filename,
               ret,
               errors);
        return -1;
    }

    if ((ret = avformat_find_stream_info(ifmt_ctx, NULL)) < 0) {
        av_strerror(ret, errors, 1024);
        av_log(NULL, AV_LOG_DEBUG, "failed to find stream information: %s, %d(%s)\n",
               in_filename,
               ret,
               errors);
        return -1;
    }

    audio_index = av_find_best_stream(ifmt_ctx, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);
    video_index = av_find_best_stream(ifmt_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
    // 解码器初始化
    dec_init(ifmt_ctx->streams[video_index]->codecpar, ifmt_ctx->streams[video_index]);
    while (av_read_frame(ifmt_ctx, &pkt) >= 0) {
        if (pkt.stream_index == audio_index) {
            AVStream *audio_stream = ifmt_ctx->streams[audio_index];
            // do nothing
        } else if (pkt.stream_index == video_index) {
            AVStream *video_stream = ifmt_ctx->streams[video_index];
            printf("video pts:%lld\n", pkt.pts);
            // decode
            decode_write(&pkt);
        }
        av_packet_unref(&pkt);
    }
    // fflush
    pkt.data = NULL;
    pkt.size = 0;
    ret = decode_write(&pkt);
    avformat_close_input(&ifmt_ctx);
    dec_uninit();
    fclose(output_file);
    return 0;
}

我的开源：

         1、Nvidia视频硬解码、渲染、软/硬编码并写入MP4文件。项目地址：https://github.com/BreakingY/Nvidia-Video-Codec
        2、Jetson Jetpack5.x视频编解码。项目地址：https://github.com/BreakingY/jetpack-dec-enc
        3、音视频(H264/H265/AAC)封装、解封装、编解码pipeline，支持NVIDIA、昇腾DVPP硬编解码。项目地址：https://github.com/BreakingY/Media-Codec-Pipeline
        4、simple rtsp server，小而高效的rtsp服务器，支持H264、H265、AAC、PCMA；支持TCP、UDP；支持鉴权。项目地址：https://github.com/BreakingY/simple-rtsp-server

5、simple rtsp client，rtsp客户端，支持TCP、UDP、H264、H265、AAC、PCMA，支持鉴权。项目地址：https://github.com/BreakingY/simple-rtsp-client

6、libflv，flv muxer/demuxer，支持H264/H265、AAC。项目地址：https://github.com/BreakingY/libflv

7、mpeg2 ts ps muxer/demuxer，支持H264/H265/MPEG1 audio/MP3/AAC/AAC_LATM/G711。项目地址：https://github.com/BreakingY/libmpeg2core