zlmediakit源码学习（扩展支持定时抽帧）

使用了很长时间的zlmediakit流媒体服务，一直对其精妙高效的设计实现十分好奇。最好的学习就是去二次开发实现一些小功能，同时摸索框架的代码结构

在参考了zlmediakit的录像功能后，分析模仿它的源码结构，实现定时抽帧的功能。

抽帧之后可以：1）进行算法分析；2）重新编码实现转码功能；3）算法分析之后再编码，实现算法结果视频流程序。

向优秀的流媒体服务zlmediakit致敬！！

--------------------------------

关键代码如下：

1.在installWebApi方法中新增开始转码和停止转码的HTTP接口

void installWebApi() {  
    *****
    api_regist("/index/api/startTranscode", [](API_ARGS_MAP_ASYNC) {
        CHECK_SECRET();
        CHECK_ARGS("type", "vhost", "app", "stream");
        auto src = MediaSource::find(allArgs["vhost"], allArgs["app"],  allArgs["stream"]);
        if (!src) {
            throw ApiRetException("can not find the stream", API::NotFound);
        }
        src->getOwnerPoller()->async([=]() mutable {
            auto result =  src->setupTranscode((Transcoder::type)allArgs["type"].as<int>(), true,  allArgs["customized_path"],
                allArgs["max_second"].as<size_t>());
            val["result"] = result;
            val["code"] = result ? API::Success : API::OtherFailed;
            val["msg"] = result ? "success" : "start record failed";
            invoker(200, headerOut, val.toStyledString());
        });
    });
    ******
    api_regist("/index/api/stopTranscode", [](API_ARGS_MAP_ASYNC) {
        CHECK_SECRET();
        CHECK_ARGS("type", "vhost", "app", "stream");
        auto src = MediaSource::find(allArgs["vhost"], allArgs["app"],  allArgs["stream"]);
        if (!src) {
            throw ApiRetException("can not find the stream", API::NotFound);
        }
        src->getOwnerPoller()->async([=]() mutable {
            auto result = src->setupTranscode(
                (Transcoder::type)allArgs["type"].as<int>(), false,  allArgs["customized_path"],
                allArgs["max_second"].as<size_t>());
            val["result"] = result;
            val["code"] = result ? API::Success : API::OtherFailed;
            val["msg"] = result ? "success" : "start record failed";
            invoker(200, headerOut, val.toStyledString());
        });
    });
    *****
}

2.在MediaSource、MediaSourceEvent、MediaSourceEventInterceptor、MultiMediaSourceMuxer中模仿setupRecord添加setupTranscode

    调用顺序是：MediaSource::setupTranscode ---> MediaSourceEventInterceptor::setupTranscode  ---> MultiMediaSourceMuxer::setupTranscode

    最终执行创建转码的对象，并赋值给MultiMediaSourceMuxer::_transcode

bool MediaSource::setupTranscode(Transcoder::type type, bool start, const string  &custom_path, size_t max_second) {
    auto listener = _listener.lock();
    if (!listener) {
        WarnL << "未设置MediaSource的事件监听者，setupRecord失败:" << getSchema() <<  "/" << getVhost() << "/"
              << getApp() << "/" << getId();
        return false;
    }
    return listener->setupTranscode(*this, type, start, custom_path, max_second);
}

bool MediaSourceEventInterceptor::setupTranscode(MediaSource &sender,  Transcoder::type type, bool start, const string &custom_path, size_t max_second) {
    auto listener = _listener.lock();
    if (!listener) {
        return false;
    }
    return listener->setupTranscode(sender, type, start, custom_path, max_second);
}

bool MultiMediaSourceMuxer::setupTranscode(MediaSource &sender, Transcoder::type  type, bool start, const std::string &custom_path, size_t max_second) {
    if (start && !_transcode) {
        //开始转码
        _transcode = makeTranscoder(sender, getTracks(), Transcoder::type_mp4,  custom_path, max_second);    //创建转码对象
    } else if (!start && _transcode) {
        //停止转码
        _transcode = nullptr;
    }
    return true;
}

3.在zlmediakit项目中模拟Record，创建Transcode相关的对象：Transcoder、FFmpegTranscoder、FFmpegMuxer

4.Transcoder::getTranscodePath。用于获取抽帧截图的文件夹目录

std::string Transcoder::getTranscodePath( type type, const std::string &vhost,  const std::string &app, const std::string &stream_id, const std::string  &customized_path) {
    GET_CONFIG(bool, enableVhost, General::kEnableVhost);
    GET_CONFIG(string, recordPath, Record::kFilePath);
    GET_CONFIG(string, recordAppName, Record::kAppName);
    string mp4FilePath;
    if (enableVhost) {
        mp4FilePath = vhost + "/" + recordAppName + "/" + app + "/" + stream_id +  "/";
    } else {
        mp4FilePath = recordAppName + "/" + app + "/" + stream_id + "/";
    }
    // Here we use the customized file path.
    if (!customized_path.empty()) {
        return File::absolutePath(mp4FilePath, customized_path);
    }
    return File::absolutePath(mp4FilePath, recordPath);
}

5.Transcoder::createTranscoder。用于创建一个FFmpegTranscoder转码对象

std::shared_ptr<MediaSinkInterface> Transcoder::createTranscoder(type type, const  std::string &vhost, const std::string &app, const std::string &stream_id,const  std::string &customized_path, size_t max_second) {
    auto path = Transcoder::getTranscodePath(type, vhost, app, stream_id,  customized_path);
    return std::make_shared<FFmpegTranscoder>(path, vhost, app, stream_id,  max_second);
}

6.FFmpegTranscoder::FFmpegTranscoder。继承自MediaSinkInterface，可以作为一个输出类型的MediaSink。保存info信息，并创建一个FFmpegMuxer封装器对象

FFmpegTranscoder::FFmpegTranscoder(const std::string &path, const std::string  &vhost, const std::string &app, const std::string &stream_id,size_t max_second) {
    _folder_path = path;    _info.app = app;
    _info.stream = stream_id;
    _info.vhost = vhost;
    _info.folder = path;
    GET_CONFIG(size_t, recordSec, Record::kFileSecond);
    _max_second = max_second ? max_second : recordSec;
    _muxer = std::make_shared<FFmpegMuxer>(_folder_path, _max_second);
}

7.FFmpegTranscoder::addTrack。继承自MediaSink::addTrack。添加音视频轨道。最终是向封装器对象_muxer中添加轨道，暂时只关注视频轨道

bool FFmpegTranscoder::addTrack(const Track::Ptr &track) {
    _tracks.emplace_back(track);
    if (track->getTrackType() == TrackVideo) {
        _have_video = true;
        _muxer->addTrack(track);
        });
    }
    return true;
}

8.FFmpegTranscoder::inputFrame。继承自MediaSink::inputFrame，接收帧数据。将帧数据写入到了_muxer

bool FFmpegTranscoder::inputFrame(const Frame::Ptr &frame) {
    if (_muxer) {
        return _muxer->inputFrame(frame);
    }
    return true;
}

9.FFmpegMuxer::addTrack。创建视频解码器，用的是zlmediakit已经封装好的FFmpegDecoder，是一个基于ffmpeg的多线程异步解码对象，并且可以硬件解码器。

    设置解码回调:解析将解码后的YUV数据帧转成图片格式并落地保存。

bool FFmpegMuxer::addTrack(const Track::Ptr &track) {
    if (track->getCodecId() == CodecH264) {
        _video_dec.reset(new FFmpegDecoder(track));
    } else if (track->getCodecId() == CodecH265) {
        _video_dec.reset(new FFmpegDecoder(track));
    } else {
    }
    if (_video_dec != nullptr) {
        _video_dec->setOnDecode([this](const FFmpegFrame::Ptr &frame) {
            // -----抽帧操作 begin
　　　　　　　time_t now = ::time(NULL);
            if (now - _last_time >= _gapTime) {
                AVFrame *avFrame = frame->get();
                int bufSize = av_image_get_buffer_size(AV_PIX_FMT_BGRA,  avFrame->width, avFrame->height, 64);
                uint8_t *buf = (uint8_t *)av_malloc(bufSize);
                int picSize = frameToImage(avFrame, AV_CODEC_ID_MJPEG, buf,  bufSize);
                if (picSize > 0) {
                    auto file_path = _folder_path + getTimeStr("%H-%M-%S_") +  std::to_string(_index) + ".jpeg";
                    auto f = fopen(file_path.c_str(), "wb+");
                    if (f) {
                        fwrite(buf, sizeof(uint8_t), bufSize, f);
                        fclose(f);
                    }
                }
                av_free(buf);
                _index++;
                _last_time = now;
            }

　　　　　　  // -----抽帧操作 end

}); } return true; }

10.FFmpegMuxer::frameToImage。利用FFmpeg将AVFrame的视频帧转成二进制数组输出

int FFmpegMuxer::frameToImage(AVFrame *frame, AVCodecID codecID, uint8_t *outbuf,  size_t outbufSize) {
    int ret = 0;
    AVPacket pkt;
    AVCodec *codec;
    AVCodecContext *ctx = NULL;
    AVFrame *rgbFrame = NULL;
    uint8_t *buffer = NULL;
    struct SwsContext *swsContext = NULL;
    av_init_packet(&pkt);
    codec = avcodec_find_encoder(codecID);
    if (!codec) {
        goto end;
    }
    if (!codec->pix_fmts) {
        goto end;
    }
    ctx = avcodec_alloc_context3(codec);
    ctx->bit_rate = 3000000;
    ctx->width = frame->width;
    ctx->height = frame->height;
    ctx->time_base.num = 1;
    ctx->time_base.den = 25;
    ctx->gop_size = 10;
    ctx->max_b_frames = 0;
    ctx->thread_count = 1;
    ctx->pix_fmt = *codec->pix_fmts;
    ret = avcodec_open2(ctx, codec, NULL);
    if (ret < 0) {
        printf("avcodec_open2 error %d", ret);
        goto end;
    }
    if (frame->format != ctx->pix_fmt) {
        rgbFrame = av_frame_alloc();
        if (rgbFrame == NULL) {
            printf("av_frame_alloc  fail");
            goto end;
        }
        swsContext = sws_getContext(
            frame->width, frame->height, (enum AVPixelFormat)frame->format,  frame->width, frame->height, ctx->pix_fmt,
            1, NULL, NULL, NULL);
        if (!swsContext) {
            printf("sws_getContext  fail");
            goto end;
        }
        int bufferSize = av_image_get_buffer_size(ctx->pix_fmt, frame->width,  frame->height, 1) * 2;
        buffer = (unsigned char *)av_malloc(bufferSize);
        if (buffer == NULL) {
            printf("buffer alloc fail:%d", bufferSize);
            goto end;
        }
        av_image_fill_arrays(rgbFrame->data, rgbFrame->linesize, buffer,  ctx->pix_fmt, frame->width, frame->height, 1);
        if ((ret = sws_scale(
                 swsContext, frame->data, frame->linesize, 0, frame->height,  rgbFrame->data, rgbFrame->linesize))
            < 0) {
            printf("sws_scale error %d", ret);
        }
        rgbFrame->format = ctx->pix_fmt;
        rgbFrame->width = ctx->width;
        rgbFrame->height = ctx->height;
        ret = avcodec_send_frame(ctx, rgbFrame);
    } else {
        ret = avcodec_send_frame(ctx, frame);
    }
    if (ret < 0) {
        printf("avcodec_send_frame error %d", ret);
        goto end;
    }
    ret = avcodec_receive_packet(ctx, &pkt);
    if (ret < 0) {
        printf("avcodec_receive_packet error %d", ret);
        goto end;
    }
    if (pkt.size > 0 && pkt.size <= outbufSize)
        memcpy(outbuf, pkt.data, pkt.size);
    ret = pkt.size;
end:
    if (swsContext) {
        sws_freeContext(swsContext);
    }
    if (rgbFrame) {
        av_frame_unref(rgbFrame);
        av_frame_free(&rgbFrame);
    }
    if (buffer) {
        av_free(buffer);
    }
    av_packet_unref(&pkt);
    if (ctx) {
        avcodec_close(ctx);
        avcodec_free_context(&ctx);
    }
    return ret;
}

 

11.FFmpegMuxer::inputFrame。向解码器中塞入H264或H265视频帧

bool FFmpegMuxer::inputFrame(const Frame::Ptr &frame) {
    if(frame->getTrackType() == TrackVideo && _video_dec != nullptr) {
        _video_dec->inputFrame(frame, true, false, false);
        if (_cb) {
            _cb(frame);
        }
    }
    return true;
}

12.MultiMediaSourceMuxer::onTrackFrame。MediaSource主对象收到视频帧时会向transcode对象中写入，从而实现抽帧

bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
    GET_CONFIG(bool, modify_stamp, General::kModifyStamp);
    auto frame = frame_in;
    if (modify_stamp) {
        //开启了时间戳覆盖
        frame = std::make_shared<FrameStamp>(frame,  _stamp[frame->getTrackType()],true);
    }
    bool ret = false;
    if (_rtmp) {
        ret = _rtmp->inputFrame(frame) ? true : ret;
    }
    if (_rtsp) {
        ret = _rtsp->inputFrame(frame) ? true : ret;
    }
    if (_ts) {
        ret = _ts->inputFrame(frame) ? true : ret;
    }
    //拷贝智能指针，目的是为了防止跨线程调用设置录像相关api导致的线程竞争问题
    //此处使用智能指针拷贝来确保线程安全，比互斥锁性能更优
    auto hls = _hls;
    if (hls) {
        ret = hls->inputFrame(frame) ? true : ret;
    }
    auto mp4 = _mp4;
    if (mp4) {
        ret = mp4->inputFrame(frame) ? true : ret;
    }
    auto transcode = _transcode;
    if (transcode) {
        ret = transcode->inputFrame(frame) ? true : ret;
    }
#if defined(ENABLE_MP4)
    if (_fmp4) {
        ret = _fmp4->inputFrame(frame) ? true : ret;
    }
#endif
#if defined(ENABLE_RTPPROXY)
    for (auto &pr : _rtp_sender) {
        ret = pr.second->inputFrame(frame) ? true : ret;
    }
#endif //ENABLE_RTPPROXY
    return ret;
}