RV1126移植并使用RKMedia中RKNN相关代码
1，首先参考sdk/external/rkmedia/examples/rkmedia_vi_rknn_venc_rstp_test.c的代码，并在同一目录下创建rkmedia_vi_venc_rtsp_md.c文件，适配自己的摄像头编写代码。这里我使用的摄像头为USB摄像头，输出图像格式为YUYV，有两种分辨率，分别为640x480,1280x720.
2，编写代码之前先在同目录下的CMakeList中加入配置语句如下：
#--------------------------
# rkmedia_vi_venc_rtsp_md
#--------------------------
link_directories(${PROJECT_SOURCE_DIR}/librtsp/)
add_executable(rkmedia_vi_venc_rtsp_md rkmedia_vi_venc_rtsp_md.c ${COMMON_SRC})
add_dependencies(rkmedia_vi_venc_rtsp_md easymedia)
target_link_libraries(rkmedia_vi_venc_rtsp_md rtsp rknn_api m easymedia rga)
target_include_directories(rkmedia_vi_venc_rtsp_md PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SYSROOT}/usr/include/rknn)
install(TARGETS rkmedia_vi_venc_rtsp_md RUNTIME DESTINATION "bin")
install(FILES rtsp-nn.cfg DESTINATION share)
install(DIRECTORY rknn_model DESTINATION share)
3,完成上述步骤开始编写代码
原文件中的图片输入格式为NV12，这里为YUYV422，因此将NV12转RGB格式的函数改为YUYV转RGB的函数即可，YUYV转RGB的函数如下
int convert_yuv_to_rgb_pixel(int y, int u, int v)
{
unsigned int pixel32 = 0;
unsigned char *pixel = (unsigned char *)&pixel32;
int r, g, b;
r = y + (1.370705 * (v-128));
g = y - (0.698001 * (v-128)) - (0.337633 * (u-128));
b = y + (1.732446 * (u-128));
if(r > 255) r = 255;
if(g > 255) g = 255;
if(b > 255) b = 255;
if(r < 0) r = 0;
if(g < 0) g = 0;
if(b < 0) b = 0;
pixel[0] = r * 220 / 256;
pixel[1] = g * 220 / 256;
pixel[2] = b * 220 / 256;
return pixel32;
}
int YUYV_to_rgb24(unsigned char *yuv, unsigned char *rgb, unsigned int width, unsigned int height)
{
unsigned int in, out = 0;
unsigned int pixel_16;
unsigned char pixel_24[3];
unsigned int pixel32;
int y0, u, y1, v;
for(in = 0; in < width * height * 2; in += 4) {
pixel_16 =
yuv[in + 3] << 24 |
yuv[in + 2] << 16 |
yuv[in + 1] << 8 |
yuv[in + 0];
y0 = (pixel_16 & 0x000000ff);
u = (pixel_16 & 0x0000ff00) >> 8;
y1 = (pixel_16 & 0x00ff0000) >> 16;
v = (pixel_16 & 0xff000000) >> 24;
pixel32 = convert_yuv_to_rgb_pixel(y0, u, v);
pixel_24[0] = (pixel32 & 0x000000ff);
pixel_24[1] = (pixel32 & 0x0000ff00) >> 8;
pixel_24[2] = (pixel32 & 0x00ff0000) >> 16;
rgb[out++] = pixel_24[0];
rgb[out++] = pixel_24[1];
rgb[out++] = pixel_24[2];
pixel32 = convert_yuv_to_rgb_pixel(y1, u, v);
pixel_24[0] = (pixel32 & 0x000000ff);
pixel_24[1] = (pixel32 & 0x0000ff00) >> 8;
pixel_24[2] = (pixel32 & 0x00ff0000) >> 16;
rgb[out++] = pixel_24[0];
rgb[out++] = pixel_24[1];
rgb[out++] = pixel_24[2];
}
return 0;
}
再将原来的NV12toRGB24函数改为以上的函数YUYV_to_rgb24即可，同时原代码中跟踪图形画框的代码也是基于NV12图像格式，这里改为YUYV格式进行处理，代码如下：
int nv12_border(char *pic, int pic_w, int pic_h, int rect_x, int rect_y,
int rect_w, int rect_h, int R, int G, int B) {
/* Set up the rectangle border size */
const int border = 15;
/* RGB convert YUV */
int Y, U, V;
Y = 0.299 * R + 0.587 * G + 0.114 * B;
U = -0.1687 * R - 0.3313 * G + 0.5 * B + 128;
V = 0.5 * R - 0.4187 * G - 0.0813 * B + 128;
/* Locking the scope of rectangle border range */
int j, k,y_index,u_index,v_index;
for (j = rect_y; j < rect_y + rect_h; j++) {
for (k = rect_x; k < rect_x + rect_w; k++) {
if (k < (rect_x + border) || k > (rect_x + rect_w - border) ||
j < (rect_y + border) || j > (rect_y + rect_h - border))
{
//y_index = j * pic_w + k;
//u_index =
//(y_index / 2 - pic_w / 2 * ((j + 1) / 2)) * 2 + pic_w * pic_h;
//v_index = u_index + 1;
/* Components of YUV's storage address index */
y_index = j * pic_w * 2 + k * 2 + pic_h *0 ;
u_index = y_index + 1;
if(k%2 == 0)
u_index = y_index + 1;
else
u_index = y_index - 1;
v_index = u_index + 2;
/* set up YUV's conponents value of rectangle border */
pic[y_index] = Y;
pic[u_index] = U;
pic[v_index] = V;
}
}
}
return 0;
}
原代码调用了双目摄像头，因此可以获取两个视频流，并创建GetMediaBuffer()与MainStream()这里要对两个函数合二为一，将MainStream中跟踪被检测物体画框的代码移植到GetMediaBuffer()最后即可，同时再Main函数中创建MainStream函数的线程也需要同时注释删除，修改后的GetMediaBuffer函数如下：
static void *GetMediaBuffer(void *arg) {
printf("#Start %s thread, arg:%p\n", __func__, arg);
rknn_context ctx;
int ret;
int model_len = 0;
unsigned char *model;
printf("Loading model ...\n");
model = load_model(g_ssd_path, &model_len);
ret = rknn_init(&ctx, model, model_len, 0);
if (ret < 0) {
printf("rknn_init fail! ret=%d\n", ret);
return NULL;
}
// Get Model Input Output Info
rknn_input_output_num io_num;
ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num));
if (ret != RKNN_SUCC) {
printf("rknn_query fail! ret=%d\n", ret);
return NULL;
}
printf("model input num: %d, output num: %d\n", io_num.n_input,
io_num.n_output);
printf("input tensors:\n");
rknn_tensor_attr input_attrs[io_num.n_input];
memset(input_attrs, 0, sizeof(input_attrs));
for (unsigned int i = 0; i < io_num.n_input; i++) {
input_attrs[i].index = i;
ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]),
sizeof(rknn_tensor_attr));
if (ret != RKNN_SUCC) {
printf("rknn_query fail! ret=%d\n", ret);
return NULL;
}
printRKNNTensor(&(input_attrs[i]));
}
printf("output tensors:\n");
rknn_tensor_attr output_attrs[io_num.n_output];
memset(output_attrs, 0, sizeof(output_attrs));
for (unsigned int i = 0; i < io_num.n_output; i++) {
output_attrs[i].index = i;
ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]),
sizeof(rknn_tensor_attr));
if (ret != RKNN_SUCC) {
printf("rknn_query fail! ret=%d\n", ret);
return NULL;
}
printRKNNTensor(&(output_attrs[i]));
}
MEDIA_BUFFER buffer = NULL;
float x_rate = (float)cfg.session_cfg[DRAW_INDEX].u32Width / MODEL_INPUT_SIZE;
float y_rate =
(float)cfg.session_cfg[DRAW_INDEX].u32Height / MODEL_INPUT_SIZE;
printf("x_rate is %f, y_rate is %f\n", x_rate, y_rate);
while (g_flag_run) {
buffer = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, 0, -1);
if (!buffer) {
printf("RK_MPI_SYS_GetMediaBuffer getmediabuffer get null buffer!\n");
break;
}
// printf("Get Frame:ptr:%p, fd:%d, size:%zu, mode:%d, channel:%d, "
// "timestamp:%lld\n",
// RK_MPI_MB_GetPtr(buffer), RK_MPI_MB_GetFD(buffer),
// RK_MPI_MB_GetSize(buffer),
// RK_MPI_MB_GetModeID(buffer), RK_MPI_MB_GetChannelID(buffer),
// RK_MPI_MB_GetTimestamp(buffer));
// nv12 to rgb24 and resize
//int rga_buffer_size = 640*360*3;
int rga_buffer_size = cfg.session_cfg[RK_NN_INDEX].u32Width *cfg.session_cfg[RK_NN_INDEX].u32Height *4; // nv12 3/2, rgb 3
int rga_buffer_model_input_size = MODEL_INPUT_SIZE * MODEL_INPUT_SIZE * 4;
unsigned char *rga_buffer = malloc(rga_buffer_size);
unsigned char *rga_buffer_model_input = malloc(rga_buffer_model_input_size);
nv12_to_rgb24(RK_MPI_MB_GetPtr(buffer), rga_buffer,
cfg.session_cfg[RK_NN_INDEX].u32Width,
cfg.session_cfg[RK_NN_INDEX].u32Height);//cfg.session_cfg[RK_NN_INDEX].u32Width,cfg.session_cfg[RK_NN_INDEX].u32Height 640,360
rgb24_resize(rga_buffer, rga_buffer_model_input,
cfg.session_cfg[RK_NN_INDEX].u32Width,
cfg.session_cfg[RK_NN_INDEX].u32Height, MODEL_INPUT_SIZE,
MODEL_INPUT_SIZE);//cfg.session_cfg[RK_NN_INDEX].u32Width,cfg.session_cfg[RK_NN_INDEX].u32Height
// Set Input Data
rknn_input inputs[1];
memset(inputs, 0, sizeof(inputs));
inputs[0].index = 0;
inputs[0].type = RKNN_TENSOR_UINT8;
inputs[0].size = rga_buffer_model_input_size;
inputs[0].fmt = RKNN_TENSOR_NHWC;
inputs[0].buf = rga_buffer_model_input;
ret = rknn_inputs_set(ctx, io_num.n_input, inputs);
if (ret < 0) {
printf("rknn_input_set fail! ret=%d\n", ret);
return NULL;
}
// Run
printf("rknn_run\n");
ret = rknn_run(ctx, NULL);
if (ret < 0) {
printf("rknn_run fail! ret=%d\n", ret);
return NULL;
}
// Get Output
rknn_output outputs[2];
memset(outputs, 0, sizeof(outputs));
outputs[0].want_float = 1;
outputs[1].want_float = 1;
ret = rknn_outputs_get(ctx, io_num.n_output, outputs, NULL);
if (ret < 0) {
printf("rknn_outputs_get fail! ret=%d\n", ret);
return NULL;
}
// Post Process
detect_result_group_t detect_result_group;
postProcessSSD((float *)(outputs[0].buf), (float *)(outputs[1].buf),
MODEL_INPUT_SIZE, MODEL_INPUT_SIZE, &detect_result_group);
// Release rknn_outputs
rknn_outputs_release(ctx, 2, outputs);
//Dump Objects
for (int i = 0; i < detect_result_group.count; i++){
detect_result_t *det_result = &(detect_result_group.results[i]);
printf("%s @ (%d %d %d %d) %f\n", det_result->name,
det_result->box.left,
det_result->box.top, det_result->box.right,
det_result->box.bottom,
det_result->prop);
}
if (detect_result_group.count > 0) {
rknn_list_push(rknn_list_, getCurrentTimeMsec(), detect_result_group);
int size = rknn_list_size(rknn_list_);
if (size >= MAX_RKNN_LIST_NUM)
rknn_list_drop(rknn_list_);
printf("size is %d\n", size);
}
// draw
if(rknn_list_size(rknn_list_)){
long time_before;
detect_result_group_t detect_result_group1;
memset(&detect_result_group1, 0, sizeof(detect_result_group1));
rknn_list_pop(rknn_list_, &time_before, &detect_result_group1);
// printf("time interval is %ld\n", getCurrentTimeMsec() - time_before);
for (int j = 0; j < detect_result_group1.count; j++) {
if (strcmp(detect_result_group1.results[j].name, "mouse"))
continue;
if (detect_result_group1.results[j].prop < 0.5)
continue;
int x = detect_result_group1.results[j].box.left * x_rate;
int y = detect_result_group1.results[j].box.top * y_rate;
int w = (detect_result_group1.results[j].box.right -
detect_result_group1.results[j].box.left) *
x_rate;
int h = (detect_result_group1.results[j].box.bottom -
detect_result_group1.results[j].box.top) *
y_rate;
if (x < 0)
x = 0;
if (y < 0)
y = 0;
while ((uint32_t)(x + w) >= cfg.session_cfg[DRAW_INDEX].u32Width) {
w -= 16;
}
while ((uint32_t)(y + h) >= cfg.session_cfg[DRAW_INDEX].u32Height) {
h -= 16;
}
printf("border=(%d %d %d %d)\n", x, y, w, h);
nv12_border((char*)RK_MPI_MB_GetPtr(buffer),
cfg.session_cfg[DRAW_INDEX].u32Width,
cfg.session_cfg[DRAW_INDEX].u32Height, x, y, w, h, 255, 0,
0);
}
}
// send from VI to VENC
RK_MPI_SYS_SendMediaBuffer(
RK_ID_VENC, cfg.session_cfg[DRAW_INDEX].stVenChn.s32ChnId, buffer);
RK_MPI_MB_ReleaseBuffer(buffer);
if (rga_buffer)
free(rga_buffer);
if (rga_buffer_model_input)
free(rga_buffer_model_input);
}
// release
if (ctx)
rknn_destroy(ctx);
if (model)
free(model);
return NULL;
}
在Main函数中，由于输入的配置文件中有关于两个摄像头的信息，这里需要只选取一个，因此将其中循环创建vi,venc通道的for循环注释，而且循环调用的语句全部更改为只需要一次，并且由于图像检测的信息在GetMediaBuffer中已经传送给了venc通道，这里，就不需要用到通道绑定的函数，RK_MPI_SYS_BIND函数一并注释，具体的Main函数代码如下：
int main(int argc, char **argv) {
RK_CHAR *pCfgPath = "/oem/usr/share/rtsp-nn.cfg";
RK_CHAR *pIqfilesPath = NULL;
RK_S32 s32CamId = 0;
#ifdef RKAIQ
RK_BOOL bMultictx = RK_FALSE;
#endif
int c;
while ((c = getopt_long(argc, argv, optstr, long_options, NULL)) != -1) {
const char *tmp_optarg = optarg;
switch (c) {
case 'a':
if (!optarg && NULL != argv[optind] && '-' != argv[optind][0]) {
tmp_optarg = argv[optind++];
}
if (tmp_optarg) {
pIqfilesPath = (char *)tmp_optarg;
} else {
pIqfilesPath = "/oem/etc/iqfiles/";
}
break;
case 'c':
pCfgPath = optarg;
break;
case 'b':
g_box_priors = optarg;
break;
case 'l':
g_labels_list = optarg;
break;
case 'p':
g_ssd_path = optarg;
break;
case 'I':
s32CamId = atoi(optarg);
break;
#ifdef RKAIQ
case 'M':
if (atoi(optarg)) {
bMultictx = RK_TRUE;
}
break;
#endif
case '?':
default:
print_usage(argv[0]);
return 0;
}
}
printf("cfg path is %s\n", pCfgPath);
printf("BOX_PRIORS_TXT_PATH is %s\n", g_box_priors);
printf("LABEL_NALE_TXT_PATH is %s\n", g_labels_list);
printf("MODEL_PATH is %s\n", g_ssd_path);
printf("#CameraIdx: %d\n\n", s32CamId);
load_cfg(pCfgPath);
signal(SIGINT, sig_proc);
if (pIqfilesPath) {
#ifdef RKAIQ
printf("xml dirpath: %s\n\n", pIqfilesPath);
printf("#bMultictx: %d\n\n", bMultictx);
int fps = 30;
rk_aiq_working_mode_t hdr_mode = RK_AIQ_WORKING_MODE_NORMAL;
SAMPLE_COMM_ISP_Init(s32CamId, hdr_mode, bMultictx, pIqfilesPath);
SAMPLE_COMM_ISP_Run(s32CamId);
SAMPLE_COMM_ISP_SetFrameRate(s32CamId, fps);
#endif
}
// init rtsp
printf("init rtsp\n");
g_rtsplive = create_rtsp_demo(554);
// init mpi
printf("init mpi\n");
RK_MPI_SYS_Init();
// create session
int i = 0; //for (int i = 0; i < cfg.session_count; i++){}
cfg.session_cfg[i].session =
rtsp_new_session(g_rtsplive, cfg.session_cfg[i].path);
// VI create
printf("VI create\n");
cfg.session_cfg[i].stViChn.enModId = RK_ID_VI;
cfg.session_cfg[i].stViChn.s32ChnId = i;
SAMPLE_COMMON_VI_Start(&cfg.session_cfg[i], VI_WORK_MODE_NORMAL, i);
// VENC create
printf("VENC create\n");
cfg.session_cfg[i].stVenChn.enModId = RK_ID_VENC;
cfg.session_cfg[i].stVenChn.s32ChnId = i;
SAMPLE_COMMON_VENC_Start(&cfg.session_cfg[i]);
//if (i == DRAW_INDEX)
RK_MPI_VI_StartStream(s32CamId, cfg.session_cfg[i].stViChn.s32ChnId);
//else
//RK_MPI_SYS_Bind(&cfg.session_cfg[i].stViChn,
//&cfg.session_cfg[i].stVenChn);
// rtsp video
printf("rtsp video\n");
switch (cfg.session_cfg[i].video_type) {
case RK_CODEC_TYPE_H264:
rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H264, NULL,
0);
break;
case RK_CODEC_TYPE_H265:
rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H265, NULL,
0);
break;
default:
printf("video codec not support.\n");
break;
}
rtsp_sync_video_ts(cfg.session_cfg[i].session, rtsp_get_reltime(),
rtsp_get_ntptime());
//}
create_rknn_list(&rknn_list_);
// Get the sub-stream buffer for humanoid recognition
pthread_t read_thread;
pthread_create(&read_thread, NULL, GetMediaBuffer, NULL);
// The mainstream draws a box asynchronously based on the recognition result
//pthread_t main_stream_thread;
//pthread_create(&main_stream_thread, NULL, MainStream, NULL);
while (g_flag_run) {
int j = 0;//for (int j = 0; j < cfg.session_count; j++) {
MEDIA_BUFFER buffer;
// send video buffer
buffer = RK_MPI_SYS_GetMediaBuffer(
RK_ID_VENC, cfg.session_cfg[j].stVenChn.s32ChnId, 0);
if (buffer) {
rtsp_tx_video(cfg.session_cfg[j].session, RK_MPI_MB_GetPtr(buffer),
RK_MPI_MB_GetSize(buffer),
RK_MPI_MB_GetTimestamp(buffer));
RK_MPI_MB_ReleaseBuffer(buffer);
}
//}
rtsp_do_event(g_rtsplive);
}
pthread_join(read_thread, NULL);
//pthread_join(main_stream_thread, NULL);
rtsp_del_demo(g_rtsplive);
int k = 0;//for (int i = 0; i < cfg.session_count; i++) {
//if (i != DRAW_INDEX)
//RK_MPI_SYS_UnBind(&cfg.session_cfg[k].stViChn,
//&cfg.session_cfg[k].stVenChn);
RK_MPI_VENC_DestroyChn(cfg.session_cfg[k].stVenChn.s32ChnId);
RK_MPI_VI_DisableChn(s32CamId, cfg.session_cfg[k].stViChn.s32ChnId);
//}
if (pIqfilesPath) {
#ifdef RKAIQ
SAMPLE_COMM_ISP_Stop(s32CamId);
#endif
}
destory_rknn_list(&rknn_list_);
return 0;
}
其他具体的更改的代码可以与一个目录下的rkmedia_vi_rknn_venc_rtsp_test.c进行比较查看，在代码中使用了rtsp-nn.cfg,box_priors,coco_labels_list,ssd_inception_v2_rv1109_rv1126文件，后三者是目标检测训练好的算法模型，更改需要重新训练人工智能，并进行更改，第一个rtsp-nn.cfg代码与本代码在同一个目录下，其中配置了摄像头的输入图像类型，像素，rtsp推流信号等参数，需要自己针对具体要求进行修改，代码配置如下：
# cfgline:
# path=%s audio_type=%d channels=%u samplerate=%u nbsample=%u alsapath=%s video_type=%s width=%u height=%u image_type=%u video_path=%s
#
# from rkmedia_common.h CODEC_TYPE_E
# audio_type list
## AAC, 0
## MP2, 1
## G711A, 3
## G711U, 4
## G726, 5
# video_type list
## H264, 6
## H265, 7
# image_type
## from rkmedia_common.h IMAGE_TYPE_E
## IMAGE_TYPE_NV12, 4
## IMAGE_TYPE_FBC0, 8
## IMAGE_TYPE_NV16, 10
#
# video_path
## rkispp_m_bypass
## rkispp_scale0
## rkispp_scale1
## rkispp_scale2
# example
path=/live/main_stream video_type=6 width=640 height=480 image_type=13 video_path=/dev/video25
#path=/live/main_stream video_type=7 width=1920 height=1080 image_type=4 video_path=rkispp_scale0
#path=/live/sub_stream video_type=6 width=720 height=576 image_type=13 video_path=/dev/video25
4，进行编译
# SDK根目录,选择环境
source envsetup.sh firefly_rv1126_rv1109
# 重编rkmedia源码
make rkmedia-dirclean && make rkmedia
# rkmedia库/程序打包到文件系统(oem.img)
./build.sh rootfs
# 重新烧写oem.img,若有其他基础包配置更新(如ffmpeg),则需要重烧rootfs.img
或者执行完前两步后，在SDK/buildroot/output/firefly_rv1126_rv1109/oem/usr/bin目录下找到rkmedia_vi_venc_rtsp_md二进制可执行程序，移动到板子某一个目录下，再进行测试。这里，我开发板挂载了Linux系统下的/home/kxq/share目录，在开发板下对应目录为/mnt/nfs,因此在本虚拟机下，将此二进制可执行程序移动到虚拟机下的/home/kxq/share目录，再在开发板下的/mnt/nfs目录执行程序，注意需要加上./与开发板中携带的程序进行区别。
5，测试
测试语句为
./rkmedia_vi_venc_rtsp_md -c /oem/usr/share/rtsp-nn.cfg -b /oem/usr/share/rknn_model/box_priors.txt -l /oem/usr/share/rknn_model/coco_labels_list.txt -p /oem/usr/share/rknn_model/ssd_inception_v2_rv1109_rv1126.rknn
vlc rtsp://192.168.137.71:554/live/main_stream
源代码如下：
https://files.cnblogs.com/files/blogs/741465/rkmedia_vi_venc_rtsp_md.zip?t=1648281776
