RV1126移植并使用RKMedia中RKNN相关代码

Deceiver_Ker·2022-03-26 16:04·2909 次阅读

RV1126移植并使用RKMedia中RKNN相关代码

1,首先参考sdk/external/rkmedia/examples/rkmedia_vi_rknn_venc_rstp_test.c的代码,并在同一目录下创建rkmedia_vi_venc_rtsp_md.c文件,适配自己的摄像头编写代码。这里我使用的摄像头为USB摄像头,输出图像格式为YUYV,有两种分辨率,分别为640x480,1280x720.
2,编写代码之前先在同目录下的CMakeList中加入配置语句如下:
Copy
#-------------------------- # rkmedia_vi_venc_rtsp_md #-------------------------- link_directories(${PROJECT_SOURCE_DIR}/librtsp/) add_executable(rkmedia_vi_venc_rtsp_md rkmedia_vi_venc_rtsp_md.c ${COMMON_SRC}) add_dependencies(rkmedia_vi_venc_rtsp_md easymedia) target_link_libraries(rkmedia_vi_venc_rtsp_md rtsp rknn_api m easymedia rga) target_include_directories(rkmedia_vi_venc_rtsp_md PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SYSROOT}/usr/include/rknn) install(TARGETS rkmedia_vi_venc_rtsp_md RUNTIME DESTINATION "bin") install(FILES rtsp-nn.cfg DESTINATION share) install(DIRECTORY rknn_model DESTINATION share)
3,完成上述步骤开始编写代码

原文件中的图片输入格式为NV12,这里为YUYV422,因此将NV12转RGB格式的函数改为YUYV转RGB的函数即可,YUYV转RGB的函数如下

Copy
int convert_yuv_to_rgb_pixel(int y, int u, int v) { unsigned int pixel32 = 0; unsigned char *pixel = (unsigned char *)&pixel32; int r, g, b; r = y + (1.370705 * (v-128)); g = y - (0.698001 * (v-128)) - (0.337633 * (u-128)); b = y + (1.732446 * (u-128)); if(r > 255) r = 255; if(g > 255) g = 255; if(b > 255) b = 255; if(r < 0) r = 0; if(g < 0) g = 0; if(b < 0) b = 0; pixel[0] = r * 220 / 256; pixel[1] = g * 220 / 256; pixel[2] = b * 220 / 256; return pixel32; } int YUYV_to_rgb24(unsigned char *yuv, unsigned char *rgb, unsigned int width, unsigned int height) { unsigned int in, out = 0; unsigned int pixel_16; unsigned char pixel_24[3]; unsigned int pixel32; int y0, u, y1, v; for(in = 0; in < width * height * 2; in += 4) { pixel_16 = yuv[in + 3] << 24 | yuv[in + 2] << 16 | yuv[in + 1] << 8 | yuv[in + 0]; y0 = (pixel_16 & 0x000000ff); u = (pixel_16 & 0x0000ff00) >> 8; y1 = (pixel_16 & 0x00ff0000) >> 16; v = (pixel_16 & 0xff000000) >> 24; pixel32 = convert_yuv_to_rgb_pixel(y0, u, v); pixel_24[0] = (pixel32 & 0x000000ff); pixel_24[1] = (pixel32 & 0x0000ff00) >> 8; pixel_24[2] = (pixel32 & 0x00ff0000) >> 16; rgb[out++] = pixel_24[0]; rgb[out++] = pixel_24[1]; rgb[out++] = pixel_24[2]; pixel32 = convert_yuv_to_rgb_pixel(y1, u, v); pixel_24[0] = (pixel32 & 0x000000ff); pixel_24[1] = (pixel32 & 0x0000ff00) >> 8; pixel_24[2] = (pixel32 & 0x00ff0000) >> 16; rgb[out++] = pixel_24[0]; rgb[out++] = pixel_24[1]; rgb[out++] = pixel_24[2]; } return 0; }

再将原来的NV12toRGB24函数改为以上的函数YUYV_to_rgb24即可,同时原代码中跟踪图形画框的代码也是基于NV12图像格式,这里改为YUYV格式进行处理,代码如下:

Copy
int nv12_border(char *pic, int pic_w, int pic_h, int rect_x, int rect_y, int rect_w, int rect_h, int R, int G, int B) { /* Set up the rectangle border size */ const int border = 15; /* RGB convert YUV */ int Y, U, V; Y = 0.299 * R + 0.587 * G + 0.114 * B; U = -0.1687 * R - 0.3313 * G + 0.5 * B + 128; V = 0.5 * R - 0.4187 * G - 0.0813 * B + 128; /* Locking the scope of rectangle border range */ int j, k,y_index,u_index,v_index; for (j = rect_y; j < rect_y + rect_h; j++) { for (k = rect_x; k < rect_x + rect_w; k++) { if (k < (rect_x + border) || k > (rect_x + rect_w - border) || j < (rect_y + border) || j > (rect_y + rect_h - border)) { //y_index = j * pic_w + k; //u_index = //(y_index / 2 - pic_w / 2 * ((j + 1) / 2)) * 2 + pic_w * pic_h; //v_index = u_index + 1; /* Components of YUV's storage address index */ y_index = j * pic_w * 2 + k * 2 + pic_h *0 ; u_index = y_index + 1; if(k%2 == 0) u_index = y_index + 1; else u_index = y_index - 1; v_index = u_index + 2; /* set up YUV's conponents value of rectangle border */ pic[y_index] = Y; pic[u_index] = U; pic[v_index] = V; } } } return 0; }

原代码调用了双目摄像头,因此可以获取两个视频流,并创建GetMediaBuffer()与MainStream()这里要对两个函数合二为一,将MainStream中跟踪被检测物体画框的代码移植到GetMediaBuffer()最后即可,同时再Main函数中创建MainStream函数的线程也需要同时注释删除,修改后的GetMediaBuffer函数如下:

Copy
static void *GetMediaBuffer(void *arg) { printf("#Start %s thread, arg:%p\n", __func__, arg); rknn_context ctx; int ret; int model_len = 0; unsigned char *model; printf("Loading model ...\n"); model = load_model(g_ssd_path, &model_len); ret = rknn_init(&ctx, model, model_len, 0); if (ret < 0) { printf("rknn_init fail! ret=%d\n", ret); return NULL; } // Get Model Input Output Info rknn_input_output_num io_num; ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num)); if (ret != RKNN_SUCC) { printf("rknn_query fail! ret=%d\n", ret); return NULL; } printf("model input num: %d, output num: %d\n", io_num.n_input, io_num.n_output); printf("input tensors:\n"); rknn_tensor_attr input_attrs[io_num.n_input]; memset(input_attrs, 0, sizeof(input_attrs)); for (unsigned int i = 0; i < io_num.n_input; i++) { input_attrs[i].index = i; ret = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr)); if (ret != RKNN_SUCC) { printf("rknn_query fail! ret=%d\n", ret); return NULL; } printRKNNTensor(&(input_attrs[i])); } printf("output tensors:\n"); rknn_tensor_attr output_attrs[io_num.n_output]; memset(output_attrs, 0, sizeof(output_attrs)); for (unsigned int i = 0; i < io_num.n_output; i++) { output_attrs[i].index = i; ret = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr)); if (ret != RKNN_SUCC) { printf("rknn_query fail! ret=%d\n", ret); return NULL; } printRKNNTensor(&(output_attrs[i])); } MEDIA_BUFFER buffer = NULL; float x_rate = (float)cfg.session_cfg[DRAW_INDEX].u32Width / MODEL_INPUT_SIZE; float y_rate = (float)cfg.session_cfg[DRAW_INDEX].u32Height / MODEL_INPUT_SIZE; printf("x_rate is %f, y_rate is %f\n", x_rate, y_rate); while (g_flag_run) { buffer = RK_MPI_SYS_GetMediaBuffer(RK_ID_VI, 0, -1); if (!buffer) { printf("RK_MPI_SYS_GetMediaBuffer getmediabuffer get null buffer!\n"); break; } // printf("Get Frame:ptr:%p, fd:%d, size:%zu, mode:%d, channel:%d, " // "timestamp:%lld\n", // RK_MPI_MB_GetPtr(buffer), RK_MPI_MB_GetFD(buffer), // RK_MPI_MB_GetSize(buffer), // RK_MPI_MB_GetModeID(buffer), RK_MPI_MB_GetChannelID(buffer), // RK_MPI_MB_GetTimestamp(buffer)); // nv12 to rgb24 and resize //int rga_buffer_size = 640*360*3; int rga_buffer_size = cfg.session_cfg[RK_NN_INDEX].u32Width *cfg.session_cfg[RK_NN_INDEX].u32Height *4; // nv12 3/2, rgb 3 int rga_buffer_model_input_size = MODEL_INPUT_SIZE * MODEL_INPUT_SIZE * 4; unsigned char *rga_buffer = malloc(rga_buffer_size); unsigned char *rga_buffer_model_input = malloc(rga_buffer_model_input_size); nv12_to_rgb24(RK_MPI_MB_GetPtr(buffer), rga_buffer, cfg.session_cfg[RK_NN_INDEX].u32Width, cfg.session_cfg[RK_NN_INDEX].u32Height);//cfg.session_cfg[RK_NN_INDEX].u32Width,cfg.session_cfg[RK_NN_INDEX].u32Height 640,360 rgb24_resize(rga_buffer, rga_buffer_model_input, cfg.session_cfg[RK_NN_INDEX].u32Width, cfg.session_cfg[RK_NN_INDEX].u32Height, MODEL_INPUT_SIZE, MODEL_INPUT_SIZE);//cfg.session_cfg[RK_NN_INDEX].u32Width,cfg.session_cfg[RK_NN_INDEX].u32Height // Set Input Data rknn_input inputs[1]; memset(inputs, 0, sizeof(inputs)); inputs[0].index = 0; inputs[0].type = RKNN_TENSOR_UINT8; inputs[0].size = rga_buffer_model_input_size; inputs[0].fmt = RKNN_TENSOR_NHWC; inputs[0].buf = rga_buffer_model_input; ret = rknn_inputs_set(ctx, io_num.n_input, inputs); if (ret < 0) { printf("rknn_input_set fail! ret=%d\n", ret); return NULL; } // Run printf("rknn_run\n"); ret = rknn_run(ctx, NULL); if (ret < 0) { printf("rknn_run fail! ret=%d\n", ret); return NULL; } // Get Output rknn_output outputs[2]; memset(outputs, 0, sizeof(outputs)); outputs[0].want_float = 1; outputs[1].want_float = 1; ret = rknn_outputs_get(ctx, io_num.n_output, outputs, NULL); if (ret < 0) { printf("rknn_outputs_get fail! ret=%d\n", ret); return NULL; } // Post Process detect_result_group_t detect_result_group; postProcessSSD((float *)(outputs[0].buf), (float *)(outputs[1].buf), MODEL_INPUT_SIZE, MODEL_INPUT_SIZE, &detect_result_group); // Release rknn_outputs rknn_outputs_release(ctx, 2, outputs); //Dump Objects for (int i = 0; i < detect_result_group.count; i++){ detect_result_t *det_result = &(detect_result_group.results[i]); printf("%s @ (%d %d %d %d) %f\n", det_result->name, det_result->box.left, det_result->box.top, det_result->box.right, det_result->box.bottom, det_result->prop); } if (detect_result_group.count > 0) { rknn_list_push(rknn_list_, getCurrentTimeMsec(), detect_result_group); int size = rknn_list_size(rknn_list_); if (size >= MAX_RKNN_LIST_NUM) rknn_list_drop(rknn_list_); printf("size is %d\n", size); } // draw if(rknn_list_size(rknn_list_)){ long time_before; detect_result_group_t detect_result_group1; memset(&detect_result_group1, 0, sizeof(detect_result_group1)); rknn_list_pop(rknn_list_, &time_before, &detect_result_group1); // printf("time interval is %ld\n", getCurrentTimeMsec() - time_before); for (int j = 0; j < detect_result_group1.count; j++) { if (strcmp(detect_result_group1.results[j].name, "mouse")) continue; if (detect_result_group1.results[j].prop < 0.5) continue; int x = detect_result_group1.results[j].box.left * x_rate; int y = detect_result_group1.results[j].box.top * y_rate; int w = (detect_result_group1.results[j].box.right - detect_result_group1.results[j].box.left) * x_rate; int h = (detect_result_group1.results[j].box.bottom - detect_result_group1.results[j].box.top) * y_rate; if (x < 0) x = 0; if (y < 0) y = 0; while ((uint32_t)(x + w) >= cfg.session_cfg[DRAW_INDEX].u32Width) { w -= 16; } while ((uint32_t)(y + h) >= cfg.session_cfg[DRAW_INDEX].u32Height) { h -= 16; } printf("border=(%d %d %d %d)\n", x, y, w, h); nv12_border((char*)RK_MPI_MB_GetPtr(buffer), cfg.session_cfg[DRAW_INDEX].u32Width, cfg.session_cfg[DRAW_INDEX].u32Height, x, y, w, h, 255, 0, 0); } } // send from VI to VENC RK_MPI_SYS_SendMediaBuffer( RK_ID_VENC, cfg.session_cfg[DRAW_INDEX].stVenChn.s32ChnId, buffer); RK_MPI_MB_ReleaseBuffer(buffer); if (rga_buffer) free(rga_buffer); if (rga_buffer_model_input) free(rga_buffer_model_input); } // release if (ctx) rknn_destroy(ctx); if (model) free(model); return NULL; }

在Main函数中,由于输入的配置文件中有关于两个摄像头的信息,这里需要只选取一个,因此将其中循环创建vi,venc通道的for循环注释,而且循环调用的语句全部更改为只需要一次,并且由于图像检测的信息在GetMediaBuffer中已经传送给了venc通道,这里,就不需要用到通道绑定的函数,RK_MPI_SYS_BIND函数一并注释,具体的Main函数代码如下:

Copy
int main(int argc, char **argv) { RK_CHAR *pCfgPath = "/oem/usr/share/rtsp-nn.cfg"; RK_CHAR *pIqfilesPath = NULL; RK_S32 s32CamId = 0; #ifdef RKAIQ RK_BOOL bMultictx = RK_FALSE; #endif int c; while ((c = getopt_long(argc, argv, optstr, long_options, NULL)) != -1) { const char *tmp_optarg = optarg; switch (c) { case 'a': if (!optarg && NULL != argv[optind] && '-' != argv[optind][0]) { tmp_optarg = argv[optind++]; } if (tmp_optarg) { pIqfilesPath = (char *)tmp_optarg; } else { pIqfilesPath = "/oem/etc/iqfiles/"; } break; case 'c': pCfgPath = optarg; break; case 'b': g_box_priors = optarg; break; case 'l': g_labels_list = optarg; break; case 'p': g_ssd_path = optarg; break; case 'I': s32CamId = atoi(optarg); break; #ifdef RKAIQ case 'M': if (atoi(optarg)) { bMultictx = RK_TRUE; } break; #endif case '?': default: print_usage(argv[0]); return 0; } } printf("cfg path is %s\n", pCfgPath); printf("BOX_PRIORS_TXT_PATH is %s\n", g_box_priors); printf("LABEL_NALE_TXT_PATH is %s\n", g_labels_list); printf("MODEL_PATH is %s\n", g_ssd_path); printf("#CameraIdx: %d\n\n", s32CamId); load_cfg(pCfgPath); signal(SIGINT, sig_proc); if (pIqfilesPath) { #ifdef RKAIQ printf("xml dirpath: %s\n\n", pIqfilesPath); printf("#bMultictx: %d\n\n", bMultictx); int fps = 30; rk_aiq_working_mode_t hdr_mode = RK_AIQ_WORKING_MODE_NORMAL; SAMPLE_COMM_ISP_Init(s32CamId, hdr_mode, bMultictx, pIqfilesPath); SAMPLE_COMM_ISP_Run(s32CamId); SAMPLE_COMM_ISP_SetFrameRate(s32CamId, fps); #endif } // init rtsp printf("init rtsp\n"); g_rtsplive = create_rtsp_demo(554); // init mpi printf("init mpi\n"); RK_MPI_SYS_Init(); // create session int i = 0; //for (int i = 0; i < cfg.session_count; i++){} cfg.session_cfg[i].session = rtsp_new_session(g_rtsplive, cfg.session_cfg[i].path); // VI create printf("VI create\n"); cfg.session_cfg[i].stViChn.enModId = RK_ID_VI; cfg.session_cfg[i].stViChn.s32ChnId = i; SAMPLE_COMMON_VI_Start(&cfg.session_cfg[i], VI_WORK_MODE_NORMAL, i); // VENC create printf("VENC create\n"); cfg.session_cfg[i].stVenChn.enModId = RK_ID_VENC; cfg.session_cfg[i].stVenChn.s32ChnId = i; SAMPLE_COMMON_VENC_Start(&cfg.session_cfg[i]); //if (i == DRAW_INDEX) RK_MPI_VI_StartStream(s32CamId, cfg.session_cfg[i].stViChn.s32ChnId); //else //RK_MPI_SYS_Bind(&cfg.session_cfg[i].stViChn, //&cfg.session_cfg[i].stVenChn); // rtsp video printf("rtsp video\n"); switch (cfg.session_cfg[i].video_type) { case RK_CODEC_TYPE_H264: rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H264, NULL, 0); break; case RK_CODEC_TYPE_H265: rtsp_set_video(cfg.session_cfg[i].session, RTSP_CODEC_ID_VIDEO_H265, NULL, 0); break; default: printf("video codec not support.\n"); break; } rtsp_sync_video_ts(cfg.session_cfg[i].session, rtsp_get_reltime(), rtsp_get_ntptime()); //} create_rknn_list(&rknn_list_); // Get the sub-stream buffer for humanoid recognition pthread_t read_thread; pthread_create(&read_thread, NULL, GetMediaBuffer, NULL); // The mainstream draws a box asynchronously based on the recognition result //pthread_t main_stream_thread; //pthread_create(&main_stream_thread, NULL, MainStream, NULL); while (g_flag_run) { int j = 0;//for (int j = 0; j < cfg.session_count; j++) { MEDIA_BUFFER buffer; // send video buffer buffer = RK_MPI_SYS_GetMediaBuffer( RK_ID_VENC, cfg.session_cfg[j].stVenChn.s32ChnId, 0); if (buffer) { rtsp_tx_video(cfg.session_cfg[j].session, RK_MPI_MB_GetPtr(buffer), RK_MPI_MB_GetSize(buffer), RK_MPI_MB_GetTimestamp(buffer)); RK_MPI_MB_ReleaseBuffer(buffer); } //} rtsp_do_event(g_rtsplive); } pthread_join(read_thread, NULL); //pthread_join(main_stream_thread, NULL); rtsp_del_demo(g_rtsplive); int k = 0;//for (int i = 0; i < cfg.session_count; i++) { //if (i != DRAW_INDEX) //RK_MPI_SYS_UnBind(&cfg.session_cfg[k].stViChn, //&cfg.session_cfg[k].stVenChn); RK_MPI_VENC_DestroyChn(cfg.session_cfg[k].stVenChn.s32ChnId); RK_MPI_VI_DisableChn(s32CamId, cfg.session_cfg[k].stViChn.s32ChnId); //} if (pIqfilesPath) { #ifdef RKAIQ SAMPLE_COMM_ISP_Stop(s32CamId); #endif } destory_rknn_list(&rknn_list_); return 0; }

其他具体的更改的代码可以与一个目录下的rkmedia_vi_rknn_venc_rtsp_test.c进行比较查看,在代码中使用了rtsp-nn.cfg,box_priors,coco_labels_list,ssd_inception_v2_rv1109_rv1126文件,后三者是目标检测训练好的算法模型,更改需要重新训练人工智能,并进行更改,第一个rtsp-nn.cfg代码与本代码在同一个目录下,其中配置了摄像头的输入图像类型,像素,rtsp推流信号等参数,需要自己针对具体要求进行修改,代码配置如下:

Copy
# cfgline: # path=%s audio_type=%d channels=%u samplerate=%u nbsample=%u alsapath=%s video_type=%s width=%u height=%u image_type=%u video_path=%s # # from rkmedia_common.h CODEC_TYPE_E # audio_type list ## AAC, 0 ## MP2, 1 ## G711A, 3 ## G711U, 4 ## G726, 5 # video_type list ## H264, 6 ## H265, 7 # image_type ## from rkmedia_common.h IMAGE_TYPE_E ## IMAGE_TYPE_NV12, 4 ## IMAGE_TYPE_FBC0, 8 ## IMAGE_TYPE_NV16, 10 # # video_path ## rkispp_m_bypass ## rkispp_scale0 ## rkispp_scale1 ## rkispp_scale2 # example path=/live/main_stream video_type=6 width=640 height=480 image_type=13 video_path=/dev/video25 #path=/live/main_stream video_type=7 width=1920 height=1080 image_type=4 video_path=rkispp_scale0 #path=/live/sub_stream video_type=6 width=720 height=576 image_type=13 video_path=/dev/video25
4,进行编译
Copy
# SDK根目录,选择环境 source envsetup.sh firefly_rv1126_rv1109 # 重编rkmedia源码 make rkmedia-dirclean && make rkmedia # rkmedia库/程序打包到文件系统(oem.img) ./build.sh rootfs # 重新烧写oem.img,若有其他基础包配置更新(如ffmpeg),则需要重烧rootfs.img

或者执行完前两步后,在SDK/buildroot/output/firefly_rv1126_rv1109/oem/usr/bin目录下找到rkmedia_vi_venc_rtsp_md二进制可执行程序,移动到板子某一个目录下,再进行测试。这里,我开发板挂载了Linux系统下的/home/kxq/share目录,在开发板下对应目录为/mnt/nfs,因此在本虚拟机下,将此二进制可执行程序移动到虚拟机下的/home/kxq/share目录,再在开发板下的/mnt/nfs目录执行程序,注意需要加上./与开发板中携带的程序进行区别。

5,测试

测试语句为
./rkmedia_vi_venc_rtsp_md -c /oem/usr/share/rtsp-nn.cfg -b /oem/usr/share/rknn_model/box_priors.txt -l /oem/usr/share/rknn_model/coco_labels_list.txt -p /oem/usr/share/rknn_model/ssd_inception_v2_rv1109_rv1126.rknn
vlc rtsp://192.168.137.71:554/live/main_stream

源代码如下:
https://files.cnblogs.com/files/blogs/741465/rkmedia_vi_venc_rtsp_md.zip?t=1648281776

posted @   Deceiver_Ker  阅读(2909)  评论(0编辑  收藏  举报
相关博文:
阅读排行:
· 微软正式发布.NET 10 Preview 1:开启下一代开发框架新篇章
· 没有源码,如何修改代码逻辑?
· NetPad:一个.NET开源、跨平台的C#编辑器
· PowerShell开发游戏 · 打蜜蜂
· 凌晨三点救火实录:Java内存泄漏的七个神坑,你至少踩过三个!
点击右上角即可分享
微信分享提示