混合编程python与C++
上个版本: 只是用到ctypes进行传输, 这次将python服务端更改为C++服务端,方便后续维护.
本文实现功能: python传输图片给C++, C++接受图片后对图片进行处理,并将结果返回给python客户端, pass image from python to C++
C++ 服务端
.h文件
注意文中的model
// .h
#pragma once
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <signal.h>
#include <opencv2/opencv.hpp>
using namespace std;
using namespace cv;
class ModelManager;
class ServerManager
{
private:
int m_port;
char *m_addr;
cv::VideoCapture m_cap;
int m_server;
int m_accept; // client conn
public:
bool initialization(const int &port, const cv::VideoCapture &cap, char *addr = nullptr);
bool initialization(const int &port, char *addr = nullptr);
bool build_connect();
bool acceptClient();
void error_print(const char *ptr);
bool free_connect();
bool send_data_frame(ModelManager& model);
bool receive_data_frame(cv::Mat &frame, ModelManager& model);
};
.cpp文件
#include "ServerManager.h"
#include "ModelManager.h"
#define BUFFER_SIZE 65538
void ServerManager::error_print(const char * ptr) {
perror(ptr);
exit(EXIT_FAILURE);
}
bool ServerManager::initialization(const int& port, const cv::VideoCapture& cap, char* addr){
m_port = htons(port);
m_addr = addr;
m_cap = cap;
return true;
}
bool ServerManager::initialization(const int& port, char* addr){
m_port = htons(port);
m_addr = addr;
return true;
}
bool ServerManager::build_connect() {
struct sockaddr_in server_addr;
bzero(&server_addr,sizeof(server_addr));
server_addr.sin_family = AF_INET;
server_addr.sin_addr.s_addr = m_addr?inet_addr(m_addr):INADDR_ANY;
server_addr.sin_port = m_port;
// create socket
m_server = socket(AF_INET, SOCK_STREAM, 0);
if(m_server < 0)
error_print("socket bind error");
// can reuse port
int on = 1;
if(setsockopt(m_server,SOL_SOCKET,SO_REUSEADDR,&on,sizeof(on)) < 0)
error_print("setsockopt error");
// bind addr
if(bind(m_server, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0)
error_print("bind error");
// listen only one client
if(listen(m_server, 1) < 0)
error_print("listen failed");
cout << "ServerManager is listening, plesae wait..." << endl;
return true;
}
bool ServerManager::acceptClient(){
struct sockaddr_in accept_addr;
socklen_t accept_len = sizeof(accept_addr);
bzero(&accept_addr,sizeof(accept_addr));
// accept client connection
if((m_accept = accept(m_server,(struct sockaddr*)&accept_addr,&accept_len)) < 0)
error_print("accept error");
std::cout << "Connection established" << std::endl;
return true;
}
bool ServerManager::send_data_frame(ModelManager& model) {
char *json_output = nullptr;
json_output = model.createJson();
if (json_output == nullptr) {
return false;
}
// printf("send data %s\n", json_output);
// just send json_output, dont memcpy new char*!!! it wastes me two hours
// send json
int result = send(m_accept, json_output, strlen(json_output), 0);
if (result == -1) {
cout << "send fail" << endl;
return false;
}
return true;
}
bool ServerManager::receive_data_frame(Mat& frame, ModelManager& model) {
// recv frame size
int data_size;
if (recv(m_accept, &data_size, sizeof(data_size), 0) != sizeof(data_size)) {
// when client close, then close connection
close(m_accept);
cout << "close connection to client" << endl;
acceptClient(); // restart a new accept, to accept new connection
return false;
}
cout << data_size << endl;
// recv frame data
// char buf[data_size];
// std::vector<uchar> decode;
// int bytes_received = 0;
// do
// {
// int nBytes = recv(m_accept, buf, data_size - bytes_received, 0);
// for (int i = 0; i < nBytes; i++) // maybe can use memcpy, maybe faster
// {
// decode.emplace_back(buf[i]);
// }
// cout << bytes_received << endl;
// bytes_received += nBytes;
// } while (bytes_received < data_size);
char *recv_char = new char[data_size];
std::vector<uchar> decode(data_size, 0);
int index = 0;
int bytes_received = 0;
int count = data_size;
while (count > 0)// if count >= 0, dead loop
{
int iRet = recv(m_accept, recv_char, count, 0);
if (index >= data_size) index = data_size;
memcpy(&decode[index], recv_char , iRet);
index += iRet;
if (!iRet) { return -1; }
count -= iRet;
}
// decode message
frame = imdecode(decode, cv::IMREAD_COLOR);
// push into Model's queueMat
model.mtxQueueImg.lock();
model.queueMat.push(frame);
model.mtxQueueImg.unlock();
return true;
}
bool ServerManager::free_connect() {
m_cap.release();
close(m_accept);
close(m_server);
return true;
}
C++ model部分代码
.h文件
#pragma once
#include "CV_Classify.h"
#include "CV_Detect.h"
#include "ServerManager.h"
#include <opencv2/opencv.hpp>
#include <mutex>
#include <queue>
#include <unistd.h> // usleep
#include <thread>
#include "cJSON.h"
#include <string>
using namespace std;
using namespace cv;
class ModelManager{
public:
Detect objdetect;
Classify objclassify;
std::mutex mtxQueueDet; // mutex for detect queue
std::mutex mtxQueueImg; // mutex for image queue
std::mutex mtxQueueCls; // mutex for classify queue
std::queue<cv::Mat> queueMat;
std::queue<ObjDetectOutput> queueDetOut;// Detect queue
std::queue<ObjClassifyOutput> queueClsOut;// Classify queue
bool DetectFlag = true;
bool ClassifyFlag = true;
bool empty_flag = false;
friend class ServerManager;
public:
void initDetectModel() ;
void initClassifyModel() ;
void DetectImg();
void ClassifyImg();
void getClsResult(ObjClassifyOutput &output);
// ObjClassifyOutput getClsResult();
char* createJson();
};
.cpp文件
部分有删减,createJson可参考使用,利用json来传递值
#include "ModelManager.h"
void ModelManager::initDetectModel()
{
std::string config_path = "DetectConfig.yaml";
objdetect.Init(config_path, 1);
}
void ModelManager::initClassifyModel()
{
std::string config_path = "ClassiflyConfig.yaml";
objclassify.Init(config_path, 1);
}
void ModelManager::DetectImg()
{
DetectInput detect_input;
DetectOutput detect_output;
cv::Mat frame;
size_t mm = 0;
while(1)
{
if (queueMat.empty())
{
if(!DetectFlag)
{
break;
}
usleep(2000);
continue;
}
// get image from queueMat
mtxQueueImg.lock();
frame = queueMat.front();
queueMat.pop();
mtxQueueImg.unlock();
// run model
objdetect.Run(detect_input, detect_output);
// push detect result into queueDetOut
mtxQueueDet.lock();
queueDetOut.push(detect_output);
// cout << "detect run !!" << endl;
mtxQueueDet.unlock();
}
return;
}
void ModelManager::ClassifyImg()
{
ObjClassifyInput input;
ObjClassifyOutput output;
cv::Mat frame;
Detoutput detect_result;
while(1)
{
if (queueDetOut.empty())
{
if(!ClassifyFlag)
{
break;
}
usleep(2000);
continue;
}
// get detect from queueDetOut
mtxQueueDet.lock();
detect_result = queueDetOut.front();
queueDetOut.pop();
mtxQueueDet.unlock();
// run model
objclassify.Run(input, output);
// push cls result into queueClsOut
mtxQueueCls.lock();
queueClsOut.push(output);
mtxQueueCls.unlock();
}
return;
}
void ModelManager::getClsResult(ObjClassifyOutput& output){
if (queueClsOut.empty()){
output.object_list.object_num = -1; // -1 is now empty;
return; // must return in thread otherwise cant use &output
}
output = queueClsOut.front();
queueClsOut.pop();
return;
}
char* ModelManager::createJson() // dont know why cant use &value, need return value
{
mtxQueueCls.lock();
ObjClassifyOutput output;
getClsResult(output);
mtxQueueCls.unlock();
if (output.object_list.object_num == -1){
return nullptr;
}
// prepare send data json
cJSON* json_object_list = NULL;
cJSON* json_ObjClassifyOutput = NULL;
json_ObjClassifyOutput = cJSON_CreateObject();
json_object_list = cJSON_CreateObject();
cJSON_AddItemToObject(json_ObjClassifyOutput, "object_list", json_object_list);
int obj_num = output.object_list.object_num;
cJSON_AddNumberToObject(json_object_list, "object_num", obj_num);
for (int i = 0; i < obj_num; ++i){
cJSON* json_object = cJSON_CreateObject();
cJSON* json_box = cJSON_CreateObject();
cJSON_AddNumberToObject(json_box,"x", output.object_list.object[i].bbox.x);
cJSON_AddNumberToObject(json_box,"y", output.object_list.object[i].bbox.y);
cJSON_AddNumberToObject(json_box,"w", output.object_list.object[i].bbox.w);
cJSON_AddNumberToObject(json_box,"h", output.object_list.object[i].bbox.h);
cJSON_AddItemToObject(json_object,"bbox", json_box);
cJSON_AddNumberToObject(json_object, "classes", output.object_list.object[i].classes);
cJSON_AddNumberToObject(json_object, "objectness", output.object_list.object[i].objectness);
// double prob = output.object_list.object[i].prob;
// cJSON_AddNumberToObject(json_object, "prob", prob); // pointer cant use?
string str = "object" + to_string(i);
cJSON_AddItemToObject(json_object_list, str.c_str(), json_object);
// printf("prob: %f", output.object_list.object[i].prob);
}
char* json_output = cJSON_Print(json_ObjClassifyOutput);
cJSON_Delete(json_ObjClassifyOutput);
return json_output;
}
C++ 服务端运行
#include <../include/ServerManager.h>
#include <../include/ModelManager.h>
#include <thread>
#define PORT 8080
void recvServer(ServerManager& s, ModelManager& model){
int idx = 0;
while (true){
// auto start = std::chrono::steady_clock::now();
cv::Mat frame;
s.receive_data_frame(frame, model);
// cal time cost
// auto end = std::chrono::steady_clock::now();
// std::chrono::duration<double, std::milli> elapsed = end - start;
// std::cout << "recv execution time: " << elapsed.count() << " ms\n";
if (frame.empty()) {
usleep(2000);
continue;
}
// cv::imwrite("image"+to_string(idx++)+".jpg", frame);
std::cout << "Image " << idx++ <<" received !!" << std::endl;
}
}
void sendServer(ServerManager& s, ModelManager& model){
while (true){
if (s.send_data_frame(model)) {
cout << "send success!!" << endl;
cout << endl;
}else{
// cout << "send fail!!" << endl;
usleep(2000);
}
}
}
int main()
{
ServerManager s;
ModelManager model;
model.initDetectModel();
model.initClassifyModel();
cout << endl;
s.initialization(PORT);
s.build_connect();
s.acceptClient();
thread recv_server(recvServer, std::ref(s), std::ref(model));
thread send_server(sendServer, std::ref(s), std::ref(model));
thread detect(&ModelManager::DetectImg, &model);
thread classfy(&ModelManager::ClassifyImg, &model);
detect.join();
classfy.join();
recv_server.join();
send_server.join();
return 0;
}
python客户端
import json
import socket
import struct
import time
from multiprocessing import JoinableQueue
from threading import Thread
import os
from natsort import ns, natsorted
host = '192.168.0.2' # '192.168.0.2' 'localhost'
port = 8080
def img_encode(img_path):
img = cv2.imread(img_path)
# img = cv2.resize(img, (500, 500), interpolation=cv2.INTER_CUBIC)
img_param = [95] # 图片压缩率0-100
_, img = cv2.imencode('.jpg', img, img_param)
img = img.tobytes()
return img
def img_product(img_queue, path, path_mode='image'):
if path_mode == 'image':
image = img_encode(path)
img_queue.put(image)
elif path_mode == 'dir':
dir_list = os.listdir(path)
files = natsorted(dir_list, alg=ns.PATH) # 顺序读取文件名
for filename in files:
img_path = path + '/' + filename
image = img_encode(img_path)
img_queue.put(image)
img_queue.put('E')
img_queue.join()
def server_consumer(img_queue):
while True:
start = int(round(time.time() * 1000))
# 1. get img from queue
img_obj = img_queue.get()
img_queue.task_done()
# get end signal
if img_obj[0] == 'E':
client.close()
break
# 2. send package(img_bytes_size, img_bytes)
pack_size = struct.pack("i", len(img_obj))
client.send(pack_size + img_obj)
end = int(round(time.time() * 1000))
data = client.recv(65536)
json_str = data.decode('utf8', 'ignore').strip(b'\x00'.decode())
results = json.loads(json_str)
end = int(round(time.time() * 1000))
end_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print('send and recv cost time: ', (end - start))
print(results)
if __name__ == '__main__':
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((host, port))
img_dir = 'data'
one_img = './data/image.jpg'
mode = 'dir'
img_jq = JoinableQueue()
producer = Thread(target=img_product, args=(img_jq, img_dir, mode,))
consumer = Thread(target=server_consumer, args=(img_jq,))
producer.daemon = True # set daemon but not set join()
producer.start()
consumer.start()
# producer.join() // 让生产者先关闭,防止close错误
consumer.join()
总结
- 其实这个项目真正做完感觉还是挺简单, 就是对socket通信不太熟悉, 以及传图片没做过.
- 实际上传图片只需要读取图片后,imencode,然后tobytes,最后发送size和data即可.而接受端只需要拼接数组,然后imdecode即可.
- 另外传输结果的话利用json传输可以让结果可读性可高, 传输也比较方便, 当时copy别人的发送代码, 没有细看,导致使用memcpy让json格式乱码,导致无法解码json.
- 如果你感觉接收端没问题,一定要看看发送端.
- 之后的新任务就是视频传输利用rtsp流,敬请期待
参考博客
- use memcpy in receive_frame function if you want,对应github地址
- a pure client and server code to create a simple demo
- 如果用ctrl+z中断,可能导致address in use,使用bg/fg
- 用户可以使用fg/bg操作继续前台或后台的任务，fg命令重新启动前台被中断的任务，bg命令把被中断的任务放在后台执行
- zmq send img
- 优化imdecode速度,本代码未使用
- 视频传输跟图片传输差不多
- Linux c++获取本地毫秒级精确时间
- 如果不想用json,可以试试struct
- 利用json来传输分类结果
