Orbbec SDK同时读取彩色、红外、深度图像并进行流同步
在上一篇文章《Orbbec SDK使用摄像头读取并展示图像》中,我们已经实现了通过Orbbec SDK读取奥比中光摄像头的深度图信息,并使用OpenCV进行图像的展示。本文将介绍如何同时彩色、红外、深度图像,并进行流同步。同时,本文还介绍了奥比中光摄像头的热拔插功能。
对相机的进一步封装
为了增强程序的可读性,我们对Orbbec SDK的相机操作做了进一步的封装。下面的Camera
类将相机的操作集成在一起,便于主业务逻辑的书写。
// camera.h
#ifndef CAMERA_H
#define CAMERA_H
#include <initializer_list>
#include <libobsensor/hpp/Frame.hpp>
#include <libobsensor/hpp/Pipeline.hpp>
#include <libobsensor/hpp/Types.hpp>
#include <opencv2/core.hpp>
class Camera {
private:
std::shared_ptr<ob::Device> device;
std::shared_ptr<ob::Pipeline> pipe;
std::shared_ptr<ob::Config> config;
void init_color();
void init_IR();
void init_depth();
public:
Camera(bool color, bool IR, bool depth);
inline ~Camera() { this->pipe->stop(); }
inline void start() { this->pipe->start(this->config); }
inline void stop() { this->pipe->stop(); }
inline std::shared_ptr<ob::FrameSet> get() { return pipe ? pipe->waitForFrames(100) : nullptr; }
static cv::Mat frame2mat(const std::shared_ptr<ob::VideoFrame> &frame);
};
#endif
Camera
类成员函数的实现如下:
// Camera.cpp
#include "camera.h"
#include <iostream>
#include <libobsensor/hpp/Context.hpp>
#include <libobsensor/hpp/Device.hpp>
#include <libobsensor/hpp/Error.hpp>
#include <libobsensor/hpp/StreamProfile.hpp>
#include <memory>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
constexpr bool FLIP = false;
void Camera::init_color() {
// 获取彩色相机的所有流配置,包括流的分辨率,帧率,以及帧的格式
auto profiles = this->pipe->getStreamProfileList(OB_SENSOR_COLOR);
std::shared_ptr<ob::VideoStreamProfile> color_profile = nullptr;
try {
// 根据指定的格式查找对应的Profile,优先选择RGB888格式
color_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_RGB888, 30);
} catch (const ob::Error &) {
// 没找到RGB888格式后不匹配格式查找对应的Profile进行开流
color_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
}
// 开启/关闭彩色相机的镜像模式
if (this->device->isPropertySupported(OB_PROP_COLOR_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
this->device->setBoolProperty(OB_PROP_COLOR_MIRROR_BOOL, FLIP);
}
// 开启彩色流
this->config->enableStream(color_profile);
}
void Camera::init_depth() {
auto profiles = this->pipe->getStreamProfileList(OB_SENSOR_DEPTH);
std::shared_ptr<ob::VideoStreamProfile> depth_profile = nullptr;
try {
// 根据指定的格式查找对应的Profile,优先查找Y16格式
depth_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_Y16, 30);
} catch (const ob::Error &) {
// 没找到Y16格式后不匹配格式查找对应的Profile进行开流
depth_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
}
// 开启/关闭深度相机的镜像模式
if (this->device->isPropertySupported(OB_PROP_DEPTH_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
this->device->setBoolProperty(OB_PROP_DEPTH_MIRROR_BOOL, FLIP);
}
// 开启深度流
this->config->enableStream(depth_profile);
}
void Camera::init_IR() {
// 获取红外相机的所有流配置,包括流的分辨率,帧率,以及帧的格式
auto profiles = pipe->getStreamProfileList(OB_SENSOR_IR);
std::shared_ptr<ob::VideoStreamProfile> ir_profile = nullptr;
try {
// 根据指定的格式查找对应的Profile,优先查找Y16格式
ir_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_Y16, 30);
} catch (const ob::Error &) {
// 没找到Y16格式后不匹配格式查找对应的Profile进行开流
ir_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
}
// 开启/关闭红外相机的镜像模式
if (this->device->isPropertySupported(OB_PROP_IR_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
this->device->setBoolProperty(OB_PROP_IR_MIRROR_BOOL, FLIP);
}
// 开启红外流
this->config->enableStream(ir_profile);
}
Camera::Camera(bool color, bool IR, bool depth)
: pipe(std::make_shared<ob::Pipeline>()), config(std::make_shared<ob::Config>()) {
this->device = pipe->getDevice();
if (color) {
this->init_color();
}
if (IR) {
this->init_IR();
}
if (depth) {
this->init_depth();
}
// 若摄像头支持多种流,则开启流同步
if (color + IR + depth > 1) {
if (device->isPropertySupported(OB_PROP_DEPTH_ALIGN_HARDWARE_BOOL, OB_PERMISSION_READ)) {
config->setAlignMode(ALIGN_D2C_HW_MODE);
} else {
config->setAlignMode(ALIGN_D2C_SW_MODE);
}
}
// 如果有新增设备,则开启流;如果有设备拔出,则关闭流。
ob::Context ctx;
ctx.setDeviceChangedCallback([this](std::shared_ptr<ob::DeviceList> removed_devices,
std::shared_ptr<ob::DeviceList> added_devices) {
if (added_devices->deviceCount() > 0) {
this->pipe = std::make_shared<ob::Pipeline>();
this->start();
} else if (removed_devices->deviceCount() > 0) {
this->stop();
}
});
}
cv::Mat Camera::frame2mat(const std::shared_ptr<ob::VideoFrame> &frame) {
if (frame == nullptr || frame->dataSize() < 1024) {
return {};
}
const OBFrameType frame_type = frame->type(); // 帧类型(彩色/深度/IR)
const OBFormat frame_format = frame->format(); // 图像格式
const int frame_height = static_cast<int>(frame->height()); // 图像高度
const int frame_width = static_cast<int>(frame->width()); // 图像宽度
void *const frame_data = frame->data(); // 帧原始数据首地址
const int data_size = static_cast<int>(frame->dataSize()); // 帧数据大小
cv::Mat result_mat;
if (frame_type == OB_FRAME_COLOR) {
// Color image
if (frame_format == OB_FORMAT_MJPG) {
const cv::Mat raw_mat(1, data_size, CV_8UC1, frame_data);
result_mat = cv::imdecode(raw_mat, 1);
} else if (frame_format == OB_FORMAT_NV21) {
const cv::Mat raw_mat(frame_height * 3 / 2, frame_width, CV_8UC1, frame_data);
cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_NV21);
} else if (frame_format == OB_FORMAT_YUYV || frame_format == OB_FORMAT_YUY2) {
const cv::Mat raw_mat(frame_height, frame_width, CV_8UC2, frame_data);
cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_YUY2);
} else if (frame_format == OB_FORMAT_RGB888) {
const cv::Mat raw_mat(frame_height, frame_width, CV_8UC3, frame_data);
cv::cvtColor(raw_mat, result_mat, cv::COLOR_RGB2BGR);
} else if (frame_format == OB_FORMAT_UYVY) {
const cv::Mat raw_mat(frame_height, frame_width, CV_8UC2, frame_data);
cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_UYVY);
}
} else if (frame_format == OB_FORMAT_Y16 || frame_format == OB_FORMAT_YUYV ||
frame_format == OB_FORMAT_YUY2) {
// IR or depth image
const cv::Mat raw_mat(frame_height, frame_width, CV_16UC1, frame_data);
const double scale =
1 / pow(2, frame->pixelAvailableBitSize() - (frame_type == OB_FRAME_DEPTH ? 10 : 8));
cv::convertScaleAbs(raw_mat, result_mat, scale);
} else if (frame_type == OB_FRAME_IR) {
// IR image
if (frame_format == OB_FORMAT_Y8) {
result_mat = cv::Mat(frame_height, frame_width, CV_8UC1, frame_data);
} else if (frame_format == OB_FORMAT_MJPG) {
const cv::Mat raw_mat(1, data_size, CV_8UC1, frame_data);
result_mat = cv::imdecode(raw_mat, 1);
}
}
return result_mat;
}
主函数的编写
主函数的逻辑与上一篇文章《Orbbec SDK使用摄像头读取并展示图像》类似:
#include <cstdlib>
#include <iostream>
#include <libobsensor/hpp/Error.hpp>
#include <memory>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include "camera.h"
int main(void) {
try {
Camera camera(true, true, true);
camera.start();
while (cv::waitKey(1) != 27) { // ESC的ASCII码是27,按ESC键可以终止程序
auto frame_set = camera.get();
if (frame_set != nullptr) {
cv::Mat img = camera.frame2mat(frame_set->colorFrame());
if (!img.empty()) {
cv::imshow("Color", img);
}
img = camera.frame2mat(frame_set->depthFrame());
if (!img.empty()) {
cv::imshow("Depth", img);
}
img = camera.frame2mat(frame_set->irFrame());
if (!img.empty()) {
cv::imshow("IR", img);
}
}
}
camera.stop();
cv::destroyAllWindows();
} catch (const ob::Error &e) {
std::cerr << "Function:" << e.getName() << "\nargs:" << e.getArgs()
<< "\nmessage:" << e.getMessage() << "\ntype:" << e.getExceptionType()
<< std::endl;
exit(EXIT_FAILURE);
} catch (const std::exception &e) {
std::cerr << e.what() << std::endl;
exit(EXIT_FAILURE);
} catch (...) {
std::cerr << "Unexpected Error!" << std::endl;
exit(EXIT_FAILURE);
}
return EXIT_SUCCESS;
}
运行结果如下图所示:
本文版权,除注明引用的部分外,归作者所有。本文严禁商业用途的转载。非商业用途的转载需在网页明显处署上作者名称及原文链接。