Fork me on github

Orbbec SDK同时读取彩色、红外、深度图像并进行流同步

在上一篇文章《Orbbec SDK使用摄像头读取并展示图像》中,我们已经实现了通过Orbbec SDK读取奥比中光摄像头的深度图信息,并使用OpenCV进行图像的展示。本文将介绍如何同时彩色、红外、深度图像,并进行流同步。同时,本文还介绍了奥比中光摄像头的热拔插功能。

对相机的进一步封装

为了增强程序的可读性,我们对Orbbec SDK的相机操作做了进一步的封装。下面的Camera类将相机的操作集成在一起,便于主业务逻辑的书写。

// camera.h
#ifndef CAMERA_H
#define CAMERA_H

#include <initializer_list>
#include <libobsensor/hpp/Frame.hpp>
#include <libobsensor/hpp/Pipeline.hpp>
#include <libobsensor/hpp/Types.hpp>
#include <opencv2/core.hpp>

class Camera {
  private:
    std::shared_ptr<ob::Device> device;
    std::shared_ptr<ob::Pipeline> pipe;
    std::shared_ptr<ob::Config> config;
    void init_color();
    void init_IR();
    void init_depth();

  public:
    Camera(bool color, bool IR, bool depth);
    inline ~Camera() { this->pipe->stop(); }
    inline void start() { this->pipe->start(this->config); }
    inline void stop() { this->pipe->stop(); }
    inline std::shared_ptr<ob::FrameSet> get() { return pipe ? pipe->waitForFrames(100) : nullptr; }
    static cv::Mat frame2mat(const std::shared_ptr<ob::VideoFrame> &frame);
};

#endif

Camera类成员函数的实现如下:

// Camera.cpp
#include "camera.h"

#include <iostream>
#include <libobsensor/hpp/Context.hpp>
#include <libobsensor/hpp/Device.hpp>
#include <libobsensor/hpp/Error.hpp>
#include <libobsensor/hpp/StreamProfile.hpp>
#include <memory>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

constexpr bool FLIP = false;

void Camera::init_color() {
    // 获取彩色相机的所有流配置,包括流的分辨率,帧率,以及帧的格式
    auto profiles = this->pipe->getStreamProfileList(OB_SENSOR_COLOR);
    std::shared_ptr<ob::VideoStreamProfile> color_profile = nullptr;
    try {
        // 根据指定的格式查找对应的Profile,优先选择RGB888格式
        color_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_RGB888, 30);
    } catch (const ob::Error &) {
        // 没找到RGB888格式后不匹配格式查找对应的Profile进行开流
        color_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
    }
    // 开启/关闭彩色相机的镜像模式
    if (this->device->isPropertySupported(OB_PROP_COLOR_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
        this->device->setBoolProperty(OB_PROP_COLOR_MIRROR_BOOL, FLIP);
    }
    // 开启彩色流
    this->config->enableStream(color_profile);
}

void Camera::init_depth() {
    auto profiles = this->pipe->getStreamProfileList(OB_SENSOR_DEPTH);
    std::shared_ptr<ob::VideoStreamProfile> depth_profile = nullptr;
    try {
        // 根据指定的格式查找对应的Profile,优先查找Y16格式
        depth_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_Y16, 30);
    } catch (const ob::Error &) {
        // 没找到Y16格式后不匹配格式查找对应的Profile进行开流
        depth_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
    }
    // 开启/关闭深度相机的镜像模式
    if (this->device->isPropertySupported(OB_PROP_DEPTH_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
        this->device->setBoolProperty(OB_PROP_DEPTH_MIRROR_BOOL, FLIP);
    }
    // 开启深度流
    this->config->enableStream(depth_profile);
}

void Camera::init_IR() {
    // 获取红外相机的所有流配置,包括流的分辨率,帧率,以及帧的格式
    auto profiles = pipe->getStreamProfileList(OB_SENSOR_IR);
    std::shared_ptr<ob::VideoStreamProfile> ir_profile = nullptr;
    try {
        // 根据指定的格式查找对应的Profile,优先查找Y16格式
        ir_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_Y16, 30);
    } catch (const ob::Error &) {
        // 没找到Y16格式后不匹配格式查找对应的Profile进行开流
        ir_profile = profiles->getVideoStreamProfile(640, 0, OB_FORMAT_UNKNOWN, 30);
    }
    // 开启/关闭红外相机的镜像模式
    if (this->device->isPropertySupported(OB_PROP_IR_MIRROR_BOOL, OB_PERMISSION_WRITE)) {
        this->device->setBoolProperty(OB_PROP_IR_MIRROR_BOOL, FLIP);
    }
    // 开启红外流
    this->config->enableStream(ir_profile);
}

Camera::Camera(bool color, bool IR, bool depth)
    : pipe(std::make_shared<ob::Pipeline>()), config(std::make_shared<ob::Config>()) {
    this->device = pipe->getDevice();
    if (color) {
        this->init_color();
    }
    if (IR) {
        this->init_IR();
    }
    if (depth) {
        this->init_depth();
    }
    // 若摄像头支持多种流,则开启流同步
    if (color + IR + depth > 1) {
        if (device->isPropertySupported(OB_PROP_DEPTH_ALIGN_HARDWARE_BOOL, OB_PERMISSION_READ)) {
            config->setAlignMode(ALIGN_D2C_HW_MODE);
        } else {
            config->setAlignMode(ALIGN_D2C_SW_MODE);
        }
    }
    // 如果有新增设备,则开启流;如果有设备拔出,则关闭流。
    ob::Context ctx;
    ctx.setDeviceChangedCallback([this](std::shared_ptr<ob::DeviceList> removed_devices,
                                        std::shared_ptr<ob::DeviceList> added_devices) {
        if (added_devices->deviceCount() > 0) {
            this->pipe = std::make_shared<ob::Pipeline>();
            this->start();
        } else if (removed_devices->deviceCount() > 0) {
            this->stop();
        }
    });
}

cv::Mat Camera::frame2mat(const std::shared_ptr<ob::VideoFrame> &frame) {
    if (frame == nullptr || frame->dataSize() < 1024) {
        return {};
    }

    const OBFrameType frame_type = frame->type();               // 帧类型(彩色/深度/IR)
    const OBFormat frame_format = frame->format();              // 图像格式
    const int frame_height = static_cast<int>(frame->height()); // 图像高度
    const int frame_width = static_cast<int>(frame->width());   // 图像宽度
    void *const frame_data = frame->data();                     // 帧原始数据首地址
    const int data_size = static_cast<int>(frame->dataSize());  // 帧数据大小

    cv::Mat result_mat;

    if (frame_type == OB_FRAME_COLOR) {
        // Color image
        if (frame_format == OB_FORMAT_MJPG) {
            const cv::Mat raw_mat(1, data_size, CV_8UC1, frame_data);
            result_mat = cv::imdecode(raw_mat, 1);
        } else if (frame_format == OB_FORMAT_NV21) {
            const cv::Mat raw_mat(frame_height * 3 / 2, frame_width, CV_8UC1, frame_data);
            cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_NV21);
        } else if (frame_format == OB_FORMAT_YUYV || frame_format == OB_FORMAT_YUY2) {
            const cv::Mat raw_mat(frame_height, frame_width, CV_8UC2, frame_data);
            cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_YUY2);
        } else if (frame_format == OB_FORMAT_RGB888) {
            const cv::Mat raw_mat(frame_height, frame_width, CV_8UC3, frame_data);
            cv::cvtColor(raw_mat, result_mat, cv::COLOR_RGB2BGR);
        } else if (frame_format == OB_FORMAT_UYVY) {
            const cv::Mat raw_mat(frame_height, frame_width, CV_8UC2, frame_data);
            cv::cvtColor(raw_mat, result_mat, cv::COLOR_YUV2BGR_UYVY);
        }
    } else if (frame_format == OB_FORMAT_Y16 || frame_format == OB_FORMAT_YUYV ||
               frame_format == OB_FORMAT_YUY2) {
        // IR or depth image
        const cv::Mat raw_mat(frame_height, frame_width, CV_16UC1, frame_data);
        const double scale =
            1 / pow(2, frame->pixelAvailableBitSize() - (frame_type == OB_FRAME_DEPTH ? 10 : 8));
        cv::convertScaleAbs(raw_mat, result_mat, scale);
    } else if (frame_type == OB_FRAME_IR) {
        // IR image
        if (frame_format == OB_FORMAT_Y8) {
            result_mat = cv::Mat(frame_height, frame_width, CV_8UC1, frame_data);
        } else if (frame_format == OB_FORMAT_MJPG) {
            const cv::Mat raw_mat(1, data_size, CV_8UC1, frame_data);
            result_mat = cv::imdecode(raw_mat, 1);
        }
    }
    return result_mat;
}

主函数的编写

主函数的逻辑与上一篇文章《Orbbec SDK使用摄像头读取并展示图像》类似:

#include <cstdlib>
#include <iostream>
#include <libobsensor/hpp/Error.hpp>
#include <memory>
#include <opencv2/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>

#include "camera.h"

int main(void) {
    try {
        Camera camera(true, true, true);
        camera.start();
        while (cv::waitKey(1) != 27) { // ESC的ASCII码是27,按ESC键可以终止程序
            auto frame_set = camera.get();
            if (frame_set != nullptr) {
                cv::Mat img = camera.frame2mat(frame_set->colorFrame());
                if (!img.empty()) {
                    cv::imshow("Color", img);
                }
                img = camera.frame2mat(frame_set->depthFrame());
                if (!img.empty()) {
                    cv::imshow("Depth", img);
                }
                img = camera.frame2mat(frame_set->irFrame());
                if (!img.empty()) {
                    cv::imshow("IR", img);
                }
            }
        }
        camera.stop();
        cv::destroyAllWindows();
    } catch (const ob::Error &e) {
        std::cerr << "Function:" << e.getName() << "\nargs:" << e.getArgs()
                  << "\nmessage:" << e.getMessage() << "\ntype:" << e.getExceptionType()
                  << std::endl;
        exit(EXIT_FAILURE);
    } catch (const std::exception &e) {
        std::cerr << e.what() << std::endl;
        exit(EXIT_FAILURE);
    } catch (...) {
        std::cerr << "Unexpected Error!" << std::endl;
        exit(EXIT_FAILURE);
    }
    return EXIT_SUCCESS;
}

运行结果如下图所示:

posted @ 2022-12-30 15:30  fang-d  阅读(1023)  评论(0编辑  收藏  举报