From 2b9d1a2ff8566129607728284342ca4117ad9aba Mon Sep 17 00:00:00 2001 From: zzuliys <68427285+zzuliys@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:23:41 +0800 Subject: [PATCH] Merge pull request #24877 from zzuliys:feature/mac MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Orbbec Camera supports MacOS，Gemini2 and Gemini2L support Y16 format #24877 note: 1.Gemini2 and Gemini2L must use the latest firmware -- https://github.com/orbbec/OrbbecFirmware; 2.Administrator privileges are necessary to run on MacOS. --- 3rdparty/orbbecsdk/orbbecsdk.cmake | 18 ++ CMakeLists.txt | 8 +- modules/videoio/CMakeLists.txt | 38 +++-- modules/videoio/cmake/detect_obsensor.cmake | 68 +++++--- modules/videoio/include/opencv2/videoio.hpp | 2 +- .../obsensor_uvc_stream_channel.cpp | 34 ++-- modules/videoio/src/cap_obsensor_capture.cpp | 16 +- modules/videoio/src/cap_obsensor_capture.hpp | 3 +- .../videoio/src/cap_obsensor_liborbbec.cpp | 154 ++++++++++++++++++ .../videoio/src/cap_obsensor_liborbbec.hpp | 74 +++++++++ samples/cpp/videocapture_obsensor.cpp | 2 +- 11 files changed, 357 insertions(+), 60 deletions(-) create mode 100644 3rdparty/orbbecsdk/orbbecsdk.cmake create mode 100644 modules/videoio/src/cap_obsensor_liborbbec.cpp create mode 100644 modules/videoio/src/cap_obsensor_liborbbec.hpp diff --git a/3rdparty/orbbecsdk/orbbecsdk.cmake b/3rdparty/orbbecsdk/orbbecsdk.cmake new file mode 100644 index 0000000000..7a553946b6 --- /dev/null +++ b/3rdparty/orbbecsdk/orbbecsdk.cmake @@ -0,0 +1,18 @@ +function(download_orbbec_sdk root_var) + set(ORBBECSDK_DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/3rdparty/orbbecsdk") + set(ORBBECSDK_FILE_HASH_CMAKE "2624c84837d3416fd8b3e95750e6e725") + ocv_download(FILENAME "v1.9.4.tar.gz" + HASH ${ORBBECSDK_FILE_HASH_CMAKE} + URL "https://github.com/orbbec/OrbbecSDK/archive/refs/tags/v1.9.4/" + DESTINATION_DIR ${ORBBECSDK_DOWNLOAD_DIR} + ID OrbbecSDK + STATUS res + UNPACK RELATIVE_URL + ) + if(${res}) + message(STATUS "orbbec sdk downloaded to: ${ORBBECSDK_DOWNLOAD_DIR}") + set(${root_var} "${ORBBECSDK_DOWNLOAD_DIR}/OrbbecSDK-1.9.4" PARENT_SCOPE) + else() + message(FATAL_ERROR "Failed to download orbbec sdk") + endif() +endfunction() \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 933aed91d6..e20f68cca0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -478,9 +478,11 @@ OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF VISIBLE_IF TRUE VERIFY HAVE_TIMVX) -# attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. -OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec RGB-D modules: Astra+/Femto)" ON - VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID) +# Attention when OBSENSOR_USE_ORBBEC_SDK set to off: +# Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. +OCV_OPTION(OBSENSOR_USE_ORBBEC_SDK "Use Orbbec SDK as backend to support more camera models and platforms (force to ON on MacOS)" OFF) +OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec 3D Cameras)" ON + VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID) OR (APPLE AND AARCH64 AND NOT IOS) VERIFY HAVE_OBSENSOR) OCV_OPTION(WITH_CANN "Include CANN support" OFF VISIBLE_IF TRUE diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index f800babdcd..776c1bb84d 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -236,25 +236,31 @@ if(TARGET ocv.3rdparty.android_native_camera) endif() if(TARGET ocv.3rdparty.obsensor) - list(APPEND videoio_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_capture.cpp) - list(APPEND videoio_hdrs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_capture.hpp - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_interface.hpp) - if(HAVE_OBSENSOR_MSMF) +if(OBSENSOR_USE_ORBBEC_SDK) list(APPEND videoio_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_msmf.cpp) - list(APPEND videoio_hdrs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.hpp - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_msmf.hpp) - elseif(HAVE_OBSENSOR_V4L2) + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_liborbbec.hpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_liborbbec.cpp) + else() list(APPEND videoio_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_v4l2.cpp) + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_capture.cpp) list(APPEND videoio_hdrs - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.hpp - ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_v4l2.hpp) + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor_capture.hpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_interface.hpp) + if(HAVE_OBSENSOR_MSMF) + list(APPEND videoio_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_msmf.cpp) + list(APPEND videoio_hdrs + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.hpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_msmf.hpp) + elseif(HAVE_OBSENSOR_V4L2) + list(APPEND videoio_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_v4l2.cpp) + list(APPEND videoio_hdrs + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_uvc_stream_channel.hpp + ${CMAKE_CURRENT_LIST_DIR}/src/cap_obsensor/obsensor_stream_channel_v4l2.hpp) + endif() endif() list(APPEND tgts ocv.3rdparty.obsensor) endif() diff --git a/modules/videoio/cmake/detect_obsensor.cmake b/modules/videoio/cmake/detect_obsensor.cmake index fe3f893b48..cf5a9063a9 100644 --- a/modules/videoio/cmake/detect_obsensor.cmake +++ b/modules/videoio/cmake/detect_obsensor.cmake @@ -1,28 +1,54 @@ # --- obsensor --- if(NOT HAVE_OBSENSOR) - if(WIN32) - check_include_file(mfapi.h HAVE_MFAPI) - check_include_file(vidcap.h HAVE_VIDCAP) - if(HAVE_MFAPI AND HAVE_VIDCAP) - set(HAVE_OBSENSOR TRUE) - set(HAVE_OBSENSOR_MSMF TRUE) - ocv_add_external_target(obsensor "" "" "HAVE_OBSENSOR;HAVE_OBSENSOR_MSMF") - else() - set(HAVE_OBSENSOR OFF) - set(HAVE_OBSENSOR_MSMF OFF) - if(NOT HAVE_MFAPI) - MESSAGE(STATUS "Could not find mfapi.h. Turning HAVE_OBSENSOR OFF") - endif() - if(NOT HAVE_VIDCAP) - MESSAGE(STATUS "Could not find vidcap.h. Turning HAVE_OBSENSOR OFF") + if(APPLE) + # force to use orbbec sdk on mac + set(OBSENSOR_USE_ORBBEC_SDK ON) + endif() + + if(OBSENSOR_USE_ORBBEC_SDK) + include(${CMAKE_SOURCE_DIR}/3rdparty/orbbecsdk/orbbecsdk.cmake) + download_orbbec_sdk(ORBBEC_SDK_ROOT_DIR) + message(STATUS "ORBBEC_SDK_ROOT_DIR: ${ORBBEC_SDK_ROOT_DIR}") + if(ORBBEC_SDK_ROOT_DIR) + set(OrbbecSDK_DIR "${ORBBEC_SDK_ROOT_DIR}") + find_package(OrbbecSDK REQUIRED) + message(STATUS "OrbbecSDK_FOUND: ${OrbbecSDK_FOUND}") + message(STATUS "OrbbecSDK_INCLUDE_DIRS: ${OrbbecSDK_INCLUDE_DIRS}") + if(OrbbecSDK_FOUND) + set(HAVE_OBSENSOR TRUE) + set(HAVE_OBSENSOR_ORBBEC_SDK TRUE) + ocv_add_external_target(obsensor "${OrbbecSDK_INCLUDE_DIRS}" "${OrbbecSDK_LIBRARY}" "HAVE_OBSENSOR;HAVE_OBSENSOR_ORBBEC_SDK") + file(COPY ${OrbbecSDK_DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/bin) + file(COPY ${OrbbecSDK_DLL_FILES} DESTINATION ${CMAKE_BINARY_DIR}/lib) + install(FILES ${OrbbecSDK_DLL_FILES} DESTINATION ${OPENCV_LIB_INSTALL_PATH}) + ocv_install_3rdparty_licenses(OrbbecSDK ${OrbbecSDK_DIR}/LICENSE.txt) endif() endif() - elseif(UNIX) - check_include_file(linux/videodev2.h HAVE_CAMV4L2_OBSENSOR) - if(HAVE_CAMV4L2_OBSENSOR) - set(HAVE_OBSENSOR TRUE) - set(HAVE_OBSENSOR_V4L2 TRUE) - ocv_add_external_target(obsensor "" "" "HAVE_OBSENSOR;HAVE_OBSENSOR_V4L2") + else() + if(WIN32) + check_include_file(mfapi.h HAVE_MFAPI) + check_include_file(vidcap.h HAVE_VIDCAP) + if(HAVE_MFAPI AND HAVE_VIDCAP) + set(HAVE_OBSENSOR TRUE) + set(HAVE_OBSENSOR_MSMF TRUE) + ocv_add_external_target(obsensor "" "" "HAVE_OBSENSOR;HAVE_OBSENSOR_MSMF") + else() + set(HAVE_OBSENSOR OFF) + set(HAVE_OBSENSOR_MSMF OFF) + if(NOT HAVE_MFAPI) + MESSAGE(STATUS "Could not find mfapi.h. Turning HAVE_OBSENSOR OFF") + endif() + if(NOT HAVE_VIDCAP) + MESSAGE(STATUS "Could not find vidcap.h. Turning HAVE_OBSENSOR OFF") + endif() + endif() + elseif(UNIX) + check_include_file(linux/videodev2.h HAVE_CAMV4L2_OBSENSOR) + if(HAVE_CAMV4L2_OBSENSOR) + set(HAVE_OBSENSOR TRUE) + set(HAVE_OBSENSOR_V4L2 TRUE) + ocv_add_external_target(obsensor "" "" "HAVE_OBSENSOR;HAVE_OBSENSOR_V4L2") + endif() endif() endif() endif() diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index ee23dbe82f..fb47036bbf 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -128,7 +128,7 @@ enum VideoCaptureAPIs { CAP_INTEL_MFX = 2300, //!< Intel MediaSDK CAP_XINE = 2400, //!< XINE engine (Linux) CAP_UEYE = 2500, //!< uEye Camera API - CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. + CAP_OBSENSOR = 2600, //!< For Orbbec 3D-Sensor device/module (Astra+, Femto, Astra2, Gemini2, Gemini2L, Gemini2XL, Femto Mega) attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. }; diff --git a/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp b/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp index 9145a010d2..eec7634eca 100644 --- a/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp +++ b/modules/videoio/src/cap_obsensor/obsensor_uvc_stream_channel.cpp @@ -316,14 +316,14 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec if(OBSENSOR_GEMINI2_PID == devInfo_.pid){ // return default param CameraParam param; - param.p0[0] = 516.652f; - param.p0[1] = 516.692f; - param.p0[2] = 322.988f; - param.p0[3] = 235.787f; - param.p1[0] = 516.652f; - param.p1[1] = 516.692f; - param.p1[2] = 322.988f; - param.p1[3] = 235.787f; + param.p0[0] = 519.342f; + param.p0[1] = 519.043f; + param.p0[2] = 319.41f; + param.p0[3] = 240.839f; + param.p1[0] = 519.342f; + param.p1[1] = 519.043f; + param.p1[2] = 319.41f; + param.p1[3] = 240.839f; param.p6[0] = 640; param.p6[1] = 480; param.p7[0] = 640; @@ -421,7 +421,7 @@ bool IUvcStreamChannel::getProperty(int propId, uint8_t* recvData, uint32_t* rec bool IUvcStreamChannel::initDepthFrameProcessor() { - if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_ASTRA2_PID == devInfo_.pid || OBSENSOR_GEMINI2L_PID == devInfo_.pid){ + if( OBSENSOR_ASTRA2_PID == devInfo_.pid){ uint8_t* rcvData; uint32_t rcvLen; @@ -433,7 +433,19 @@ bool IUvcStreamChannel::initDepthFrameProcessor() depthFrameProcessor_ = makePtr(); return true; - }else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){ + } + else if(OBSENSOR_GEMINI2_PID == devInfo_.pid || OBSENSOR_GEMINI2L_PID == devInfo_.pid){ + uint8_t* rcvData; + uint32_t rcvLen; + + setXu(2, OB_EXT_CMD7, sizeof(OB_EXT_CMD7)); + getXu(2, &rcvData, &rcvLen); + + setXu(2, OB_EXT_CMD9, sizeof(OB_EXT_CMD9)); + getXu(2, &rcvData, &rcvLen); + return true; + } + else if(OBSENSOR_GEMINI2XL_PID == devInfo_.pid){ uint8_t* rcvData; uint32_t rcvLen; @@ -445,7 +457,7 @@ bool IUvcStreamChannel::initDepthFrameProcessor() return true; } - else if (streamType_ == OBSENSOR_STREAM_DEPTH && setXu(2, OB_EXT_CMD4, sizeof(OB_EXT_CMD4))) + else if(streamType_ == OBSENSOR_STREAM_DEPTH && setXu(2, OB_EXT_CMD4, sizeof(OB_EXT_CMD4))) { uint8_t* rcvData; uint32_t rcvLen; diff --git a/modules/videoio/src/cap_obsensor_capture.cpp b/modules/videoio/src/cap_obsensor_capture.cpp index 21db152fc3..4c64faee11 100644 --- a/modules/videoio/src/cap_obsensor_capture.cpp +++ b/modules/videoio/src/cap_obsensor_capture.cpp @@ -23,7 +23,8 @@ #include "cap_obsensor_capture.hpp" #include "cap_obsensor/obsensor_stream_channel_interface.hpp" -#ifdef HAVE_OBSENSOR + +#if defined(HAVE_OBSENSOR) && !defined(HAVE_OBSENSOR_ORBBEC_SDK) namespace cv { Ptr create_obsensor_capture(int index) { @@ -34,13 +35,13 @@ VideoCapture_obsensor::VideoCapture_obsensor(int index) : isOpened_(false) { static const obsensor::StreamProfile colorProfile = { 640, 480, 30, obsensor::FRAME_FORMAT_MJPG }; static const obsensor::StreamProfile depthProfile = {640, 480, 30, obsensor::FRAME_FORMAT_Y16}; - static const obsensor::StreamProfile gemini2DepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14}; + static const obsensor::StreamProfile gemini2DepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y16}; static const obsensor::StreamProfile astra2ColorProfile = {800, 600, 30, obsensor::FRAME_FORMAT_MJPG}; static const obsensor::StreamProfile astra2DepthProfile = {800, 600, 30, obsensor::FRAME_FORMAT_Y14}; static const obsensor::StreamProfile megaColorProfile = {1280, 720, 30, obsensor::FRAME_FORMAT_MJPG}; static const obsensor::StreamProfile megaDepthProfile = {640, 576, 30, obsensor::FRAME_FORMAT_Y16}; static const obsensor::StreamProfile gemini2lColorProfile = { 1280, 720, 30, obsensor::FRAME_FORMAT_MJPG}; - static const obsensor::StreamProfile gemini2lDepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y14}; + static const obsensor::StreamProfile gemini2lDepthProfile = {1280, 800, 30, obsensor::FRAME_FORMAT_Y16}; static const obsensor::StreamProfile gemini2XlColorProfile = { 1280, 800, 10, obsensor::FRAME_FORMAT_MJPG}; static const obsensor::StreamProfile gemini2XlDepthProfile = {1280, 800, 10, obsensor::FRAME_FORMAT_Y16}; @@ -143,19 +144,22 @@ bool VideoCapture_obsensor::retrieveFrame(int outputType, OutputArray frame) if (!grabbedDepthFrame_.empty()) { if(OBSENSOR_GEMINI2_PID == streamChannelGroup_.front()->getPid()){ - grabbedDepthFrame_ = grabbedDepthFrame_*0.8; + const double DepthValueScaleGemini2 = 0.2; + grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleGemini2; Rect rect(320, 160, 640, 480); grabbedDepthFrame_(rect).copyTo(frame); } else if(OBSENSOR_ASTRA2_PID == streamChannelGroup_.front()->getPid()){ - grabbedDepthFrame_ = grabbedDepthFrame_*0.8; + const double DepthValueScaleAstra2 = 0.8; + grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleAstra2; grabbedDepthFrame_.copyTo(frame); } else if(OBSENSOR_FEMTO_MEGA_PID == streamChannelGroup_.front()->getPid()){ Rect rect(0, 0, 640, 360); grabbedDepthFrame_(rect).copyTo(frame); }else if(OBSENSOR_GEMINI2L_PID == streamChannelGroup_.front()->getPid()){ - grabbedDepthFrame_ = grabbedDepthFrame_*0.8; + const double DepthValueScaleGemini2L = 0.2; + grabbedDepthFrame_ = grabbedDepthFrame_*DepthValueScaleGemini2L; Rect rect(0, 40, 1280, 720); grabbedDepthFrame_(rect).copyTo(frame); }else if(OBSENSOR_GEMINI2XL_PID == streamChannelGroup_.front()->getPid()){ diff --git a/modules/videoio/src/cap_obsensor_capture.hpp b/modules/videoio/src/cap_obsensor_capture.hpp index 821e6193a0..89ab403bb2 100644 --- a/modules/videoio/src/cap_obsensor_capture.hpp +++ b/modules/videoio/src/cap_obsensor_capture.hpp @@ -28,7 +28,8 @@ #include "cap_obsensor/obsensor_stream_channel_interface.hpp" -#ifdef HAVE_OBSENSOR +#if defined(HAVE_OBSENSOR) && !defined(HAVE_OBSENSOR_ORBBEC_SDK) + namespace cv { class VideoCapture_obsensor : public IVideoCapture { diff --git a/modules/videoio/src/cap_obsensor_liborbbec.cpp b/modules/videoio/src/cap_obsensor_liborbbec.cpp new file mode 100644 index 0000000000..ac581d81dc --- /dev/null +++ b/modules/videoio/src/cap_obsensor_liborbbec.cpp @@ -0,0 +1,154 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +/* +* Copyright(C) 2024 by ORBBEC Technology., Inc. +* Authors: +* Huang Zhenchang +* Yu Shuai +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#include "precomp.hpp" + +#if defined(HAVE_OBSENSOR) && defined(HAVE_OBSENSOR_ORBBEC_SDK) +#include "libobsensor/ObSensor.hpp" +#include "cap_obsensor_liborbbec.hpp" + +namespace cv +{ +Ptr create_obsensor_capture(int index) +{ + return makePtr(index); +} + +VideoCapture_obsensor::VideoCapture_obsensor(int) +{ + ob::Context::setLoggerToFile(OB_LOG_SEVERITY_OFF, ""); + config = std::make_shared(); + pipe = std::make_shared(); + auto colorProfiles = pipe->getStreamProfileList(OB_SENSOR_COLOR); + auto colorProfile = colorProfiles->getProfile(OB_PROFILE_DEFAULT); + config->enableStream(colorProfile->as()); + + auto depthProfiles = pipe->getStreamProfileList(OB_SENSOR_DEPTH); + auto depthProfile = depthProfiles->getProfile(OB_PROFILE_DEFAULT); + config->enableStream(depthProfile->as()); + + config->setAlignMode(ALIGN_D2C_SW_MODE); + + pipe->start(config, [&](std::shared_ptr frameset) { + std::unique_lock lk(videoFrameMutex); + colorFrame = frameset->colorFrame(); + depthFrame = frameset->depthFrame(); + }); + + auto param = pipe->getCameraParam(); + camParam.p1[0] = param.rgbIntrinsic.fx; + camParam.p1[1] = param.rgbIntrinsic.fy; + camParam.p1[2] = param.rgbIntrinsic.cx; + camParam.p1[3] = param.rgbIntrinsic.cy; +} + +VideoCapture_obsensor::~VideoCapture_obsensor(){ + pipe->stop(); +} + +double VideoCapture_obsensor::getProperty(int propIdx) const +{ + double rst = 0.0; + propIdx = propIdx & (~CAP_OBSENSOR_GENERATORS_MASK); + switch (propIdx) + { + case CAP_PROP_OBSENSOR_INTRINSIC_FX: + rst = camParam.p1[0]; + break; + case CAP_PROP_OBSENSOR_INTRINSIC_FY: + rst = camParam.p1[1]; + break; + case CAP_PROP_OBSENSOR_INTRINSIC_CX: + rst = camParam.p1[2]; + break; + case CAP_PROP_OBSENSOR_INTRINSIC_CY: + rst = camParam.p1[3]; + break; + } + return rst; +} + +bool VideoCapture_obsensor::setProperty(int, double) +{ + return false; +} + +bool VideoCapture_obsensor::grabFrame() +{ + std::unique_lock lk(videoFrameMutex); + grabbedColorFrame = colorFrame; + grabbedDepthFrame = depthFrame; + + return grabbedColorFrame || grabbedDepthFrame; +} + +bool VideoCapture_obsensor::retrieveFrame(int outputType, cv::OutputArray frame) +{ + switch (outputType) + { + case CAP_OBSENSOR_BGR_IMAGE: + if(grabbedColorFrame != nullptr){ + auto format = grabbedColorFrame->format(); + if(format != OB_FORMAT_MJPEG){ + CV_LOG_WARNING(NULL, "Unsupported color frame format"); + return false; + } + auto mjpgMat = Mat(1, grabbedColorFrame->dataSize() , CV_8UC1, grabbedColorFrame->data()).clone(); + auto bgrMat = imdecode(mjpgMat, IMREAD_COLOR); + if(bgrMat.empty()){ + CV_LOG_WARNING(NULL, "Failed to decode color frame"); + return false; + } + bgrMat.copyTo(frame); + return true; + } + break; + case CAP_OBSENSOR_DEPTH_MAP: + if(grabbedDepthFrame != nullptr){ + auto format = grabbedDepthFrame->format(); + if(format != OB_FORMAT_Y16){ + CV_LOG_WARNING(NULL, "Unsupported depth frame format"); + return false; + } + Mat(grabbedDepthFrame->height(), grabbedDepthFrame->width(), CV_16UC1, grabbedDepthFrame->data()).copyTo(frame); + return true; + } + break; + default: + return false; + } + + return false; +} + +int VideoCapture_obsensor::getCaptureDomain() +{ + return CAP_OBSENSOR; +} + +bool VideoCapture_obsensor::isOpened() const +{ + return true; +} + +} +#endif diff --git a/modules/videoio/src/cap_obsensor_liborbbec.hpp b/modules/videoio/src/cap_obsensor_liborbbec.hpp new file mode 100644 index 0000000000..13dbf413ca --- /dev/null +++ b/modules/videoio/src/cap_obsensor_liborbbec.hpp @@ -0,0 +1,74 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +/* +* Copyright(C) 2024 by ORBBEC Technology., Inc. +* Authors: +* Huang Zhenchang +* Yu Shuai +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +#ifndef _CAP_LIBORBBEC_HPP_ +#define _CAP_LIBORBBEC_HPP_ + +#if defined(HAVE_OBSENSOR) && defined(HAVE_OBSENSOR_ORBBEC_SDK) + +#include +#include + +namespace cv +{ + +struct CameraParam +{ + float p0[4]; + float p1[4]; + float p2[9]; + float p3[3]; + float p4[5]; + float p5[5]; + uint32_t p6[2]; + uint32_t p7[2]; +}; + +class VideoCapture_obsensor : public IVideoCapture +{ +public: + VideoCapture_obsensor(int index); + virtual ~VideoCapture_obsensor(); + + virtual double getProperty(int propIdx) const CV_OVERRIDE; + virtual bool setProperty(int propIdx, double propVal) CV_OVERRIDE; + + virtual bool grabFrame() CV_OVERRIDE; + virtual bool retrieveFrame(int outputType, OutputArray frame) CV_OVERRIDE; + virtual int getCaptureDomain() CV_OVERRIDE; + virtual bool isOpened() const CV_OVERRIDE; + +protected: + std::mutex videoFrameMutex; + std::shared_ptr colorFrame; + std::shared_ptr depthFrame; + std::shared_ptr grabbedColorFrame; + std::shared_ptr grabbedDepthFrame; + std::shared_ptr pipe; + std::shared_ptr config; + CameraParam camParam; +}; + +} + +#endif +#endif diff --git a/samples/cpp/videocapture_obsensor.cpp b/samples/cpp/videocapture_obsensor.cpp index 7dda3386ce..ce71a1808b 100644 --- a/samples/cpp/videocapture_obsensor.cpp +++ b/samples/cpp/videocapture_obsensor.cpp @@ -1,5 +1,5 @@ /** - * attention: Astra2, Gemini2, and Gemini2L cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. + * attention: Astra2 cameras currently only support Windows and Linux kernel versions no higher than 4.15, and higher versions of Linux kernel may have exceptions. */ #include