Add MultiSense AI support

README.md

@@ -251,7 +251,7 @@ installed via pip into a Python virtual environment or a local Python project.
 
 To install the LibMultiSense Python bindings
 
-    > sudo apt install build-essential pybind11-dev nlohmann-json3-dev
+    > sudo apt install build-essential pybind11-dev nlohmann-json3-dev libcpp-httplib-dev
 
     > git clone https://github.com/carnegierobotics/LibMultiSense.git
     > cd LibMultiSense

python/bindings.cc

@@ -202,6 +202,13 @@ PYBIND11_MODULE(_libmultisense, m) {
                              strides));
         });
 
+    // BufferWrapper
+    py::class_<multisense::BufferWrapper>(m, "BufferWrapper")
+        .def(py::init<std::shared_ptr<const std::vector<uint8_t>> , size_t>())
+        .def(py::init<const uint8_t* , size_t>())
+        .def("data", &multisense::BufferWrapper::data)
+        .def("size", &multisense::BufferWrapper::data);
+
     // Image
     py::class_<multisense::Image>(m, "Image")
         .def(py::init<>())
@@ -211,12 +218,10 @@ PYBIND11_MODULE(_libmultisense, m) {
             if (image.format == multisense::Image::PixelFormat::H264 ||
                 image.format == multisense::Image::PixelFormat::JPEG)
             {
-                const SSizeVector shape = {
-                    static_cast<py::ssize_t>(image.raw_data->size() - image.image_data_offset)
-                };
+                const SSizeVector shape = { static_cast<py::ssize_t>(image.raw_data->size()) };
                 const SSizeVector strides = {static_cast<py::ssize_t>(sizeof(uint8_t))};
                 return py::array(py::buffer_info(
-                                 const_cast<uint8_t*>(image.raw_data->data() + image.image_data_offset),
+                                 const_cast<uint8_t*>(image.raw_data->data()),
                                  static_cast<py::ssize_t>(sizeof(uint8_t)),
                                  py::format_descriptor<uint8_t>::format(),
                                  1,
@@ -270,7 +275,7 @@ PYBIND11_MODULE(_libmultisense, m) {
 
             // Map the cv::Mat to a NumPy array without copying the data
             return py::array(py::buffer_info(
-                             const_cast<uint8_t*>(image.raw_data->data() + image.image_data_offset),
+                             const_cast<uint8_t*>(image.raw_data->data()),
                              element_size,
                              format,
                              static_cast<py::ssize_t>(shape.size()),
@@ -283,7 +288,8 @@ PYBIND11_MODULE(_libmultisense, m) {
         .def_readonly("camera_timestamp", &multisense::Image::camera_timestamp)
         .def_readonly("ptp_timestamp", &multisense::Image::ptp_timestamp)
         .def_readonly("source", &multisense::Image::source)
-        .def_readonly("calibration", &multisense::Image::calibration);
+        .def_readonly("calibration", &multisense::Image::calibration)
+        .def_readonly("pixel_scale", &multisense::Image::pixel_scale);
 
     // ImageHistogram
     py::class_<multisense::ImageHistogram>(m, "ImageHistogram")
@@ -830,7 +836,7 @@ PYBIND11_MODULE(_libmultisense, m) {
 
     // Utilities
     py::class_<multisense::QMatrix>(m, "QMatrix")
-        .def(py::init<const multisense::CameraCalibration &, const multisense::CameraCalibration &>())
+        .def(py::init<const multisense::CameraCalibration &, double, double>())
         .def("reproject", &multisense::QMatrix::reproject)
         .def("matrix", &multisense::QMatrix::matrix);
 

source/LibMultiSense/CMakeLists.txt

@@ -9,6 +9,11 @@ endif()
 set(DETAILS_SRC details/factory.cc
                 details/multi_channel.cc
                 details/utilities.cc
+                details/amb/channel.cc
+                details/amb/info.cc
+                details/amb/http.cc
+                details/amb/utilities.cc
+                details/amb/webrtc.cc
                 details/legacy/calibration.cc
                 details/legacy/channel.cc
                 details/legacy/configuration.cc
@@ -69,11 +74,15 @@ target_include_directories(MultiSense PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_S
 set_target_properties(MultiSense PROPERTIES PUBLIC_HEADER "${MULTISENSE_HEADERS}")
 
 #
-# We want to link against our child libraries.
+# TODO (malvarado): Properly handle this
 #
-
+find_package(multisense-webrtc-client REQUIRED)
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(HTTPLIB REQUIRED IMPORTED_TARGET cpp-httplib)
+find_package(OpenSSL REQUIRED)
 find_package(Threads REQUIRED)
-target_link_libraries(MultiSense PUBLIC Threads::Threads)
+target_link_libraries(MultiSense PUBLIC Threads::Threads PkgConfig::HTTPLIB OpenSSL::SSL OpenSSL::Crypto multisense-webrtc-client)
+
 if(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "WindowsStore")
     target_link_libraries(MultiSense PRIVATE ws2_32)
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")

source/LibMultiSense/MultiSenseConfig.cmake.in

@@ -27,6 +27,12 @@ if (MULTISENSE_COMPILER_CHECKS_VIOLATED)
     message(FATAL_ERROR "The new MultiSense API requires at minimum a C++17 standard")
 endif()
 
+# TODO (malvarado): Make this conditional
+find_dependency(multisense-webrtc-client)
+find_dependency(OpenSSL)
+find_dependency(PkgConfig)
+pkg_check_modules(HTTPLIB REQUIRED IMPORTED_TARGET cpp-httplib)
+
 include("${CMAKE_CURRENT_LIST_DIR}/MultiSenseTargets.cmake")
 
 # The exported target is namespaced as `MultiSense::MultiSense`.  Historically,