V3 RGBD Node

.github/workflows/python-main.yml

@@ -52,12 +52,16 @@ jobs:
         run: |
           sudo apt update
           python -m pip install --upgrade pip
-          sudo apt install libusb-1.0-0-dev pkg-config bison autoconf libtool libxi-dev libxtst-dev libxrandr-dev libx11-dev libxft-dev libxext-dev nasm flex libudev-dev
+          sudo apt install libusb-1.0-0-dev pkg-config bison autoconf libtool libxi-dev libxtst-dev libxrandr-dev libx11-dev libxft-dev libxext-dev nasm flex libudev-dev automake libltdl-dev
           python -m pip install -r bindings/python/docs/requirements_mkdoc.txt
       - name: Configure project
         run: cmake -S . -B build -DDEPTHAI_BUILD_PYTHON=ON -DDEPTHAI_PYTHON_FORCE_DOCSTRINGS=ON -DDEPTHAI_BASALT_SUPPORT=ON -DDEPTHAI_PCL_SUPPORT=ON -DDEPTHAI_RTABMAP_SUPPORT=ON -DDEPTHAI_PYTHON_DOCSTRINGS_OUTPUT="$PWD/bindings/python/docstrings/depthai_python_docstring.hpp"
       - name: Build target 'pybind11_mkdoc'
         run: cmake --build build --target pybind11_mkdoc --parallel 4
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: bash ./bindings/python/ci/show_vcpkg_logs.sh
+
       - name: Upload docstring artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -140,6 +144,9 @@ jobs:
         run: |
           cmake -S . -B build -DDEPTHAI_BUILD_PYTHON=ON -D CMAKE_BUILD_TYPE=Release -D DEPTHAI_PYTHON_DOCSTRINGS_INPUT=$PWD/bindings/python/docstrings/depthai_python_docstring.hpp -D DEPTHAI_PYTHON_ENABLE_TESTS=ON
           cmake --build build --parallel 4
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
       - name: Test
         run: |
           cmake --build build --target pytest --config Release
@@ -207,6 +214,7 @@ jobs:
       DEPTHAI_BUILD_BASALT: OFF
       DEPTHAI_BUILD_PCL: ON
       DEPTHAI_BUILD_RTABMAP: ON
+      DEPTHAI_BUILD_KOMPUTE: ON
       VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
     steps:
       - name: Cache .hunter folder
@@ -247,6 +255,9 @@ jobs:
           python -m pip install --upgrade pip
       - name: Building wheels
         run: cd bindings/python && python -m pip wheel . -w ./wheelhouse/audited/ --verbose
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
       - name: Archive wheel artifacts
         uses: actions/upload-artifact@v3
         with:
@@ -274,6 +285,7 @@ jobs:
       DEPTHAI_BUILD_BASALT: ON
       DEPTHAI_BUILD_PCL: ON
       DEPTHAI_BUILD_RTABMAP: ON
+      DEPTHAI_BUILD_KOMPUTE: ON
     steps:
       - name: Cache .hunter folder
         uses: actions/cache@v3
@@ -317,6 +329,9 @@ jobs:
           python -m pip install delocate
       - name: Building wheels
         run: cd bindings/python && python -m pip wheel . -w ./wheelhouse/ --verbose
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
       - name: Auditing wheels
         run: cd bindings/python && ci/repair-whl-macos.sh `pwd`/wheelhouse/* `pwd`/wheelhouse/audited
       - name: Archive wheel artifacts
@@ -410,6 +425,7 @@ jobs:
       DEPTHAI_BUILD_BASALT: ON
       DEPTHAI_BUILD_PCL: ON
       DEPTHAI_BUILD_RTABMAP: ON
+      DEPTHAI_BUILD_KOMPUTE: ON
       VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
     steps:
       - name: Cache .hunter folder
@@ -428,11 +444,6 @@ jobs:
           submodules: 'recursive'
       - name: Installing libusb1-devel dependency
         run: yum install -y libusb1-devel perl-core curl zip unzip tar ninja-build  zlib-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel libXrandr-devel libXtst-devel libudev-devel lapack-devel nasm libtool autoconf automake
-      - name: Installing cmake dependency
-        run: |
-          cd bindings/python
-          /opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
-          ln -s /opt/python/cp38-cp38/bin/cmake /bin/
       - name: Create folder structure
         run: cd bindings/python && mkdir -p wheelhouse/audited/
 
@@ -455,6 +466,9 @@ jobs:
       - name: Build wheels
         run: |
           cd bindings/python && for PYBIN in /opt/python/${{ matrix.python-set }}/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
 
       - name: Audit wheels
         run: cd bindings/python && for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
@@ -493,6 +507,7 @@ jobs:
       DEPTHAI_BUILD_BASALT: ON
       DEPTHAI_BUILD_PCL: ON
       DEPTHAI_BUILD_RTABMAP: ON
+      DEPTHAI_BUILD_KOMPUTE: ON
       VCPKG_MAX_CONCURRENCY: "2"
     steps:
       - name: Export GitHub Actions cache environment variables
@@ -505,12 +520,7 @@ jobs:
         with:
           submodules: 'recursive'
       - name: Installing libusb1-devel dependency
-        run: yum install -y libusb1-devel perl-core curl zip unzip tar  zlib-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel libXrandr-devel libXtst-devel libudev-devel lapack-devel nasm libtool autoconf automake
-      - name: Installing cmake dependency
-        run: |
-          cd bindings/python
-          /opt/python/cp38-cp38/bin/python3.8 -m pip install cmake
-          ln -s /opt/python/cp38-cp38/bin/cmake /bin/
+        run: yum install -y libusb1-devel perl-core curl zip unzip tar zlib-devel curl-devel libxcb-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel libXrandr-devel libXtst-devel libudev-devel lapack-devel nasm libtool autoconf automake libX11-devel pkgconfig 
       - name: Setup ninja required for arm64 builds
         run: |
           git clone https://github.com/ninja-build/ninja.git
@@ -534,6 +544,10 @@ jobs:
       - name: Building wheels
         run: |
           cd bindings/python && for PYBIN in /opt/python/${{ matrix.python-set }}/bin; do "${PYBIN}/pip" wheel . -w ./wheelhouse/ --verbose; done
+      - name: Print out vcpkg logs if building port fails
+        if: failure()    # Only run this if the build step fails
+        run: cd bindings/python && bash ./ci/show_vcpkg_logs.sh
+
       - name: Auditing wheels
         run: cd bindings/python && for whl in wheelhouse/*.whl; do auditwheel repair "$whl" --plat $PLAT -w wheelhouse/audited/; done
       - name: Archive wheel artifacts

.github/workflows/test.workflow.yml

@@ -80,7 +80,7 @@ jobs:
         export DISPLAY=:99
         xdpyinfo -display $DISPLAY >/dev/null 2>&1 || (Xvfb $DISPLAY &)
         source venv/bin/activate  # Activate virtual environment
-        python3 -m pip install jinja2
+        python3 -m pip install jinja2 rerun-sdk open3d
         cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_${{ matrix.flavor }} -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON
         cmake --build build --parallel 4 --config Release
         cd tests

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 3.20)
 # Early options
 option(DEPTHAI_ENABLE_LIBUSB "Enable usage of libusb and interaction with USB devices" ON)
 # Build AprilTag node code (note, that even if set to OFF, an AprilTagNode can be used with setRunOnHost(false))
@@ -9,6 +9,7 @@ option(DEPTHAI_ENABLE_PROTOBUF "Enable Protobuf support" ON)
 option(DEPTHAI_BUILD_PYTHON "Build python bindings" OFF)
 option(DEPTHAI_BUILD_TESTS "Build tests" OFF)
 option(DEPTHAI_OPENCV_SUPPORT "Enable optional OpenCV support" ON)
+OPTION(DEPTHAI_ENABLE_KOMPUTE "Enable Kompute support" OFF)
 option(DEPTHAI_PCL_SUPPORT "Enable optional PCL support" OFF)
 option(DEPTHAI_BOOTSTRAP_VCPKG "Automatically bootstrap VCPKG" ON)
 option(DEPTHAI_MERGED_TARGET "Enable merged target build" ON)
@@ -51,6 +52,9 @@ endif()
 if(DEPTHAI_PCL_SUPPORT)
 list(APPEND VCPKG_MANIFEST_FEATURES "pcl-support")
 endif()
+if(DEPTHAI_ENABLE_KOMPUTE)
+list(APPEND VCPKG_MANIFEST_FEATURES "kompute-support")
+endif()
 
 # Enable backward stack printing on crash
 if(ANDROID OR EMSCRIPTEN)
@@ -275,13 +279,20 @@ include(depthaiDependencies)
 # Add threads preference
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
+
 # TODO Remove shared naming
 set(DEPTHAI_SHARED_3RDPARTY_INCLUDE
     ${CMAKE_CURRENT_LIST_DIR}/include/3rdparty/
 )
 # Add depthai-bootloader-shared
 include(${CMAKE_CURRENT_LIST_DIR}/shared/depthai-bootloader-shared.cmake)
 
+if(DEPTHAI_ENABLE_KOMPUTE)
+    add_subdirectory(shaders)
+
+    list(APPEND targets_to_export shaders)
+endif()
+
 # Add flags helpers
 include(Flags)
 
@@ -388,6 +399,7 @@ set(TARGET_CORE_SOURCES
     src/pipeline/node/host/XLinkInHost.cpp
     src/pipeline/node/host/XLinkOutHost.cpp
     src/pipeline/node/host/HostNode.cpp
+    src/pipeline/node/host/RGBD.cpp
     src/pipeline/datatype/DatatypeEnum.cpp
     src/pipeline/node/PointCloud.cpp
     src/pipeline/datatype/Buffer.cpp
@@ -418,6 +430,7 @@ set(TARGET_CORE_SOURCES
     src/pipeline/datatype/BenchmarkReport.cpp
     src/pipeline/datatype/PointCloudConfig.cpp
     src/pipeline/datatype/PointCloudData.cpp
+    src/pipeline/datatype/RGBDData.cpp
     src/pipeline/datatype/MessageGroup.cpp
     src/pipeline/datatype/TransformData.cpp
     src/utility/H26xParsers.cpp
@@ -502,6 +515,10 @@ set_target_properties(${TARGET_CORE_NAME} PROPERTIES EXPORT_NAME ${TARGET_CORE_A
 # Add to list of targets to export and install
 list(APPEND targets_to_export ${TARGET_CORE_NAME})
 
+if(DEPTHAI_ENABLE_KOMPUTE)
+    target_link_libraries(${TARGET_CORE_NAME} PRIVATE shaders kompute::kompute)
+    target_compile_definitions(${TARGET_CORE_NAME} PRIVATE DEPTHAI_ENABLE_KOMPUTE)
+endif()
 # Add model_zoo helper binary
 add_executable(zoo_helper src/modelzoo/zoo_helper.cpp)
 target_compile_definitions(zoo_helper PRIVATE DEPTHAI_TARGET_CORE)

bindings/python/CMakeLists.txt

@@ -108,6 +108,7 @@ set(SOURCE_LIST
     src/pipeline/node/RecordBindings.cpp
     src/pipeline/node/ReplayBindings.cpp
     src/pipeline/node/ImageAlignBindings.cpp
+    src/pipeline/node/RGBDBindings.cpp
 
     src/pipeline/datatype/ADatatypeBindings.cpp
     src/pipeline/datatype/AprilTagConfigBindings.cpp

bindings/python/ci/show_vcpkg_logs.sh

@@ -0,0 +1,51 @@
+echo "Searching vcpkg-manifest-install.log for failing port logs..."
+LOGFILE=$(find . -name "vcpkg-manifest-install.log" | head -n 1)
+if [ -z "$LOGFILE" ]; then
+    echo "No vcpkg-manifest-install.log found!"
+    exit 1
+else
+    echo "Found log file: $LOGFILE"
+fi
+
+# 1) Grab lines following "See logs for more information:"
+# 2) From those lines, extract only the file paths ending in ".log"
+# 3) Remove any leading spaces
+grep -A50 "See logs for more information:" $LOGFILE \
+    | grep "\.log" \
+    | sed 's/^[[:space:]]*//' \
+    > failed_logs.txt
+
+# Check if we found any logs
+# If not, print a message and exit
+# Otherwise, print the log paths
+if [ ! -s failed_logs.txt ]; then
+    echo "No failed logs found!"
+    exit 1
+else
+    echo "Found the following failed logs:"
+    cat failed_logs.txt
+fi
+
+# Now read each log path we found, and print it
+while IFS= read -r log; do
+    echo "==== Showing log: $log ===="
+    cat "$log" || true
+echo
+
+# Also show CMakeOutput.log (and/or CMakeError.log) within the same port’s directory
+# Often it’s in the same or neighboring folder (e.g. arm64-linux-rel/CMakeFiles/CMakeOutput.log).
+# We'll "walk up" one directory and search for CMakeOutput.log in CMakeFiles/.
+port_dir="$(dirname "$log")"
+# In some cases (e.g. config-arm64-linux-out.log), you may need to go up another level:
+# port_dir="$(dirname "$port_dir")"
+
+# We'll now look for any CMakeOutput.log within this port’s subdirectories
+found_outputs=$(find "$port_dir" -name "CMakeOutput.log" -o -name "CMakeError.log" -print 2>/dev/null)
+if [ -n "$found_outputs" ]; then
+    for cof in $found_outputs; do
+        echo "==== Showing $cof ===="
+        cat "$cof" || true
+        echo
+    done
+fi
+done < failed_logs.txt

bindings/python/setup.py

@@ -125,6 +125,8 @@ def build_extension(self, ext):
             cmake_args += ['-DDEPTHAI_PCL_SUPPORT=ON']
         if env.get('DEPTHAI_BUILD_RTABMAP') == 'ON': 
             cmake_args += ['-DDEPTHAI_RTABMAP_SUPPORT=ON']
+        if env.get('DEPTHAI_BUILD_KOMPUTE') == 'ON':
+            cmake_args += ['-DDEPTHAI_KOMPUTE_SUPPORT=ON']
         build_args += ['--target=depthai']
 
         # Specify output directory and python executable

bindings/python/src/pipeline/datatype/RGBDDataBindings.cpp

@@ -0,0 +1,39 @@
+
+#include "DatatypeBindings.hpp"
+#include "pipeline/CommonBindings.hpp"
+#include <unordered_map>
+#include <memory>
+
+// depthai
+#include "depthai/pipeline/datatype/RGBDData.hpp"
+
+//pybind
+#include <pybind11/chrono.h>
+#include <pybind11/numpy.h>
+
+void bind_transformdata(pybind11::module& m, void* pCallstack){
+
+    using namespace dai;
+
+    py::class_<RGBDData, Py<RGBDData>, Buffer, std::shared_ptr<RGBDData>> rgbdData(m, "RGBDData", DOC(dai, RGBDData));
+
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    // Call the rest of the type defines, then perform the actual bindings
+    Callstack* callstack = (Callstack*) pCallstack;
+    auto cb = callstack->top();
+    callstack->pop();
+    cb(m, pCallstack);
+    // Actual bindings
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+
+    // Metadata / raw
+    rgbdData
+        .def(py::init<>())
+        .def("__repr__", &RGBDData::str)
+        .def_readwrite("rgbFrame", &RGBDData::rgbFrame, DOC(dai, RGBDData, rgbFrame))
+        .def_readwrite("depthFrame", &RGBDData::depthFrame, DOC(dai, RGBDData, depthFrame));
+}

bindings/python/src/pipeline/node/NodeBindings.cpp

@@ -161,6 +161,7 @@ void bind_hostnode(pybind11::module& m, void* pCallstack);
 void bind_record(pybind11::module& m, void* pCallstack);
 void bind_replay(pybind11::module& m, void* pCallstack);
 void bind_imagealign(pybind11::module& m, void* pCallstack);
+void bind_rgbd(pybind11::module& m, void* pCallstack);
 #ifdef DEPTHAI_HAVE_BASALT_SUPPORT
 void bind_basaltnode(pybind11::module& m, void* pCallstack);
 #endif
@@ -208,6 +209,7 @@ void NodeBindings::addToCallstack(std::deque<StackFunction>& callstack) {
     callstack.push_front(bind_record);
     callstack.push_front(bind_replay);
     callstack.push_front(bind_imagealign);
+    callstack.push_front(bind_rgbd);
 #ifdef DEPTHAI_HAVE_BASALT_SUPPORT
     callstack.push_front(bind_basaltnode);
 #endif

bindings/python/src/pipeline/node/RGBDBindings.cpp

@@ -0,0 +1,49 @@
+
+#include <pybind11/eval.h>
+
+#include "Common.hpp"
+#include "NodeBindings.hpp"
+#include "depthai/pipeline/ThreadedHostNode.hpp"
+#include "depthai/pipeline/node/host/RGBD.hpp"
+
+extern py::handle daiNodeModule;
+
+void bind_rgbd(pybind11::module& m, void* pCallstack) {
+    using namespace dai;
+    using namespace dai::node;
+
+    // declare upfront
+    auto rgbdNode = ADD_NODE_DERIVED(RGBD, ThreadedHostNode);
+
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    // Call the rest of the type defines, then perform the actual bindings
+    Callstack* callstack = (Callstack*)pCallstack;
+    auto cb = callstack->top();
+    callstack->pop();
+    cb(m, pCallstack);
+    // Actual bindings
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
+
+    // RGBD Node
+    rgbdNode
+        .def_property_readonly(
+            "inColor", [](RGBD& node) { return &node.inColor; }, py::return_value_policy::reference_internal)
+        .def_property_readonly(
+            "inDepth", [](RGBD& node) { return &node.inDepth; }, py::return_value_policy::reference_internal)
+        .def_readonly("pcl", &RGBD::pcl, DOC(dai, node, RGBD, pcl))
+        .def("build", static_cast<std::shared_ptr<RGBD> (RGBD::*)()>(&RGBD::build))
+        .def("build",
+             static_cast<std::shared_ptr<RGBD> (RGBD::*)(bool, std::pair<int, int>)>(&RGBD::build),
+             py::arg("autocreate"),
+             py::arg("size"),
+             DOC(dai, node, RGBD, build, 2))
+        .def("setDepthUnits", &RGBD::setDepthUnit, py::arg("units"), DOC(dai, node, RGBD, setDepthUnit))
+        .def("useCPU", &RGBD::useCPU, DOC(dai, node, RGBD, useCPU))
+        .def("useCPUMT", &RGBD::useCPUMT, py::arg("numThreads") = 2, DOC(dai, node, RGBD, useCPUMT))
+        .def("useGPU", &RGBD::useGPU, py::arg("device") = 0, DOC(dai, node, RGBD, useGPU))
+        .def("printDevices", &RGBD::printDevices, DOC(dai, node, RGBD, printDevices));
+}

cmake/depthaiDependencies.cmake

@@ -42,6 +42,10 @@ if(NOT CONFIG_MODE OR (CONFIG_MODE AND NOT DEPTHAI_SHARED_LIBS))
     find_path(FP16_INCLUDE_DIR NAMES fp16.h)
 
     find_package(PNG  REQUIRED)
+
+    if(DEPTHAI_ENABLE_KOMPUTE)
+        find_package(kompute ${_QUIET} CONFIG REQUIRED)
+    endif()
     # libarchive for firmware packages
     find_package(LibArchive ${_QUIET}  REQUIRED)
     find_package(liblzma ${_QUIET} CONFIG REQUIRED)