Add enableUndistortion flag to ImgFrameCapabilities

bindings/python/src/capabilities/ImgFrameCapabilityBindings.cpp

@@ -27,6 +27,7 @@ void ImgFrameCapabilityBindings::bind(pybind11::module& m, void* pCallstack) {
         .def_readwrite("fps", &ImgFrameCapability::fps)
         .def_readwrite("type", &ImgFrameCapability::type)
         .def_readwrite("resizeMode", &ImgFrameCapability::resizeMode)
+        .def_readwrite("enableUndistortion", &ImgFrameCapability::enableUndistortion)
 
         ;
 }

bindings/python/src/pipeline/datatype/ImgFrameBindings.cpp

@@ -144,9 +144,20 @@ void bind_imgframe(pybind11::module& m, void* pCallstack) {
         .def("getSourceIntrinsicMatrixInv", &ImgTransformation::getSourceIntrinsicMatrixInv)
         .def("getIntrinsicMatrix", &ImgTransformation::getIntrinsicMatrix)
         .def("getIntrinsicMatrixInv", &ImgTransformation::getIntrinsicMatrixInv)
+        .def("getDistortionModel", &ImgTransformation::getDistortionModel, DOC(dai, ImgTransformation, getDistortionModel))
+        .def("getDistortionCoefficients", &ImgTransformation::getDistortionCoefficients, DOC(dai, ImgTransformation, getDistortionCoefficients))
+        .def("getSrcCrops", &ImgTransformation::getSrcCrops, DOC(dai, ImgTransformation, getSrcCrops))
+        .def("getSrcMaskPt", &ImgTransformation::getSrcMaskPt, py::arg("x"), py::arg("y"), DOC(dai, ImgTransformation, getSrcMaskPt))
+        .def("getDstMaskPt", &ImgTransformation::getDstMaskPt, py::arg("x"), py::arg("y"), DOC(dai, ImgTransformation, getDstMaskPt))
         .def("getDFov", &ImgTransformation::getDFov, py::arg("source") = false)
         .def("getHFov", &ImgTransformation::getHFov, py::arg("source") = false)
         .def("getVFov", &ImgTransformation::getVFov, py::arg("source") = false)
+        .def("setIntrinsicMatrix", &ImgTransformation::setIntrinsicMatrix, py::arg("intrinsicMatrix"), DOC(dai, ImgTransformation, setIntrinsicMatrix))
+        .def("setDistortionModel", &ImgTransformation::setDistortionModel, py::arg("model"), DOC(dai, ImgTransformation, setDistortionModel))
+        .def("setDistortionCoefficients",
+             &ImgTransformation::setDistortionCoefficients,
+             py::arg("coefficients"),
+             DOC(dai, ImgTransformation, setDistortionCoefficients))
         .def("addTransformation", &ImgTransformation::addTransformation, py::arg("matrix"), DOC(dai, ImgTransformation, addTransformation))
         .def("addCrop", &ImgTransformation::addCrop, py::arg("x"), py::arg("y"), py::arg("width"), py::arg("height"), DOC(dai, ImgTransformation, addCrop))
         .def("addPadding",
@@ -201,6 +212,7 @@ void bind_imgframe(pybind11::module& m, void* pCallstack) {
         .def("getSourceWidth", &ImgFrame::getSourceWidth, DOC(dai, ImgFrame, getSourceWidth))
         .def("getSourceHeight", &ImgFrame::getSourceHeight, DOC(dai, ImgFrame, getSourceHeight))
         .def("getTransformation", [](ImgFrame& msg) {return msg.transformation;})
+        .def("validateTransformations", &ImgFrame::validateTransformations, DOC(dai, ImgFrame, validateTransformations))
 
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
         // The cast function itself does a copy, so we can avoid two copies by always not copying

bindings/python/src/pipeline/node/CameraBindings.cpp

@@ -27,11 +27,12 @@ void bind_camera(pybind11::module& m, void* pCallstack) {
         // .def("setCamera", &Camera::setCamera, "name"_a, DOC(dai, node, Camera, setCamera))
         // .def("getCamera", &Camera::getCamera, DOC(dai, node, Camera, getCamera))
         .def("requestOutput",
-             py::overload_cast<std::pair<uint32_t, uint32_t>, std::optional<ImgFrame::Type>, ImgResizeMode, float>(&Camera::requestOutput),
+             py::overload_cast<std::pair<uint32_t, uint32_t>, std::optional<ImgFrame::Type>, ImgResizeMode, float, std::optional<bool>>(&Camera::requestOutput),
              "size"_a,
              "type"_a = std::nullopt,
              "resizeMode"_a = dai::ImgResizeMode::CROP,
              "fps"_a = 30,
+             "enableUndistortion"_a = std::nullopt,
              py::return_value_policy::reference_internal,
              DOC(dai, node, Camera, requestOutput))
         .def("requestOutput",

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
 # set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+670797e3b8cbc185c7d457e24382495200486573")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+84c5c0523a6da38977bed5703d52cf4f1474ff8b")

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "fa759809823266407a39cd7cc77d98175234a7f6")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "108b7c0bd0a963401dedf929417eeee5acd4bdb2")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/cpp/CMakeLists.txt

@@ -273,6 +273,9 @@ dai_add_example(record_imu RVC2/Record/record_imu.cpp OFF OFF)
 dai_add_example(replay_video_meta RVC2/Replay/replay_video_meta.cpp OFF OFF)
 dai_add_example(replay_imu RVC2/Replay/replay_imu.cpp OFF OFF)
 
+# StereoDepth
+dai_add_example(depth_preview StereoDepth/depth_preview.cpp OFF OFF)
+
 # Thermal
 dai_add_example(thermal RVC2/Thermal/thermal.cpp OFF OFF)
 

examples/cpp/ImageManip/image_manip_v2_mod.cpp

@@ -14,13 +14,10 @@ int main(int argc, char** argv) {
     }
     dai::Pipeline pipeline(device);
 
-    auto camRgb = pipeline.create<dai::node::ColorCamera>();
+    auto camRgb = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_A);
     auto display = pipeline.create<dai::node::Display>();
     auto manip = pipeline.create<dai::node::ImageManipV2>();
 
-    camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A);
-    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
-
     manip->setMaxOutputFrameSize(4000000);
     manip->initialConfig.setOutputSize(1280, 720, dai::ImageManipConfigV2::ResizeMode::LETTERBOX);
     manip->initialConfig.setBackgroundColor(100, 100, 100);
@@ -29,7 +26,8 @@ int main(int argc, char** argv) {
     manip->initialConfig.addFlipVertical();
     manip->initialConfig.setFrameType(dai::ImgFrame::Type::RGB888p);
 
-    camRgb->video.link(manip->inputImage);
+    auto* rgbOut = camRgb->requestOutput({1920, 1080});
+    rgbOut->link(manip->inputImage);
     manip->out.link(display->input);
 
     pipeline.start();

examples/cpp/StereoDepth/depth_preview.cpp

@@ -0,0 +1,65 @@
+#include <iostream>
+
+// Includes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+#include "depthai/pipeline/datatype/StereoDepthConfig.hpp"
+#include "depthai/pipeline/node/StereoDepth.hpp"
+#include "depthai/properties/StereoDepthProperties.hpp"
+
+// Closer-in minimum depth, disparity range is doubled (from 95 to 190):
+static std::atomic<bool> extended_disparity{false};
+// Better accuracy for longer distance, fractional disparity 32-levels:
+static std::atomic<bool> subpixel{false};
+// Better handling for occlusions:
+static std::atomic<bool> lr_check{true};
+
+int main() {
+    // Create pipeline
+    dai::Pipeline pipeline;
+
+    // Define sources and outputs
+    auto monoLeft = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_B);
+    auto monoRight = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_C);
+    auto depth = pipeline.create<dai::node::StereoDepth>();
+
+    // Properties
+    auto* lout = monoLeft->requestOutput({640, 400});
+    auto* rout = monoRight->requestOutput({640, 400});
+
+    // Create a node that will produce the depth map (using disparity output as it's easier to visualize depth this way)
+    depth->build(*lout, *rout, dai::node::StereoDepth::PresetMode::HIGH_DENSITY);
+    // Options: MEDIAN_OFF, KERNEL_3x3, KERNEL_5x5, KERNEL_7x7 (default)
+    depth->initialConfig.setMedianFilter(dai::StereoDepthConfig::MedianFilter::KERNEL_7x7);
+    depth->setLeftRightCheck(lr_check);
+    depth->setExtendedDisparity(extended_disparity);
+    depth->setSubpixel(subpixel);
+
+    // Output queue will be used to get the disparity frames from the outputs defined above
+    auto q = depth->disparity.createOutputQueue();
+    auto qleft = lout->createOutputQueue();
+
+    pipeline.start();
+
+    while(true) {
+        auto inDepth = q->get<dai::ImgFrame>();
+        auto inLeft = qleft->get<dai::ImgFrame>();
+        auto frame = inDepth->getFrame();
+        // Normalization for better visualization
+        frame.convertTo(frame, CV_8UC1, 255 / depth->initialConfig.getMaxDisparity());
+
+        std::cout << "Left type: " << inLeft->fb.str() << std::endl;
+
+        cv::imshow("disparity", frame);
+
+        // Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
+        cv::applyColorMap(frame, frame, cv::COLORMAP_JET);
+        cv::imshow("disparity_color", frame);
+
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') {
+            break;
+        }
+    }
+    pipeline.stop();
+    return 0;
+}

examples/python/CMakeLists.txt

@@ -147,6 +147,8 @@ dai_set_example_test_labels(april_tags_replay ondevice rvc2_all rvc4 ci)
 ## Detection network
 add_python_example(detection_network DetectionNetwork/detection_network.py)
 dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci)
+add_python_example(detection_network_remap DetectionNetwork/detection_network_remap.py)
+dai_set_example_test_labels(detection_network ondevice rvc2_all rvc4 ci)
 
 ## Host nodes
 add_python_example(display HostNodes/display.py)
@@ -162,6 +164,9 @@ dai_set_example_test_labels(image_manip_mod ondevice rvc2_all rvc4 ci)
 add_python_example(image_manip_resize ImageManip/image_manip_v2_resize.py)
 dai_set_example_test_labels(image_manip_resize ondevice rvc2_all rvc4 ci)
 
+add_python_example(image_manip_remap ImageManip/image_manip_v2_remap.py)
+dai_set_example_test_labels(image_manip_remap ondevice rvc2_all rvc4 ci)
+
 ## Misc
 add_python_example(reconnect_callback Misc/AutoReconnect/reconnect_callback.py)
 dai_set_example_test_labels(reconnect_callback ondevice rvc2_all rvc4 ci)

examples/python/RVC4/DetectionNetwork/detection_network_remap.py → examples/python/DetectionNetwork/detection_network_remap.py

@@ -4,6 +4,33 @@
 import depthai as dai
 import numpy as np
 
+def colorizeDepth(frameDepth):
+    invalidMask = frameDepth == 0
+    # Log the depth, minDepth and maxDepth
+    try:
+        minDepth = np.percentile(frameDepth[frameDepth != 0], 3)
+        maxDepth = np.percentile(frameDepth[frameDepth != 0], 95)
+        logDepth = np.log(frameDepth, where=frameDepth != 0)
+        logMinDepth = np.log(minDepth)
+        logMaxDepth = np.log(maxDepth)
+        np.nan_to_num(logDepth, copy=False, nan=logMinDepth)
+        # Clip the values to be in the 0-255 range
+        logDepth = np.clip(logDepth, logMinDepth, logMaxDepth)
+
+        # Interpolate only valid logDepth values, setting the rest based on the mask
+        depthFrameColor = np.interp(logDepth, (logMinDepth, logMaxDepth), (0, 255))
+        depthFrameColor = np.nan_to_num(depthFrameColor)
+        depthFrameColor = depthFrameColor.astype(np.uint8)
+        depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_JET)
+        # Set invalid depth pixels to black
+        depthFrameColor[invalidMask] = 0
+    except IndexError:
+        # Frame is likely empty
+        depthFrameColor = np.zeros((frameDepth.shape[0], frameDepth.shape[1], 3), dtype=np.uint8)
+    except Exception as e:
+        raise e
+    return depthFrameColor
+
 # Create pipeline
 with dai.Pipeline() as pipeline:
     cameraNode = pipeline.create(dai.node.Camera).build()
@@ -35,8 +62,7 @@ def displayFrame(name: str, frame: dai.ImgFrame, imgDetections: dai.ImgDetection
         assert imgDetections.getTransformation() is not None
         cvFrame = frame.getFrame() if frame.getType() == dai.ImgFrame.Type.RAW16 else frame.getCvFrame()
         if(frame.getType() == dai.ImgFrame.Type.RAW16):
-            cvFrame = (cvFrame * (255 / stereo.initialConfig.getMaxDisparity())).astype(np.uint8)
-            cvFrame = cv2.applyColorMap(cvFrame, cv2.COLORMAP_JET)
+            cvFrame = colorizeDepth(cvFrame)
         for detection in imgDetections.detections:
             # Get the shape of the frame from which the detections originated for denormalization
             normShape = imgDetections.getTransformation().getSize()

examples/python/ImageAlign/depth_align.py

@@ -25,15 +25,7 @@ def getFps(self):
             return 0
         return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0])
 
-device = dai.Device()
-
-calibrationHandler = device.readCalibration()
-rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET)
-distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET)
-if distortionModel != dai.CameraModel.Perspective:
-    raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.")
-
-pipeline = dai.Pipeline(device)
+pipeline = dai.Pipeline()
 
 platform = pipeline.getDefaultDevice().getPlatform()
 
@@ -47,7 +39,6 @@ def getFps(self):
     align = pipeline.create(dai.node.ImageAlign)
 
 stereo.setExtendedDisparity(True)
-
 sync.setSyncThreshold(timedelta(seconds=1/(2*FPS)))
 
 rgbOut = camRgb.requestOutput(size = (1280, 960), fps = FPS)
@@ -142,14 +133,10 @@ def updateBlendWeights(percentRgb):
         # Blend when both received
         if frameDepth is not None:
             cvFrame = frameRgb.getCvFrame()
-
-            # Undistort the rgb frame
-            rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(cvFrame.shape[1]), int(cvFrame.shape[0]))
-
             cvFrameUndistorted = cv2.undistort(
                 cvFrame,
-                np.array(rgbIntrinsics),
-                np.array(rgbDistortion),
+                np.array(frameRgb.getTransformation().getIntrinsicMatrix()),
+                np.array(frameRgb.getTransformation().getDistortionCoefficients()),
             )
             # Colorize the aligned depth
             alignedDepthColorized = colorizeDepth(frameDepth.getFrame())

examples/python/ImageManip/image_manip_v2_remap.py

@@ -0,0 +1,70 @@
+import depthai as dai
+import cv2
+import numpy as np
+
+def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
+    """
+    Draws a rotated rectangle on the given frame.
+
+    Args:
+        frame (numpy.ndarray): The image/frame to draw on.
+        center (tuple): The (x, y) coordinates of the rectangle's center.
+        size (tuple): The (width, height) of the rectangle.
+        angle (float): The rotation angle of the rectangle in degrees (counter-clockwise).
+        color (tuple): The color of the rectangle in BGR format (e.g., (0, 255, 0) for green).
+        thickness (int): The thickness of the rectangle edges. Default is 2.
+    """
+    # Create a rotated rectangle
+    rect = ((center[0], center[1]), (size[0], size[1]), angle)
+
+    # Get the four vertices of the rotated rectangle
+    box = cv2.boxPoints(rect)
+    box = np.intp(box)  # Convert to integer coordinates
+
+    # Draw the rectangle on the frame
+    cv2.polylines(frame, [box], isClosed=True, color=color, thickness=thickness)
+
+with dai.Pipeline() as pipeline:
+    cam = pipeline.create(dai.node.Camera).build()
+    camOut = cam.requestOutput((640, 400), dai.ImgFrame.Type.BGR888i, fps = 30.0)
+    manip1 = pipeline.create(dai.node.ImageManipV2)
+    manip2 = pipeline.create(dai.node.ImageManipV2)
+
+    camOut.link(manip1.inputImage)
+    manip1.out.link(manip2.inputImage)
+
+    manip1.initialConfig.addRotateDeg(90)
+    manip1.initialConfig.setOutputSize(200, 320)
+
+    manip2.initialConfig.addRotateDeg(90)
+    manip2.initialConfig.setOutputSize(320, 200)
+    manip2.setRunOnHost(True)
+
+    outQcam = camOut.createOutputQueue()
+    outQ1 = manip1.out.createOutputQueue()
+    outQ2 = manip2.out.createOutputQueue()
+
+    pipeline.start()
+
+    while True:
+        camFrame: dai.ImgFrame = outQcam.get()
+        manip1Frame: dai.ImgFrame = outQ1.get()
+        manip2Frame: dai.ImgFrame = outQ2.get()
+
+        camCv = camFrame.getCvFrame()
+        manip1Cv = manip1Frame.getCvFrame()
+        manip2Cv = manip2Frame.getCvFrame()
+
+        rect2 = dai.RotatedRect(dai.Rect(dai.Point2f(100, 100), dai.Point2f(200, 150)), 0)
+        rect1 = manip2Frame.getTransformation().remapRectTo(manip1Frame.getTransformation(), rect2)
+        rectcam = manip1Frame.getTransformation().remapRectTo(camFrame.getTransformation(), rect1)
+
+        draw_rotated_rectangle(manip2Cv, (rect2.center.x, rect2.center.y), (rect2.size.width, rect2.size.height), rect2.angle, (255, 0, 0))
+        draw_rotated_rectangle(manip1Cv, (rect1.center.x, rect1.center.y), (rect1.size.width, rect1.size.height), rect1.angle, (255, 0, 0))
+        draw_rotated_rectangle(camCv, (rectcam.center.x, rectcam.center.y), (rectcam.size.width, rectcam.size.height), rectcam.angle, (255, 0, 0))
+
+        cv2.imshow("cam", camCv)
+        cv2.imshow("manip1", manip1Cv)
+        cv2.imshow("manip2", manip2Cv)
+        if cv2.waitKey(1) == ord('q'):
+            break

examples/python/RVC2/Camera/camera_undistortion.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import cv2
+import depthai as dai
+
+# Create pipeline
+with dai.Pipeline() as pipeline:
+    # Define source and output
+    cam = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)
+    croppedQueue = cam.requestOutput((300,300), resizeMode=dai.ImgResizeMode.CROP, enableUndistortion=True).createOutputQueue()
+    stretchedQueue = cam.requestOutput((300,300), resizeMode=dai.ImgResizeMode.STRETCH, enableUndistortion=True).createOutputQueue()
+    letterBoxedQueue = cam.requestOutput((300,300), resizeMode=dai.ImgResizeMode.LETTERBOX, enableUndistortion=True).createOutputQueue()
+
+    # Connect to device and start pipeline
+    pipeline.start()
+    while pipeline.isRunning():
+        croppedIn = croppedQueue.get()
+        assert isinstance(croppedIn, dai.ImgFrame)
+        cv2.imshow("cropped undistorted", croppedIn.getCvFrame())
+
+        stretchedIn = stretchedQueue.get()
+        assert isinstance(stretchedIn, dai.ImgFrame)
+        cv2.imshow("stretched undistorted", stretchedIn.getCvFrame())
+
+        letterBoxedIn = letterBoxedQueue.get()
+        assert isinstance(letterBoxedIn, dai.ImgFrame)
+        cv2.imshow("letterboxed undistorted", letterBoxedIn.getCvFrame())
+
+        if cv2.waitKey(1) == ord("q"):
+            break