RVC2 img transformations

bindings/python/src/pipeline/datatype/ImgFrameBindings.cpp

@@ -144,9 +144,20 @@ void bind_imgframe(pybind11::module& m, void* pCallstack) {
         .def("getSourceIntrinsicMatrixInv", &ImgTransformation::getSourceIntrinsicMatrixInv)
         .def("getIntrinsicMatrix", &ImgTransformation::getIntrinsicMatrix)
         .def("getIntrinsicMatrixInv", &ImgTransformation::getIntrinsicMatrixInv)
+        .def("getDistortionModel", &ImgTransformation::getDistortionModel, DOC(dai, ImgTransformation, getDistortionModel))
+        .def("getDistortionCoefficients", &ImgTransformation::getDistortionCoefficients, DOC(dai, ImgTransformation, getDistortionCoefficients))
+        .def("getSrcCrops", &ImgTransformation::getSrcCrops, DOC(dai, ImgTransformation, getSrcCrops))
+        .def("getSrcMaskPt", &ImgTransformation::getSrcMaskPt, py::arg("x"), py::arg("y"), DOC(dai, ImgTransformation, getSrcMaskPt))
+        .def("getDstMaskPt", &ImgTransformation::getDstMaskPt, py::arg("x"), py::arg("y"), DOC(dai, ImgTransformation, getDstMaskPt))
         .def("getDFov", &ImgTransformation::getDFov, py::arg("source") = false)
         .def("getHFov", &ImgTransformation::getHFov, py::arg("source") = false)
         .def("getVFov", &ImgTransformation::getVFov, py::arg("source") = false)
+        .def("setIntrinsicMatrix", &ImgTransformation::setIntrinsicMatrix, py::arg("intrinsicMatrix"), DOC(dai, ImgTransformation, setIntrinsicMatrix))
+        .def("setDistortionModel", &ImgTransformation::setDistortionModel, py::arg("model"), DOC(dai, ImgTransformation, setDistortionModel))
+        .def("setDistortionCoefficients",
+             &ImgTransformation::setDistortionCoefficients,
+             py::arg("coefficients"),
+             DOC(dai, ImgTransformation, setDistortionCoefficients))
         .def("addTransformation", &ImgTransformation::addTransformation, py::arg("matrix"), DOC(dai, ImgTransformation, addTransformation))
         .def("addCrop", &ImgTransformation::addCrop, py::arg("x"), py::arg("y"), py::arg("width"), py::arg("height"), DOC(dai, ImgTransformation, addCrop))
         .def("addPadding",
@@ -201,6 +212,7 @@ void bind_imgframe(pybind11::module& m, void* pCallstack) {
         .def("getSourceWidth", &ImgFrame::getSourceWidth, DOC(dai, ImgFrame, getSourceWidth))
         .def("getSourceHeight", &ImgFrame::getSourceHeight, DOC(dai, ImgFrame, getSourceHeight))
         .def("getTransformation", [](ImgFrame& msg) {return msg.transformation;})
+        .def("validateTransformations", &ImgFrame::validateTransformations, DOC(dai, ImgFrame, validateTransformations))
 
 #ifdef DEPTHAI_HAVE_OPENCV_SUPPORT
         // The cast function itself does a copy, so we can avoid two copies by always not copying

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
 # set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+6fc71e8674fe7a520b93f4370cb157805f0bc0f2")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+38fbf795d3d5335f409db0247c9cfbada0e66c49")

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "052ee5648d9d5a49e6367c4a81d14a9b9ae8bfcf")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "435d0b10d52cb3e78b30f64b9a6e2e28da28cc76")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/cpp/CMakeLists.txt

@@ -273,6 +273,9 @@ dai_add_example(record_imu RVC2/Record/record_imu.cpp OFF OFF)
 dai_add_example(replay_video_meta RVC2/Replay/replay_video_meta.cpp OFF OFF)
 dai_add_example(replay_imu RVC2/Replay/replay_imu.cpp OFF OFF)
 
+# StereoDepth
+dai_add_example(depth_preview StereoDepth/depth_preview.cpp OFF OFF)
+
 # Thermal
 dai_add_example(thermal RVC2/Thermal/thermal.cpp OFF OFF)
 

examples/cpp/ImageManip/image_manip_v2_mod.cpp

@@ -14,13 +14,10 @@ int main(int argc, char** argv) {
     }
     dai::Pipeline pipeline(device);
 
-    auto camRgb = pipeline.create<dai::node::ColorCamera>();
+    auto camRgb = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_A);
     auto display = pipeline.create<dai::node::Display>();
     auto manip = pipeline.create<dai::node::ImageManipV2>();
 
-    camRgb->setBoardSocket(dai::CameraBoardSocket::CAM_A);
-    camRgb->setResolution(dai::ColorCameraProperties::SensorResolution::THE_1080_P);
-
     manip->setMaxOutputFrameSize(4000000);
     manip->initialConfig.setOutputSize(1280, 720, dai::ImageManipConfigV2::ResizeMode::LETTERBOX);
     manip->initialConfig.setBackgroundColor(100, 100, 100);
@@ -29,7 +26,8 @@ int main(int argc, char** argv) {
     manip->initialConfig.addFlipVertical();
     manip->initialConfig.setFrameType(dai::ImgFrame::Type::RGB888p);
 
-    camRgb->video.link(manip->inputImage);
+    auto* rgbOut = camRgb->requestOutput({1920, 1080});
+    rgbOut->link(manip->inputImage);
     manip->out.link(display->input);
 
     pipeline.start();

examples/cpp/StereoDepth/depth_preview.cpp

@@ -0,0 +1,65 @@
+#include <iostream>
+
+// Includes common necessary includes for development using depthai library
+#include "depthai/depthai.hpp"
+#include "depthai/pipeline/datatype/StereoDepthConfig.hpp"
+#include "depthai/pipeline/node/StereoDepth.hpp"
+#include "depthai/properties/StereoDepthProperties.hpp"
+
+// Closer-in minimum depth, disparity range is doubled (from 95 to 190):
+static std::atomic<bool> extended_disparity{false};
+// Better accuracy for longer distance, fractional disparity 32-levels:
+static std::atomic<bool> subpixel{false};
+// Better handling for occlusions:
+static std::atomic<bool> lr_check{true};
+
+int main() {
+    // Create pipeline
+    dai::Pipeline pipeline;
+
+    // Define sources and outputs
+    auto monoLeft = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_B);
+    auto monoRight = pipeline.create<dai::node::Camera>()->build(dai::CameraBoardSocket::CAM_C);
+    auto depth = pipeline.create<dai::node::StereoDepth>();
+
+    // Properties
+    auto* lout = monoLeft->requestOutput({640, 400});
+    auto* rout = monoRight->requestOutput({640, 400});
+
+    // Create a node that will produce the depth map (using disparity output as it's easier to visualize depth this way)
+    depth->build(*lout, *rout, dai::node::StereoDepth::PresetMode::HIGH_DENSITY);
+    // Options: MEDIAN_OFF, KERNEL_3x3, KERNEL_5x5, KERNEL_7x7 (default)
+    depth->initialConfig.setMedianFilter(dai::StereoDepthConfig::MedianFilter::KERNEL_7x7);
+    depth->setLeftRightCheck(lr_check);
+    depth->setExtendedDisparity(extended_disparity);
+    depth->setSubpixel(subpixel);
+
+    // Output queue will be used to get the disparity frames from the outputs defined above
+    auto q = depth->disparity.createOutputQueue();
+    auto qleft = lout->createOutputQueue();
+
+    pipeline.start();
+
+    while(true) {
+        auto inDepth = q->get<dai::ImgFrame>();
+        auto inLeft = qleft->get<dai::ImgFrame>();
+        auto frame = inDepth->getFrame();
+        // Normalization for better visualization
+        frame.convertTo(frame, CV_8UC1, 255 / depth->initialConfig.getMaxDisparity());
+
+        std::cout << "Left type: " << inLeft->fb.str() << std::endl;
+
+        cv::imshow("disparity", frame);
+
+        // Available color maps: https://docs.opencv.org/3.4/d3/d50/group__imgproc__colormap.html
+        cv::applyColorMap(frame, frame, cv::COLORMAP_JET);
+        cv::imshow("disparity_color", frame);
+
+        int key = cv::waitKey(1);
+        if(key == 'q' || key == 'Q') {
+            break;
+        }
+    }
+    pipeline.stop();
+    return 0;
+}

examples/python/ImageAlign/aligned_remap.py

@@ -0,0 +1,65 @@
+import numpy as np
+import depthai as dai
+import cv2
+
+def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
+    """
+    Draws a rotated rectangle on the given frame.
+
+    Args:
+        frame (numpy.ndarray): The image/frame to draw on.
+        center (tuple): The (x, y) coordinates of the rectangle's center.
+        size (tuple): The (width, height) of the rectangle.
+        angle (float): The rotation angle of the rectangle in degrees (counter-clockwise).
+        color (tuple): The color of the rectangle in BGR format (e.g., (0, 255, 0) for green).
+        thickness (int): The thickness of the rectangle edges. Default is 2.
+    """
+    # Create a rotated rectangle
+    rect = ((center[0], center[1]), (size[0], size[1]), angle)
+
+    # Get the four vertices of the rotated rectangle
+    box = cv2.boxPoints(rect)
+    box = np.int0(box)  # Convert to integer coordinates
+
+    # Draw the rectangle on the frame
+    cv2.polylines(frame, [box], isClosed=True, color=color, thickness=thickness)
+
+with dai.Pipeline() as pipeline:
+    monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)
+    monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C)
+    align = pipeline.create(dai.node.ImageAlign)
+
+    monoLeftOut = monoLeft.requestOutput((640, 480))
+    monoRightOut = monoRight.requestOutput((640, 480))
+
+    monoLeftOut.link(align.input)
+    monoRightOut.link(align.inputAlignTo)
+
+    rightOut = monoRightOut.createOutputQueue()
+    alignOut = align.outputAligned.createOutputQueue()
+
+    pipeline.start()
+    while pipeline.isRunning():
+        rightFrame = rightOut.get()
+        alignedFrame = alignOut.get()
+
+        print(alignedFrame)
+
+        assert rightFrame.validateTransformations()
+        assert alignedFrame.validateTransformations()
+
+        right = rightFrame.getCvFrame()
+        depth = alignedFrame.getCvFrame()
+
+        rect = dai.RotatedRect(dai.Point2f(200, 100), dai.Size2f(200, 100), 10)
+        remappedRect = rightFrame.getTransformation().remapRectTo(alignedFrame.getTransformation(), rect)
+
+        draw_rotated_rectangle(right, (rect.center.x, rect.center.y), (rect.size.width, rect.size.height), rect.angle, (255, 0, 0))
+        draw_rotated_rectangle(depth, (remappedRect.center.x, remappedRect.center.y), (remappedRect.size.width, remappedRect.size.height), remappedRect.angle, (255, 0, 0))
+
+        cv2.imshow("right", right)
+        cv2.imshow("aligned", depth)
+
+        if cv2.waitKey(1) == ord('q'):
+            break
+    pipeline.stop()

examples/python/ImageAlign/depth_align.py

@@ -25,15 +25,7 @@ def getFps(self):
             return 0
         return (len(self.frameTimes) - 1) / (self.frameTimes[-1] - self.frameTimes[0])
 
-device = dai.Device()
-
-calibrationHandler = device.readCalibration()
-rgbDistortion = calibrationHandler.getDistortionCoefficients(RGB_SOCKET)
-distortionModel = calibrationHandler.getDistortionModel(RGB_SOCKET)
-if distortionModel != dai.CameraModel.Perspective:
-    raise RuntimeError("Unsupported distortion model for RGB camera. This example supports only Perspective model.")
-
-pipeline = dai.Pipeline(device)
+pipeline = dai.Pipeline()
 
 platform = pipeline.getDefaultDevice().getPlatform()
 
@@ -47,7 +39,6 @@ def getFps(self):
     align = pipeline.create(dai.node.ImageAlign)
 
 stereo.setExtendedDisparity(True)
-
 sync.setSyncThreshold(timedelta(seconds=1/(2*FPS)))
 
 rgbOut = camRgb.requestOutput(size = (1280, 960), fps = FPS)
@@ -142,14 +133,10 @@ def updateBlendWeights(percentRgb):
         # Blend when both received
         if frameDepth is not None:
             cvFrame = frameRgb.getCvFrame()
-
-            # Undistort the rgb frame
-            rgbIntrinsics = calibrationHandler.getCameraIntrinsics(RGB_SOCKET, int(cvFrame.shape[1]), int(cvFrame.shape[0]))
-
             cvFrameUndistorted = cv2.undistort(
                 cvFrame,
-                np.array(rgbIntrinsics),
-                np.array(rgbDistortion),
+                np.array(frameRgb.getTransformation().getIntrinsicMatrix()),
+                np.array(frameRgb.getTransformation().getDistortionCoefficients()),
             )
             # Colorize the aligned depth
             alignedDepthColorized = colorizeDepth(frameDepth.getFrame())

examples/python/StereoDepth/stereo_depth_remap.py

@@ -0,0 +1,84 @@
+import depthai as dai
+import cv2
+import numpy as np
+
+def draw_rotated_rectangle(frame, center, size, angle, color, thickness=2):
+    """
+    Draws a rotated rectangle on the given frame.
+
+    Args:
+        frame (numpy.ndarray): The image/frame to draw on.
+        center (tuple): The (x, y) coordinates of the rectangle's center.
+        size (tuple): The (width, height) of the rectangle.
+        angle (float): The rotation angle of the rectangle in degrees (counter-clockwise).
+        color (tuple): The color of the rectangle in BGR format (e.g., (0, 255, 0) for green).
+        thickness (int): The thickness of the rectangle edges. Default is 2.
+    """
+    # Create a rotated rectangle
+    rect = ((center[0], center[1]), (size[0], size[1]), angle)
+
+    # Get the four vertices of the rotated rectangle
+    box = cv2.boxPoints(rect)
+    box = np.intp(box)  # Convert to integer coordinates
+
+    # Draw the rectangle on the frame
+    cv2.polylines(frame, [box], isClosed=True, color=color, thickness=thickness)
+
+def processDepthFrame(depthFrame):
+    depth_downscaled = depthFrame[::4]
+    if np.all(depth_downscaled == 0):
+        min_depth = 0
+    else:
+        min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1)
+    max_depth = np.percentile(depth_downscaled, 99)
+    depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8)
+    return cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT)
+
+with dai.Pipeline() as pipeline:
+    color = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A)
+    monoLeft = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_B)
+    monoRight = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_C)
+    stereo = pipeline.create(dai.node.StereoDepth)
+
+    stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)
+    # stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)
+    # stereo.setOutputSize(640, 400)
+
+    colorCamOut = color.requestOutput((640, 480))
+
+    monoLeftOut = monoLeft.requestOutput((640, 480))
+    monoRightOut = monoRight.requestOutput((640, 480))
+
+    monoLeftOut.link(stereo.left)
+    monoRightOut.link(stereo.right)
+
+    colorOut = colorCamOut.createOutputQueue()
+    rightOut = monoRightOut.createOutputQueue()
+    stereoOut = stereo.depth.createOutputQueue()
+
+    pipeline.start()
+    while pipeline.isRunning():
+        colorFrame = colorOut.get()
+        stereoFrame = stereoOut.get()
+
+        assert colorFrame.validateTransformations()
+        assert stereoFrame.validateTransformations()
+
+        clr = colorFrame.getCvFrame()
+        depth = processDepthFrame(stereoFrame.getCvFrame())
+
+        rect = dai.RotatedRect(dai.Point2f(300, 200), dai.Size2f(200, 100), 10)
+        remappedRect = colorFrame.getTransformation().remapRectTo(stereoFrame.getTransformation(), rect)
+
+        print(f"Original rect x: {rect.center.x} y: {rect.center.y} width: {rect.size.width} height: {rect.size.height} angle: {rect.angle}")
+        print(f"Remapped rect x: {remappedRect.center.x} y: {remappedRect.center.y} width: {remappedRect.size.width} height: {remappedRect.size.height} angle: {remappedRect.angle}")
+
+        draw_rotated_rectangle(clr, (rect.center.x, rect.center.y), (rect.size.width, rect.size.height), rect.angle, (255, 0, 0))
+        draw_rotated_rectangle(depth, (remappedRect.center.x, remappedRect.center.y), (remappedRect.size.width, remappedRect.size.height), remappedRect.angle, (255, 0, 0))
+
+        cv2.imshow("color", clr)
+        cv2.imshow("depth", depth)
+
+        if cv2.waitKey(1) == ord('q'):
+            break
+    pipeline.stop()

include/depthai/common/ImgTransformations.hpp

@@ -177,12 +177,12 @@ struct ImgTransformation {
     ImgTransformation& addCrop(int x, int y, int width, int height);
     /**
      * Add a pad transformation. Works like crop, but in reverse.
-     * @param x Padding on the left. The x coordinate of the top-left corner in the new image.
-     * @param y Padding on the top. The y coordinate of the top-left corner in the new image.
-     * @param width New image width
-     * @param height New image height
+     * @param top Padding on the top
+     * @param bottom Padding on the bottom
+     * @param left Padding on the left
+     * @param right Padding on the right
      */
-    ImgTransformation& addPadding(int x, int y, int width, int height);
+    ImgTransformation& addPadding(int top, int bottom, int left, int right);
     /**
      * Add a vertical flip transformation.
      */
@@ -204,6 +204,11 @@ struct ImgTransformation {
      */
     ImgTransformation& addScale(float scaleX, float scaleY);
     ImgTransformation& addSrcCrops(const std::vector<dai::RotatedRect>& crops);
+    ImgTransformation& setSize(size_t width, size_t height);
+    ImgTransformation& setSourceSize(size_t width, size_t height);
+    ImgTransformation& setIntrinsicMatrix(std::array<std::array<float, 3>, 3> intrinsicMatrix);
+    ImgTransformation& setDistortionModel(CameraModel model);
+    ImgTransformation& setDistortionCoefficients(std::vector<float> coefficients);
 
     /**
      * Remap a point from this transformation to another. If the intrinsics are different (e.g. different camera), the function will also use the intrinsics to

include/depthai/utility/ImageManipV2Impl.hpp

@@ -2811,15 +2811,12 @@ void Warp<ImageManipBuffer, ImageManipData>::build(const FrameSpecs srcFrameSpec
     sourceMaxY = inHeight;
     for(const auto& corners : srcCorners) {
         auto [minx, maxx, miny, maxy] = getOuterRect(std::vector<std::array<float, 2>>(corners.begin(), corners.end()));
-        minx = std::max(minx, 0.0f);
-        maxx = std::min(maxx, (float)inWidth);
-        miny = std::max(miny, 0.0f);
-        maxy = std::min(maxy, (float)inHeight);
-        sourceMinX = std::max(sourceMinX, (size_t)std::floor(minx));
-        sourceMinY = std::max(sourceMinY, (size_t)std::floor(miny));
+        sourceMinX = std::max(sourceMinX, (size_t)std::floor(std::max(minx, 0.f)));
+        sourceMinY = std::max(sourceMinY, (size_t)std::floor(std::max(miny, 0.f)));
         sourceMaxX = std::min(sourceMaxX, (size_t)std::ceil(maxx));
         sourceMaxY = std::min(sourceMaxY, (size_t)std::ceil(maxy));
     }
+    if(sourceMinX >= sourceMaxX || sourceMinY >= sourceMaxY) throw std::runtime_error("Initial crop is outside the source image");
 
 #if !DEPTHAI_IMAGEMANIPV2_OPENCV && !DEPTHAI_IMAGEMANIPV2_FASTCV || !defined(DEPTHAI_HAVE_OPENCV_SUPPORT) && !defined(DEPTHAI_HAVE_FASTCV_SUPPORT)
     const uint32_t outWidth = dstFrameSpecs.width;