Sync issue in Remote mapping between RPi5 and Ubuntu Linux

[rdlynx19]

Hello,
I am trying to remote mapping using an Oak D Lite Camera.

Camera Platform: Raspberry Pi5 w Ubuntu 24.04 and ROS2 Jazzy
Remote Station: System 76 Laptop w Ubuntu 24.04 and ROS2 Jazzy

I tried running the example depthai.launch.py on my laptop (remote station) and it works fine. Now, I'm trying to do remote mapping by running the camera nodes on the RPi5, and the mapping nodes on my remote laptop. Both the systems are using rmw_zenoh_cpp as the ROS2 DDS to discover the topics and nodes. I have verified that the topics and nodes running on the RPi are listed by running ros2 topic list on my remote station.

However, when I use this launch file on the RPi5 ( ideally only the camera and sync nodes should be run here)

import os

from ament_index_python.packages import get_package_share_directory

from launch import LaunchDescription
from launch_ros.actions import Node
from launch.actions import IncludeLaunchDescription
from launch.launch_description_sources import PythonLaunchDescriptionSource


def generate_launch_description():
    parameters=[{'frame_id':'oak-d-base-frame',
                 'subscribe_rgbd':True,
                 'subscribe_odom_info':True,
                 'approx_sync':False,
                 'wait_imu_to_init':True}]

    remappings=[('imu', '/imu/data')]

    return LaunchDescription([

        # Launch camera driver
        IncludeLaunchDescription(
            PythonLaunchDescriptionSource([os.path.join(
                get_package_share_directory('depthai_examples'), 'launch'),
               '/stereo_inertial_node.launch.py']),
                launch_arguments={'depth_aligned': 'false',
                                  'enableRviz': 'false',
                                  'monoResolution': '400p'}.items(),
        ),

        # Sync right/depth/camera_info together
        Node(   
           package='rtabmap_sync', executable='rgbd_sync', output='screen',
            parameters=parameters,
            remappings=[('rgb/image', '/right/image_rect'),
                        ('rgb/camera_info', '/right/camera_info'),
                        ('depth/image', '/stereo/depth')]),

         Compute quaternion of the IMU
        Node(
            package='imu_filter_madgwick', executable='imu_filter_madgwick_node', output='screen',
            parameters=[{'use_mag': False, 
                         'world_frame':'enu', 
                         'publish_tf':False}],
          remappings=[('imu/data_raw', '/imu')]),

        # Visual odometry
        #Node(
        #   package='rtabmap_odom', executable='rgbd_odometry', output='screen',
        #    parameters=parameters,
        #   remappings=remappings),

        # VSLAM
       #Node(
       #     package='rtabmap_slam', executable='rtabmap', output='screen',
       #    parameters=parameters,
       #      remappings=remappings,
       #    arguments=['-d']),

      # Visualization
      #  Node(
      #      package='rtabmap_viz', executable='rtabmap_viz', output='screen',
      #      parameters=parameters,
      #      remappings=remappings)
    ])

And this launch file on my laptop (all the mapping nodes and visualisation should be run here)

import os

from ament_index_python.packages import get_package_share_directory

from launch import LaunchDescription
from launch_ros.actions import Node
from launch.actions import IncludeLaunchDescription
from launch.launch_description_sources import PythonLaunchDescriptionSource


def generate_launch_description():
    parameters=[{'frame_id':'oak-d-base-frame',
                 'subscribe_rgbd':True,
                 'subscribe_odom_info':True,
                 'approx_sync':False,
                 'wait_imu_to_init':True}]

    remappings=[('imu', '/imu/data')]

    return LaunchDescription([

        # Launch camera driver
        #IncludeLaunchDescription(
        #    PythonLaunchDescriptionSource([os.path.join(
        #        get_package_share_directory('depthai_examples'), 'launch'),
        #        '/stereo_inertial_node.launch.py']),
        #        launch_arguments={'depth_aligned': 'false',
        #                          'enableRviz': 'false',
        #                          'monoResolution': '400p'}.items(),
        #),

        # Sync right/depth/camera_info together
        #Node(   
        #    package='rtabmap_sync', executable='rgbd_sync', output='screen',
        #    parameters=parameters,
        #    remappings=[('rgb/image', '/right/image_rect'),
        #                ('rgb/camera_info', '/right/camera_info'),
        #                ('depth/image', '/stereo/depth')]),

        # Compute quaternion of the IMU
        #Node(
        #    package='imu_filter_madgwick', executable='imu_filter_madgwick_node', output='screen',
        #    parameters=[{'use_mag': False, 
        #                 'world_frame':'enu', 
        #                 'publish_tf':False}],
        #    remappings=[('imu/data_raw', '/imu')]),

        # Visual odometry
        Node(
            package='rtabmap_odom', executable='rgbd_odometry', output='screen',
            parameters=parameters,
            remappings=remappings),

        # VSLAM
        Node(
            package='rtabmap_slam', executable='rtabmap', output='screen',
            parameters=parameters,
            remappings=remappings,
            arguments=['-d']),

        # Visualization
        Node(
            package='rtabmap_viz', executable='rtabmap_viz', output='screen',
            parameters=parameters,
            remappings=remappings)
    ])

I get these sync issues. I can see they suggest to change the queue_size parameters but I don't understand which launch file uses them.

[rtabmap-2] [WARN] [1738387989.642794450] [rtabmap]: rtabmap: Already running!                                                                                         
[rtabmap_viz-3] [INFO] [1738387989.644068305] [rtabmap_viz]: rtabmap_viz started.                                                                                      
[rgbd_odometry-1] [WARN] [1738387989.645072410] [rgbd_odometry]: Odometry: Already running!                                                                            
[rgbd_odometry-1] [WARN] [1738387993.465017903] [rgbd_odometry]: rgbd_odometry: Did not receive data since 5 seconds! Make sure the input topics are published ("$ rost
opic hz my_topic") and the timestamps in their header are set. Parameter "approx_sync" is false, which means that input topics should have all the exact timestamp for 
the callback to be called.                                                                                                                                             
[rgbd_odometry-1] rgbd_odometry subscribed to:                                                                                                                         
[rgbd_odometry-1]    /rgbd_image                                                                                                                                       
[rtabmap-2] [WARN] [1738387993.716843672] [rtabmap]: rtabmap: Did not receive data since 5 seconds! Make sure the input topics are published ("$ ros2 topic hz my_topic
") and the timestamps in their header are set. If topics are coming from different computers, make sure the clocks of the computers are synchronized ("ntpdate"). Ajust
ing topic_queue_size (10) and sync_queue_size (10) can also help for better synchronization if framerates and/or delays are different. Parameter "approx_sync" is false
, which means that input topics should have all the exact timestamp for the callback to be called.
[rtabmap-2] rtabmap subscribed to (exact sync):
[rtabmap-2]    /odom \
[rtabmap-2]    /rgbd_image \
[rtabmap-2]    /odom_info
[rtabmap_viz-3] [WARN] [1738387994.644295687] [rtabmap_viz]: rtabmap_viz: Did not receive data since 5 seconds! Make sure the input topics are published ("$ ros2 topic
 hz my_topic") and the timestamps in their header are set. If topics are coming from different computers, make sure the clocks of the computers are synchronized ("ntpd
ate"). Ajusting topic_queue_size (10) and sync_queue_size (10) can also help for better synchronization if framerates and/or delays are different. Parameter "approx_sy
nc" is false, which means that input topics should have all the exact timestamp for the callback to be called.
[rtabmap_viz-3] rtabmap_viz subscribed to (exact sync):
[rtabmap_viz-3]    /odom \
[rtabmap_viz-3]    /rgbd_image \
[rtabmap_viz-3]    /odom_info

Let me know if anymore info is needed to debug this. Thanks!