Make it easy to log custom arrow data in C++ (#8880)

Co-authored-by: Clement Rey <[email protected]>
rerun-io · Jan 31, 2025 · 5949a22 · 5949a22
1 parent 80acaa7
commit 5949a22
Show file tree

Hide file tree

Showing 135 changed files with 693 additions and 960 deletions.
diff --git a/crates/build/re_types_builder/src/codegen/cpp/mod.rs b/crates/build/re_types_builder/src/codegen/cpp/mod.rs
@@ -641,7 +641,7 @@ impl QuotedObject {
         Partitions the component data into multiple sub-batches.
 
         Specifically, this transforms the existing `ComponentBatch` data into `ComponentColumn`s
-        instead, via `ComponentColumn::from_batch_with_lengths`.
+        instead, via `ComponentBatch::partitioned`.
 
         This makes it possible to use `RecordingStream::send_columns` to send columnar data directly into Rerun.
 
@@ -659,9 +659,7 @@ impl QuotedObject {
                     let field_ident = field_name_ident(field);
                     quote! {
                         if (#field_ident.has_value()) {
-                            columns.push_back(ComponentColumn::from_batch_with_lengths(
-                                #field_ident.value(), lengths_
-                            ).value_or_throw());
+                            columns.push_back(#field_ident.value().partitioned(lengths_).value_or_throw());
                         }
                     }
                 });

diff --git a/docs/snippets/CMakeLists.txt b/docs/snippets/CMakeLists.txt
@@ -33,4 +33,10 @@ foreach(SOURCE_PATH ${sources_list})
 endforeach()
 
 # `text_log_integration` uses `loguru` as the example text logging library.
-target_link_libraries(text_log_integration PRIVATE rerun_sdk loguru::loguru)
+target_link_libraries(text_log_integration PRIVATE loguru::loguru)
+
+# `any_values`-style examples use `arrow` directly.
+target_link_libraries(any_batch_value_send_columns PRIVATE rerun_arrow_target)
+target_link_libraries(any_values PRIVATE rerun_arrow_target)
+target_link_libraries(any_values_send_columns PRIVATE rerun_arrow_target)
+target_link_libraries(extra_values PRIVATE rerun_arrow_target)
diff --git a/docs/snippets/INDEX.md b/docs/snippets/INDEX.md
diff --git a/docs/snippets/all/howto/any_batch_value_send_columns.cpp b/docs/snippets/all/howto/any_batch_value_send_columns.cpp
@@ -0,0 +1,56 @@
+// Use `send_column` to send an entire column of custom data to Rerun.
+
+#include <rerun.hpp>
+
+#include <arrow/array/builder_primitive.h>
+#include <cmath>
+#include <cstdio>
+#include <numeric>
+
+arrow::Status run_main() {
+    const auto rec = rerun::RecordingStream("rerun_example_any_batch_value_send_columns");
+    rec.spawn().exit_on_failure();
+
+    constexpr int64_t STEPS = 64;
+
+    std::vector<int64_t> times(STEPS);
+    std::iota(times.begin(), times.end(), 0);
+
+    std::shared_ptr<arrow::Array> arrow_array;
+
+    arrow::DoubleBuilder one_per_timestamp_builder;
+    for (int64_t i = 0; i < STEPS; i++) {
+        ARROW_RETURN_NOT_OK(one_per_timestamp_builder.Append(sin(static_cast<double>(i) / 10.0)));
+    }
+    ARROW_RETURN_NOT_OK(one_per_timestamp_builder.Finish(&arrow_array));
+    auto one_per_timestamp =
+        rerun::ComponentBatch::from_arrow_array(std::move(arrow_array), "custom_component_single")
+            .value_or_throw();
+
+    arrow::DoubleBuilder ten_per_timestamp_builder;
+    for (int64_t i = 0; i < STEPS * 10; i++) {
+        ARROW_RETURN_NOT_OK(ten_per_timestamp_builder.Append(cos(static_cast<double>(i) / 100.0)));
+    }
+    ARROW_RETURN_NOT_OK(ten_per_timestamp_builder.Finish(&arrow_array));
+    auto ten_per_timestamp =
+        rerun::ComponentBatch::from_arrow_array(std::move(arrow_array), "custom_component_multi")
+            .value_or_throw();
+
+    rec.send_columns(
+        "/",
+        rerun::TimeColumn::from_sequence_points("step", std::move(times)),
+        one_per_timestamp.partitioned().value_or_throw(),
+        ten_per_timestamp.partitioned(std::vector<uint32_t>(STEPS, 10)).value_or_throw()
+    );
+
+    return arrow::Status::OK();
+}
+
+int main() {
+    arrow::Status status = run_main();
+    if (!status.ok()) {
+        printf("%s\n", status.ToString().c_str());
+        return 1;
+    }
+    return 0;
+}
diff --git a/docs/snippets/all/howto/any_values_send_columns.cpp b/docs/snippets/all/howto/any_values_send_columns.cpp
@@ -0,0 +1,54 @@
+// Use `send_column` to send entire columns of custom data to Rerun.
+
+#include <rerun.hpp>
+
+#include <arrow/array/builder_primitive.h>
+#include <cmath>
+#include <cstdio>
+#include <numeric>
+
+arrow::Status run_main() {
+    const auto rec = rerun::RecordingStream("rerun_example_any_values_send_columns");
+    rec.spawn().exit_on_failure();
+
+    constexpr int64_t STEPS = 64;
+
+    std::vector<int64_t> times(STEPS);
+    std::iota(times.begin(), times.end(), 0);
+
+    std::shared_ptr<arrow::Array> arrow_array;
+
+    arrow::DoubleBuilder sin_builder;
+    for (int64_t i = 0; i < STEPS; i++) {
+        ARROW_RETURN_NOT_OK(sin_builder.Append(sin(static_cast<double>(i) / 10.0)));
+    }
+    ARROW_RETURN_NOT_OK(sin_builder.Finish(&arrow_array));
+    auto sin =
+        rerun::ComponentBatch::from_arrow_array(std::move(arrow_array), "sin").value_or_throw();
+
+    arrow::DoubleBuilder cos_builder;
+    for (int64_t i = 0; i < STEPS; i++) {
+        ARROW_RETURN_NOT_OK(cos_builder.Append(cos(static_cast<double>(i) / 10.0)));
+    }
+    ARROW_RETURN_NOT_OK(cos_builder.Finish(&arrow_array));
+    auto cos =
+        rerun::ComponentBatch::from_arrow_array(std::move(arrow_array), "cos").value_or_throw();
+
+    rec.send_columns(
+        "/",
+        rerun::TimeColumn::from_sequence_points("step", std::move(times)),
+        sin.partitioned().value_or_throw(),
+        cos.partitioned().value_or_throw()
+    );
+
+    return arrow::Status::OK();
+}
+
+int main() {
+    arrow::Status status = run_main();
+    if (!status.ok()) {
+        printf("%s\n", status.ToString().c_str());
+        return 1;
+    }
+    return 0;
+}