liria/dora-zed
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

add typed Arrow payloads and a dora-image-view dataflow

Browse Source 
dora-image-view takes std/media/v1/Image, and dora's compatibility graph
has no edge from std/core/v1/UInt8 to it, so wiring the two failed
`dora validate` whatever the node actually sent. Its flat-buffer fallback
would not have decoded these frames either: it reads width/height
metadata, and this node only published shape.

image and depth now go out as the Image struct, camera_info and
point_cloud as Float32 arrays and camera_state as a string, all built
against the Arrow C data interface in src/arrow so the node still links
no Arrow C++. Shape, dtype and encoding metadata are dropped where the
payload describes itself; intrinsics, frame_id and both timestamps are
unchanged. tests/run.sh covers the packing without a camera or the SDK,
round-tripping each array through pyarrow and rebuilding it under the
sanitizers.

Breaking: image_bgr is renamed to image and every payload changes shape.
dataflow_plot.yml is replaced by dataflow_view.yml, which wires the
viewer from a tag and repeats its contract in input_types so validate
checks both sides of the wiring.
This commit is contained in:
cristhian
2026-09-15 10:52:08 -05:00
parent d4c9f153c1
commit c1a7d70ba6
13 changed files with 648 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.cc
+42 -43
View File
@@ -1,12 +1,15 @@
#include "arrow/array.h"
#include "zed/capture.h"
#include <dora-node-api.h>
#include <chrono>
#include <cstdint>
#include <cstdlib>
#include <initializer_list>
#include <iostream>
#include <string>
#include <utility>
#include <vector>
namespace {
@@ -47,9 +50,18 @@ void set_frame_stamps(Metadata& metadata, const zed::Frame& frame) {
metadata.set_timestamp("host_timestamp_ns", frame.host_ns);
}
void report(const char* output, const DoraResult& result) {
// Publish one Arrow array. dora moves both structs out and releases them once
// the payload is in shared memory, so the buffers they borrow only have to
// outlive this call - which the frame the caller is holding does.
void send(DoraNode& dora_node, const char* id, arrow_c::Exported payload,
rust::Box<Metadata> metadata) {
auto result = send_arrow_output(dora_node.send_output, id,
reinterpret_cast<uint8_t*>(&payload.array),
reinterpret_cast<uint8_t*>(&payload.schema),
std::move(metadata));
if (!std::string(result.error).empty()) {
std::cerr << "Error sending " << output << ": " << std::string(result.error) << std::endl;
std::cerr << "Error sending " << id << ": " << std::string(result.error) << std::endl;
}
}
@@ -90,17 +102,11 @@ int main() {
std::string state;
if (capture.take_state_change(state)) {
rust::Slice<const uint8_t> state_slice{
reinterpret_cast<const uint8_t*>(state.data()), state.size()};
auto state_metadata = new_metadata();
state_metadata->set_string("dtype", "string");
state_metadata->set_string("encoding", "utf8");
state_metadata->set_timestamp("host_timestamp_ns", host_timestamp_ns());
report("camera_state",
send_output_with_metadata(dora_node.send_output, "camera_state", state_slice,
std::move(state_metadata)));
send(dora_node, "camera_state", arrow_c::string(state.data(), state.size()),
std::move(state_metadata));
}
// No new frame since the last tick: the camera is slower than the timer,
@@ -109,44 +115,41 @@ int main() {
continue;
}
// --- image_bgr ---
rust::Slice<const uint8_t> image_slice{frame.image_bgr.data(), frame.image_bgr.size()};
auto width = static_cast<uint32_t>(frame.width);
auto height = static_cast<uint32_t>(frame.height);
// --- image ---
// The payload carries its own width, height and encoding, so none of
// that travels as metadata: what is left is what describes the frame
// rather than the picture.
auto image_metadata = new_metadata();
image_metadata->set_string("dtype", "uint8");
image_metadata->set_string("encoding", "bgr8");
image_metadata->set_string("layout", "HWC");
image_metadata->set_list_int("shape", make_vector_int({frame.height, frame.width, 3}));
set_intrinsics(*image_metadata, frame.calib);
set_frame_stamps(*image_metadata, frame);
report("image_bgr", send_output_with_metadata(dora_node.send_output, "image_bgr",
image_slice, std::move(image_metadata)));
send(dora_node, "image",
arrow_c::image(width, height, "bgr8", frame.image_bgr.data(), frame.image_bgr.size()),
std::move(image_metadata));
// --- depth ---
rust::Slice<const uint8_t> depth_slice{
reinterpret_cast<const uint8_t*>(frame.depth.data()),
frame.depth.size() * sizeof(float)};
// A depth map is an image whose pixels are distances, which is how ROS
// spells it too: 32FC1 is one float32 per pixel, and `units` says what
// those floats measure.
auto depth_metadata = new_metadata();
depth_metadata->set_string("dtype", "float32");
depth_metadata->set_string("layout", "HW");
depth_metadata->set_string("units", "mm");
depth_metadata->set_list_int("shape", make_vector_int({frame.height, frame.width}));
set_intrinsics(*depth_metadata, frame.calib);
set_frame_stamps(*depth_metadata, frame);
report("depth", send_output_with_metadata(dora_node.send_output, "depth", depth_slice,
std::move(depth_metadata)));
send(dora_node, "depth",
arrow_c::image(width, height, "32FC1",
reinterpret_cast<const uint8_t*>(frame.depth.data()),
frame.depth.size() * sizeof(float)),
std::move(depth_metadata));
// --- point_cloud ---
if (frame.has_point_cloud) {
rust::Slice<const uint8_t> pc_slice{
reinterpret_cast<const uint8_t*>(frame.point_cloud.data()),
frame.point_cloud.size() * sizeof(float)};
// Unlike the two images, a cloud is a flat array of values: its
// grid shape is only in the metadata.
auto pc_metadata = new_metadata();
pc_metadata->set_string("dtype", "float32");
pc_metadata->set_string("channels", "XYZRGBA");
pc_metadata->set_string("units", "mm");
pc_metadata->set_string("layout", "HWC");
@@ -155,26 +158,22 @@ int main() {
set_intrinsics(*pc_metadata, frame.calib);
set_frame_stamps(*pc_metadata, frame);
report("point_cloud", send_output_with_metadata(dora_node.send_output, "point_cloud",
pc_slice, std::move(pc_metadata)));
send(dora_node, "point_cloud",
arrow_c::float32(frame.point_cloud.data(), frame.point_cloud.size()),
std::move(pc_metadata));
}
// --- camera_info ---
if (frame.camera_info_first) {
float camera_data[4] = {frame.calib.fx, frame.calib.fy, frame.calib.cx,
frame.calib.cy};
rust::Slice<const uint8_t> cam_slice{
reinterpret_cast<const uint8_t*>(camera_data), sizeof(camera_data)};
const float camera_data[4] = {frame.calib.fx, frame.calib.fy, frame.calib.cx,
frame.calib.cy};
auto cam_metadata = new_metadata();
cam_metadata->set_string("dtype", "float32");
cam_metadata->set_string("layout", "C");
cam_metadata->set_list_int("shape", make_vector_int({4}));
cam_metadata->set_list_float("distortion", make_distortion(frame.calib.distortion));
set_frame_stamps(*cam_metadata, frame);
report("camera_info", send_output_with_metadata(dora_node.send_output, "camera_info",
cam_slice, std::move(cam_metadata)));
send(dora_node, "camera_info", arrow_c::float32(camera_data, 4),
std::move(cam_metadata));
}
}