liria/dora-zed
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

remove detection from the node

Browse Source 
dora-zed publishes camera data only. The two engines under src/engines/, the
objects output, the ZED_ENGINE/ZED_DETECTION_*/ZED_ONNX_* variables and the
ZED_PC_MODE=ROI crop are gone, along with the standalone engine test and the
model export script.

This drops the unconditional ONNX Runtime requirement from CMakeLists, the
Dockerfile and the install scripts, so the node now configures on a machine
with no ONNX Runtime — which is what blocked the Jetson build, since there is
no official ARM64 GPU binary.

zed_native is dropped rather than kept: it is bound to the live camera handle
and cannot run in a node that receives images over a dataflow. The removed
code remains reachable at tag v0.1.0.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y7g2cBJ5e3r15ZbJ9WZn8c
This commit is contained in:
cesar
2026-09-08 09:12:40 -03:00
co-authored by Claude Opus 5
parent a734692b60
commit 126dbf4f32
17 changed files with 18 additions and 1242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engines/engine.h
-27
View File
@@ -1,27 +0,0 @@
#pragma once
#include <sl/Camera.hpp>
#include <string>
#include <vector>
namespace engines {
struct DetectedObject {
int label_id;
float confidence;
float position[3]; // center x, y, z (mm)
float bounding_box_2d[8]; // 4 corners × 2 coords (pixels)
float bounding_box_3d[24]; // 8 corners × 3 coords (mm)
};
class Engine {
public:
virtual ~Engine() = default;
virtual void enable(sl::Camera& camera) = 0;
virtual void detect(sl::Camera& camera) = 0;
virtual const std::vector<DetectedObject>& objects() const = 0;
virtual std::string label_names() const = 0;
};
} // namespace engines
src/engines/yolo_helpers.h
-136
View File
@@ -1,136 +0,0 @@
#pragma once
#include <opencv2/opencv.hpp>
#include <sl/Camera.hpp>
#include <algorithm>
#include <cmath>
#include <vector>
namespace engines::detail {
// --- NMS ---
struct RawDetection {
float x1, y1, x2, y2;
float score;
int class_id;
};
inline float iou(const RawDetection& a, const RawDetection& b) {
float inter_x1 = std::max(a.x1, b.x1);
float inter_y1 = std::max(a.y1, b.y1);
float inter_x2 = std::min(a.x2, b.x2);
float inter_y2 = std::min(a.y2, b.y2);
float inter_w = std::max(0.0f, inter_x2 - inter_x1);
float inter_h = std::max(0.0f, inter_y2 - inter_y1);
float inter_area = inter_w * inter_h;
float area_a = (a.x2 - a.x1) * (a.y2 - a.y1);
float area_b = (b.x2 - b.x1) * (b.y2 - b.y1);
float union_area = area_a + area_b - inter_area;
return union_area > 0.0f ? inter_area / union_area : 0.0f;
}
inline std::vector<RawDetection> nms(std::vector<RawDetection> dets, float threshold) {
std::sort(dets.begin(), dets.end(),
[](const RawDetection& a, const RawDetection& b) {
return a.score > b.score;
});
std::vector<bool> suppressed(dets.size(), false);
std::vector<RawDetection> result;
for (size_t i = 0; i < dets.size(); i++) {
if (suppressed[i]) continue;
result.push_back(dets[i]);
for (size_t j = i + 1; j < dets.size(); j++) {
if (!suppressed[j] && iou(dets[i], dets[j]) > threshold)
suppressed[j] = true;
}
}
return result;
}
// --- Letterbox resize ---
inline cv::Mat letterbox(const cv::Mat& src, int size,
float& scale, float& pad_x, float& pad_y) {
float scale_w = static_cast<float>(size) / src.cols;
float scale_h = static_cast<float>(size) / src.rows;
scale = std::min(scale_w, scale_h);
int new_w = static_cast<int>(std::round(src.cols * scale));
int new_h = static_cast<int>(std::round(src.rows * scale));
pad_x = (size - new_w) / 2.0f;
pad_y = (size - new_h) / 2.0f;
cv::Mat resized;
cv::resize(src, resized, {new_w, new_h});
cv::Mat canvas(size, size, CV_8UC3, cv::Scalar(114, 114, 114));
resized.copyTo(canvas(cv::Rect(
static_cast<int>(pad_x), static_cast<int>(pad_y), new_w, new_h)));
return canvas;
}
// --- Point cloud fusion ---
struct FusionResult {
float cx, cy, cz;
float depth_near, depth_far;
bool valid = false;
};
inline FusionResult fuse_bbox(const sl::Mat& pc, int x1, int y1, int x2, int y2) {
FusionResult result{};
int w = static_cast<int>(pc.getWidth());
int h = static_cast<int>(pc.getHeight());
x1 = std::max(x1, 0);
y1 = std::max(y1, 0);
x2 = std::min(x2, w - 1);
y2 = std::min(y2, h - 1);
std::vector<float> xs, ys, zs;
xs.reserve((x2 - x1) * (y2 - y1));
ys.reserve(xs.capacity());
zs.reserve(xs.capacity());
for (int y = y1; y <= y2; y++) {
for (int x = x1; x <= x2; x++) {
sl::float4 pt;
pc.getValue(x, y, &pt);
if (!std::isnan(pt.x) && !std::isnan(pt.y) && !std::isnan(pt.z) &&
!std::isinf(pt.x) && !std::isinf(pt.y) && !std::isinf(pt.z)) {
xs.push_back(pt.x);
ys.push_back(pt.y);
zs.push_back(pt.z);
}
}
}
if (xs.empty()) return result;
std::sort(xs.begin(), xs.end());
std::sort(ys.begin(), ys.end());
std::sort(zs.begin(), zs.end());
size_t n = xs.size();
size_t mid = n / 2;
size_t p25 = n / 4;
size_t p75 = 3 * n / 4;
result.cx = xs[mid];
result.cy = ys[mid];
result.cz = zs[mid];
result.depth_near = zs[p25];
result.depth_far = zs[p75];
result.valid = true;
return result;
}
inline void backproject(float u, float v, float z,
float fx, float fy, float cx, float cy,
float& X, float& Y, float& Z) {
Z = z;
X = (u - cx) * z / fx;
Y = (v - cy) * z / fy;
}
} // namespace engines::detail
src/engines/yolo_onnx.cc
-369
View File
@@ -1,369 +0,0 @@
#include "engines/yolo_onnx.h"
#include "engines/yolo_helpers.h"
#include "util/env.h"
#include <onnxruntime_cxx_api.h>
#include <opencv2/opencv.hpp>
#include <sl/Camera.hpp>
#include <cmath>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
namespace engines {
using namespace detail;
// --- Impl ---
struct YoloOnnx::Impl {
Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "yolo_onnx"};
Ort::SessionOptions session_options;
std::unique_ptr<Ort::Session> session;
Ort::AllocatorWithDefaultOptions allocator;
std::string input_name;
std::string output_name;
int num_classes = 0;
YoloOutputLayout resolved_layout = YoloOutputLayout::NC_FIRST;
std::vector<RawDetection> run_inference(const YoloOnnxConfig& config,
const cv::Mat& image_bgr);
};
// --- Impl::run_inference ---
std::vector<RawDetection> YoloOnnx::Impl::run_inference(
const YoloOnnxConfig& config, const cv::Mat& image_bgr)
{
if (!session) return {};
int orig_w = image_bgr.cols;
int orig_h = image_bgr.rows;
cv::Mat rgb;
cv::cvtColor(image_bgr, rgb, cv::COLOR_BGR2RGB);
float scale, pad_x, pad_y;
cv::Mat lb = letterbox(rgb, config.input_size, scale, pad_x, pad_y);
cv::Mat lb_float;
lb.convertTo(lb_float, CV_32F, 1.0f / 255.0f);
int sz = config.input_size;
std::vector<float> input_data(3 * sz * sz);
for (int c = 0; c < 3; c++) {
for (int h = 0; h < sz; h++) {
for (int w = 0; w < sz; w++) {
input_data[c * sz * sz + h * sz + w] =
lb_float.at<cv::Vec3f>(h, w)[c];
}
}
}
auto mem_info = Ort::MemoryInfo::CreateCpu(
OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault);
std::vector<int64_t> image_shape{1, 3, sz, sz};
Ort::Value image_tensor = Ort::Value::CreateTensor<float>(
mem_info, input_data.data(), input_data.size(),
image_shape.data(), image_shape.size());
const char* input_name_cstr = input_name.c_str();
const char* output_name_cstr = output_name.c_str();
std::vector<Ort::Value> outputs;
try {
outputs = session->Run(
Ort::RunOptions{nullptr},
&input_name_cstr, &image_tensor, 1,
&output_name_cstr, 1);
} catch (const Ort::Exception& e) {
std::cerr << "YoloOnnx: inference failed: " << e.what() << std::endl;
return {};
}
auto out_shape = outputs[0].GetTensorTypeAndShapeInfo().GetShape();
if (out_shape.size() < 3) return {};
const float* out = outputs[0].GetTensorData<float>();
int nc = num_classes;
if (nc <= 0) return {};
std::vector<RawDetection> candidates;
if (resolved_layout == YoloOutputLayout::NC_FIRST) {
// (1, 4+N, anchors) — YOLOv8, YOLOv11
int num_anchors = static_cast<int>(out_shape[2]);
for (int a = 0; a < num_anchors; a++) {
float cx_lb = out[0 * num_anchors + a];
float cy_lb = out[1 * num_anchors + a];
float w_lb = out[2 * num_anchors + a];
float h_lb = out[3 * num_anchors + a];
float best_score = 0.0f;
int best_class = 0;
for (int c = 0; c < nc; c++) {
float s = out[(4 + c) * num_anchors + a];
if (s > best_score) { best_score = s; best_class = c; }
}
if (best_score < config.confidence_threshold) continue;
float x1_lb = cx_lb - w_lb / 2.0f;
float y1_lb = cy_lb - h_lb / 2.0f;
float x2_lb = cx_lb + w_lb / 2.0f;
float y2_lb = cy_lb + h_lb / 2.0f;
float x1 = std::max(0.0f, std::min((x1_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y1 = std::max(0.0f, std::min((y1_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
float x2 = std::max(0.0f, std::min((x2_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y2 = std::max(0.0f, std::min((y2_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
candidates.push_back({x1, y1, x2, y2, best_score, best_class});
}
} else {
// (1, anchors, 4+N) or (1, anchors, 5+N)
int num_anchors = static_cast<int>(out_shape[1]);
int row_stride = static_cast<int>(out_shape[2]);
int cls_offset = (resolved_layout == YoloOutputLayout::NC_LAST_OBJ) ? 5 : 4;
for (int a = 0; a < num_anchors; a++) {
const float* row = out + a * row_stride;
float cx_lb = row[0], cy_lb = row[1], w_lb = row[2], h_lb = row[3];
float obj_conf = (resolved_layout == YoloOutputLayout::NC_LAST_OBJ) ? row[4] : 1.0f;
float best_score = 0.0f;
int best_class = 0;
for (int c = 0; c < nc; c++) {
float s = obj_conf * row[cls_offset + c];
if (s > best_score) { best_score = s; best_class = c; }
}
if (best_score < config.confidence_threshold) continue;
float x1_lb = cx_lb - w_lb / 2.0f;
float y1_lb = cy_lb - h_lb / 2.0f;
float x2_lb = cx_lb + w_lb / 2.0f;
float y2_lb = cy_lb + h_lb / 2.0f;
float x1 = std::max(0.0f, std::min((x1_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y1 = std::max(0.0f, std::min((y1_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
float x2 = std::max(0.0f, std::min((x2_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y2 = std::max(0.0f, std::min((y2_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
candidates.push_back({x1, y1, x2, y2, best_score, best_class});
}
}
return nms(std::move(candidates), config.nms_threshold);
}
// --- Config ---
YoloOnnxConfig YoloOnnxConfig::from_env() {
YoloOnnxConfig c;
c.model_path = util::get_env_str("ZED_ONNX_MODEL_PATH", c.model_path);
c.confidence_threshold = util::get_env_float("ZED_ONNX_CONFIDENCE", c.confidence_threshold);
c.nms_threshold = util::get_env_float("ZED_ONNX_NMS_THRESHOLD", c.nms_threshold);
c.input_size = util::get_env_int("ZED_ONNX_INPUT_SIZE", c.input_size);
c.labels = util::get_env_str("ZED_ONNX_LABELS", c.labels);
c.labels_file = util::get_env_str("ZED_ONNX_LABELS_FILE", c.labels_file);
std::string fmt = util::get_env_str("ZED_ONNX_OUTPUT_FORMAT", "");
if (fmt == "nc_first" || fmt == "yolov8" || fmt == "yolov11")
c.output_layout = YoloOutputLayout::NC_FIRST;
else if (fmt == "nc_last")
c.output_layout = YoloOutputLayout::NC_LAST;
else if (fmt == "nc_last_obj" || fmt == "yolov5")
c.output_layout = YoloOutputLayout::NC_LAST_OBJ;
return c;
}
// --- YoloOnnx ---
YoloOnnx::YoloOnnx(const YoloOnnxConfig& config)
: config_(config), impl_(std::make_unique<Impl>()) {}
YoloOnnx::~YoloOnnx() = default;
void YoloOnnx::enable(sl::Camera& /* camera */) {
if (config_.model_path.empty()) {
std::cerr << "YoloOnnx: ZED_ONNX_MODEL_PATH is not set." << std::endl;
return;
}
OrtCUDAProviderOptions cuda_opts{};
cuda_opts.device_id = 0;
try {
impl_->session_options.AppendExecutionProvider_CUDA(cuda_opts);
} catch (const Ort::Exception& e) {
std::cerr << "YoloOnnx: CUDA unavailable (" << e.what() << "), using CPU." << std::endl;
}
impl_->session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
std::cout << "YoloOnnx: loading model (" << config_.model_path << ")..." << std::endl;
try {
impl_->session = std::make_unique<Ort::Session>(
impl_->env, config_.model_path.c_str(), impl_->session_options);
} catch (const Ort::Exception& e) {
std::cerr << "YoloOnnx: failed to load model: " << e.what() << std::endl;
return;
}
impl_->input_name = impl_->session->GetInputNameAllocated(0, impl_->allocator).get();
impl_->output_name = impl_->session->GetOutputNameAllocated(0, impl_->allocator).get();
size_t num_inputs = impl_->session->GetInputCount();
if (num_inputs >= 2) {
std::cerr << "YoloOnnx: model has " << num_inputs << " inputs — "
"this engine only supports single-input models."
<< std::endl;
}
auto out_info = impl_->session->GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo();
auto shape = out_info.GetShape();
if (shape.size() == 3) {
int64_t dim1 = shape[1];
int64_t dim2 = shape[2];
YoloOutputLayout resolved;
if (config_.output_layout == YoloOutputLayout::AUTO) {
// dim1 < dim2 → NC_FIRST (e.g. 84 vs 8400 for YOLOv8/11)
// dim1 > dim2 → NC_LAST_OBJ (e.g. 25200 vs 85 for YOLOv5)
resolved = (dim1 < dim2) ? YoloOutputLayout::NC_FIRST
: YoloOutputLayout::NC_LAST_OBJ;
} else {
resolved = config_.output_layout;
}
impl_->resolved_layout = resolved;
if (resolved == YoloOutputLayout::NC_FIRST) {
impl_->num_classes = static_cast<int>(dim1) - 4;
} else if (resolved == YoloOutputLayout::NC_LAST) {
impl_->num_classes = static_cast<int>(dim2) - 4;
} else { // NC_LAST_OBJ
impl_->num_classes = static_cast<int>(dim2) - 5;
}
}
if (!config_.labels_file.empty() && config_.labels.empty()) {
std::ifstream lf(config_.labels_file);
if (!lf) {
std::cerr << "YoloOnnx: cannot open ZED_ONNX_LABELS_FILE: "
<< config_.labels_file << std::endl;
} else {
std::string line, joined;
while (std::getline(lf, line)) {
size_t l = line.find_first_not_of(" \t\r");
size_t r = line.find_last_not_of(" \t\r");
if (l == std::string::npos) continue;
if (!joined.empty()) joined += ',';
joined += line.substr(l, r - l + 1);
}
if (!joined.empty()) {
config_.labels = joined;
std::cout << "YoloOnnx: loaded labels from file ("
<< config_.labels_file << ")" << std::endl;
}
}
}
std::cout << "YoloOnnx: model loaded."
<< " Classes=" << impl_->num_classes
<< " Layout=" << (impl_->resolved_layout == YoloOutputLayout::NC_FIRST ? "NC_FIRST"
: impl_->resolved_layout == YoloOutputLayout::NC_LAST ? "NC_LAST"
: "NC_LAST_OBJ")
<< std::endl;
}
void YoloOnnx::detect(sl::Camera& camera) {
detected_.clear();
if (!impl_->session) return;
sl::Mat img_mat;
camera.retrieveImage(img_mat, sl::VIEW::LEFT);
sl::Mat pc_mat;
camera.retrieveMeasure(pc_mat, sl::MEASURE::XYZRGBA);
sl::CameraInformation info = camera.getCameraInformation();
auto calib = info.camera_configuration.calibration_parameters.left_cam;
float fx = calib.fx, fy = calib.fy, cx = calib.cx, cy = calib.cy;
int orig_w = static_cast<int>(img_mat.getWidth());
int orig_h = static_cast<int>(img_mat.getHeight());
cv::Mat rgba(orig_h, orig_w, CV_8UC4, img_mat.getPtr<sl::uchar1>());
cv::Mat bgr;
cv::cvtColor(rgba, bgr, cv::COLOR_RGBA2BGR);
auto kept = impl_->run_inference(config_, bgr);
for (const auto& det : kept) {
FusionResult fusion = fuse_bbox(pc_mat,
static_cast<int>(det.x1), static_cast<int>(det.y1),
static_cast<int>(det.x2), static_cast<int>(det.y2));
DetectedObject obj{};
obj.label_id = det.class_id;
obj.confidence = det.score * 100.0f;
if (fusion.valid) {
obj.position[0] = fusion.cx;
obj.position[1] = fusion.cy;
obj.position[2] = fusion.cz;
}
obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1;
obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1;
obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2;
obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2;
if (fusion.valid) {
float corners_u[4] = {det.x1, det.x2, det.x2, det.x1};
float corners_v[4] = {det.y1, det.y1, det.y2, det.y2};
float depths[2] = {fusion.depth_near, fusion.depth_far};
for (int face = 0; face < 2; face++) {
for (int c = 0; c < 4; c++) {
float X, Y, Z;
backproject(corners_u[c], corners_v[c], depths[face],
fx, fy, cx, cy, X, Y, Z);
int idx = (face * 4 + c) * 3;
obj.bounding_box_3d[idx + 0] = X;
obj.bounding_box_3d[idx + 1] = Y;
obj.bounding_box_3d[idx + 2] = Z;
}
}
}
detected_.push_back(obj);
}
}
void YoloOnnx::detect(const cv::Mat& image_bgr) {
detected_.clear();
if (!impl_->session) return;
auto kept = impl_->run_inference(config_, image_bgr);
for (const auto& det : kept) {
DetectedObject obj{};
obj.label_id = det.class_id;
obj.confidence = det.score * 100.0f;
obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1;
obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1;
obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2;
obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2;
detected_.push_back(obj);
}
}
const std::vector<DetectedObject>& YoloOnnx::objects() const {
return detected_;
}
std::string YoloOnnx::label_names() const {
return config_.labels;
}
} // namespace engines
src/engines/yolo_onnx.h
-48
View File
@@ -1,48 +0,0 @@
#pragma once
#include "engines/engine.h"
#include <opencv2/core.hpp>
#include <memory>
#include <string>
namespace engines {
enum class YoloOutputLayout {
AUTO, // detect from output tensor shape at load time
NC_FIRST, // (1, 4+N, anchors) — YOLOv8, YOLOv11
NC_LAST, // (1, anchors, 4+N) — rows-first, no objectness
NC_LAST_OBJ // (1, anchors, 5+N) — YOLOv5, rows-first with objectness
};
struct YoloOnnxConfig {
std::string model_path; // ZED_ONNX_MODEL_PATH (required)
float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE
float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD
int input_size = 640; // ZED_ONNX_INPUT_SIZE
std::string labels; // ZED_ONNX_LABELS (comma-separated)
std::string labels_file; // ZED_ONNX_LABELS_FILE (one class name per line)
YoloOutputLayout output_layout = YoloOutputLayout::AUTO;
static YoloOnnxConfig from_env();
};
class YoloOnnx : public Engine {
public:
explicit YoloOnnx(const YoloOnnxConfig& config);
~YoloOnnx() override;
void enable(sl::Camera& camera) override;
void detect(sl::Camera& camera) override;
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
const std::vector<DetectedObject>& objects() const override;
std::string label_names() const override;
private:
YoloOnnxConfig config_;
struct Impl;
std::unique_ptr<Impl> impl_;
std::vector<DetectedObject> detected_;
};
} // namespace engines
src/engines/zed_native.cc
-99
View File
@@ -1,99 +0,0 @@
#include "engines/zed_native.h"
#include "util/env.h"
#include <iostream>
#include <string>
namespace {
sl::OBJECT_DETECTION_MODEL parse_detection_model(const std::string& model) {
if (model == "MULTI_CLASS_BOX_FAST") {
return sl::OBJECT_DETECTION_MODEL::MULTI_CLASS_BOX_FAST;
}
if (model == "MULTI_CLASS_BOX_ACCURATE") {
return sl::OBJECT_DETECTION_MODEL::MULTI_CLASS_BOX_ACCURATE;
}
return sl::OBJECT_DETECTION_MODEL::MULTI_CLASS_BOX_MEDIUM;
}
} // namespace
namespace engines {
DetectionConfig DetectionConfig::from_env() {
DetectionConfig c;
c.model = util::get_env_str("ZED_DETECTION_MODEL", c.model);
c.confidence_threshold = util::get_env_float("ZED_DETECTION_CONFIDENCE", c.confidence_threshold);
c.tracking = util::get_env_bool("ZED_DETECTION_TRACKING", c.tracking);
return c;
}
ZedNative::ZedNative(const DetectionConfig& config) : config_(config) {
det_params_.detection_model = parse_detection_model(config_.model);
det_params_.enable_tracking = config_.tracking;
det_params_.enable_segmentation = false;
det_rt_params_.detection_confidence_threshold = config_.confidence_threshold;
}
void ZedNative::enable(sl::Camera& camera) {
if (config_.tracking) {
sl::PositionalTrackingParameters tracking_params;
sl::ERROR_CODE err = camera.enablePositionalTracking(tracking_params);
if (err != sl::ERROR_CODE::SUCCESS) {
std::cerr << "Failed to enable positional tracking: "
<< sl::toString(err) << std::endl;
}
}
std::cout << "Object Detection: loading model (" << config_.model << ")..." << std::endl;
sl::ERROR_CODE err = camera.enableObjectDetection(det_params_);
if (err != sl::ERROR_CODE::SUCCESS) {
std::cerr << "Failed to enable object detection: "
<< sl::toString(err) << std::endl;
return;
}
std::cout << "Object Detection: model loaded." << std::endl;
}
void ZedNative::detect(sl::Camera& camera) {
detected_.clear();
sl::ERROR_CODE err = camera.retrieveObjects(raw_objects_, det_rt_params_);
if (err != sl::ERROR_CODE::SUCCESS) {
return;
}
for (const auto& obj : raw_objects_.object_list) {
DetectedObject det{};
det.label_id = static_cast<int>(obj.label);
det.confidence = obj.confidence;
det.position[0] = obj.position.x;
det.position[1] = obj.position.y;
det.position[2] = obj.position.z;
for (int i = 0; i < 4 && i < static_cast<int>(obj.bounding_box_2d.size()); i++) {
det.bounding_box_2d[i * 2 + 0] = obj.bounding_box_2d[i].x;
det.bounding_box_2d[i * 2 + 1] = obj.bounding_box_2d[i].y;
}
for (int i = 0; i < 8 && i < static_cast<int>(obj.bounding_box.size()); i++) {
det.bounding_box_3d[i * 3 + 0] = obj.bounding_box[i].x;
det.bounding_box_3d[i * 3 + 1] = obj.bounding_box[i].y;
det.bounding_box_3d[i * 3 + 2] = obj.bounding_box[i].z;
}
detected_.push_back(det);
}
}
const std::vector<DetectedObject>& ZedNative::objects() const {
return detected_;
}
std::string ZedNative::label_names() const {
return "PERSON,VEHICLE,BAG,ANIMAL,ELECTRONICS,FRUIT_VEGETABLE,SPORT";
}
} // namespace engines
src/engines/zed_native.h
-34
View File
@@ -1,34 +0,0 @@
#pragma once
#include "engines/engine.h"
#include <string>
namespace engines {
struct DetectionConfig {
std::string model = "MULTI_CLASS_BOX_MEDIUM";
float confidence_threshold = 40.0f;
bool tracking = false;
static DetectionConfig from_env();
};
class ZedNative : public Engine {
public:
explicit ZedNative(const DetectionConfig& config);
void enable(sl::Camera& camera) override;
void detect(sl::Camera& camera) override;
const std::vector<DetectedObject>& objects() const override;
std::string label_names() const override;
private:
DetectionConfig config_;
sl::ObjectDetectionParameters det_params_;
sl::ObjectDetectionRuntimeParameters det_rt_params_;
sl::Objects raw_objects_;
std::vector<DetectedObject> detected_;
};
} // namespace engines
src/main.cc
+2 -99
View File
@@ -1,6 +1,3 @@
#include "engines/engine.h"
#include "engines/yolo_onnx.h"
#include "engines/zed_native.h"
#include "util/env.h"
#include "zed/camera.h"
@@ -8,18 +5,14 @@
#include <opencv2/opencv.hpp>
#include <chrono>
#include <cstring>
#include <initializer_list>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
namespace {
constexpr int FIELDS_PER_OBJECT = 37;
enum class PcMode { FULL, HALF, ROI };
enum class PcMode { FULL, HALF };
rust::Vec<int64_t> make_shape(int height, int width, int channels) {
rust::Vec<int64_t> shape;
@@ -45,22 +38,6 @@ rust::Vec<int64_t> make_vector_int(std::initializer_list<int64_t> values) {
return out;
}
std::vector<float> serialize_objects(const std::vector<engines::DetectedObject>& objects) {
std::vector<float> buffer(objects.size() * FIELDS_PER_OBJECT);
for (size_t i = 0; i < objects.size(); i++) {
float* row = buffer.data() + i * FIELDS_PER_OBJECT;
const auto& obj = objects[i];
row[0] = static_cast<float>(obj.label_id);
row[1] = obj.confidence;
row[2] = obj.position[0];
row[3] = obj.position[1];
row[4] = obj.position[2];
std::memcpy(row + 5, obj.bounding_box_2d, sizeof(obj.bounding_box_2d));
std::memcpy(row + 13, obj.bounding_box_3d, sizeof(obj.bounding_box_3d));
}
return buffer;
}
} // namespace
int main() {
@@ -72,21 +49,8 @@ int main() {
camera.warmup();
auto calib = camera.calibration();
std::string engine_name = util::get_env_str("ZED_ENGINE", "native");
std::unique_ptr<engines::Engine> detector;
if (engine_name == "yolo_onnx" || engine_name == "onnx") {
auto cfg = engines::YoloOnnxConfig::from_env();
detector = std::make_unique<engines::YoloOnnx>(cfg);
} else {
auto cfg = engines::DetectionConfig::from_env();
detector = std::make_unique<engines::ZedNative>(cfg);
}
detector->enable(camera.handle());
std::string pc_mode_str = util::get_env_str("ZED_PC_MODE", "FULL");
PcMode pc_mode = (pc_mode_str == "HALF") ? PcMode::HALF
: (pc_mode_str == "ROI") ? PcMode::ROI
: PcMode::FULL;
PcMode pc_mode = (pc_mode_str == "HALF") ? PcMode::HALF : PcMode::FULL;
int64_t frame_id = 0;
while (true) {
@@ -106,12 +70,9 @@ int main() {
camera.close();
camera.open();
calib = camera.calibration();
detector->enable(camera.handle());
continue;
}
detector->detect(camera.handle());
sl::Mat& image_mat = camera.image();
sl::Mat& point_cloud_mat = camera.point_cloud();
@@ -185,30 +146,6 @@ int main() {
pc_metadata->set_list_int("shape", make_shape(hh, hw, 4));
pc_data_ptr = reinterpret_cast<const uint8_t*>(pc_buf.data());
pc_data_size = pc_buf.size() * sizeof(float);
} else if (pc_mode == PcMode::ROI) {
const auto& objs = detector->objects();
for (const auto& obj : objs) {
int x1 = std::max(0, static_cast<int>(obj.bounding_box_2d[0]));
int y1 = std::max(0, static_cast<int>(obj.bounding_box_2d[1]));
int x2 = std::min(pc_width - 1, static_cast<int>(obj.bounding_box_2d[4]));
int y2 = std::min(pc_height - 1, static_cast<int>(obj.bounding_box_2d[5]));
for (int y = y1; y <= y2; y++) {
for (int x = x1; x <= x2; x++) {
sl::float4 pt;
point_cloud_mat.getValue(x, y, &pt);
if (!std::isnan(pt.x) && !std::isinf(pt.x)) {
pc_buf.push_back(pt.x); pc_buf.push_back(pt.y);
pc_buf.push_back(pt.z); pc_buf.push_back(pt.w);
}
}
}
}
int N = static_cast<int>(pc_buf.size() / 4);
pc_metadata->set_string("layout", "NC");
pc_metadata->set_string("mode", "roi");
pc_metadata->set_list_int("shape", make_vector_int({N, 4}));
pc_data_ptr = reinterpret_cast<const uint8_t*>(pc_buf.data());
pc_data_size = pc_buf.size() * sizeof(float);
} else {
// FULL — send raw sl::Mat pointer directly (no copy)
pc_data_size = static_cast<size_t>(pc_width * pc_height * sizeof(sl::float4));
@@ -240,40 +177,6 @@ int main() {
if (!std::string(cam_result.error).empty()) {
std::cerr << "Error sending camera_info: " << std::string(cam_result.error) << std::endl;
}
// --- objects ---
const auto& objects = detector->objects();
auto obj_buffer = serialize_objects(objects);
int num_objects = static_cast<int>(objects.size());
const size_t obj_size = obj_buffer.size() * sizeof(float);
rust::Slice<const uint8_t> obj_slice{
reinterpret_cast<const uint8_t*>(obj_buffer.data()), obj_size};
auto obj_metadata = new_metadata();
obj_metadata->set_string("dtype", "float32");
obj_metadata->set_string("layout", "RC");
obj_metadata->set_string("units", "mm");
obj_metadata->set_list_int("shape", make_vector_int({num_objects, FIELDS_PER_OBJECT}));
obj_metadata->set_string("fields",
"label_id,confidence,x,y,z,"
"bb2d_x0,bb2d_y0,bb2d_x1,bb2d_y1,bb2d_x2,bb2d_y2,bb2d_x3,bb2d_y3,"
"bb3d_x0,bb3d_y0,bb3d_z0,bb3d_x1,bb3d_y1,bb3d_z1,"
"bb3d_x2,bb3d_y2,bb3d_z2,bb3d_x3,bb3d_y3,bb3d_z3,"
"bb3d_x4,bb3d_y4,bb3d_z4,bb3d_x5,bb3d_y5,bb3d_z5,"
"bb3d_x6,bb3d_y6,bb3d_z6,bb3d_x7,bb3d_y7,bb3d_z7");
obj_metadata->set_string("label_names", detector->label_names());
obj_metadata->set_string("engine", engine_name);
obj_metadata->set_int("frame_id", current_frame_id);
obj_metadata->set_int("timestamp_ns", timestamp_ns);
obj_metadata->set_int("image_width", width);
obj_metadata->set_int("image_height", height);
auto obj_result = send_output_with_metadata(
dora_node.send_output, "objects", obj_slice, std::move(obj_metadata));
if (!std::string(obj_result.error).empty()) {
std::cerr << "Error sending objects: " << std::string(obj_result.error) << std::endl;
}
}
return 0;
src/test_yolo_onnx.cc
-113
View File
@@ -1,113 +0,0 @@
// Standalone test for the YoloOnnx (closed-vocabulary) detection engine.
// No dora framework, no live ZED camera — loads an image from disk.
//
// Usage:
// bin/test_yolo_onnx [image_path]
// image_path default: test/assets/yolo.png
//
// Required env vars:
// ZED_ONNX_MODEL_PATH path to YOLO ONNX model
//
// Optional env vars:
// ZED_ONNX_LABELS comma-separated class names
// ZED_ONNX_LABELS_FILE path to newline-separated class names file
// ZED_ONNX_CONFIDENCE detection threshold (default 0.25)
// ZED_ONNX_NMS_THRESHOLD NMS IoU threshold (default 0.45)
// ZED_ONNX_INPUT_SIZE model input size (default 640)
// ZED_ONNX_OUTPUT_FORMAT nc_first|nc_last|nc_last_obj|yolov5|yolov8|yolov11|auto
#include "engines/yolo_onnx.h"
#include "util/env.h"
#include <opencv2/opencv.hpp>
#include <sl/Camera.hpp>
#include <filesystem>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
namespace fs = std::filesystem;
namespace {
std::vector<std::string> parse_csv(const std::string& s) {
std::vector<std::string> out;
std::istringstream ss(s);
std::string tok;
while (std::getline(ss, tok, ',')) {
size_t l = tok.find_first_not_of(" \t");
size_t r = tok.find_last_not_of(" \t");
if (l != std::string::npos)
out.push_back(tok.substr(l, r - l + 1));
}
return out;
}
void draw_detections(cv::Mat& img,
const std::vector<engines::DetectedObject>& objects,
const std::vector<std::string>& labels) {
for (const auto& obj : objects) {
int x1 = static_cast<int>(obj.bounding_box_2d[0]);
int y1 = static_cast<int>(obj.bounding_box_2d[1]);
int x2 = static_cast<int>(obj.bounding_box_2d[4]);
int y2 = static_cast<int>(obj.bounding_box_2d[5]);
std::string name = (obj.label_id >= 0 && obj.label_id < static_cast<int>(labels.size()))
? labels[obj.label_id] : std::to_string(obj.label_id);
char buf[64];
std::snprintf(buf, sizeof(buf), " %.2f", obj.confidence / 100.0f);
cv::rectangle(img, {x1, y1}, {x2, y2}, {0, 200, 0}, 2);
cv::putText(img, name + buf, {x1, std::max(y1 - 6, 14)},
cv::FONT_HERSHEY_SIMPLEX, 0.55, {0, 200, 0}, 2, cv::LINE_AA);
}
}
} // namespace
int main(int argc, char* argv[]) {
std::string image_path = (argc > 1) ? argv[1] : "test/assets/yolo.png";
auto config = engines::YoloOnnxConfig::from_env();
sl::Camera dummy_cam;
engines::YoloOnnx engine(config);
engine.enable(dummy_cam);
cv::Mat img = cv::imread(image_path);
if (img.empty()) {
std::cerr << "Error: cannot read image: " << image_path << std::endl;
return 1;
}
engine.detect(img);
const auto& objects = engine.objects();
auto labels = parse_csv(engine.label_names());
std::cout << "[" << image_path << "] " << objects.size() << " detection(s):" << std::endl;
for (const auto& obj : objects) {
std::string name = (obj.label_id >= 0 && obj.label_id < static_cast<int>(labels.size()))
? labels[obj.label_id] : std::to_string(obj.label_id);
std::cout << " " << name
<< " conf=" << obj.confidence / 100.0f
<< " bbox=["
<< obj.bounding_box_2d[0] << "," << obj.bounding_box_2d[1]
<< " -> "
<< obj.bounding_box_2d[4] << "," << obj.bounding_box_2d[5]
<< "]" << std::endl;
}
cv::Mat annotated = img.clone();
draw_detections(annotated, objects, labels);
fs::path stem = fs::path(image_path).stem();
fs::path parent = fs::path(image_path).parent_path();
std::string result_path = (parent / (stem.string() + "_result.png")).string();
cv::imwrite(result_path, annotated);
std::cout << "Saved: " << result_path << std::endl;
return 0;
}