liria/dora-zed
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

remove the YOLO-World open-vocabulary engine

Browse Source 
Deletes the yoloworld_onnx engine, the CLIP BPE tokenizer, the standalone
test binary and both export scripts. Drops the update_vocabulary input and
the yoloworld_onnx/open_vocab dispatch from main.cc, along with the now
unused Engine::set_labels. Documentation follows.
This commit is contained in:
cristhian
2026-09-07 05:10:59 -03:00
parent 44eec29b83
commit f1934c9b4a
14 changed files with 8 additions and 1132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/engines/engine.h
-1
View File
@@ -20,7 +20,6 @@ public:
virtual void enable(sl::Camera& camera) = 0;
virtual void detect(sl::Camera& camera) = 0;
virtual void set_labels(const std::string& /*labels*/) {}
virtual const std::vector<DetectedObject>& objects() const = 0;
virtual std::string label_names() const = 0;
};
src/engines/yolo_onnx.cc
+1 -5
View File
@@ -216,7 +216,7 @@ void YoloOnnx::enable(sl::Camera& /* camera */) {
size_t num_inputs = impl_->session->GetInputCount();
if (num_inputs >= 2) {
std::cerr << "YoloOnnx: model has " << num_inputs << " inputs — "
"this looks like a YOLO-World model. Use ZED_ENGINE=yoloworld_onnx instead."
"this engine only supports single-input models."
<< std::endl;
}
@@ -358,10 +358,6 @@ void YoloOnnx::detect(const cv::Mat& image_bgr) {
}
}
void YoloOnnx::set_labels(const std::string& labels) {
config_.labels = labels;
}
const std::vector<DetectedObject>& YoloOnnx::objects() const {
return detected_;
}
src/engines/yolo_onnx.h
-1
View File
@@ -35,7 +35,6 @@ public:
void enable(sl::Camera& camera) override;
void detect(sl::Camera& camera) override;
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
void set_labels(const std::string& labels) override;
const std::vector<DetectedObject>& objects() const override;
std::string label_names() const override;
src/engines/yoloworld_onnx.cc
-415
View File
@@ -1,415 +0,0 @@
#include "engines/yoloworld_onnx.h"
#include "engines/yolo_helpers.h"
#include "util/clip_tokenizer.h"
#include "util/env.h"
#include <onnxruntime_cxx_api.h>
#include <opencv2/opencv.hpp>
#include <sl/Camera.hpp>
#include <algorithm>
#include <cmath>
#include <iostream>
#include <string>
#include <vector>
namespace engines {
using namespace detail;
// --- Impl ---
struct YoloWorldOnnx::Impl {
// Visual model
Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "yoloworld_onnx"};
Ort::SessionOptions session_options;
std::unique_ptr<Ort::Session> session;
Ort::AllocatorWithDefaultOptions allocator;
std::string input_name; // image input
std::string text_input_name; // text_feats input
std::string output_name;
int num_classes = 0;
// Text encoder
Ort::SessionOptions text_session_options;
std::unique_ptr<Ort::Session> text_session;
std::string text_enc_input_name; // "input_ids"
std::string text_enc_output_name; // "text_embeds"
// Tokenizer + embedding cache
util::ClipTokenizer tokenizer;
std::vector<float> cached_text_feats; // (num_classes * 512) flat, row-major
std::string cached_labels;
std::vector<RawDetection> run_inference(const YoloWorldOnnxConfig& config,
const cv::Mat& image_bgr);
};
// --- Impl::run_inference ---
std::vector<RawDetection> YoloWorldOnnx::Impl::run_inference(
const YoloWorldOnnxConfig& config, const cv::Mat& image_bgr)
{
if (!session) return {};
if (cached_text_feats.empty()) return {};
int orig_w = image_bgr.cols;
int orig_h = image_bgr.rows;
cv::Mat rgb;
cv::cvtColor(image_bgr, rgb, cv::COLOR_BGR2RGB);
float scale, pad_x, pad_y;
cv::Mat lb = letterbox(rgb, config.input_size, scale, pad_x, pad_y);
cv::Mat lb_float;
lb.convertTo(lb_float, CV_32F, 1.0f / 255.0f);
int sz = config.input_size;
std::vector<float> input_data(3 * sz * sz);
for (int c = 0; c < 3; c++) {
for (int h = 0; h < sz; h++) {
for (int w = 0; w < sz; w++) {
input_data[c * sz * sz + h * sz + w] =
lb_float.at<cv::Vec3f>(h, w)[c];
}
}
}
auto mem_info = Ort::MemoryInfo::CreateCpu(
OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault);
std::vector<int64_t> image_shape{1, 3, sz, sz};
Ort::Value image_tensor = Ort::Value::CreateTensor<float>(
mem_info, input_data.data(), input_data.size(),
image_shape.data(), image_shape.size());
std::vector<int64_t> text_shape{1, num_classes, 512};
Ort::Value text_tensor = Ort::Value::CreateTensor<float>(
mem_info,
cached_text_feats.data(),
cached_text_feats.size(),
text_shape.data(), text_shape.size());
const char* input_names[2] = {input_name.c_str(), text_input_name.c_str()};
Ort::Value input_tensors[2] = {std::move(image_tensor), std::move(text_tensor)};
const char* output_name_cstr = output_name.c_str();
std::vector<Ort::Value> outputs;
try {
outputs = session->Run(
Ort::RunOptions{nullptr},
input_names, input_tensors, 2,
&output_name_cstr, 1);
} catch (const Ort::Exception& e) {
std::cerr << "YoloWorldOnnx: inference failed: " << e.what() << std::endl;
return {};
}
// Output: (1, 4+num_classes, num_anchors) — NC_FIRST only
auto out_shape = outputs[0].GetTensorTypeAndShapeInfo().GetShape();
if (out_shape.size() < 3) return {};
int num_anchors = static_cast<int>(out_shape[2]);
int nc = num_classes;
if (nc <= 0) return {};
const float* out = outputs[0].GetTensorData<float>();
std::vector<RawDetection> candidates;
for (int a = 0; a < num_anchors; a++) {
float cx_lb = out[0 * num_anchors + a];
float cy_lb = out[1 * num_anchors + a];
float w_lb = out[2 * num_anchors + a];
float h_lb = out[3 * num_anchors + a];
float best_score = 0.0f;
int best_class = 0;
for (int c = 0; c < nc; c++) {
float s = out[(4 + c) * num_anchors + a];
if (s > best_score) { best_score = s; best_class = c; }
}
if (best_score < config.confidence_threshold) continue;
float x1_lb = cx_lb - w_lb / 2.0f;
float y1_lb = cy_lb - h_lb / 2.0f;
float x2_lb = cx_lb + w_lb / 2.0f;
float y2_lb = cy_lb + h_lb / 2.0f;
float x1 = std::max(0.0f, std::min((x1_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y1 = std::max(0.0f, std::min((y1_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
float x2 = std::max(0.0f, std::min((x2_lb - pad_x) / scale, static_cast<float>(orig_w - 1)));
float y2 = std::max(0.0f, std::min((y2_lb - pad_y) / scale, static_cast<float>(orig_h - 1)));
candidates.push_back({x1, y1, x2, y2, best_score, best_class});
}
return nms(std::move(candidates), config.nms_threshold);
}
// --- Config ---
YoloWorldOnnxConfig YoloWorldOnnxConfig::from_env() {
YoloWorldOnnxConfig c;
c.model_path = util::get_env_str("ZED_ONNX_MODEL_PATH", c.model_path);
c.confidence_threshold = util::get_env_float("ZED_ONNX_CONFIDENCE", c.confidence_threshold);
c.nms_threshold = util::get_env_float("ZED_ONNX_NMS_THRESHOLD", c.nms_threshold);
c.input_size = util::get_env_int("ZED_ONNX_INPUT_SIZE", c.input_size);
c.labels = util::get_env_str("ZED_ONNX_LABELS", c.labels);
c.text_encoder_path = util::get_env_str("ZED_ONNX_TEXT_ENCODER_PATH", c.text_encoder_path);
c.clip_vocab_path = util::get_env_str("ZED_ONNX_CLIP_VOCAB", c.clip_vocab_path);
c.clip_merges_path = util::get_env_str("ZED_ONNX_CLIP_MERGES", c.clip_merges_path);
return c;
}
// --- YoloWorldOnnx ---
YoloWorldOnnx::YoloWorldOnnx(const YoloWorldOnnxConfig& config)
: config_(config), impl_(std::make_unique<Impl>()) {}
YoloWorldOnnx::~YoloWorldOnnx() = default;
void YoloWorldOnnx::enable(sl::Camera& /* camera */) {
if (config_.model_path.empty()) {
std::cerr << "YoloWorldOnnx: ZED_ONNX_MODEL_PATH is not set." << std::endl;
return;
}
// --- Load visual model ---
OrtCUDAProviderOptions cuda_opts{};
cuda_opts.device_id = 0;
try {
impl_->session_options.AppendExecutionProvider_CUDA(cuda_opts);
} catch (const Ort::Exception& e) {
std::cerr << "YoloWorldOnnx: CUDA unavailable (" << e.what() << "), using CPU." << std::endl;
}
impl_->session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
std::cout << "YoloWorldOnnx: loading visual model (" << config_.model_path << ")..." << std::endl;
try {
impl_->session = std::make_unique<Ort::Session>(
impl_->env, config_.model_path.c_str(), impl_->session_options);
} catch (const Ort::Exception& e) {
std::cerr << "YoloWorldOnnx: failed to load visual model: " << e.what() << std::endl;
return;
}
size_t num_inputs = impl_->session->GetInputCount();
if (num_inputs < 2) {
std::cerr << "YoloWorldOnnx: model has only " << num_inputs << " input(s) — "
"expected a two-input YOLO-World model. Use ZED_ENGINE=yolo_onnx for single-input models."
<< std::endl;
return;
}
impl_->input_name = impl_->session->GetInputNameAllocated(0, impl_->allocator).get();
impl_->text_input_name = impl_->session->GetInputNameAllocated(1, impl_->allocator).get();
impl_->output_name = impl_->session->GetOutputNameAllocated(0, impl_->allocator).get();
auto out_info = impl_->session->GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo();
auto shape = out_info.GetShape();
if (shape.size() == 3 && shape[1] > 4) {
impl_->num_classes = static_cast<int>(shape[1]) - 4;
}
std::cout << "YoloWorldOnnx: visual model loaded."
<< " Classes=" << impl_->num_classes
<< std::endl;
// --- Load text encoder + tokenizer ---
if (config_.text_encoder_path.empty()) {
std::cerr << "YoloWorldOnnx: ZED_ONNX_TEXT_ENCODER_PATH is not set." << std::endl;
return;
}
impl_->text_session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL);
std::cout << "YoloWorldOnnx: loading text encoder (" << config_.text_encoder_path << ")..." << std::endl;
try {
impl_->text_session = std::make_unique<Ort::Session>(
impl_->env, config_.text_encoder_path.c_str(), impl_->text_session_options);
} catch (const Ort::Exception& e) {
std::cerr << "YoloWorldOnnx: failed to load text encoder: " << e.what() << std::endl;
return;
}
impl_->text_enc_input_name =
impl_->text_session->GetInputNameAllocated(0, impl_->allocator).get();
impl_->text_enc_output_name =
impl_->text_session->GetOutputNameAllocated(0, impl_->allocator).get();
if (!config_.clip_vocab_path.empty() && !config_.clip_merges_path.empty()) {
try {
impl_->tokenizer.load(config_.clip_vocab_path, config_.clip_merges_path);
std::cout << "YoloWorldOnnx: tokenizer loaded." << std::endl;
} catch (const std::exception& e) {
std::cerr << "YoloWorldOnnx: failed to load tokenizer: " << e.what() << std::endl;
return;
}
} else {
std::cerr << "YoloWorldOnnx: ZED_ONNX_CLIP_VOCAB / ZED_ONNX_CLIP_MERGES not set." << std::endl;
return;
}
if (!config_.labels.empty()) {
encode_labels(config_.labels);
}
}
void YoloWorldOnnx::encode_labels(const std::string& labels_str) {
if (!impl_->text_session || !impl_->tokenizer.is_loaded()) return;
auto label_vec = split_labels(labels_str);
int N = impl_->num_classes;
impl_->cached_text_feats.assign(static_cast<size_t>(N) * 512, 0.0f);
auto mem_info = Ort::MemoryInfo::CreateCpu(
OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault);
int encode_n = std::min(static_cast<int>(label_vec.size()), N);
for (int i = 0; i < encode_n; i++) {
auto tokens = impl_->tokenizer.tokenize(label_vec[i]);
std::vector<int64_t> input_shape{1, util::ClipTokenizer::SEQ_LEN};
Ort::Value token_tensor = Ort::Value::CreateTensor<int64_t>(
mem_info, tokens.data(), tokens.size(),
input_shape.data(), input_shape.size());
const char* in = impl_->text_enc_input_name.c_str();
const char* out = impl_->text_enc_output_name.c_str();
std::vector<Ort::Value> outputs;
try {
outputs = impl_->text_session->Run(
Ort::RunOptions{nullptr}, &in, &token_tensor, 1, &out, 1);
} catch (const Ort::Exception& e) {
std::cerr << "YoloWorldOnnx: text encoder failed for \"" << label_vec[i]
<< "\": " << e.what() << std::endl;
continue;
}
const float* emb = outputs[0].GetTensorData<float>();
float norm = 0.0f;
for (int j = 0; j < 512; j++) norm += emb[j] * emb[j];
norm = std::sqrt(norm);
float inv_norm = (norm > 1e-8f) ? 1.0f / norm : 0.0f;
for (int j = 0; j < 512; j++)
impl_->cached_text_feats[i * 512 + j] = emb[j] * inv_norm;
}
impl_->cached_labels = labels_str;
std::cout << "YoloWorldOnnx: encoded " << encode_n << " label(s): " << labels_str << std::endl;
}
void YoloWorldOnnx::detect(sl::Camera& camera) {
detected_.clear();
if (!impl_->session) return;
// Re-encode if labels changed (on-the-fly vocabulary switching)
const char* env_labels = std::getenv("ZED_ONNX_LABELS");
std::string current_labels = env_labels ? env_labels : config_.labels;
if (current_labels != impl_->cached_labels) {
encode_labels(current_labels);
}
if (impl_->cached_text_feats.empty()) return;
sl::Mat img_mat;
camera.retrieveImage(img_mat, sl::VIEW::LEFT);
sl::Mat pc_mat;
camera.retrieveMeasure(pc_mat, sl::MEASURE::XYZRGBA);
sl::CameraInformation info = camera.getCameraInformation();
auto calib = info.camera_configuration.calibration_parameters.left_cam;
float fx = calib.fx, fy = calib.fy, cx = calib.cx, cy = calib.cy;
int orig_w = static_cast<int>(img_mat.getWidth());
int orig_h = static_cast<int>(img_mat.getHeight());
cv::Mat rgba(orig_h, orig_w, CV_8UC4, img_mat.getPtr<sl::uchar1>());
cv::Mat bgr;
cv::cvtColor(rgba, bgr, cv::COLOR_RGBA2BGR);
auto kept = impl_->run_inference(config_, bgr);
for (const auto& det : kept) {
FusionResult fusion = fuse_bbox(pc_mat,
static_cast<int>(det.x1), static_cast<int>(det.y1),
static_cast<int>(det.x2), static_cast<int>(det.y2));
DetectedObject obj{};
obj.label_id = det.class_id;
obj.confidence = det.score * 100.0f;
if (fusion.valid) {
obj.position[0] = fusion.cx;
obj.position[1] = fusion.cy;
obj.position[2] = fusion.cz;
}
obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1;
obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1;
obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2;
obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2;
if (fusion.valid) {
float corners_u[4] = {det.x1, det.x2, det.x2, det.x1};
float corners_v[4] = {det.y1, det.y1, det.y2, det.y2};
float depths[2] = {fusion.depth_near, fusion.depth_far};
for (int face = 0; face < 2; face++) {
for (int c = 0; c < 4; c++) {
float X, Y, Z;
backproject(corners_u[c], corners_v[c], depths[face],
fx, fy, cx, cy, X, Y, Z);
int idx = (face * 4 + c) * 3;
obj.bounding_box_3d[idx + 0] = X;
obj.bounding_box_3d[idx + 1] = Y;
obj.bounding_box_3d[idx + 2] = Z;
}
}
}
detected_.push_back(obj);
}
}
void YoloWorldOnnx::detect(const cv::Mat& image_bgr) {
detected_.clear();
if (!impl_->session) return;
const char* env_labels = std::getenv("ZED_ONNX_LABELS");
std::string current_labels = env_labels ? env_labels : config_.labels;
if (current_labels != impl_->cached_labels) {
encode_labels(current_labels);
}
if (impl_->cached_text_feats.empty()) return;
auto kept = impl_->run_inference(config_, image_bgr);
for (const auto& det : kept) {
DetectedObject obj{};
obj.label_id = det.class_id;
obj.confidence = det.score * 100.0f;
obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1;
obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1;
obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2;
obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2;
detected_.push_back(obj);
}
}
void YoloWorldOnnx::set_labels(const std::string& labels) {
config_.labels = labels;
encode_labels(labels);
}
const std::vector<DetectedObject>& YoloWorldOnnx::objects() const {
return detected_;
}
std::string YoloWorldOnnx::label_names() const {
return impl_->cached_labels.empty() ? config_.labels : impl_->cached_labels;
}
} // namespace engines
src/engines/yoloworld_onnx.h
-47
View File
@@ -1,47 +0,0 @@
#pragma once
#include "engines/engine.h"
#include <opencv2/core.hpp>
#include <memory>
#include <string>
namespace engines {
struct YoloWorldOnnxConfig {
std::string model_path; // ZED_ONNX_MODEL_PATH (required)
float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE
float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD
int input_size = 640; // ZED_ONNX_INPUT_SIZE
std::string labels; // ZED_ONNX_LABELS (comma-separated)
// CLIP text encoder (required)
std::string text_encoder_path; // ZED_ONNX_TEXT_ENCODER_PATH
std::string clip_vocab_path; // ZED_ONNX_CLIP_VOCAB
std::string clip_merges_path; // ZED_ONNX_CLIP_MERGES
static YoloWorldOnnxConfig from_env();
};
class YoloWorldOnnx : public Engine {
public:
explicit YoloWorldOnnx(const YoloWorldOnnxConfig& config);
~YoloWorldOnnx() override;
void enable(sl::Camera& camera) override;
void detect(sl::Camera& camera) override;
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
void set_labels(const std::string& labels) override;
const std::vector<DetectedObject>& objects() const override;
std::string label_names() const override;
private:
YoloWorldOnnxConfig config_;
struct Impl;
std::unique_ptr<Impl> impl_;
std::vector<DetectedObject> detected_;
void encode_labels(const std::string& labels_str);
};
} // namespace engines