add open-vocab YOLO-World engine with CLIP text encoder

- Add OpenVocab engine: YOLO-World ONNX detection with runtime
  vocabulary via CLIP text embeddings (two-input model support)
- Add header-only C++ CLIP BPE tokenizer (no external deps)
- Add detect(cv::Mat) overload to OpenVocab for camera-free testing
- Add standalone test binary (test_open_vocab) using test/assets images
- Add export scripts for YOLO-World and CLIP text encoder (uv/PEP 723)
- Add installation scripts for Linux x64, ARM (Jetson), and Windows
- Update CMakeLists to use find_package(onnxruntime) instead of FetchContent
- Update README and AGENTS with env vars, setup workflow, and test usage
This commit is contained in:
cristhian aguilera
2026-03-28 10:41:11 -03:00
parent 6e6dc8ef27
commit fd6993e294
18 changed files with 1611 additions and 37 deletions
+19 -8
View File
@@ -1,4 +1,7 @@
#include "engines/engine.h"
#include "engines/open_vocab.h"
#include "engines/zed_native.h"
#include "util/env.h"
#include "zed/camera.h"
#include <dora-node-api.h>
@@ -8,6 +11,7 @@
#include <cstring>
#include <initializer_list>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
@@ -66,9 +70,16 @@ int main() {
camera.warmup();
auto calib = camera.calibration();
auto det_config = engines::DetectionConfig::from_env();
engines::ZedNative detector(det_config);
detector.enable(camera.handle());
std::string engine_name = util::get_env_str("ZED_ENGINE", "native");
std::unique_ptr<engines::Engine> detector;
if (engine_name == "open_vocab") {
auto cfg = engines::OpenVocabConfig::from_env();
detector = std::make_unique<engines::OpenVocab>(cfg);
} else {
auto cfg = engines::DetectionConfig::from_env();
detector = std::make_unique<engines::ZedNative>(cfg);
}
detector->enable(camera.handle());
int64_t frame_id = 0;
while (true) {
@@ -91,11 +102,11 @@ int main() {
camera.close();
camera.open();
calib = camera.calibration();
detector.enable(camera.handle());
detector->enable(camera.handle());
continue;
}
detector.detect(camera.handle());
detector->detect(camera.handle());
sl::Mat& image_mat = camera.image();
sl::Mat& point_cloud_mat = camera.point_cloud();
@@ -181,7 +192,7 @@ int main() {
}
// --- objects ---
const auto& objects = detector.objects();
const auto& objects = detector->objects();
auto obj_buffer = serialize_objects(objects);
int num_objects = static_cast<int>(objects.size());
@@ -201,8 +212,8 @@ int main() {
"bb3d_x2,bb3d_y2,bb3d_z2,bb3d_x3,bb3d_y3,bb3d_z3,"
"bb3d_x4,bb3d_y4,bb3d_z4,bb3d_x5,bb3d_y5,bb3d_z5,"
"bb3d_x6,bb3d_y6,bb3d_z6,bb3d_x7,bb3d_y7,bb3d_z7");
obj_metadata->set_string("label_names",
"PERSON,VEHICLE,BAG,ANIMAL,ELECTRONICS,FRUIT_VEGETABLE,SPORT");
obj_metadata->set_string("label_names", detector->label_names());
obj_metadata->set_string("engine", engine_name);
obj_metadata->set_int("frame_id", current_frame_id);
obj_metadata->set_int("timestamp_ns", timestamp_ns);
obj_metadata->set_int("image_width", width);