add open-vocab YOLO-World engine with CLIP text encoder
- Add OpenVocab engine: YOLO-World ONNX detection with runtime vocabulary via CLIP text embeddings (two-input model support) - Add header-only C++ CLIP BPE tokenizer (no external deps) - Add detect(cv::Mat) overload to OpenVocab for camera-free testing - Add standalone test binary (test_open_vocab) using test/assets images - Add export scripts for YOLO-World and CLIP text encoder (uv/PEP 723) - Add installation scripts for Linux x64, ARM (Jetson), and Windows - Update CMakeLists to use find_package(onnxruntime) instead of FetchContent - Update README and AGENTS with env vars, setup workflow, and test usage
This commit is contained in:
+19
-8
@@ -1,4 +1,7 @@
|
||||
#include "engines/engine.h"
|
||||
#include "engines/open_vocab.h"
|
||||
#include "engines/zed_native.h"
|
||||
#include "util/env.h"
|
||||
#include "zed/camera.h"
|
||||
|
||||
#include <dora-node-api.h>
|
||||
@@ -8,6 +11,7 @@
|
||||
#include <cstring>
|
||||
#include <initializer_list>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -66,9 +70,16 @@ int main() {
|
||||
camera.warmup();
|
||||
auto calib = camera.calibration();
|
||||
|
||||
auto det_config = engines::DetectionConfig::from_env();
|
||||
engines::ZedNative detector(det_config);
|
||||
detector.enable(camera.handle());
|
||||
std::string engine_name = util::get_env_str("ZED_ENGINE", "native");
|
||||
std::unique_ptr<engines::Engine> detector;
|
||||
if (engine_name == "open_vocab") {
|
||||
auto cfg = engines::OpenVocabConfig::from_env();
|
||||
detector = std::make_unique<engines::OpenVocab>(cfg);
|
||||
} else {
|
||||
auto cfg = engines::DetectionConfig::from_env();
|
||||
detector = std::make_unique<engines::ZedNative>(cfg);
|
||||
}
|
||||
detector->enable(camera.handle());
|
||||
|
||||
int64_t frame_id = 0;
|
||||
while (true) {
|
||||
@@ -91,11 +102,11 @@ int main() {
|
||||
camera.close();
|
||||
camera.open();
|
||||
calib = camera.calibration();
|
||||
detector.enable(camera.handle());
|
||||
detector->enable(camera.handle());
|
||||
continue;
|
||||
}
|
||||
|
||||
detector.detect(camera.handle());
|
||||
detector->detect(camera.handle());
|
||||
|
||||
sl::Mat& image_mat = camera.image();
|
||||
sl::Mat& point_cloud_mat = camera.point_cloud();
|
||||
@@ -181,7 +192,7 @@ int main() {
|
||||
}
|
||||
|
||||
// --- objects ---
|
||||
const auto& objects = detector.objects();
|
||||
const auto& objects = detector->objects();
|
||||
auto obj_buffer = serialize_objects(objects);
|
||||
int num_objects = static_cast<int>(objects.size());
|
||||
|
||||
@@ -201,8 +212,8 @@ int main() {
|
||||
"bb3d_x2,bb3d_y2,bb3d_z2,bb3d_x3,bb3d_y3,bb3d_z3,"
|
||||
"bb3d_x4,bb3d_y4,bb3d_z4,bb3d_x5,bb3d_y5,bb3d_z5,"
|
||||
"bb3d_x6,bb3d_y6,bb3d_z6,bb3d_x7,bb3d_y7,bb3d_z7");
|
||||
obj_metadata->set_string("label_names",
|
||||
"PERSON,VEHICLE,BAG,ANIMAL,ELECTRONICS,FRUIT_VEGETABLE,SPORT");
|
||||
obj_metadata->set_string("label_names", detector->label_names());
|
||||
obj_metadata->set_string("engine", engine_name);
|
||||
obj_metadata->set_int("frame_id", current_frame_id);
|
||||
obj_metadata->set_int("timestamp_ns", timestamp_ns);
|
||||
obj_metadata->set_int("image_width", width);
|
||||
|
||||
Reference in New Issue
Block a user