add open-vocab YOLO-World engine with CLIP text encoder

- Add OpenVocab engine: YOLO-World ONNX detection with runtime
  vocabulary via CLIP text embeddings (two-input model support)
- Add header-only C++ CLIP BPE tokenizer (no external deps)
- Add detect(cv::Mat) overload to OpenVocab for camera-free testing
- Add standalone test binary (test_open_vocab) using test/assets images
- Add export scripts for YOLO-World and CLIP text encoder (uv/PEP 723)
- Add installation scripts for Linux x64, ARM (Jetson), and Windows
- Update CMakeLists to use find_package(onnxruntime) instead of FetchContent
- Update README and AGENTS with env vars, setup workflow, and test usage
This commit is contained in:
cristhian aguilera
2026-03-28 10:41:11 -03:00
parent 6e6dc8ef27
commit fd6993e294
18 changed files with 1611 additions and 37 deletions
+46
View File
@@ -0,0 +1,46 @@
#pragma once
#include "engines/engine.h"
#include <opencv2/core.hpp>
#include <memory>
#include <string>
namespace engines {
struct OpenVocabConfig {
std::string model_path; // ZED_ONNX_MODEL_PATH (required)
float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE
float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD
int input_size = 640; // ZED_ONNX_INPUT_SIZE
std::string labels; // ZED_ONNX_LABELS (comma-separated)
// Open-vocabulary text encoder (required when model has two inputs)
std::string text_encoder_path; // ZED_ONNX_TEXT_ENCODER_PATH
std::string clip_vocab_path; // ZED_ONNX_CLIP_VOCAB
std::string clip_merges_path; // ZED_ONNX_CLIP_MERGES
static OpenVocabConfig from_env();
};
class OpenVocab : public Engine {
public:
explicit OpenVocab(const OpenVocabConfig& config);
~OpenVocab() override;
void enable(sl::Camera& camera) override;
void detect(sl::Camera& camera) override;
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
const std::vector<DetectedObject>& objects() const override;
std::string label_names() const override;
private:
OpenVocabConfig config_;
struct Impl;
std::unique_ptr<Impl> impl_;
std::vector<DetectedObject> detected_;
void encode_labels(const std::string& labels_str);
};
} // namespace engines