add open-vocab YOLO-World engine with CLIP text encoder
- Add OpenVocab engine: YOLO-World ONNX detection with runtime vocabulary via CLIP text embeddings (two-input model support) - Add header-only C++ CLIP BPE tokenizer (no external deps) - Add detect(cv::Mat) overload to OpenVocab for camera-free testing - Add standalone test binary (test_open_vocab) using test/assets images - Add export scripts for YOLO-World and CLIP text encoder (uv/PEP 723) - Add installation scripts for Linux x64, ARM (Jetson), and Windows - Update CMakeLists to use find_package(onnxruntime) instead of FetchContent - Update README and AGENTS with env vars, setup workflow, and test usage
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
#pragma once
|
||||
|
||||
#include "engines/engine.h"
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace engines {
|
||||
|
||||
struct OpenVocabConfig {
|
||||
std::string model_path; // ZED_ONNX_MODEL_PATH (required)
|
||||
float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE
|
||||
float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD
|
||||
int input_size = 640; // ZED_ONNX_INPUT_SIZE
|
||||
std::string labels; // ZED_ONNX_LABELS (comma-separated)
|
||||
|
||||
// Open-vocabulary text encoder (required when model has two inputs)
|
||||
std::string text_encoder_path; // ZED_ONNX_TEXT_ENCODER_PATH
|
||||
std::string clip_vocab_path; // ZED_ONNX_CLIP_VOCAB
|
||||
std::string clip_merges_path; // ZED_ONNX_CLIP_MERGES
|
||||
|
||||
static OpenVocabConfig from_env();
|
||||
};
|
||||
|
||||
class OpenVocab : public Engine {
|
||||
public:
|
||||
explicit OpenVocab(const OpenVocabConfig& config);
|
||||
~OpenVocab() override;
|
||||
|
||||
void enable(sl::Camera& camera) override;
|
||||
void detect(sl::Camera& camera) override;
|
||||
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
|
||||
const std::vector<DetectedObject>& objects() const override;
|
||||
std::string label_names() const override;
|
||||
|
||||
private:
|
||||
OpenVocabConfig config_;
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
std::vector<DetectedObject> detected_;
|
||||
|
||||
void encode_labels(const std::string& labels_str);
|
||||
};
|
||||
|
||||
} // namespace engines
|
||||
Reference in New Issue
Block a user