cristhian aguilera
2d10a68527
The open_vocab engine now accepts an update_vocabulary input event carrying a UTF-8 comma-separated label string. Labels are re-encoded immediately via the CLIP text encoder; the next tick uses the new vocabulary. ZedNative ignores the input via a no-op base class method.
48 lines
1.5 KiB
C++
48 lines
1.5 KiB
C++
#pragma once
|
|
|
|
#include "engines/engine.h"
|
|
|
|
#include <opencv2/core.hpp>
|
|
#include <memory>
|
|
#include <string>
|
|
|
|
namespace engines {
|
|
|
|
struct OpenVocabConfig {
|
|
std::string model_path; // ZED_ONNX_MODEL_PATH (required)
|
|
float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE
|
|
float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD
|
|
int input_size = 640; // ZED_ONNX_INPUT_SIZE
|
|
std::string labels; // ZED_ONNX_LABELS (comma-separated)
|
|
|
|
// Open-vocabulary text encoder (required when model has two inputs)
|
|
std::string text_encoder_path; // ZED_ONNX_TEXT_ENCODER_PATH
|
|
std::string clip_vocab_path; // ZED_ONNX_CLIP_VOCAB
|
|
std::string clip_merges_path; // ZED_ONNX_CLIP_MERGES
|
|
|
|
static OpenVocabConfig from_env();
|
|
};
|
|
|
|
class OpenVocab : public Engine {
|
|
public:
|
|
explicit OpenVocab(const OpenVocabConfig& config);
|
|
~OpenVocab() override;
|
|
|
|
void enable(sl::Camera& camera) override;
|
|
void detect(sl::Camera& camera) override;
|
|
void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion
|
|
void set_labels(const std::string& labels) override;
|
|
const std::vector<DetectedObject>& objects() const override;
|
|
std::string label_names() const override;
|
|
|
|
private:
|
|
OpenVocabConfig config_;
|
|
struct Impl;
|
|
std::unique_ptr<Impl> impl_;
|
|
std::vector<DetectedObject> detected_;
|
|
|
|
void encode_labels(const std::string& labels_str);
|
|
};
|
|
|
|
} // namespace engines
|