From 2d10a685274cb291faaae85d9e920be61d92b827 Mon Sep 17 00:00:00 2001 From: cristhian aguilera Date: Sat, 28 Mar 2026 11:03:18 -0300 Subject: [PATCH] add update_vocabulary dora input for runtime label changes The open_vocab engine now accepts an update_vocabulary input event carrying a UTF-8 comma-separated label string. Labels are re-encoded immediately via the CLIP text encoder; the next tick uses the new vocabulary. ZedNative ignores the input via a no-op base class method. --- AGENTS.md | 2 +- README.md | 7 +++++++ dataflow.yml | 1 + src/engines/engine.h | 1 + src/engines/open_vocab.cc | 5 +++++ src/engines/open_vocab.h | 3 ++- src/main.cc | 13 ++++++++++++- 7 files changed, 29 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 086a9fd..79e605f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Camera and detection behavior is configured entirely through environment variabl - **`src/engines/zed_native.h/cc`** — `engines::ZedNative`: wraps ZED SDK's TensorRT-optimized object detection module. - **`src/engines/open_vocab.h/cc`** — `engines::OpenVocab`: YOLO-World open-vocabulary detection via ONNX Runtime. Supports two modes: single-input (fixed vocab, legacy) and two-input (open-vocab, `images` + `text_feats`). In two-input mode, reads `ZED_ONNX_LABELS` on every tick; when the labels change, re-tokenizes and re-encodes via the CLIP text encoder ONNX session, then passes the resulting `(1, N, 512)` L2-normalized embeddings to the visual model. Preprocesses images (letterbox, normalize, CHW), runs inference, applies NMS, then fuses 2D detections with the ZED point cloud to compute 3D positions and bounding boxes. - **`src/util/clip_tokenizer.h`** — Header-only C++ CLIP BPE tokenizer (no external deps). Implements the GPT-2 `bytes_to_unicode()` byte encoder + iterative BPE merge algorithm. Loads `clip_vocab.txt` and `clip_merges.txt` generated by `scripts/export_clip_text_encoder.py`. Returns 77 `int64` token IDs (BOS=49406, EOS=49407, padded with 0). -- **`src/main.cc`** — Dora event loop. Selects engine via `ZED_ENGINE` env var, holds a `std::unique_ptr`. Receives `tick` inputs, calls `camera.grab()` and `detector->detect()`, then sends `image_bgr`, `point_cloud`, `camera_info`, and `objects` outputs with shared `frame_id`. Handles reconnection on grab failure. +- **`src/main.cc`** — Dora event loop. Selects engine via `ZED_ENGINE` env var, holds a `std::unique_ptr`. On `tick` input: calls `camera.grab()` and `detector->detect()`, then sends `image_bgr`, `point_cloud`, `camera_info`, and `objects` outputs. On `update_vocabulary` input: decodes payload as a UTF-8 label string and calls `detector->set_labels()` for immediate re-encoding. Handles reconnection on grab failure. - **`DoraTargets.cmake`** — Handles building or fetching dora-rs and generating the CXX bridge files. Two paths: local (`DORA_ROOT_DIR`) or git clone. Produces `dora_node_api_cxx` library and the `node_bridge.cc` glue. - **`dataflow.yml`** — Example dora dataflow definition using the `git:` source format. - **`scripts/export_yoloworld.py`** — PEP 723 uv script. Exports a YOLOv8-worldv2 model to ONNX with N class slots (labels arbitrary at runtime). Run: `uv run scripts/export_yoloworld.py --model s --num-classes 10`. diff --git a/README.md b/README.md index 4b84ffa..2c64edc 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,13 @@ Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass | `ZED_ONNX_CLIP_VOCAB` | _(required for open-vocab)_ | (`open_vocab`) Path to `clip_vocab.txt` BPE vocabulary file | | `ZED_ONNX_CLIP_MERGES` | _(required for open-vocab)_ | (`open_vocab`) Path to `clip_merges.txt` BPE merge rules file | +## Inputs + +| Input | Description | +|---|---| +| `tick` | Required. Triggers a camera grab and detection on each message. Connect to `dora/timer/millis/N`. | +| `update_vocabulary` | Optional (`open_vocab` engine only). Payload: raw UTF-8 bytes of a comma-separated label string (e.g. `"person,cup,chair"`). Immediately re-encodes CLIP embeddings; the next `tick` uses the new labels. | + ## Outputs ### `image_bgr` diff --git a/dataflow.yml b/dataflow.yml index fecd9a3..656d51f 100644 --- a/dataflow.yml +++ b/dataflow.yml @@ -5,6 +5,7 @@ nodes: path: bin/dora_zed inputs: tick: dora/timer/millis/100 + # update_vocabulary: vocabulary-node/labels # optional: send new comma-separated labels at runtime outputs: - image_bgr - point_cloud diff --git a/src/engines/engine.h b/src/engines/engine.h index eb3f07a..50d5d4b 100644 --- a/src/engines/engine.h +++ b/src/engines/engine.h @@ -20,6 +20,7 @@ public: virtual void enable(sl::Camera& camera) = 0; virtual void detect(sl::Camera& camera) = 0; + virtual void set_labels(const std::string& /*labels*/) {} virtual const std::vector& objects() const = 0; virtual std::string label_names() const = 0; }; diff --git a/src/engines/open_vocab.cc b/src/engines/open_vocab.cc index b1851fd..5df2b1b 100644 --- a/src/engines/open_vocab.cc +++ b/src/engines/open_vocab.cc @@ -576,6 +576,11 @@ void OpenVocab::detect(const cv::Mat& image_bgr) { } } +void OpenVocab::set_labels(const std::string& labels) { + config_.labels = labels; + encode_labels(labels); +} + const std::vector& OpenVocab::objects() const { return detected_; } diff --git a/src/engines/open_vocab.h b/src/engines/open_vocab.h index 8e586e9..252f57f 100644 --- a/src/engines/open_vocab.h +++ b/src/engines/open_vocab.h @@ -30,7 +30,8 @@ public: void enable(sl::Camera& camera) override; void detect(sl::Camera& camera) override; - void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion + void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion + void set_labels(const std::string& labels) override; const std::vector& objects() const override; std::string label_names() const override; diff --git a/src/main.cc b/src/main.cc index 2668927..3aa6735 100644 --- a/src/main.cc +++ b/src/main.cc @@ -95,7 +95,18 @@ int main() { } auto input = event_as_input(std::move(event)); - (void)input; + std::string input_id = std::string(input.id); + + if (input_id == "update_vocabulary") { + if (!input.data.empty()) { + std::string labels( + reinterpret_cast(input.data.data()), + input.data.size()); + detector->set_labels(labels); + std::cout << "Vocabulary updated: " << labels << std::endl; + } + continue; + } if (!camera.grab()) { std::cerr << "Grab failed; attempting to reconnect..." << std::endl;