diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b3cea90 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git/ +build/ +bin/ +tmp/ +models/ diff --git a/AGENTS.md b/AGENTS.md index 3f7e5e4..310a548 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,8 @@ The binary is installed to `bin/dora_zed` relative to the repo root. The first build clones dora-rs from GitHub and compiles `dora-node-api-cxx` via Cargo. This takes a while. Subsequent builds are incremental. If `DORA_ROOT_DIR` is set to a local dora checkout, it builds against that instead. +For a build without installing the SDK stack on the host, see the `Dockerfile` (Ubuntu 24.04 — the newest release with a ZED SDK; there is no Stereolabs build for 26.04). Run the compile with `--gpus all`. + ## Run ```bash @@ -34,24 +36,20 @@ Camera and detection behavior is configured entirely through environment variabl - **`src/zed/camera.cc`** — Implementation. SDK enum parsers live in an anonymous namespace. - **`src/engines/engine.h`** — `engines::DetectedObject` struct and `engines::Engine` abstract base class (`enable`, `detect`, `objects`, `label_names`). - **`src/engines/zed_native.h/cc`** — `engines::ZedNative`: wraps ZED SDK's TensorRT-optimized object detection module. -- **`src/engines/yolo_helpers.h`** — Header-only shared helpers used by both ONNX engines: `RawDetection`, `nms()`, `letterbox()`, `fuse_bbox()`, `backproject()`, `split_labels()`. Lives in `engines::detail` namespace. +- **`src/engines/yolo_helpers.h`** — Header-only shared helpers used by the ONNX engine: `RawDetection`, `nms()`, `letterbox()`, `fuse_bbox()`, `backproject()`. Lives in `engines::detail` namespace. - **`src/engines/yolo_onnx.h/cc`** — `engines::YoloOnnx` / `YoloOnnxConfig`: closed-vocabulary detection for any single-input YOLO ONNX model (YOLOv8, YOLOv11, YOLOv5, etc.). Auto-detects output layout (`NC_FIRST` / `NC_LAST` / `NC_LAST_OBJ`) from tensor shape; override with `ZED_ONNX_OUTPUT_FORMAT`. Labels loaded from `ZED_ONNX_LABELS` or `ZED_ONNX_LABELS_FILE`. -- **`src/engines/yoloworld_onnx.h/cc`** — `engines::YoloWorldOnnx` / `YoloWorldOnnxConfig`: open-vocabulary detection via YOLO-World ONNX (two inputs: `images` + `text_feats`). Reads `ZED_ONNX_LABELS` on every tick; when labels change, re-tokenizes and re-encodes via the CLIP text encoder ONNX session, passing `(1, N, 512)` L2-normalized embeddings to the visual model. Output is always NC_FIRST `(1, 4+N, anchors)`. -- **`src/util/clip_tokenizer.h`** — Header-only C++ CLIP BPE tokenizer (no external deps). Implements the GPT-2 `bytes_to_unicode()` byte encoder + iterative BPE merge algorithm. Loads `clip_vocab.txt` and `clip_merges.txt` generated by `scripts/export_clip_text_encoder.py`. Returns 77 `int64` token IDs (BOS=49406, EOS=49407, padded with 0). -- **`src/main.cc`** — Dora event loop. Selects engine via `ZED_ENGINE` env var (`yolo_onnx`/`onnx`, `yoloworld_onnx`/`open_vocab`, or `native`), holds a `std::unique_ptr`. Parses `ZED_PC_MODE` (`FULL`/`HALF`/`ROI`) once before the loop. On `tick` input: calls `camera.grab()` and `detector->detect()`, then sends `image_bgr`, `point_cloud`, `camera_info`, and `objects` outputs. Point cloud output varies by mode: `FULL` sends the raw `sl::Mat` pointer (no copy); `HALF` downsamples by 2 in each dimension; `ROI` emits only valid points within detected bounding boxes as a flat `[N, 4]` array. On `update_vocabulary` input: decodes payload as a UTF-8 label string and calls `detector->set_labels()` for immediate re-encoding. Handles reconnection on grab failure. +- **`src/main.cc`** — Dora event loop. Selects engine via `ZED_ENGINE` env var (`yolo_onnx`/`onnx` or `native`), holds a `std::unique_ptr`. Parses `ZED_PC_MODE` (`FULL`/`HALF`/`ROI`) once before the loop. On any input event (intended to be driven by a tick timer; the id is not inspected): calls `camera.grab()` and `detector->detect()`, then sends `image_bgr`, `point_cloud`, `camera_info`, and `objects` outputs. Point cloud output varies by mode: `FULL` sends the raw `sl::Mat` pointer (no copy); `HALF` downsamples by 2 in each dimension; `ROI` emits only valid points within detected bounding boxes as a flat `[N, 4]` array. Handles reconnection on grab failure. - **`DoraTargets.cmake`** — Handles building or fetching dora-rs and generating the CXX bridge files. Two paths: local (`DORA_ROOT_DIR`) or git clone. Produces `dora_node_api_cxx` library and the `node_bridge.cc` glue. - **`dataflow.yml`** — Example dora dataflow definition using the `git:` source format. +- **`Dockerfile`** — Ubuntu 24.04 x86-64 build environment (CUDA 12.8, ZED SDK 5.2, ONNX Runtime, Rust). No ARM64/Jetson image; build natively there. No camera needed; run the compile with `--gpus all`. Sets `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` — the `video` capability is what makes the container runtime mount NVDEC/NVENC, which `libsl_zed.so` links against. ONNX Runtime's CMake package files are relocated to `/usr/local/lib/cmake` because Ubuntu's multiarch CMake does not search `lib64`. Rust lives at `/usr/local/{rustup,cargo}` rather than `/root` so the build can run as the invoking user. `docker build -t dora-zed-build . && docker run --rm --gpus all --user "$(id -u):$(id -g)" -v "$PWD:/src" -v dora-zed-cargo:/usr/local/cargo/registry dora-zed-build`. - **`scripts/export_yolo.py`** — PEP 723 uv script. Exports a standard YOLOv11 model to ONNX for closed-vocabulary detection. Run: `uv run scripts/export_yolo.py --model n --output-dir models/`. -- **`scripts/export_yoloworld.py`** — PEP 723 uv script. Exports a YOLOv8-worldv2 model to ONNX with N class slots (labels arbitrary at runtime). Run: `uv run scripts/export_yoloworld.py --model s --num-classes 10`. -- **`scripts/export_clip_text_encoder.py`** — PEP 723 uv script. Exports CLIP ViT-B/32 text encoder to ONNX and saves `clip_vocab.txt` / `clip_merges.txt`. Run once; reused across all YOLO-World model sizes. Run: `uv run scripts/export_clip_text_encoder.py --output-dir models/`. - **`src/test_yolo_onnx.cc`** — Standalone C++ test binary (`bin/test_yolo_onnx`). Exercises `YoloOnnx` with an image (default `test/assets/yolo.png`). No dora and no live camera required. Uses the same env vars as the main binary. Saves an annotated result image alongside the input. -- **`src/test_yoloworld_onnx.cc`** — Standalone C++ test binary (`bin/test_yoloworld_onnx`). Exercises `YoloWorldOnnx` with an image (default `test/assets/yoloworld.png`). Same interface as `test_yolo_onnx`. ## Key APIs - **dora C++ API** (``): `init_dora_node()`, `dora_node.events->next()`, `event_type()`, `send_output_with_metadata()`, `new_metadata()`. - **ZED SDK** (``): `sl::Camera`, `sl::InitParameters`, `sl::RuntimeParameters`, `sl::Mat`, `sl::ObjectDetectionParameters`, `sl::Objects`. -- **ONNX Runtime** (``): `Ort::Env`, `Ort::Session`, `Ort::SessionOptions`, `Ort::Value`. Used by the `open_vocab` engine. +- **ONNX Runtime** (``): `Ort::Env`, `Ort::Session`, `Ort::SessionOptions`, `Ort::Value`. Used by the `yolo_onnx` engine. - Output data is sent as raw byte slices (`rust::Slice`) with typed metadata (shape, dtype, encoding). ## Tools diff --git a/CMakeLists.txt b/CMakeLists.txt index 72b4e48..71f3cd6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,6 @@ add_executable(dora_zed src/zed/camera.cc src/engines/zed_native.cc src/engines/yolo_onnx.cc - src/engines/yoloworld_onnx.cc ${node_bridge} ) add_dependencies(dora_zed Dora_cxx) @@ -65,21 +64,3 @@ target_link_libraries(test_yolo_onnx PRIVATE ${ONNXRUNTIME_LIBRARIES} ) install(TARGETS test_yolo_onnx DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin) - -add_executable(test_yoloworld_onnx - src/test_yoloworld_onnx.cc - src/engines/yoloworld_onnx.cc -) -target_include_directories(test_yoloworld_onnx PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${ZED_INCLUDE_DIRS} - ${OpenCV_INCLUDE_DIRS} - ${CUDAToolkit_INCLUDE_DIRS} - ${ONNXRUNTIME_INCLUDE_DIRS} -) -target_link_libraries(test_yoloworld_onnx PRIVATE - ${ZED_LIBRARIES} - ${OpenCV_LIBS} - ${ONNXRUNTIME_LIBRARIES} -) -install(TARGETS test_yoloworld_onnx DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2e30c15 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,75 @@ +# Reproducible build environment for dora-zed. +# +# Compiles only — no camera is needed. Requires an NVIDIA driver and the NVIDIA +# Container Toolkit on the host, as the ZED SDK does anyway. +# +# NVIDIA_DRIVER_CAPABILITIES below includes `video` because libsl_zed.so links +# against NVDEC (libnvcuvid) and NVENC (libnvidia-encode); the container runtime +# mounts those from the host only when `video` is in the list. +# +# Ubuntu 24.04 is the newest release Stereolabs ships a ZED SDK for. +# +# docker build -t dora-zed-build . +# docker run --rm --gpus all --user "$(id -u):$(id -g)" \ +# -v "$PWD:/src" -v dora-zed-cargo:/usr/local/cargo/registry dora-zed-build +# +# --user keeps build/ and bin/ owned by you instead of root. +# +# The first run clones dora-rs and compiles it with Cargo, which takes a while; +# the named volume keeps the registry cache across runs. + +FROM nvidia/cuda:12.8.1-devel-ubuntu24.04 + +ARG ZED_VERSION=5.2 +ARG ORT_VERSION=1.24.4 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake git curl ca-certificates pkg-config \ + libopencv-dev zstd udev sudo \ + && rm -rf /var/lib/apt/lists/* + +# The toolchain lives outside /root so the container can run as the invoking +# user (--user) and still write the cargo caches. registry/ is created here so +# that a named volume mounted over it inherits these permissions rather than +# being created root-owned. +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:${PATH} +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --profile minimal --default-toolchain stable \ + && mkdir -p ${CARGO_HOME}/registry \ + && chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME} + +# The installer leaves /usr/local/zed as drwxrwx--- root:zed, which an arbitrary +# --user uid cannot read, so relax it to make find_package(zed) work for one. +RUN curl -fSL -o /tmp/ZED_SDK.run \ + "https://download.stereolabs.com/zedsdk/${ZED_VERSION}/cu12/ubuntu24" \ + && chmod +x /tmp/ZED_SDK.run \ + && /tmp/ZED_SDK.run -- silent skip_python skip_tools skip_drivers skip_cuda \ + && chmod -R a+rX /usr/local/zed \ + && rm -f /tmp/ZED_SDK.run + +# The tarball puts its CMake package files under lib/, but onnxruntimeTargets.cmake +# resolves the library as /lib64. Ubuntu is multiarch, so CMake does not +# search lib64 for packages and ld.so does not search /usr/local/lib64 at all: +# keep the libraries in lib64, move only the package files to lib/cmake, and add +# lib64 to the linker path. +RUN curl -fSL -o /tmp/ort.tgz \ + "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-gpu-${ORT_VERSION}.tgz" \ + && tar -xzf /tmp/ort.tgz -C /tmp \ + && mkdir -p /usr/local/include/onnxruntime /usr/local/lib64 \ + && cp -r /tmp/onnxruntime-linux-x64-gpu-${ORT_VERSION}/include/* /usr/local/include/onnxruntime/ \ + && cp -r /tmp/onnxruntime-linux-x64-gpu-${ORT_VERSION}/lib/* /usr/local/lib64/ \ + && mkdir -p /usr/local/lib/cmake \ + && mv /usr/local/lib64/cmake/onnxruntime /usr/local/lib/cmake/ \ + && rmdir /usr/local/lib64/cmake \ + && echo /usr/local/lib64 > /etc/ld.so.conf.d/onnxruntime.conf \ + && ldconfig \ + && rm -rf /tmp/ort.tgz /tmp/onnxruntime-linux-x64-gpu-${ORT_VERSION} + +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,video + +WORKDIR /src +CMD ["bash", "-lc", "mkdir -p build && cd build && cmake .. && make -j$(nproc) install"] diff --git a/DoraTargets.cmake b/DoraTargets.cmake index 836f09c..2c25324 100644 --- a/DoraTargets.cmake +++ b/DoraTargets.cmake @@ -5,6 +5,16 @@ set(dora_c_include_dir "${CMAKE_CURRENT_BINARY_DIR}/include/c") set(dora_cxx_include_dir "${CMAKE_CURRENT_BINARY_DIR}/include/cxx") set(node_bridge "${CMAKE_CURRENT_BINARY_DIR}/node_bridge.cc") +# Match the cargo profile to the CMake build type, so a release build does not +# link unoptimized dora bindings. +if(CMAKE_BUILD_TYPE MATCHES "^(Release|RelWithDebInfo|MinSizeRel)$") + set(_cargo_profile_flag --release) + set(_cargo_target_subdir release) +else() + set(_cargo_profile_flag "") + set(_cargo_target_subdir debug) +endif() + if(DORA_ROOT_DIR AND EXISTS "${DORA_ROOT_DIR}/Cargo.toml") include(ExternalProject) ExternalProject_Add(Dora @@ -12,10 +22,10 @@ if(DORA_ROOT_DIR AND EXISTS "${DORA_ROOT_DIR}/Cargo.toml") BUILD_IN_SOURCE True CONFIGURE_COMMAND "" BUILD_COMMAND - cargo build + cargo build ${_cargo_profile_flag} --package dora-node-api-c && - cargo build + cargo build ${_cargo_profile_flag} --package dora-node-api-cxx INSTALL_COMMAND "" ) @@ -37,39 +47,53 @@ if(DORA_ROOT_DIR AND EXISTS "${DORA_ROOT_DIR}/Cargo.toml") add_custom_target(Dora_c DEPENDS ${dora_c_include_dir}) add_custom_target(Dora_cxx DEPENDS ${node_bridge} ${dora_cxx_include_dir}) - set(dora_link_dirs ${DORA_ROOT_DIR}/target/debug) + set(dora_link_dirs ${DORA_ROOT_DIR}/target/${_cargo_target_subdir}) else() include(ExternalProject) - set(DORA_GIT_TAG "main") - find_program(DORA_CLI dora) - if(DORA_CLI) - execute_process( - COMMAND ${DORA_CLI} --version - RESULT_VARIABLE _dora_version_result - OUTPUT_VARIABLE _dora_version_out - ERROR_VARIABLE _dora_version_err - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - if(_dora_version_result EQUAL 0) - string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+)" _dora_version_match "${_dora_version_out}") - if(_dora_version_match) - set(DORA_GIT_TAG "v${CMAKE_MATCH_1}") + + # This node targets dora 1.0+. Set DORA_GIT_TAG to pin a tag explicitly; + # left empty it follows the installed dora CLI so the bindings match the + # runtime, and falls back to the default below when no CLI is present. + set(DORA_GIT_TAG "" CACHE STRING "dora-rs git tag to build against (empty: follow the dora CLI)") + set(_dora_tag "${DORA_GIT_TAG}") + + if(NOT _dora_tag) + set(_dora_tag "v1.0.1") + find_program(DORA_CLI dora) + if(DORA_CLI) + execute_process( + COMMAND ${DORA_CLI} --version + RESULT_VARIABLE _dora_version_result + OUTPUT_VARIABLE _dora_version_out + ERROR_VARIABLE _dora_version_err + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(_dora_version_result EQUAL 0) + string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+)" _dora_version_match "${_dora_version_out}") + if(_dora_version_match) + set(_dora_tag "v${CMAKE_MATCH_1}") + if(CMAKE_MATCH_1 VERSION_LESS 1.0) + message(WARNING "dora CLI ${CMAKE_MATCH_1} is older than the 1.0 this node targets") + endif() + endif() endif() endif() endif() + message(STATUS "dora-rs: building C++ bindings from ${_dora_tag}") + ExternalProject_Add(Dora PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dora GIT_REPOSITORY https://github.com/dora-rs/dora.git - GIT_TAG ${DORA_GIT_TAG} + GIT_TAG ${_dora_tag} BUILD_IN_SOURCE True CONFIGURE_COMMAND "" BUILD_COMMAND - cargo build + cargo build ${_cargo_profile_flag} --package dora-node-api-c --target-dir ${CMAKE_CURRENT_BINARY_DIR}/dora/src/Dora/target && - cargo build + cargo build ${_cargo_profile_flag} --package dora-node-api-cxx --target-dir ${CMAKE_CURRENT_BINARY_DIR}/dora/src/Dora/target INSTALL_COMMAND "" @@ -90,7 +114,7 @@ else() cp ../apis/c/node ${CMAKE_CURRENT_BINARY_DIR}/include/c -r ) - set(dora_link_dirs ${CMAKE_CURRENT_BINARY_DIR}/dora/src/Dora/target/debug) + set(dora_link_dirs ${CMAKE_CURRENT_BINARY_DIR}/dora/src/Dora/target/${_cargo_target_subdir}) add_custom_target(Dora_c DEPENDS ${dora_c_include_dir}) add_custom_target(Dora_cxx DEPENDS ${node_bridge} ${dora_cxx_include_dir}) diff --git a/README.md b/README.md index a12ee3b..2bfe4cf 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ See [`dataflow.yml`](dataflow.yml) for a complete runnable example with a camera | [ZED SDK](https://www.stereolabs.com/developers/release) | 5.x | Bundles OpenCV; default path `/usr/local/zed` | | [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) | — | Required by ZED SDK | | [Rust toolchain](https://rustup.rs) | stable | Builds dora-rs C++ bindings | -| [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) | 1.20+ | Required only for `open_vocab` engine | +| [ONNX Runtime](https://github.com/microsoft/onnxruntime/releases) | 1.20+ | Required at configure time; used at runtime only by `yolo_onnx` | ### Installation scripts @@ -42,7 +42,7 @@ See [`dataflow.yml`](dataflow.yml) for a complete runnable example with a camera | NVIDIA Jetson (ARM64) | `sudo scripts/install_linux_arm.sh` | | Windows x64 (PowerShell admin) | `scripts/install_windows.ps1` | -> **Jetson:** No official ONNX Runtime GPU binary for ARM64. The `native` engine works without it. The `open_vocab` engine requires [building from source](https://onnxruntime.ai/docs/build/eps.html#tensorrt). +> **Jetson:** No official ONNX Runtime GPU binary for ARM64. It is still needed to configure the build; only the `yolo_onnx` engine uses it at runtime. Requires [building from source](https://onnxruntime.ai/docs/build/eps.html#tensorrt). > > **Windows:** Install ZED SDK manually from [stereolabs.com](https://www.stereolabs.com/developers/release), then pass the ONNX Runtime path: `cmake .. -DCMAKE_PREFIX_PATH=C:\onnxruntime` @@ -56,6 +56,29 @@ mkdir -p build && cd build && cmake .. && make install Binary installed to `bin/dora_zed`. +### Containerized build + +[`Dockerfile`](Dockerfile) provides the full toolchain — CUDA, ZED SDK, ONNX Runtime and Rust — on Ubuntu 24.04, the newest release Stereolabs ships a ZED SDK for. No camera is needed. + +```bash +docker build -t dora-zed-build . +docker run --rm --gpus all --user "$(id -u):$(id -g)" \ + -v "$PWD:/src" -v dora-zed-cargo:/usr/local/cargo/registry dora-zed-build +``` + +The first run clones dora-rs and compiles it with Cargo; the named volume keeps that cache across runs. +`--user` keeps `build/` and `bin/` owned by you rather than root. + +x86-64 only. On Jetson, run [`scripts/install_linux_arm.sh`](scripts/install_linux_arm.sh) on the +device and build natively — there is no ARM64 image here. + +Needs an NVIDIA driver and the +[NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) +on the host, as the ZED SDK does anyway. The image sets +`NVIDIA_DRIVER_CAPABILITIES=compute,utility,video`: `libsl_zed.so` links against NVDEC +(`libnvcuvid`) and NVENC (`libnvidia-encode`), which the container runtime mounts from the host +only when `video` is in the capability list. + ## Configuration All behavior is controlled via environment variables. @@ -82,7 +105,6 @@ Set `ZED_ENGINE` to select the detection backend: |---|---|---| | `native` (default) | No — ZED SDK built-in TensorRT | Fixed: 7 categories | | `yolo_onnx` (alias: `onnx`) | Yes — any single-input YOLO ONNX (v5/v8/v11) | Fixed labels from file or env var | -| `yoloworld_onnx` (alias: `open_vocab`) | Yes — YOLO-World two-input ONNX | Defined at runtime via CLIP | #### `native` engine @@ -131,49 +153,6 @@ dora start dataflow.yml | `ZED_ONNX_INPUT_SIZE` | `640` | Model input size (px) | | `ZED_ONNX_OUTPUT_FORMAT` | `auto` | Output layout: `auto`, `nc_first`/`yolov8`/`yolov11`, `nc_last`, `nc_last_obj`/`yolov5`. `auto` detects from tensor shape. | -#### `yoloworld_onnx` engine (open-vocabulary) - -For YOLO-World two-input ONNX models with a CLIP text encoder. Labels are defined at runtime — no recompilation required. Use `ZED_ENGINE=open_vocab` as a legacy alias. - -Export models once: - -```bash -uv run scripts/export_yoloworld.py --model s --num-classes 10 --output-dir models/ -uv run scripts/export_clip_text_encoder.py --output-dir models/ -``` - -Then run: - -```bash -ZED_ENGINE=yoloworld_onnx \ -ZED_ONNX_MODEL_PATH=models/yolov8s-worldv2_n10_i640.onnx \ -ZED_ONNX_TEXT_ENCODER_PATH=models/clip_text_encoder.onnx \ -ZED_ONNX_CLIP_VOCAB=models/clip_vocab.txt \ -ZED_ONNX_CLIP_MERGES=models/clip_merges.txt \ -ZED_ONNX_LABELS="person,car,chair" \ -dora start dataflow.yml -``` - -**Required:** - -| Variable | Description | -|---|---| -| `ZED_ONNX_MODEL_PATH` | Path to YOLO-World ONNX model | -| `ZED_ONNX_TEXT_ENCODER_PATH` | Path to `clip_text_encoder.onnx` | -| `ZED_ONNX_CLIP_VOCAB` | Path to `clip_vocab.txt` | -| `ZED_ONNX_CLIP_MERGES` | Path to `clip_merges.txt` | -| `ZED_ONNX_LABELS` | Comma-separated labels, e.g. `person,car,chair` | - -**Optional:** - -| Variable | Default | Description | -|---|---|---| -| `ZED_ONNX_CONFIDENCE` | `0.25` | Confidence threshold (0.0–1.0) | -| `ZED_ONNX_NMS_THRESHOLD` | `0.45` | NMS IoU threshold | -| `ZED_ONNX_INPUT_SIZE` | `640` | Model input size (px) | - -Labels can be updated at runtime without restarting — send a new comma-separated string to the `update_vocabulary` input. - ### Point cloud | `ZED_PC_MODE` | Shape | Description | @@ -187,7 +166,6 @@ Labels can be updated at runtime without restarting — send a new comma-separat | Input | Description | |---|---| | `tick` | Triggers a camera grab and detection. Connect to `dora/timer/millis/N`. | -| `update_vocabulary` | (`open_vocab` only) UTF-8 comma-separated label string. Re-encodes CLIP embeddings immediately. | ## Outputs diff --git a/scripts/export_clip_text_encoder.py b/scripts/export_clip_text_encoder.py deleted file mode 100644 index d6216cc..0000000 --- a/scripts/export_clip_text_encoder.py +++ /dev/null @@ -1,105 +0,0 @@ -# /// script -# requires-python = ">=3.10" -# dependencies = ["transformers>=4.30", "torch", "onnx"] -# /// -"""Export the CLIP ViT-B/32 text encoder to ONNX and save vocab files. - -Compatible with all YOLOv8-worldv2 sizes (s/m/l/x) — run once and reuse. - -The C++ node uses these three files to compute text embeddings at runtime: - clip_text_encoder.onnx input: input_ids (1, 77) int64 - output: text_embeds (1, 512) float32 (L2-normalize in C++) - clip_vocab.txt token → id mapping (tab-separated, one per line) - clip_merges.txt BPE merge rules (space-separated pairs, one per line) - -Usage: - uv run scripts/export_clip_text_encoder.py - uv run scripts/export_clip_text_encoder.py --output-dir models/ -""" - -import argparse -import sys -from pathlib import Path - -CLIP_MODEL = "openai/clip-vit-base-patch32" - - -def parse_args(): - p = argparse.ArgumentParser(description="Export CLIP ViT-B/32 text encoder to ONNX") - p.add_argument( - "--output-dir", "-o", - default="models", - help="Directory for output files. Default: models/", - ) - return p.parse_args() - - -def main(): - args = parse_args() - - try: - import torch - from transformers import CLIPTextModelWithProjection, CLIPTokenizer - except ImportError as e: - print(f"Error: {e}\nRun: uv run scripts/export_clip_text_encoder.py", file=sys.stderr) - sys.exit(1) - - out = Path(args.output_dir) - out.mkdir(parents=True, exist_ok=True) - - print(f"Loading {CLIP_MODEL} ...") - model = CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL) - tokenizer = CLIPTokenizer.from_pretrained(CLIP_MODEL) - model.eval() - - # --- Export ONNX text encoder --- - # Input: input_ids (1, 77) int64 — BPE token IDs, padded/truncated to 77 - # Output: text_embeds (1, 512) float32 — projected features (NOT normalized; C++ normalizes) - onnx_path = out / "clip_text_encoder.onnx" - print(f"Exporting ONNX → {onnx_path} ...") - dummy = torch.zeros(1, 77, dtype=torch.long) - torch.onnx.export( - model, - dummy, - str(onnx_path), - input_names=["input_ids"], - output_names=["text_embeds"], - dynamic_axes={ - "input_ids": {0: "batch_size"}, - "text_embeds": {0: "batch_size"}, - }, - opset_version=14, - ) - - # --- Save vocab as tab-separated text (token\tid) --- - # Avoids JSON parsing in C++. Tokens with special chars are escaped. - vocab_path = out / "clip_vocab.txt" - print(f"Saving vocab → {vocab_path} ...") - vocab = tokenizer.get_vocab() - with open(vocab_path, "w", encoding="utf-8") as f: - for token, idx in sorted(vocab.items(), key=lambda x: x[1]): - token_esc = (token - .replace("\\", "\\\\") - .replace("\t", "\\t") - .replace("\n", "\\n")) - f.write(f"{token_esc}\t{idx}\n") - - # --- Save BPE merge rules --- - merges_path = out / "clip_merges.txt" - print(f"Saving merges → {merges_path} ...") - with open(merges_path, "w", encoding="utf-8") as f: - for (a, b) in tokenizer.bpe_ranks.keys(): - f.write(f"{a} {b}\n") - - print(f"\nDone. Files written to {out.resolve()}/") - print(f" clip_text_encoder.onnx — ONNX model") - print(f" clip_vocab.txt — {len(vocab)} tokens") - print(f" clip_merges.txt — {len(tokenizer.bpe_ranks)} merge rules") - print(f"\nSet env vars:") - print(f" ZED_ONNX_TEXT_ENCODER_PATH={out.resolve()}/clip_text_encoder.onnx") - print(f" ZED_ONNX_CLIP_VOCAB={out.resolve()}/clip_vocab.txt") - print(f" ZED_ONNX_CLIP_MERGES={out.resolve()}/clip_merges.txt") - - -if __name__ == "__main__": - main() diff --git a/scripts/export_yoloworld.py b/scripts/export_yoloworld.py deleted file mode 100644 index df28594..0000000 --- a/scripts/export_yoloworld.py +++ /dev/null @@ -1,124 +0,0 @@ -# /// script -# requires-python = ">=3.10" -# dependencies = ["ultralytics>=8.0"] -# /// -"""Export a YOLOv8-worldv2 model to ONNX with N open-vocabulary class slots. - -The exported model has two inputs: - images (1, 3, imgsz, imgsz) float32 - text_feats (1, N, 512) float32 ← CLIP embeddings, set at runtime - -N is fixed per export. Use export_clip_text_encoder.py to generate the text -encoder and vocab files used to compute embeddings at runtime in the C++ node. - -Usage: - uv run scripts/export_yoloworld.py - uv run scripts/export_yoloworld.py --model m --num-classes 20 - uv run scripts/export_yoloworld.py --model l --num-classes 5 --output-dir models/ -""" - -import argparse -import shutil -import sys -from pathlib import Path - -MODELS = { - "s": "yolov8s-worldv2", - "m": "yolov8m-worldv2", - "l": "yolov8l-worldv2", - "x": "yolov8x-worldv2", -} - - -def parse_args(): - p = argparse.ArgumentParser( - description="Export YOLOv8-worldv2 to ONNX with open-vocabulary text inputs" - ) - p.add_argument( - "--model", "-m", - choices=list(MODELS), - default="s", - help="Model size: s (~14 MB), m (~30 MB), l (~50 MB), x (~100 MB). Default: s", - ) - p.add_argument( - "--num-classes", "-n", - type=int, - default=10, - help="Number of concurrent class slots (must match number of labels at runtime). Default: 10", - ) - p.add_argument( - "--imgsz", - type=int, - default=640, - help="Input image size in pixels (square). Default: 640", - ) - p.add_argument( - "--output-dir", "-o", - default=".", - help="Output directory for the .onnx file. Default: current directory", - ) - return p.parse_args() - - -def verify_inputs(onnx_path: Path, expected_inputs: int): - try: - import onnx - model = onnx.load(str(onnx_path)) - actual = len(model.graph.input) - if actual != expected_inputs: - print(f"\nWarning: expected {expected_inputs} inputs, got {actual}.") - print("The visual model may have baked text embeddings (ultralytics version dependent).") - print("In that case, use a fixed vocabulary instead of the text encoder.") - else: - inputs = [(i.name, [d.dim_value for d in i.type.tensor_type.shape.dim]) - for i in model.graph.input] - print(f" Inputs: {inputs}") - except ImportError: - print(" (install 'onnx' to verify model inputs)") - - -def main(): - args = parse_args() - - try: - from ultralytics import YOLO - except ImportError: - print("Error: ultralytics not installed. Run: uv run scripts/export_yoloworld.py", file=sys.stderr) - sys.exit(1) - - model_name = MODELS[args.model] - out_dir = Path(args.output_dir) - out_dir.mkdir(parents=True, exist_ok=True) - out_name = f"{model_name}_n{args.num_classes}_i{args.imgsz}.onnx" - dst = out_dir / out_name - - print(f"Model: {model_name}.pt") - print(f"Class slots: {args.num_classes}") - print(f"Input size: {args.imgsz}") - print(f"Output: {dst}") - print() - - model = YOLO(f"{model_name}.pt") - # Set N dummy class slots — actual label meanings are provided at runtime via text_feats - model.set_classes(["object"] * args.num_classes) - exported = Path(model.export(format="onnx", imgsz=args.imgsz)) - - if exported.resolve() != dst.resolve(): - shutil.move(str(exported), str(dst)) - - print(f"\nVerifying exported model...") - verify_inputs(dst, expected_inputs=2) - - print(f"\nExported: {dst.resolve()}") - print(f"\nNext step — export the CLIP text encoder (once, reused for all model sizes):") - print(f" uv run scripts/export_clip_text_encoder.py --output-dir models/") - print(f"\nRun with:") - print(f" ZED_ENGINE=open_vocab \\") - print(f" ZED_ONNX_MODEL_PATH={dst.resolve()} \\") - print(f" ZED_ONNX_TEXT_ENCODER_PATH=models/clip_text_encoder.onnx \\") - print(f' ZED_ONNX_LABELS="person,car,bag,... (exactly {args.num_classes} labels)" \\') - print(f" dora start dataflow.yml") - - -if __name__ == "__main__": - main() diff --git a/scripts/install_linux_arm.sh b/scripts/install_linux_arm.sh index 8faa713..f6d6afd 100755 --- a/scripts/install_linux_arm.sh +++ b/scripts/install_linux_arm.sh @@ -11,8 +11,8 @@ # JetPack 7.1 (L4T 38.4) # # ONNX Runtime: Microsoft does not publish official GPU binaries for ARM64. -# The native engine (ZED SDK built-in TensorRT) works without ONNX Runtime. -# To use the open_vocab engine, build ONNX Runtime from source: +# ONNX Runtime is still needed to configure the build; only the yolo_onnx +# engine uses it at runtime. Build it from source: # https://onnxruntime.ai/docs/build/eps.html#tensorrt set -euo pipefail @@ -66,8 +66,8 @@ fi # --- ONNX Runtime --- echo "" echo ">>> ONNX Runtime (ARM64 GPU): no official pre-built binary available." -echo " The native ZED engine works without ONNX Runtime." -echo " To use the open_vocab engine, build from source:" +echo " Needed to configure the build; only yolo_onnx uses it at runtime." +echo " Build from source:" echo " https://onnxruntime.ai/docs/build/eps.html#tensorrt" echo "" diff --git a/scripts/install_linux_x64.sh b/scripts/install_linux_x64.sh index cd10f5d..ff05d0a 100755 --- a/scripts/install_linux_x64.sh +++ b/scripts/install_linux_x64.sh @@ -35,7 +35,7 @@ else fi # --- ONNX Runtime (C++ GPU, installs to /usr/local) --- -if [ -f /usr/local/lib64/libonnxruntime.so ] && [ -d /usr/local/include/onnxruntime ]; then +if [ -f /usr/local/lib64/libonnxruntime.so ] && [ -d /usr/local/lib/cmake/onnxruntime ]; then echo ">>> ONNX Runtime already installed — skipping." else echo ">>> Downloading ONNX Runtime ${ORT_VERSION} GPU..." @@ -49,6 +49,15 @@ else sudo cp -r "${TMP_DIR}/onnxruntime-linux-x64-gpu-${ORT_VERSION}/include/"* /usr/local/include/onnxruntime/ sudo mkdir -p /usr/local/lib64 sudo cp -r "${TMP_DIR}/onnxruntime-linux-x64-gpu-${ORT_VERSION}/lib/"* /usr/local/lib64/ + # onnxruntimeTargets.cmake resolves the library as /lib64, but Ubuntu + # is multiarch: CMake does not search lib64 for packages and ld.so does not + # search /usr/local/lib64 at all. Move only the package files to lib/cmake + # and put lib64 on the linker path. + sudo mkdir -p /usr/local/lib/cmake + sudo rm -rf /usr/local/lib/cmake/onnxruntime + sudo mv /usr/local/lib64/cmake/onnxruntime /usr/local/lib/cmake/ + sudo rmdir /usr/local/lib64/cmake + echo /usr/local/lib64 | sudo tee /etc/ld.so.conf.d/onnxruntime.conf > /dev/null sudo ldconfig fi diff --git a/src/engines/engine.h b/src/engines/engine.h index 50d5d4b..eb3f07a 100644 --- a/src/engines/engine.h +++ b/src/engines/engine.h @@ -20,7 +20,6 @@ public: virtual void enable(sl::Camera& camera) = 0; virtual void detect(sl::Camera& camera) = 0; - virtual void set_labels(const std::string& /*labels*/) {} virtual const std::vector& objects() const = 0; virtual std::string label_names() const = 0; }; diff --git a/src/engines/yolo_helpers.h b/src/engines/yolo_helpers.h index 85eb673..1f4645e 100644 --- a/src/engines/yolo_helpers.h +++ b/src/engines/yolo_helpers.h @@ -5,8 +5,6 @@ #include #include -#include -#include #include namespace engines::detail { @@ -135,19 +133,4 @@ inline void backproject(float u, float v, float z, Y = (v - cy) * z / fy; } -// --- Label parsing --- - -inline std::vector split_labels(const std::string& s) { - std::vector out; - std::stringstream ss(s); - std::string tok; - while (std::getline(ss, tok, ',')) { - size_t l = tok.find_first_not_of(" \t"); - size_t r = tok.find_last_not_of(" \t"); - if (l != std::string::npos) - out.push_back(tok.substr(l, r - l + 1)); - } - return out; -} - } // namespace engines::detail diff --git a/src/engines/yolo_onnx.cc b/src/engines/yolo_onnx.cc index 8e96a62..f170edd 100644 --- a/src/engines/yolo_onnx.cc +++ b/src/engines/yolo_onnx.cc @@ -216,7 +216,7 @@ void YoloOnnx::enable(sl::Camera& /* camera */) { size_t num_inputs = impl_->session->GetInputCount(); if (num_inputs >= 2) { std::cerr << "YoloOnnx: model has " << num_inputs << " inputs — " - "this looks like a YOLO-World model. Use ZED_ENGINE=yoloworld_onnx instead." + "this engine only supports single-input models." << std::endl; } @@ -358,10 +358,6 @@ void YoloOnnx::detect(const cv::Mat& image_bgr) { } } -void YoloOnnx::set_labels(const std::string& labels) { - config_.labels = labels; -} - const std::vector& YoloOnnx::objects() const { return detected_; } diff --git a/src/engines/yolo_onnx.h b/src/engines/yolo_onnx.h index 524ebb7..8dec6b4 100644 --- a/src/engines/yolo_onnx.h +++ b/src/engines/yolo_onnx.h @@ -35,7 +35,6 @@ public: void enable(sl::Camera& camera) override; void detect(sl::Camera& camera) override; void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion - void set_labels(const std::string& labels) override; const std::vector& objects() const override; std::string label_names() const override; diff --git a/src/engines/yoloworld_onnx.cc b/src/engines/yoloworld_onnx.cc deleted file mode 100644 index fa83461..0000000 --- a/src/engines/yoloworld_onnx.cc +++ /dev/null @@ -1,415 +0,0 @@ -#include "engines/yoloworld_onnx.h" -#include "engines/yolo_helpers.h" -#include "util/clip_tokenizer.h" -#include "util/env.h" - -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace engines { - -using namespace detail; - -// --- Impl --- - -struct YoloWorldOnnx::Impl { - // Visual model - Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "yoloworld_onnx"}; - Ort::SessionOptions session_options; - std::unique_ptr session; - Ort::AllocatorWithDefaultOptions allocator; - - std::string input_name; // image input - std::string text_input_name; // text_feats input - std::string output_name; - int num_classes = 0; - - // Text encoder - Ort::SessionOptions text_session_options; - std::unique_ptr text_session; - std::string text_enc_input_name; // "input_ids" - std::string text_enc_output_name; // "text_embeds" - - // Tokenizer + embedding cache - util::ClipTokenizer tokenizer; - std::vector cached_text_feats; // (num_classes * 512) flat, row-major - std::string cached_labels; - - std::vector run_inference(const YoloWorldOnnxConfig& config, - const cv::Mat& image_bgr); -}; - -// --- Impl::run_inference --- - -std::vector YoloWorldOnnx::Impl::run_inference( - const YoloWorldOnnxConfig& config, const cv::Mat& image_bgr) -{ - if (!session) return {}; - if (cached_text_feats.empty()) return {}; - - int orig_w = image_bgr.cols; - int orig_h = image_bgr.rows; - - cv::Mat rgb; - cv::cvtColor(image_bgr, rgb, cv::COLOR_BGR2RGB); - - float scale, pad_x, pad_y; - cv::Mat lb = letterbox(rgb, config.input_size, scale, pad_x, pad_y); - - cv::Mat lb_float; - lb.convertTo(lb_float, CV_32F, 1.0f / 255.0f); - - int sz = config.input_size; - std::vector input_data(3 * sz * sz); - for (int c = 0; c < 3; c++) { - for (int h = 0; h < sz; h++) { - for (int w = 0; w < sz; w++) { - input_data[c * sz * sz + h * sz + w] = - lb_float.at(h, w)[c]; - } - } - } - - auto mem_info = Ort::MemoryInfo::CreateCpu( - OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault); - - std::vector image_shape{1, 3, sz, sz}; - Ort::Value image_tensor = Ort::Value::CreateTensor( - mem_info, input_data.data(), input_data.size(), - image_shape.data(), image_shape.size()); - - std::vector text_shape{1, num_classes, 512}; - Ort::Value text_tensor = Ort::Value::CreateTensor( - mem_info, - cached_text_feats.data(), - cached_text_feats.size(), - text_shape.data(), text_shape.size()); - - const char* input_names[2] = {input_name.c_str(), text_input_name.c_str()}; - Ort::Value input_tensors[2] = {std::move(image_tensor), std::move(text_tensor)}; - const char* output_name_cstr = output_name.c_str(); - - std::vector outputs; - try { - outputs = session->Run( - Ort::RunOptions{nullptr}, - input_names, input_tensors, 2, - &output_name_cstr, 1); - } catch (const Ort::Exception& e) { - std::cerr << "YoloWorldOnnx: inference failed: " << e.what() << std::endl; - return {}; - } - - // Output: (1, 4+num_classes, num_anchors) — NC_FIRST only - auto out_shape = outputs[0].GetTensorTypeAndShapeInfo().GetShape(); - if (out_shape.size() < 3) return {}; - - int num_anchors = static_cast(out_shape[2]); - int nc = num_classes; - if (nc <= 0) return {}; - - const float* out = outputs[0].GetTensorData(); - - std::vector candidates; - for (int a = 0; a < num_anchors; a++) { - float cx_lb = out[0 * num_anchors + a]; - float cy_lb = out[1 * num_anchors + a]; - float w_lb = out[2 * num_anchors + a]; - float h_lb = out[3 * num_anchors + a]; - - float best_score = 0.0f; - int best_class = 0; - for (int c = 0; c < nc; c++) { - float s = out[(4 + c) * num_anchors + a]; - if (s > best_score) { best_score = s; best_class = c; } - } - if (best_score < config.confidence_threshold) continue; - - float x1_lb = cx_lb - w_lb / 2.0f; - float y1_lb = cy_lb - h_lb / 2.0f; - float x2_lb = cx_lb + w_lb / 2.0f; - float y2_lb = cy_lb + h_lb / 2.0f; - - float x1 = std::max(0.0f, std::min((x1_lb - pad_x) / scale, static_cast(orig_w - 1))); - float y1 = std::max(0.0f, std::min((y1_lb - pad_y) / scale, static_cast(orig_h - 1))); - float x2 = std::max(0.0f, std::min((x2_lb - pad_x) / scale, static_cast(orig_w - 1))); - float y2 = std::max(0.0f, std::min((y2_lb - pad_y) / scale, static_cast(orig_h - 1))); - - candidates.push_back({x1, y1, x2, y2, best_score, best_class}); - } - - return nms(std::move(candidates), config.nms_threshold); -} - -// --- Config --- - -YoloWorldOnnxConfig YoloWorldOnnxConfig::from_env() { - YoloWorldOnnxConfig c; - c.model_path = util::get_env_str("ZED_ONNX_MODEL_PATH", c.model_path); - c.confidence_threshold = util::get_env_float("ZED_ONNX_CONFIDENCE", c.confidence_threshold); - c.nms_threshold = util::get_env_float("ZED_ONNX_NMS_THRESHOLD", c.nms_threshold); - c.input_size = util::get_env_int("ZED_ONNX_INPUT_SIZE", c.input_size); - c.labels = util::get_env_str("ZED_ONNX_LABELS", c.labels); - c.text_encoder_path = util::get_env_str("ZED_ONNX_TEXT_ENCODER_PATH", c.text_encoder_path); - c.clip_vocab_path = util::get_env_str("ZED_ONNX_CLIP_VOCAB", c.clip_vocab_path); - c.clip_merges_path = util::get_env_str("ZED_ONNX_CLIP_MERGES", c.clip_merges_path); - return c; -} - -// --- YoloWorldOnnx --- - -YoloWorldOnnx::YoloWorldOnnx(const YoloWorldOnnxConfig& config) - : config_(config), impl_(std::make_unique()) {} - -YoloWorldOnnx::~YoloWorldOnnx() = default; - -void YoloWorldOnnx::enable(sl::Camera& /* camera */) { - if (config_.model_path.empty()) { - std::cerr << "YoloWorldOnnx: ZED_ONNX_MODEL_PATH is not set." << std::endl; - return; - } - - // --- Load visual model --- - OrtCUDAProviderOptions cuda_opts{}; - cuda_opts.device_id = 0; - try { - impl_->session_options.AppendExecutionProvider_CUDA(cuda_opts); - } catch (const Ort::Exception& e) { - std::cerr << "YoloWorldOnnx: CUDA unavailable (" << e.what() << "), using CPU." << std::endl; - } - impl_->session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); - - std::cout << "YoloWorldOnnx: loading visual model (" << config_.model_path << ")..." << std::endl; - try { - impl_->session = std::make_unique( - impl_->env, config_.model_path.c_str(), impl_->session_options); - } catch (const Ort::Exception& e) { - std::cerr << "YoloWorldOnnx: failed to load visual model: " << e.what() << std::endl; - return; - } - - size_t num_inputs = impl_->session->GetInputCount(); - if (num_inputs < 2) { - std::cerr << "YoloWorldOnnx: model has only " << num_inputs << " input(s) — " - "expected a two-input YOLO-World model. Use ZED_ENGINE=yolo_onnx for single-input models." - << std::endl; - return; - } - - impl_->input_name = impl_->session->GetInputNameAllocated(0, impl_->allocator).get(); - impl_->text_input_name = impl_->session->GetInputNameAllocated(1, impl_->allocator).get(); - impl_->output_name = impl_->session->GetOutputNameAllocated(0, impl_->allocator).get(); - - auto out_info = impl_->session->GetOutputTypeInfo(0).GetTensorTypeAndShapeInfo(); - auto shape = out_info.GetShape(); - if (shape.size() == 3 && shape[1] > 4) { - impl_->num_classes = static_cast(shape[1]) - 4; - } - - std::cout << "YoloWorldOnnx: visual model loaded." - << " Classes=" << impl_->num_classes - << std::endl; - - // --- Load text encoder + tokenizer --- - if (config_.text_encoder_path.empty()) { - std::cerr << "YoloWorldOnnx: ZED_ONNX_TEXT_ENCODER_PATH is not set." << std::endl; - return; - } - - impl_->text_session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); - - std::cout << "YoloWorldOnnx: loading text encoder (" << config_.text_encoder_path << ")..." << std::endl; - try { - impl_->text_session = std::make_unique( - impl_->env, config_.text_encoder_path.c_str(), impl_->text_session_options); - } catch (const Ort::Exception& e) { - std::cerr << "YoloWorldOnnx: failed to load text encoder: " << e.what() << std::endl; - return; - } - - impl_->text_enc_input_name = - impl_->text_session->GetInputNameAllocated(0, impl_->allocator).get(); - impl_->text_enc_output_name = - impl_->text_session->GetOutputNameAllocated(0, impl_->allocator).get(); - - if (!config_.clip_vocab_path.empty() && !config_.clip_merges_path.empty()) { - try { - impl_->tokenizer.load(config_.clip_vocab_path, config_.clip_merges_path); - std::cout << "YoloWorldOnnx: tokenizer loaded." << std::endl; - } catch (const std::exception& e) { - std::cerr << "YoloWorldOnnx: failed to load tokenizer: " << e.what() << std::endl; - return; - } - } else { - std::cerr << "YoloWorldOnnx: ZED_ONNX_CLIP_VOCAB / ZED_ONNX_CLIP_MERGES not set." << std::endl; - return; - } - - if (!config_.labels.empty()) { - encode_labels(config_.labels); - } -} - -void YoloWorldOnnx::encode_labels(const std::string& labels_str) { - if (!impl_->text_session || !impl_->tokenizer.is_loaded()) return; - - auto label_vec = split_labels(labels_str); - int N = impl_->num_classes; - impl_->cached_text_feats.assign(static_cast(N) * 512, 0.0f); - - auto mem_info = Ort::MemoryInfo::CreateCpu( - OrtAllocatorType::OrtArenaAllocator, OrtMemTypeDefault); - - int encode_n = std::min(static_cast(label_vec.size()), N); - for (int i = 0; i < encode_n; i++) { - auto tokens = impl_->tokenizer.tokenize(label_vec[i]); - - std::vector input_shape{1, util::ClipTokenizer::SEQ_LEN}; - Ort::Value token_tensor = Ort::Value::CreateTensor( - mem_info, tokens.data(), tokens.size(), - input_shape.data(), input_shape.size()); - - const char* in = impl_->text_enc_input_name.c_str(); - const char* out = impl_->text_enc_output_name.c_str(); - std::vector outputs; - try { - outputs = impl_->text_session->Run( - Ort::RunOptions{nullptr}, &in, &token_tensor, 1, &out, 1); - } catch (const Ort::Exception& e) { - std::cerr << "YoloWorldOnnx: text encoder failed for \"" << label_vec[i] - << "\": " << e.what() << std::endl; - continue; - } - - const float* emb = outputs[0].GetTensorData(); - - float norm = 0.0f; - for (int j = 0; j < 512; j++) norm += emb[j] * emb[j]; - norm = std::sqrt(norm); - float inv_norm = (norm > 1e-8f) ? 1.0f / norm : 0.0f; - for (int j = 0; j < 512; j++) - impl_->cached_text_feats[i * 512 + j] = emb[j] * inv_norm; - } - - impl_->cached_labels = labels_str; - std::cout << "YoloWorldOnnx: encoded " << encode_n << " label(s): " << labels_str << std::endl; -} - -void YoloWorldOnnx::detect(sl::Camera& camera) { - detected_.clear(); - if (!impl_->session) return; - - // Re-encode if labels changed (on-the-fly vocabulary switching) - const char* env_labels = std::getenv("ZED_ONNX_LABELS"); - std::string current_labels = env_labels ? env_labels : config_.labels; - if (current_labels != impl_->cached_labels) { - encode_labels(current_labels); - } - if (impl_->cached_text_feats.empty()) return; - - sl::Mat img_mat; - camera.retrieveImage(img_mat, sl::VIEW::LEFT); - - sl::Mat pc_mat; - camera.retrieveMeasure(pc_mat, sl::MEASURE::XYZRGBA); - - sl::CameraInformation info = camera.getCameraInformation(); - auto calib = info.camera_configuration.calibration_parameters.left_cam; - float fx = calib.fx, fy = calib.fy, cx = calib.cx, cy = calib.cy; - - int orig_w = static_cast(img_mat.getWidth()); - int orig_h = static_cast(img_mat.getHeight()); - - cv::Mat rgba(orig_h, orig_w, CV_8UC4, img_mat.getPtr()); - cv::Mat bgr; - cv::cvtColor(rgba, bgr, cv::COLOR_RGBA2BGR); - - auto kept = impl_->run_inference(config_, bgr); - - for (const auto& det : kept) { - FusionResult fusion = fuse_bbox(pc_mat, - static_cast(det.x1), static_cast(det.y1), - static_cast(det.x2), static_cast(det.y2)); - - DetectedObject obj{}; - obj.label_id = det.class_id; - obj.confidence = det.score * 100.0f; - - if (fusion.valid) { - obj.position[0] = fusion.cx; - obj.position[1] = fusion.cy; - obj.position[2] = fusion.cz; - } - - obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1; - obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1; - obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2; - obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2; - - if (fusion.valid) { - float corners_u[4] = {det.x1, det.x2, det.x2, det.x1}; - float corners_v[4] = {det.y1, det.y1, det.y2, det.y2}; - float depths[2] = {fusion.depth_near, fusion.depth_far}; - for (int face = 0; face < 2; face++) { - for (int c = 0; c < 4; c++) { - float X, Y, Z; - backproject(corners_u[c], corners_v[c], depths[face], - fx, fy, cx, cy, X, Y, Z); - int idx = (face * 4 + c) * 3; - obj.bounding_box_3d[idx + 0] = X; - obj.bounding_box_3d[idx + 1] = Y; - obj.bounding_box_3d[idx + 2] = Z; - } - } - } - - detected_.push_back(obj); - } -} - -void YoloWorldOnnx::detect(const cv::Mat& image_bgr) { - detected_.clear(); - if (!impl_->session) return; - - const char* env_labels = std::getenv("ZED_ONNX_LABELS"); - std::string current_labels = env_labels ? env_labels : config_.labels; - if (current_labels != impl_->cached_labels) { - encode_labels(current_labels); - } - if (impl_->cached_text_feats.empty()) return; - - auto kept = impl_->run_inference(config_, image_bgr); - - for (const auto& det : kept) { - DetectedObject obj{}; - obj.label_id = det.class_id; - obj.confidence = det.score * 100.0f; - obj.bounding_box_2d[0] = det.x1; obj.bounding_box_2d[1] = det.y1; - obj.bounding_box_2d[2] = det.x2; obj.bounding_box_2d[3] = det.y1; - obj.bounding_box_2d[4] = det.x2; obj.bounding_box_2d[5] = det.y2; - obj.bounding_box_2d[6] = det.x1; obj.bounding_box_2d[7] = det.y2; - detected_.push_back(obj); - } -} - -void YoloWorldOnnx::set_labels(const std::string& labels) { - config_.labels = labels; - encode_labels(labels); -} - -const std::vector& YoloWorldOnnx::objects() const { - return detected_; -} - -std::string YoloWorldOnnx::label_names() const { - return impl_->cached_labels.empty() ? config_.labels : impl_->cached_labels; -} - -} // namespace engines diff --git a/src/engines/yoloworld_onnx.h b/src/engines/yoloworld_onnx.h deleted file mode 100644 index f18e23b..0000000 --- a/src/engines/yoloworld_onnx.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include "engines/engine.h" - -#include -#include -#include - -namespace engines { - -struct YoloWorldOnnxConfig { - std::string model_path; // ZED_ONNX_MODEL_PATH (required) - float confidence_threshold = 0.25f; // ZED_ONNX_CONFIDENCE - float nms_threshold = 0.45f; // ZED_ONNX_NMS_THRESHOLD - int input_size = 640; // ZED_ONNX_INPUT_SIZE - std::string labels; // ZED_ONNX_LABELS (comma-separated) - - // CLIP text encoder (required) - std::string text_encoder_path; // ZED_ONNX_TEXT_ENCODER_PATH - std::string clip_vocab_path; // ZED_ONNX_CLIP_VOCAB - std::string clip_merges_path; // ZED_ONNX_CLIP_MERGES - - static YoloWorldOnnxConfig from_env(); -}; - -class YoloWorldOnnx : public Engine { -public: - explicit YoloWorldOnnx(const YoloWorldOnnxConfig& config); - ~YoloWorldOnnx() override; - - void enable(sl::Camera& camera) override; - void detect(sl::Camera& camera) override; - void detect(const cv::Mat& image_bgr); // image-only; no 3D fusion - void set_labels(const std::string& labels) override; - const std::vector& objects() const override; - std::string label_names() const override; - -private: - YoloWorldOnnxConfig config_; - struct Impl; - std::unique_ptr impl_; - std::vector detected_; - - void encode_labels(const std::string& labels_str); -}; - -} // namespace engines diff --git a/src/main.cc b/src/main.cc index 505829e..c06c814 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,5 @@ #include "engines/engine.h" #include "engines/yolo_onnx.h" -#include "engines/yoloworld_onnx.h" #include "engines/zed_native.h" #include "util/env.h" #include "zed/camera.h" @@ -78,9 +77,6 @@ int main() { if (engine_name == "yolo_onnx" || engine_name == "onnx") { auto cfg = engines::YoloOnnxConfig::from_env(); detector = std::make_unique(cfg); - } else if (engine_name == "yoloworld_onnx" || engine_name == "open_vocab") { - auto cfg = engines::YoloWorldOnnxConfig::from_env(); - detector = std::make_unique(cfg); } else { auto cfg = engines::DetectionConfig::from_env(); detector = std::make_unique(cfg); @@ -105,20 +101,6 @@ int main() { continue; } - auto input = event_as_input(std::move(event)); - std::string input_id = std::string(input.id); - - if (input_id == "update_vocabulary") { - if (!input.data.empty()) { - std::string labels( - reinterpret_cast(input.data.data()), - input.data.size()); - detector->set_labels(labels); - std::cout << "Vocabulary updated: " << labels << std::endl; - } - continue; - } - if (!camera.grab()) { std::cerr << "Grab failed; attempting to reconnect..." << std::endl; camera.close(); diff --git a/src/test_yoloworld_onnx.cc b/src/test_yoloworld_onnx.cc deleted file mode 100644 index 45f3f7a..0000000 --- a/src/test_yoloworld_onnx.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Standalone test for the YoloWorldOnnx (open-vocabulary) detection engine. -// No dora framework, no live ZED camera — loads an image from disk. -// -// Usage: -// bin/test_yoloworld_onnx [image_path] -// image_path default: test/assets/yoloworld.png -// -// Required env vars: -// ZED_ONNX_MODEL_PATH path to YOLO-World ONNX model (two-input) -// ZED_ONNX_TEXT_ENCODER_PATH path to clip_text_encoder.onnx -// ZED_ONNX_CLIP_VOCAB path to clip_vocab.txt -// ZED_ONNX_CLIP_MERGES path to clip_merges.txt -// ZED_ONNX_LABELS comma-separated class labels -// -// Optional env vars: -// ZED_ONNX_CONFIDENCE detection threshold (default 0.25) -// ZED_ONNX_NMS_THRESHOLD NMS IoU threshold (default 0.45) -// ZED_ONNX_INPUT_SIZE model input size (default 640) - -#include "engines/yoloworld_onnx.h" -#include "util/env.h" - -#include -#include - -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; - -namespace { - -std::vector parse_csv(const std::string& s) { - std::vector out; - std::istringstream ss(s); - std::string tok; - while (std::getline(ss, tok, ',')) { - size_t l = tok.find_first_not_of(" \t"); - size_t r = tok.find_last_not_of(" \t"); - if (l != std::string::npos) - out.push_back(tok.substr(l, r - l + 1)); - } - return out; -} - -void draw_detections(cv::Mat& img, - const std::vector& objects, - const std::vector& labels) { - for (const auto& obj : objects) { - int x1 = static_cast(obj.bounding_box_2d[0]); - int y1 = static_cast(obj.bounding_box_2d[1]); - int x2 = static_cast(obj.bounding_box_2d[4]); - int y2 = static_cast(obj.bounding_box_2d[5]); - - std::string name = (obj.label_id >= 0 && obj.label_id < static_cast(labels.size())) - ? labels[obj.label_id] : std::to_string(obj.label_id); - - char buf[64]; - std::snprintf(buf, sizeof(buf), " %.2f", obj.confidence / 100.0f); - - cv::rectangle(img, {x1, y1}, {x2, y2}, {0, 200, 0}, 2); - cv::putText(img, name + buf, {x1, std::max(y1 - 6, 14)}, - cv::FONT_HERSHEY_SIMPLEX, 0.55, {0, 200, 0}, 2, cv::LINE_AA); - } -} - -} // namespace - -int main(int argc, char* argv[]) { - std::string image_path = (argc > 1) ? argv[1] : "test/assets/yoloworld.png"; - - auto config = engines::YoloWorldOnnxConfig::from_env(); - - sl::Camera dummy_cam; - engines::YoloWorldOnnx engine(config); - engine.enable(dummy_cam); - - cv::Mat img = cv::imread(image_path); - if (img.empty()) { - std::cerr << "Error: cannot read image: " << image_path << std::endl; - return 1; - } - - engine.detect(img); - const auto& objects = engine.objects(); - auto labels = parse_csv(engine.label_names()); - - std::cout << "[" << image_path << "] " << objects.size() << " detection(s):" << std::endl; - for (const auto& obj : objects) { - std::string name = (obj.label_id >= 0 && obj.label_id < static_cast(labels.size())) - ? labels[obj.label_id] : std::to_string(obj.label_id); - std::cout << " " << name - << " conf=" << obj.confidence / 100.0f - << " bbox=[" - << obj.bounding_box_2d[0] << "," << obj.bounding_box_2d[1] - << " -> " - << obj.bounding_box_2d[4] << "," << obj.bounding_box_2d[5] - << "]" << std::endl; - } - - cv::Mat annotated = img.clone(); - draw_detections(annotated, objects, labels); - - fs::path stem = fs::path(image_path).stem(); - fs::path parent = fs::path(image_path).parent_path(); - std::string result_path = (parent / (stem.string() + "_result.png")).string(); - cv::imwrite(result_path, annotated); - std::cout << "Saved: " << result_path << std::endl; - - return 0; -} diff --git a/src/util/clip_tokenizer.h b/src/util/clip_tokenizer.h deleted file mode 100644 index 9834a2b..0000000 --- a/src/util/clip_tokenizer.h +++ /dev/null @@ -1,226 +0,0 @@ -#pragma once -// Minimal CLIP ViT-B/32 BPE tokenizer — header-only, C++17, no external deps. -// -// Loads clip_vocab.txt (token\tid, tab-separated) and clip_merges.txt -// (a b, space-separated BPE pairs) produced by export_clip_text_encoder.py. -// Returns 77 int64 token IDs compatible with clip_text_encoder.onnx. - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace util { - -class ClipTokenizer { -public: - static constexpr int SEQ_LEN = 77; - static constexpr int64_t BOS = 49406; - static constexpr int64_t EOS = 49407; - static constexpr int64_t PAD = 0; - - void load(const std::string& vocab_path, const std::string& merges_path) { - load_vocab(vocab_path); - load_merges(merges_path); - init_byte_encoder(); - } - - bool is_loaded() const { return !vocab_.empty(); } - - // Tokenize a label string. Returns exactly SEQ_LEN int64 token IDs. - std::vector tokenize(const std::string& text) const { - std::vector ids; - ids.reserve(SEQ_LEN); - ids.push_back(BOS); - - std::string lower = text; - std::transform(lower.begin(), lower.end(), lower.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - - // Split on whitespace — sufficient for typical detection label names - std::istringstream ss(lower); - std::string word; - while (ss >> word) { - if (static_cast(ids.size()) >= SEQ_LEN - 1) break; - - // Byte-encode the word via the GPT-2 byte encoder - std::string encoded; - encoded.reserve(word.size() * 3); - for (unsigned char c : word) { - auto it = byte_encoder_.find(c); - if (it != byte_encoder_.end()) - encoded += utf8_of(it->second); - } - if (encoded.empty()) continue; - - // Apply BPE - auto tokens = bpe_encode(encoded); - for (const auto& tok : tokens) { - if (static_cast(ids.size()) >= SEQ_LEN - 1) break; - auto vit = vocab_.find(tok); - if (vit != vocab_.end()) - ids.push_back(static_cast(vit->second)); - } - } - - ids.push_back(EOS); - ids.resize(SEQ_LEN, PAD); - return ids; - } - -private: - std::unordered_map vocab_; // token string → id - std::unordered_map merge_ranks_; // "a b" → rank - std::unordered_map byte_encoder_; // byte → unicode codepoint - - // ----------------------------------------------------------------- - // Byte encoder — matches Python's bytes_to_unicode() in OpenAI CLIP - // ----------------------------------------------------------------- - void init_byte_encoder() { - // Printable ASCII + Latin-1 supplement ranges that map to themselves - std::vector bs; - for (int i = '!'; i <= '~'; i++) bs.push_back(i); - for (int i = 0xA1; i <= 0xAC; i++) bs.push_back(i); - for (int i = 0xAE; i <= 0xFF; i++) bs.push_back(i); - - std::vector cs; - cs.reserve(bs.size()); - for (int b : bs) cs.push_back(static_cast(b)); - - // Remaining bytes map to codepoints starting at U+0100 - int n = 0; - for (int b = 0; b < 256; b++) { - if (std::find(bs.begin(), bs.end(), b) == bs.end()) { - bs.push_back(b); - cs.push_back(256u + static_cast(n)); - ++n; - } - } - - byte_encoder_.clear(); - for (size_t i = 0; i < bs.size(); i++) - byte_encoder_[static_cast(bs[i])] = cs[i]; - } - - // Encode a unicode codepoint as a UTF-8 std::string - static std::string utf8_of(uint32_t cp) { - std::string s; - if (cp < 0x80u) { - s += static_cast(cp); - } else if (cp < 0x800u) { - s += static_cast(0xC0u | (cp >> 6)); - s += static_cast(0x80u | (cp & 0x3Fu)); - } else if (cp < 0x10000u) { - s += static_cast(0xE0u | (cp >> 12)); - s += static_cast(0x80u | ((cp >> 6) & 0x3Fu)); - s += static_cast(0x80u | (cp & 0x3Fu)); - } else { - s += static_cast(0xF0u | (cp >> 18)); - s += static_cast(0x80u | ((cp >> 12) & 0x3Fu)); - s += static_cast(0x80u | ((cp >> 6) & 0x3Fu)); - s += static_cast(0x80u | (cp & 0x3Fu)); - } - return s; - } - - // ----------------------------------------------------------------- - // BPE - // ----------------------------------------------------------------- - - // Split a UTF-8 string into individual Unicode character substrings - static std::vector split_utf8_chars(const std::string& s) { - std::vector chars; - for (size_t i = 0; i < s.size(); ) { - unsigned char c = static_cast(s[i]); - int len = (c < 0x80) ? 1 : (c < 0xE0) ? 2 : (c < 0xF0) ? 3 : 4; - chars.push_back(s.substr(i, len)); - i += len; - } - return chars; - } - - // BPE-encode a single byte-encoded word, returning the token strings - std::vector bpe_encode(const std::string& word_str) const { - std::vector word = split_utf8_chars(word_str); - if (word.size() <= 1) return word; - - while (true) { - // Find the pair with the lowest merge rank - int best_rank = INT_MAX; - int best_i = -1; - for (int i = 0; i + 1 < static_cast(word.size()); i++) { - auto it = merge_ranks_.find(word[i] + " " + word[i + 1]); - if (it != merge_ranks_.end() && it->second < best_rank) { - best_rank = it->second; - best_i = i; - } - } - if (best_i < 0) break; - - // Apply merge at best_i - std::string merged = word[best_i] + word[best_i + 1]; - word.erase(word.begin() + best_i, word.begin() + best_i + 2); - word.insert(word.begin() + best_i, std::move(merged)); - } - - return word; - } - - // ----------------------------------------------------------------- - // File loading - // ----------------------------------------------------------------- - - void load_vocab(const std::string& path) { - std::ifstream f(path); - if (!f.is_open()) - throw std::runtime_error("ClipTokenizer: cannot open vocab file: " + path); - - std::string line; - while (std::getline(f, line)) { - auto tab = line.rfind('\t'); - if (tab == std::string::npos) continue; - std::string token = unescape(line.substr(0, tab)); - int id = std::stoi(line.substr(tab + 1)); - vocab_[token] = id; - } - } - - void load_merges(const std::string& path) { - std::ifstream f(path); - if (!f.is_open()) - throw std::runtime_error("ClipTokenizer: cannot open merges file: " + path); - - std::string line; - int rank = 0; - while (std::getline(f, line)) { - if (line.empty() || line[0] == '#') continue; - auto sp = line.find(' '); - if (sp == std::string::npos) continue; - merge_ranks_[line] = rank++; // key is "a b" as written in the file - } - } - - static std::string unescape(const std::string& s) { - std::string r; - r.reserve(s.size()); - for (size_t i = 0; i < s.size(); i++) { - if (s[i] == '\\' && i + 1 < s.size()) { - switch (s[++i]) { - case '\\': r += '\\'; break; - case 'n': r += '\n'; break; - case 't': r += '\t'; break; - default: r += '\\'; r += s[i]; break; - } - } else { - r += s[i]; - } - } - return r; - } -}; - -} // namespace util