From 8a7bda586763fdb1f03f38405f2fa8f9ffdaa333 Mon Sep 17 00:00:00 2001 From: Cristhian Date: Tue, 15 Sep 2026 05:27:40 -0300 Subject: [PATCH] use std/media/v1/Image for image and depth Both inputs now declare dora's Arrow struct {width, height, encoding, data} rather than a flat buffer plus loose metadata. The payload describes itself, so nothing about its shape has to be agreed out of band, and the frame travels as one Arrow value instead of a buffer whose meaning lives in a sidecar dict. Depth uses the same type. dora's registry has no depth type, and a depth map is an image whose pixels are distances -- the reading ROS takes as well -- so it arrives with a single-channel encoding: 32FC1/float32 or 16UC1/mono16/uint16. `units` stays in message metadata because the struct has no field for it. decode.IMAGE_TYPE is the single definition of the struct; the example source and the tests build payloads from it rather than respelling the schema. The older shapes are still decoded -- a flat UInt8 buffer with width/height/encoding for image, a flat Float32 buffer with shape/units for depth -- so a producer's payload can move before its manifest does. That is only half a migration path: type compatibility is directional, so a producer still declaring std/core/v1/UInt8 fails validate against an std/media/v1/Image input regardless of what it sends, and has to update its manifest too. Verified: 41 tests, validate clean with 4 contracts injected, the dataflow runs to a clean exit, and a composite rendered through the struct path is identical to the one rendered before the migration. --- AGENTS.md | 6 +++-- README.md | 32 +++++++++++++---------- dataflow.yml | 4 +-- dora-node.yml | 25 ++++++++++-------- dora_image_view/decode.py | 18 +++++++++---- dora_image_view/depth.py | 37 +++++++++++++++++++++++++-- examples/source.py | 36 ++++++++++++++++---------- tests/test_dora_image_view.py | 48 ++++++++++++++++++++++++++++++----- 8 files changed, 153 insertions(+), 53 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4ae4065..2f2cd02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,9 +26,11 @@ CI=true SOURCE_FRAMES=30 dora run dataflow.yml --uv # the same, headless - **`dora_image_view/main.py`** — Argument parsing and the dora event loop, nothing else. The env var wins over the CLI flag for `VIEW_WIDTH`/`VIEW_HEIGHT`. The upstream node spells these `PLOT_WIDTH`/`PLOT_HEIGHT`; they were renamed with the node, so a dataflow copied from upstream needs its env keys changed. `Node(args.name)` rather than `Node()` so the node also runs standalone as a dynamic node. - **`dora_image_view/decode.py`** — `to_bgr()`: every supported encoding to a writable BGR array. `unpack()` accepts two payload shapes — the `std/media/v1/Image` struct (`width`, `height`, `encoding`, `data`), and the flat `UInt8` buffer described by message metadata that a `std/core/v1/UInt8` producer sends. A struct describes itself, so its metadata is ignored; everything after unpacking is common. - **The declared type is still `std/core/v1/UInt8` and the reason is interop.** Type compatibility is directional: `Image` satisfies a `Bytes` input, but a `std/core/v1/UInt8` output wired into an `std/media/v1/Image` input is a validate error. dora-zed declares `image_bgr` as `std/core/v1/UInt8`, so flipping this manifest to `Image` breaks the camera until dora-zed is migrated. Accepting both shapes first is what makes that migration orderable: producers move, then the contract. Two behaviours are deliberate, not bugs: `bgr8` **copies** (the payload may be a read-only view on shared memory, and the frame is annotated in place), and `yuv420` converts with `COLOR_YUV420p2RGB` rather than `...2BGR` — the upstream node converts to RGB and then displays it as BGR, and that channel swap is part of the behaviour callers already see. + `IMAGE_TYPE` is the one definition of that struct; the example source and the tests build payloads from it rather than respelling the schema. + + **Interop cost of the contract being `Image`.** Type compatibility is directional: `Image` satisfies a `Bytes` input, but a `std/core/v1/UInt8` output wired into an `std/media/v1/Image` input is a validate error. A producer still declaring `std/core/v1/UInt8` — dora-zed's `image_bgr` at time of writing — has to change its manifest, not just its payload, before it can feed this node. The decoder accepting the flat form does not rescue it: the failure is at validate, before anything runs. Two behaviours are deliberate, not bugs: `bgr8` **copies** (the payload may be a read-only view on shared memory, and the frame is annotated in place), and `yuv420` converts with `COLOR_YUV420p2RGB` rather than `...2BGR` — the upstream node converts to RGB and then displays it as BGR, and that channel swap is part of the behaviour callers already see. - **`dora_image_view/view.py`** — `View` (the panels, the overlay state and the window) and `Bboxes`. `headless` (from `CI=true`) skips only the window: every input is still decoded and composed, which is what makes the node safe in a pipeline test. Panels are tiled in the fixed order in `PANELS`, not in arrival order, so a panel does not jump when a stream drops a frame. Boxes are drawn on the `image` panel before scaling, because they are in image pixels; the caption is drawn on the composite, because the panel it would otherwise land on may not be the leftmost one. -- **`dora_image_view/depth.py`** — `to_millimetres()` and `colorize()`. Decoding normalises to float32 millimetres with 0 for missing, so the colouring only ever sees one representation. One thing is deliberate: the auto range is the valid pixels' full min/max, **not** a percentile. Trimming the tails looks safer and breaks the exact scene this node exists for — a small object in front of a large wall is a few percent of the pixels, so both tails land on the wall and the object clips away. A rendered check caught that; do not reintroduce percentile ranging. +- **`dora_image_view/depth.py`** — `to_millimetres()` and `colorize()`. Decoding normalises to float32 millimetres with 0 for missing, so the colouring only ever sees one representation. Depth arrives as `std/media/v1/Image` with a single-channel encoding (`DEPTH_DTYPES`) because dora's registry has no depth type and a depth map *is* an image of distances — the same reading ROS takes. `units` stays in message metadata: the struct has no field for it. One thing is deliberate: the auto range is the valid pixels' full min/max, **not** a percentile. Trimming the tails looks safer and breaks the exact scene this node exists for — a small object in front of a large wall is a few percent of the pixels, so both tails land on the wall and the object clips away. A rendered check caught that; do not reintroduce percentile ranging. - **`dora_image_view/payload.py`** — `as_float32()`: reinterprets a `uint8` payload rather than converting it, because a C++ node sending a raw byte slice produces a `UInt8` arrow array despite declaring `Float32`. Shared by `depth` and `intrinsics`, which both meet it. - **`dora_image_view/intrinsics.py`** — `Intrinsics`, `parse()` for the `camera_info` payload and `from_metadata()` for the copy senders attach to each frame. Both exist because dora-zed publishes the same numbers twice, and the per-frame copy is the one that survives a viewer starting after the camera did — `camera_info` goes out **once per session**. A nine-element row-major K is accepted alongside the compact four values, since that is how ROS spells the same model. The node needs none of this to draw; it is held for whatever turns depth into 3D, and drawn so a wrong camera model is visible rather than merely plausible. - **`dora_image_view/bbox.py`** — Parses the `bbox` struct into xyxy corners. `xywh` is centre-based (x, y are the box centre). `_flat()` unwraps the single-element list that `pa.array([{...}])` produces around each field, and reads a field stored flat as-is. diff --git a/README.md b/README.md index b9bb0df..e785a5b 100644 --- a/README.md +++ b/README.md @@ -106,25 +106,31 @@ no window. That is what makes it safe to run in a pipeline test. | Input | Payload | Metadata | |---|---|---| -| `image` | `std/media/v1/Image` struct, **or** a flat `UInt8` array | none for the struct; `width`, `height`, `encoding` for the flat form | -| `depth` | Flat `Float32` array, `[H, W]` | `shape`, `units` | +| `image` | `std/media/v1/Image` struct | none — the struct describes itself | +| `depth` | `std/media/v1/Image` struct, one channel | `units` (defaults to `mm`) | | `camera_info` | Flat `Float32` array, `[fx, fy, cx, cy]` or a 9-element K | `distortion`, `distortion_model` | | `bbox` | Arrow struct array, element `[0]` with fields `bbox`, `conf`, `labels` | `format` | | `text` | Arrow array of size 1 | — | -Two image payload shapes are accepted. The typed one is `std/media/v1/Image`, an Arrow -struct of `width: UInt32`, `height: UInt32`, `encoding: Utf8` and `data: LargeBinary`, -which carries inside the payload what the flat form carries beside it; when it is -received the message metadata is not consulted at all. The flat form is a raw `UInt8` -buffer described by `width`, `height` and `encoding` metadata, which is what a node -declaring `std/core/v1/UInt8` sends. +`image` and `depth` are both `std/media/v1/Image`: an Arrow struct of `width: UInt32`, +`height: UInt32`, `encoding: Utf8` and `data: LargeBinary`. The payload describes +itself, so nothing about its shape has to travel as message metadata. -The declared contract is still `std/core/v1/UInt8`, so a producer sending the struct -should keep declaring that until every producer in the dataflow has moved: dora's type -compatibility is directional, and a `std/core/v1/UInt8` output wired into an -`std/media/v1/Image` input fails `dora validate`. +Depth is an image whose pixels are distances — dora's type registry has no depth type, +and this is how ROS spells it too. Its encodings are `32FC1`/`float32` and +`16UC1`/`mono16`/`uint16`, one channel per pixel, and `units` metadata still switches +millimetres to metres. -Supported `encoding` values: `bgr8`, `rgb8`, `jpeg`/`jpg`/`jpe`/`bmp`/`webp`/`png` +Older payloads are still decoded, so a producer can move at its own pace: a flat `UInt8` +buffer with `width`/`height`/`encoding` metadata for `image`, and a flat `Float32` +buffer with `shape`/`units` metadata for `depth`. + +**Migrating a producer.** dora's type compatibility is directional — `Image` satisfies a +`Bytes` input, but a `std/core/v1/UInt8` output wired into an `std/media/v1/Image` input +is a `dora validate` error. A producer still declaring `std/core/v1/UInt8` or +`std/core/v1/Float32` therefore has to change its manifest as well as its payload. + +Supported `image` `encoding` values: `bgr8`, `rgb8`, `jpeg`/`jpg`/`jpe`/`bmp`/`webp`/`png` (encoded byte stream), `yuv420`, `avif`. `width` and `height` are only needed by the raw encodings — the rest carry their own dimensions. An unsupported encoding raises. diff --git a/dataflow.yml b/dataflow.yml index b6331fd..28ade4b 100644 --- a/dataflow.yml +++ b/dataflow.yml @@ -40,9 +40,9 @@ nodes: # bbox is deliberately absent: it is untyped on the viewer's side too, # because senders disagree on 32- vs 64-bit numeric fields. output_types: - image: std/core/v1/UInt8 + image: std/media/v1/Image text: std/core/v1/String - depth: std/core/v1/Float32 + depth: std/media/v1/Image camera_info: std/core/v1/Float32 - id: view diff --git a/dora-node.yml b/dora-node.yml index 0c29cad..beb1a55 100644 --- a/dora-node.yml +++ b/dora-node.yml @@ -21,13 +21,14 @@ dora: ">=1.0" inputs: image: - type: std/core/v1/UInt8 + type: std/media/v1/Image required: false description: >- - Base image to draw on: a flat UInt8 buffer described by the metadata - {width, height, encoding}. Supported encodings: bgr8, rgb8, - jpeg/jpg/jpe/bmp/webp/png (encoded bytes), yuv420, avif. Receiving this - input triggers a redraw and display. + Base image to draw on: an Arrow struct {width, height, encoding, data}. + Supported encodings: bgr8, rgb8, jpeg/jpg/jpe/bmp/webp/png (encoded + bytes), yuv420, avif. Receiving this input triggers a redraw and display. + A flat UInt8 buffer described by {width, height, encoding} metadata is + still decoded, for producers that have not moved to the struct. bbox: required: false description: >- @@ -53,13 +54,17 @@ inputs: this input -- they are picked up from there, which is what covers a camera that published its intrinsics before this node started. depth: - type: std/core/v1/Float32 + type: std/media/v1/Image required: false description: >- - Depth map as a flat Float32 buffer, described by the metadata - {shape: [height, width], units}. Millimetres unless `units` says `m`; - 0 and non-finite values are missing depth and draw black. Colour-mapped - into its own panel, which receiving this input redraws. + Depth map as an Arrow struct {width, height, encoding, data}, one channel + per pixel. Supported encodings: 32FC1/float32 and 16UC1/mono16/uint16 -- + dora has no depth type, and a depth map is an image whose pixels are + distances, which is how ROS spells it too. Millimetres unless the `units` + metadata says `m`; 0 and non-finite values are missing depth and draw + black. Colour-mapped into its own panel, which receiving this input + redraws. A flat Float32 buffer with {shape, units} metadata is still + decoded, for producers that have not moved to the struct. outputs: {} diff --git a/dora_image_view/decode.py b/dora_image_view/decode.py index a1745b4..f491a8d 100644 --- a/dora_image_view/decode.py +++ b/dora_image_view/decode.py @@ -13,6 +13,7 @@ import io import cv2 import numpy as np import pillow_avif # noqa: F401 # registers the AVIF plugin with Pillow +import pyarrow as pa from PIL import Image # Encodings that carry their own dimensions, so `width`/`height` metadata is not @@ -22,9 +23,16 @@ SELF_DESCRIBING = frozenset({"jpeg", "jpg", "jpe", "bmp", "webp", "png"}) SUPPORTED = SELF_DESCRIBING | {"bgr8", "rgb8", "yuv420", "avif"} -# The fields of std/media/v1/Image. A payload carrying them describes itself and -# its metadata is not consulted. -IMAGE_FIELDS = ("width", "height", "encoding", "data") +# std/media/v1/Image. A payload carrying these fields describes itself, so its +# message metadata is not consulted. Exported as an Arrow type so a producer -- +# the example source, the tests -- builds it from one definition. +IMAGE_TYPE = pa.struct([ + ("width", pa.uint32()), + ("height", pa.uint32()), + ("encoding", pa.utf8()), + ("data", pa.large_binary()), +]) +IMAGE_FIELDS = tuple(field.name for field in IMAGE_TYPE) def to_bgr(storage, metadata: dict) -> np.ndarray: @@ -68,7 +76,7 @@ def to_bgr(storage, metadata: dict) -> np.ndarray: def unpack(storage, metadata: dict) -> tuple[np.ndarray, str, int | None, int | None]: """Split a payload of either shape into buffer, encoding, width and height.""" - if _is_image_struct(storage): + if is_image_struct(storage): # One image per message: the struct is a single-row array. row = storage[0].as_py() return ( @@ -86,7 +94,7 @@ def unpack(storage, metadata: dict) -> tuple[np.ndarray, str, int | None, int | ) -def _is_image_struct(storage) -> bool: +def is_image_struct(storage) -> bool: """Whether the payload is an Arrow struct with the std/media/v1/Image fields.""" fields = getattr(getattr(storage, "type", None), "fields", None) diff --git a/dora_image_view/depth.py b/dora_image_view/depth.py index a75aeed..380c138 100644 --- a/dora_image_view/depth.py +++ b/dora_image_view/depth.py @@ -11,6 +11,7 @@ import os import cv2 import numpy as np +from dora_image_view.decode import is_image_struct from dora_image_view.payload import as_float32 # Perceptually uniform and readable in both directions, unlike JET. @@ -18,6 +19,18 @@ COLORMAP = cv2.COLORMAP_TURBO MISSING = 0.0 +# Single-channel depth encodings and the pixel type each names. The ROS spellings +# are the ones senders use; a depth map is an image whose pixels are distances, +# which is why it arrives as std/media/v1/Image rather than a type of its own -- +# dora's registry has no depth type. +DEPTH_DTYPES = { + "32fc1": np.float32, + "float32": np.float32, + "16uc1": np.uint16, + "mono16": np.uint16, + "uint16": np.uint16, +} + def to_millimetres(storage, metadata: dict) -> np.ndarray: """Decode one `depth` payload into a float32 array of millimetres. @@ -25,8 +38,7 @@ def to_millimetres(storage, metadata: dict) -> np.ndarray: Missing depth is 0. Non-finite values are normalised to 0 as well, so everything downstream only has to test for it. """ - depth = as_float32(storage) - height, width = _hw(metadata, depth.size) + depth, height, width = _values(storage, metadata) if depth.size < height * width: raise RuntimeError( @@ -87,6 +99,27 @@ def resolve_range(depth_mm: np.ndarray) -> tuple[float, float]: return float(low), float(high) +def _values(storage, metadata: dict) -> tuple[np.ndarray, int, int]: + """Read the payload of either shape into float32 values plus its shape.""" + if is_image_struct(storage): + row = storage[0].as_py() + encoding = str(row["encoding"]).lower() + dtype = DEPTH_DTYPES.get(encoding) + + if dtype is None: + raise RuntimeError( + f"unsupported depth encoding: {encoding}; " + f"expected one of {', '.join(sorted(DEPTH_DTYPES))}", + ) + + values = np.frombuffer(row["data"], dtype=dtype).astype(np.float32) + return values, int(row["height"]), int(row["width"]) + + values = as_float32(storage) + height, width = _hw(metadata, values.size) + return values, height, width + + def _hw(metadata: dict, size: int) -> tuple[int, int]: shape = metadata.get("shape") diff --git a/examples/source.py b/examples/source.py index 75935f7..7dde418 100644 --- a/examples/source.py +++ b/examples/source.py @@ -16,6 +16,8 @@ import numpy as np import pyarrow as pa from dora import Node +from dora_image_view.decode import IMAGE_TYPE + WIDTH = int(os.getenv("SOURCE_WIDTH", "640")) HEIGHT = int(os.getenv("SOURCE_HEIGHT", "480")) # 0 runs until the viewer quits; a positive count is what lets CI run the @@ -55,11 +57,7 @@ def main() -> None: ) frame, corners = render(sent) - node.send_output( - "image", - pa.array(frame.ravel()), - {"width": WIDTH, "height": HEIGHT, "encoding": "bgr8", **INTRINSICS_METADATA}, - ) + node.send_output("image", image_struct(frame, "bgr8"), INTRINSICS_METADATA) node.send_output( "bbox", pa.array([{ @@ -72,14 +70,8 @@ def main() -> None: depth = render_depth(corners) node.send_output( "depth", - pa.array(depth.ravel()), - { - "shape": [HEIGHT, WIDTH], - "dtype": "float32", - "layout": "HW", - "units": "mm", - **INTRINSICS_METADATA, - }, + image_struct(depth, "32FC1"), + {"units": "mm", **INTRINSICS_METADATA}, ) node.send_output("text", pa.array([CAPTION])) @@ -88,6 +80,24 @@ def main() -> None: break +def image_struct(frame: np.ndarray, encoding: str) -> pa.Array: + """Pack a frame as std/media/v1/Image. + + The struct carries the width, height and encoding a receiver needs, so none + of it has to travel as message metadata. `frame` may be any dtype: the + encoding names how to read the bytes back. + """ + return pa.array( + [{ + "width": frame.shape[1], + "height": frame.shape[0], + "encoding": encoding, + "data": frame.tobytes(), + }], + type=IMAGE_TYPE, + ) + + def render(index: int) -> tuple[np.ndarray, np.ndarray]: """Draw frame `index` and return it with the xyxy corners of the square in it.""" frame = np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8) diff --git a/tests/test_dora_image_view.py b/tests/test_dora_image_view.py index fb15416..b45a510 100644 --- a/tests/test_dora_image_view.py +++ b/tests/test_dora_image_view.py @@ -298,12 +298,7 @@ def _image_struct(frame: np.ndarray, encoding: str = "bgr8"): "encoding": encoding, "data": frame.tobytes(), }], - type=pa.struct([ - ("width", pa.uint32()), - ("height", pa.uint32()), - ("encoding", pa.utf8()), - ("data", pa.large_binary()), - ]), + type=decode.IMAGE_TYPE, ) @@ -336,3 +331,44 @@ def test_a_decoded_image_struct_frame_is_writable(frame): decoded[0, 0] = (1, 2, 3) assert tuple(decoded[0, 0]) == (1, 2, 3) + + +def test_depth_is_decoded_from_an_image_struct(): + values = np.array([[100.0, 200.0], [300.0, 400.0]], dtype=np.float32) + + # No shape metadata: the struct carries its own dimensions. + decoded = depth.to_millimetres(_image_struct(values, "32FC1"), {}) + + np.testing.assert_allclose(decoded, values) + + +def test_depth_image_struct_reads_a_uint16_encoding(): + values = np.array([[1500, 2500]], dtype=np.uint16) + + decoded = depth.to_millimetres(_image_struct(values, "16UC1"), {}) + + np.testing.assert_allclose(decoded, [[1500.0, 2500.0]]) + + +def test_depth_image_struct_still_honours_the_units_metadata(): + values = np.array([[1.5, 2.0]], dtype=np.float32) + + decoded = depth.to_millimetres(_image_struct(values, "32FC1"), {"units": "m"}) + + np.testing.assert_allclose(decoded, [[1500.0, 2000.0]]) + + +def test_depth_image_struct_with_an_unsupported_encoding_is_rejected(): + values = np.zeros((1, 2), dtype=np.float32) + + with pytest.raises(RuntimeError, match="unsupported depth encoding"): + depth.to_millimetres(_image_struct(values, "rgb8"), {}) + + +def test_a_decoded_depth_struct_is_writable(): + values = np.array([[100.0, 0.0]], dtype=np.float32) + + decoded = depth.to_millimetres(_image_struct(values, "32FC1"), {}) + decoded[0, 0] = 42.0 + + assert decoded[0, 0] == 42.0