accept std/media/v1/Image payloads

dora's typed image is an Arrow struct of width, height, encoding and data --
it carries inside the payload what this node has been reading out of message
metadata. The decoder now takes either shape: a struct describes itself and
its metadata is ignored, a flat UInt8 buffer is described by width/height/
encoding beside it, and everything after unpacking is common.

The declared contract stays std/core/v1/UInt8, deliberately. Type
compatibility is directional -- Image satisfies a Bytes input, but a
std/core/v1/UInt8 output wired into an std/media/v1/Image input is a validate
error -- and dora-zed declares image_bgr as std/core/v1/UInt8. Flipping this
manifest now would break the camera. Accepting both shapes first is what lets
the producers move before the contract does.

output_framing is not part of this. It selects the wire format, raw or
arrow-ipc, and is not required for images: a struct payload was verified to
arrive identically under both framings, and validate does not ask for
arrow-ipc when an output is annotated std/media/v1/Image. It earns its keep
when the wire must be self-describing, such as crossing languages or
recording.
This commit is contained in:
2026-09-15 05:25:18 -03:00
parent 92abe8a446
commit a6fa3ce32b
5 changed files with 133 additions and 24 deletions
+64 -11
View File
@@ -4,6 +4,7 @@ import io
import cv2
import numpy as np
import pyarrow as pa
import pytest
from PIL import Image
@@ -13,6 +14,11 @@ from dora_image_view.view import Bboxes, View
WIDTH, HEIGHT = 12, 8
def _meta(encoding: str, width: int | None = None, height: int | None = None) -> dict:
"""Build message metadata as a node sending a flat buffer attaches it."""
return {"encoding": encoding, "width": width, "height": height}
class FakeStorage:
"""Stands in for the Arrow array dora hands to the node."""
@@ -30,7 +36,7 @@ def frame() -> np.ndarray:
def test_bgr8_round_trips_unchanged(frame):
decoded = decode.to_bgr(FakeStorage(frame.ravel()), "bgr8", WIDTH, HEIGHT)
decoded = decode.to_bgr(FakeStorage(frame.ravel()), _meta("bgr8", WIDTH, HEIGHT))
np.testing.assert_array_equal(decoded, frame)
@@ -38,7 +44,7 @@ def test_bgr8_round_trips_unchanged(frame):
def test_rgb8_round_trips_unchanged(frame):
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
decoded = decode.to_bgr(FakeStorage(rgb.ravel()), "rgb8", WIDTH, HEIGHT)
decoded = decode.to_bgr(FakeStorage(rgb.ravel()), _meta("rgb8", WIDTH, HEIGHT))
np.testing.assert_array_equal(decoded, frame)
@@ -53,7 +59,7 @@ def test_self_describing_encodings_need_no_dimensions(frame, suffix, encoding):
ok, buffer = cv2.imencode(suffix, frame)
assert ok
decoded = decode.to_bgr(FakeStorage(buffer.ravel()), encoding, None, None)
decoded = decode.to_bgr(FakeStorage(buffer.ravel()), _meta(encoding))
assert decoded.shape == (HEIGHT, WIDTH, 3)
@@ -61,7 +67,7 @@ def test_self_describing_encodings_need_no_dimensions(frame, suffix, encoding):
def test_yuv420_is_decoded(frame):
planes = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
decoded = decode.to_bgr(FakeStorage(planes.ravel()), "yuv420", WIDTH, HEIGHT)
decoded = decode.to_bgr(FakeStorage(planes.ravel()), _meta("yuv420", WIDTH, HEIGHT))
assert decoded.shape == (HEIGHT, WIDTH, 3)
@@ -71,7 +77,7 @@ def test_avif_is_decoded(frame):
Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).save(buffer, format="AVIF")
payload = np.frombuffer(buffer.getvalue(), dtype=np.uint8)
decoded = decode.to_bgr(FakeStorage(payload), "avif", None, None)
decoded = decode.to_bgr(FakeStorage(payload), _meta("avif"))
assert decoded.shape == (HEIGHT, WIDTH, 3)
@@ -79,7 +85,7 @@ def test_avif_is_decoded(frame):
def test_decoded_frame_is_writable():
storage = FakeStorage(np.arange(2 * 3 * 3, dtype=np.uint8))
decoded = decode.to_bgr(storage, "bgr8", width=3, height=2)
decoded = decode.to_bgr(storage, _meta("bgr8", 3, 2))
# The frame is annotated in place, so it must not be a read-only view.
decoded[0, 0] = 0
@@ -88,24 +94,22 @@ def test_decoded_frame_is_writable():
def test_raw_encodings_need_dimensions():
storage = FakeStorage(np.zeros(18, dtype=np.uint8))
with pytest.raises(RuntimeError, match="width"):
decode.to_bgr(storage, "bgr8", width=None, height=None)
decode.to_bgr(storage, _meta("bgr8"))
def test_short_payload_is_rejected():
storage = FakeStorage(np.zeros(4, dtype=np.uint8))
with pytest.raises(RuntimeError, match="expected at least"):
decode.to_bgr(storage, "bgr8", width=3, height=2)
decode.to_bgr(storage, _meta("bgr8", 3, 2))
def test_unsupported_encoding_is_rejected():
storage = FakeStorage(np.zeros(4, dtype=np.uint8))
with pytest.raises(RuntimeError, match="unsupported image encoding"):
decode.to_bgr(storage, "tiff", width=None, height=None)
decode.to_bgr(storage, _meta("tiff"))
def test_xywh_is_converted_to_corners():
import pyarrow as pa
value = pa.array(
[
{
@@ -283,3 +287,52 @@ def test_the_camera_model_is_drawn_on_the_composite():
with_model = view.compose()
assert not np.array_equal(without, with_model), "the model should be drawn"
def _image_struct(frame: np.ndarray, encoding: str = "bgr8"):
"""Build a payload shaped like std/media/v1/Image."""
return pa.array(
[{
"width": frame.shape[1],
"height": frame.shape[0],
"encoding": encoding,
"data": frame.tobytes(),
}],
type=pa.struct([
("width", pa.uint32()),
("height", pa.uint32()),
("encoding", pa.utf8()),
("data", pa.large_binary()),
]),
)
def test_image_struct_is_decoded(frame):
# No metadata at all: the struct describes itself.
decoded = decode.to_bgr(_image_struct(frame), {})
np.testing.assert_array_equal(decoded, frame)
def test_image_struct_wins_over_message_metadata(frame):
# Metadata disagreeing with the struct must not be consulted.
decoded = decode.to_bgr(_image_struct(frame), _meta("rgb8", 1, 1))
np.testing.assert_array_equal(decoded, frame)
def test_image_struct_carrying_an_encoded_frame_is_decoded(frame):
ok, encoded = cv2.imencode(".png", frame)
assert ok
payload = np.frombuffer(encoded.tobytes(), dtype=np.uint8).reshape(1, -1)
decoded = decode.to_bgr(_image_struct(payload, "png"), {})
np.testing.assert_array_equal(decoded, frame)
def test_a_decoded_image_struct_frame_is_writable(frame):
decoded = decode.to_bgr(_image_struct(frame), {})
decoded[0, 0] = (1, 2, 3)
assert tuple(decoded[0, 0]) == (1, 2, 3)