accept std/media/v1/Image payloads
dora's typed image is an Arrow struct of width, height, encoding and data -- it carries inside the payload what this node has been reading out of message metadata. The decoder now takes either shape: a struct describes itself and its metadata is ignored, a flat UInt8 buffer is described by width/height/ encoding beside it, and everything after unpacking is common. The declared contract stays std/core/v1/UInt8, deliberately. Type compatibility is directional -- Image satisfies a Bytes input, but a std/core/v1/UInt8 output wired into an std/media/v1/Image input is a validate error -- and dora-zed declares image_bgr as std/core/v1/UInt8. Flipping this manifest now would break the camera. Accepting both shapes first is what lets the producers move before the contract does. output_framing is not part of this. It selects the wire format, raw or arrow-ipc, and is not required for images: a struct payload was verified to arrive identically under both framings, and validate does not ask for arrow-ipc when an output is annotated std/media/v1/Image. It earns its keep when the wire must be self-describing, such as crossing languages or recording.
This commit is contained in:
@@ -4,6 +4,7 @@ import io
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
@@ -13,6 +14,11 @@ from dora_image_view.view import Bboxes, View
|
||||
WIDTH, HEIGHT = 12, 8
|
||||
|
||||
|
||||
def _meta(encoding: str, width: int | None = None, height: int | None = None) -> dict:
|
||||
"""Build message metadata as a node sending a flat buffer attaches it."""
|
||||
return {"encoding": encoding, "width": width, "height": height}
|
||||
|
||||
|
||||
class FakeStorage:
|
||||
"""Stands in for the Arrow array dora hands to the node."""
|
||||
|
||||
@@ -30,7 +36,7 @@ def frame() -> np.ndarray:
|
||||
|
||||
|
||||
def test_bgr8_round_trips_unchanged(frame):
|
||||
decoded = decode.to_bgr(FakeStorage(frame.ravel()), "bgr8", WIDTH, HEIGHT)
|
||||
decoded = decode.to_bgr(FakeStorage(frame.ravel()), _meta("bgr8", WIDTH, HEIGHT))
|
||||
|
||||
np.testing.assert_array_equal(decoded, frame)
|
||||
|
||||
@@ -38,7 +44,7 @@ def test_bgr8_round_trips_unchanged(frame):
|
||||
def test_rgb8_round_trips_unchanged(frame):
|
||||
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
decoded = decode.to_bgr(FakeStorage(rgb.ravel()), "rgb8", WIDTH, HEIGHT)
|
||||
decoded = decode.to_bgr(FakeStorage(rgb.ravel()), _meta("rgb8", WIDTH, HEIGHT))
|
||||
|
||||
np.testing.assert_array_equal(decoded, frame)
|
||||
|
||||
@@ -53,7 +59,7 @@ def test_self_describing_encodings_need_no_dimensions(frame, suffix, encoding):
|
||||
ok, buffer = cv2.imencode(suffix, frame)
|
||||
assert ok
|
||||
|
||||
decoded = decode.to_bgr(FakeStorage(buffer.ravel()), encoding, None, None)
|
||||
decoded = decode.to_bgr(FakeStorage(buffer.ravel()), _meta(encoding))
|
||||
|
||||
assert decoded.shape == (HEIGHT, WIDTH, 3)
|
||||
|
||||
@@ -61,7 +67,7 @@ def test_self_describing_encodings_need_no_dimensions(frame, suffix, encoding):
|
||||
def test_yuv420_is_decoded(frame):
|
||||
planes = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV_I420)
|
||||
|
||||
decoded = decode.to_bgr(FakeStorage(planes.ravel()), "yuv420", WIDTH, HEIGHT)
|
||||
decoded = decode.to_bgr(FakeStorage(planes.ravel()), _meta("yuv420", WIDTH, HEIGHT))
|
||||
|
||||
assert decoded.shape == (HEIGHT, WIDTH, 3)
|
||||
|
||||
@@ -71,7 +77,7 @@ def test_avif_is_decoded(frame):
|
||||
Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).save(buffer, format="AVIF")
|
||||
payload = np.frombuffer(buffer.getvalue(), dtype=np.uint8)
|
||||
|
||||
decoded = decode.to_bgr(FakeStorage(payload), "avif", None, None)
|
||||
decoded = decode.to_bgr(FakeStorage(payload), _meta("avif"))
|
||||
|
||||
assert decoded.shape == (HEIGHT, WIDTH, 3)
|
||||
|
||||
@@ -79,7 +85,7 @@ def test_avif_is_decoded(frame):
|
||||
def test_decoded_frame_is_writable():
|
||||
storage = FakeStorage(np.arange(2 * 3 * 3, dtype=np.uint8))
|
||||
|
||||
decoded = decode.to_bgr(storage, "bgr8", width=3, height=2)
|
||||
decoded = decode.to_bgr(storage, _meta("bgr8", 3, 2))
|
||||
|
||||
# The frame is annotated in place, so it must not be a read-only view.
|
||||
decoded[0, 0] = 0
|
||||
@@ -88,24 +94,22 @@ def test_decoded_frame_is_writable():
|
||||
def test_raw_encodings_need_dimensions():
|
||||
storage = FakeStorage(np.zeros(18, dtype=np.uint8))
|
||||
with pytest.raises(RuntimeError, match="width"):
|
||||
decode.to_bgr(storage, "bgr8", width=None, height=None)
|
||||
decode.to_bgr(storage, _meta("bgr8"))
|
||||
|
||||
|
||||
def test_short_payload_is_rejected():
|
||||
storage = FakeStorage(np.zeros(4, dtype=np.uint8))
|
||||
with pytest.raises(RuntimeError, match="expected at least"):
|
||||
decode.to_bgr(storage, "bgr8", width=3, height=2)
|
||||
decode.to_bgr(storage, _meta("bgr8", 3, 2))
|
||||
|
||||
|
||||
def test_unsupported_encoding_is_rejected():
|
||||
storage = FakeStorage(np.zeros(4, dtype=np.uint8))
|
||||
with pytest.raises(RuntimeError, match="unsupported image encoding"):
|
||||
decode.to_bgr(storage, "tiff", width=None, height=None)
|
||||
decode.to_bgr(storage, _meta("tiff"))
|
||||
|
||||
|
||||
def test_xywh_is_converted_to_corners():
|
||||
import pyarrow as pa
|
||||
|
||||
value = pa.array(
|
||||
[
|
||||
{
|
||||
@@ -283,3 +287,52 @@ def test_the_camera_model_is_drawn_on_the_composite():
|
||||
with_model = view.compose()
|
||||
|
||||
assert not np.array_equal(without, with_model), "the model should be drawn"
|
||||
|
||||
|
||||
def _image_struct(frame: np.ndarray, encoding: str = "bgr8"):
|
||||
"""Build a payload shaped like std/media/v1/Image."""
|
||||
return pa.array(
|
||||
[{
|
||||
"width": frame.shape[1],
|
||||
"height": frame.shape[0],
|
||||
"encoding": encoding,
|
||||
"data": frame.tobytes(),
|
||||
}],
|
||||
type=pa.struct([
|
||||
("width", pa.uint32()),
|
||||
("height", pa.uint32()),
|
||||
("encoding", pa.utf8()),
|
||||
("data", pa.large_binary()),
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
def test_image_struct_is_decoded(frame):
|
||||
# No metadata at all: the struct describes itself.
|
||||
decoded = decode.to_bgr(_image_struct(frame), {})
|
||||
|
||||
np.testing.assert_array_equal(decoded, frame)
|
||||
|
||||
|
||||
def test_image_struct_wins_over_message_metadata(frame):
|
||||
# Metadata disagreeing with the struct must not be consulted.
|
||||
decoded = decode.to_bgr(_image_struct(frame), _meta("rgb8", 1, 1))
|
||||
|
||||
np.testing.assert_array_equal(decoded, frame)
|
||||
|
||||
|
||||
def test_image_struct_carrying_an_encoded_frame_is_decoded(frame):
|
||||
ok, encoded = cv2.imencode(".png", frame)
|
||||
assert ok
|
||||
payload = np.frombuffer(encoded.tobytes(), dtype=np.uint8).reshape(1, -1)
|
||||
|
||||
decoded = decode.to_bgr(_image_struct(payload, "png"), {})
|
||||
|
||||
np.testing.assert_array_equal(decoded, frame)
|
||||
|
||||
|
||||
def test_a_decoded_image_struct_frame_is_writable(frame):
|
||||
decoded = decode.to_bgr(_image_struct(frame), {})
|
||||
decoded[0, 0] = (1, 2, 3)
|
||||
|
||||
assert tuple(decoded[0, 0]) == (1, 2, 3)
|
||||
|
||||
Reference in New Issue
Block a user