add a depth map panel

The node showed a colour image only, so debugging a depth pipeline meant
either trusting the numbers or writing a throwaway viewer. It now keeps one
panel per frame stream -- image and depth -- and tiles them side by side,
scaled to a common height. Wiring one is a plain viewer; wiring both shows
them together. bbox and text stay overlays rather than panels.

The contract follows what dora-zed already publishes: float32 [H, W] in
millimetres with 0 for missing. It is sent there as a raw byte slice, so the
payload arrives as UInt8 despite the Float32 contract and is reinterpreted
rather than converted.

Auto-ranging the colormap over the valid pixels' full extent, rather than over
a percentile of them, is deliberate and was not the first attempt: trimming
the tails collapsed on the scene the node exists for. A small object in front
of a large wall is a few percent of the pixels, so both tails landed on the
wall and the object clipped away. Rendering a frame and looking at it found
that; every test passed either way.

Verified with dora-cli 1.0.1: validate passes with both contracts injected,
the dataflow runs both nodes to a clean exit, and a composed frame was
rendered and inspected.

dora_image_view/depth.py

@@ -0,0 +1,126 @@
+"""Decoding and colouring of the `depth` input.
+
+A depth map is a single-channel distance image, not a picture: it is turned into
+something displayable by mapping a distance range onto a colormap. Decoding
+normalises to float32 millimetres with 0 for missing, so the colouring only ever
+sees one representation.
+"""
+
+import os
+
+import cv2
+import numpy as np
+
+# Perceptually uniform and readable in both directions, unlike JET.
+COLORMAP = cv2.COLORMAP_TURBO
+
+MISSING = 0.0
+
+
+def to_millimetres(storage, metadata: dict) -> np.ndarray:
+    """Decode one `depth` payload into a float32 array of millimetres.
+
+    Missing depth is 0. Non-finite values are normalised to 0 as well, so
+    everything downstream only has to test for it.
+    """
+    depth = as_float32(storage)
+    height, width = _hw(metadata, depth.size)
+
+    if depth.size < height * width:
+        raise RuntimeError(
+            f"depth payload holds {depth.size} values, "
+            f"expected at least {height * width} for the declared shape",
+        )
+
+    depth = depth[: height * width].reshape(height, width).copy()
+    depth[~np.isfinite(depth)] = MISSING
+
+    if units(metadata) == "m":
+        depth *= 1000.0
+
+    return depth
+
+
+def colorize(depth_mm: np.ndarray) -> np.ndarray:
+    """Map millimetres onto the colormap, leaving missing depth black."""
+    low, high = resolve_range(depth_mm)
+
+    normalised = np.clip((depth_mm - low) / (high - low), 0.0, 1.0)
+    frame = cv2.applyColorMap((normalised * 255).astype(np.uint8), COLORMAP)
+    frame[depth_mm <= MISSING] = 0
+    return frame
+
+
+def resolve_range(depth_mm: np.ndarray) -> tuple[float, float]:
+    """Pick the millimetre range to map onto the colormap.
+
+    Each end is taken from the environment if it is set there and auto-ranged
+    over the valid pixels otherwise, so pinning one end does not force the
+    other. Auto-ranging re-reads the frame every time, which is what makes an
+    unconfigured sensor legible and also what makes the colours shift between
+    frames.
+
+    The auto range is the full extent of the valid pixels, not a percentile of
+    them. Trimming the tails reads as the safer choice but breaks the scene
+    this viewer exists for: a small object in front of a large flat wall is a
+    few percent of the pixels, so both tails land on the wall, the range
+    collapses, and the object -- the thing being looked at -- clips to one
+    colour with everything else.
+    """
+    low = _env("VIEW_DEPTH_MIN_MM")
+    high = _env("VIEW_DEPTH_MAX_MM")
+
+    if low is None or high is None:
+        valid = depth_mm[depth_mm > MISSING]
+        auto_low, auto_high = (
+            (float(valid.min()), float(valid.max())) if valid.size else (0.0, 1.0)
+        )
+        low = auto_low if low is None else low
+        high = auto_high if high is None else high
+
+    # An empty or single-valued frame would otherwise divide by zero.
+    if high <= low:
+        high = low + 1.0
+
+    return float(low), float(high)
+
+
+def as_float32(storage) -> np.ndarray:
+    """Read the payload as float32, whether it arrived as floats or as bytes.
+
+    A node that sends the map as a raw byte slice -- dora-zed does -- produces a
+    UInt8 arrow array even though its manifest declares Float32, so the buffer
+    has to be reinterpreted rather than converted.
+    """
+    buffer = storage.to_numpy()
+
+    if buffer.dtype == np.uint8:
+        return buffer.view(np.float32)
+
+    return buffer.astype(np.float32, copy=False)
+
+
+def _hw(metadata: dict, size: int) -> tuple[int, int]:
+    shape = metadata.get("shape")
+
+    if shape is not None and len(shape) >= 2:
+        return int(shape[0]), int(shape[1])
+
+    height, width = metadata.get("height"), metadata.get("width")
+    if height and width:
+        return int(height), int(width)
+
+    raise RuntimeError(
+        "depth needs a `shape` [height, width] or `height`/`width` metadata; "
+        f"got a payload of {size} values and neither",
+    )
+
+
+def units(metadata: dict) -> str:
+    """Read the distance unit the sender declared, defaulting to millimetres."""
+    return str(metadata.get("units", "mm")).lower()
+
+
+def _env(name: str) -> float | None:
+    value = os.getenv(name)
+    return float(value) if value else None

dora_image_view/main.py

@@ -1,15 +1,16 @@
 """A dora display sink: overlays bounding boxes and text on an image input.
 
-An `image` event decodes the frame and immediately redraws. `bbox` and `text`
-events only update stored state, which the next `image` event draws.
+`image` and `depth` each fill one panel of the window and redraw. `bbox` and
+`text` events only update stored state, which the next redraw draws.
 """
 
 import argparse
 import os
 
+import numpy as np
 from dora import Node
 
-from dora_image_view import bbox, decode
+from dora_image_view import bbox, decode, depth
 from dora_image_view.view import View
 
 
@@ -36,14 +37,12 @@ def main() -> None:
         event_id = event["id"]
         metadata = event["metadata"]
 
-        if event_id == "image":
-            frame = decode.to_bgr(
-                event["value"],
-                metadata.get("encoding", "bgr8").lower(),
-                metadata.get("width"),
-                metadata.get("height"),
-            )
-            if view.show(frame):
+        panel = _decode_panel(event_id, event["value"], metadata)
+
+        if panel is not None:
+            name, frame = panel
+            view.panels[name] = frame
+            if view.show():
                 break
         elif event_id == "bbox":
             view.bboxes = bbox.parse(event["value"], metadata.get("format", "xyxy").lower())
@@ -53,9 +52,29 @@ def main() -> None:
             print(f"ignoring unexpected input `{event_id}`")
 
 
+def _decode_panel(event_id: str, value, metadata: dict) -> tuple[str, np.ndarray] | None:
+    """Decode a frame-bearing input into the panel name and frame it fills.
+
+    Returns None for an input that carries no frame, which is what tells the
+    event loop to treat it as overlay state instead.
+    """
+    if event_id == "image":
+        return "image", decode.to_bgr(
+            value,
+            metadata.get("encoding", "bgr8").lower(),
+            metadata.get("width"),
+            metadata.get("height"),
+        )
+
+    if event_id == "depth":
+        return "depth", depth.colorize(depth.to_millimetres(value, metadata))
+
+    return None
+
+
 def _parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
-        description="OpenCV plotter: draws text and bounding boxes on an image.",
+        description="OpenCV viewer: image and depth, with bbox and text overlays.",
     )
     parser.add_argument(
         "--name",

dora_image_view/view.py

@@ -1,4 +1,4 @@
-"""The display state: the frame, the overlays drawn on it, and the window."""
+"""The display state: the panels, the overlays drawn on them, and the window."""
 
 from dataclasses import dataclass, field
 
@@ -11,6 +11,10 @@ WHITE = (255, 255, 255)
 FONT = cv2.FONT_HERSHEY_SIMPLEX
 FONT_SCALE = 0.5
 
+# Left to right in the composite. Fixed rather than insertion-ordered so a panel
+# does not move when a stream drops a frame.
+PANELS = ("image", "depth")
+
 
 @dataclass
 class Bboxes:
@@ -23,9 +27,11 @@ class Bboxes:
 
 @dataclass
 class View:
-    """Overlay state, plus the window it is displayed in.
+    """Panel state, plus the window the panels are displayed in.
 
-    `headless` suppresses the window; the node still decodes every input.
+    Each panel holds the most recent frame of one stream, as BGR. They are
+    tiled left to right, so wiring only `depth` shows only depth. `headless`
+    suppresses the window; the node still decodes every input.
     """
 
     width: int | None = None
@@ -33,13 +39,17 @@ class View:
     headless: bool = False
     bboxes: Bboxes = field(default_factory=Bboxes)
     text: str = ""
+    panels: dict[str, np.ndarray] = field(default_factory=dict)
 
-    def show(self, frame: np.ndarray) -> bool:
-        """Draw the overlays on `frame`, resize it, and display it.
+    def show(self) -> bool:
+        """Draw the overlays, tile the panels, and display them.
 
         Returns whether the viewer asked to quit by pressing `q`.
         """
-        self._draw(frame)
+        frame = self.compose()
+
+        if frame is None:
+            return False
 
         if self.width and self.height:
             frame = cv2.resize(frame, (self.width, self.height))
@@ -50,7 +60,42 @@ class View:
         cv2.imshow(WINDOW, frame)
         return cv2.waitKey(1) & 0xFF == ord("q")
 
-    def _draw(self, frame: np.ndarray) -> None:
+    def compose(self) -> np.ndarray | None:
+        """Tile the populated panels into one frame, tallest panel setting the height."""
+        panels = [(name, self.panels[name]) for name in PANELS if name in self.panels]
+
+        if not panels:
+            return None
+
+        # The boxes are in image pixels, so they are drawn before any scaling.
+        if "image" in self.panels:
+            self._draw_bboxes(self.panels["image"])
+
+        tallest = max(panel.shape[0] for _, panel in panels)
+        scaled = [self._label(_scale(panel, tallest), name) for name, panel in panels]
+
+        frame = scaled[0] if len(scaled) == 1 else cv2.hconcat(scaled)
+
+        # On the composite, not on a panel: the caption describes the frame as a
+        # whole, and a panel it was drawn on might not be the leftmost one.
+        cv2.putText(frame, self.text, (20, 20), FONT, FONT_SCALE, WHITE, 1)
+        return frame
+
+    def _label(self, panel: np.ndarray, name: str) -> np.ndarray:
+        """Name a panel, so two side by side are not read as one picture."""
+        if len(self.panels) > 1:
+            cv2.putText(
+                panel,
+                name,
+                (10, panel.shape[0] - 10),
+                FONT,
+                FONT_SCALE,
+                WHITE,
+                1,
+            )
+        return panel
+
+    def _draw_bboxes(self, frame: np.ndarray) -> None:
         for corners, conf, label in zip(
             self.bboxes.corners,
             self.bboxes.conf,
@@ -69,4 +114,11 @@ class View:
                 1,
             )
 
-        cv2.putText(frame, self.text, (20, 20), FONT, FONT_SCALE, WHITE, 1)
+
+def _scale(panel: np.ndarray, height: int) -> np.ndarray:
+    """Match a panel to the composite height, keeping its aspect ratio."""
+    if panel.shape[0] == height:
+        return panel
+
+    width = max(round(panel.shape[1] * height / panel.shape[0]), 1)
+    return cv2.resize(panel, (width, height))