add a depth map panel

The node showed a colour image only, so debugging a depth pipeline meant
either trusting the numbers or writing a throwaway viewer. It now keeps one
panel per frame stream -- image and depth -- and tiles them side by side,
scaled to a common height. Wiring one is a plain viewer; wiring both shows
them together. bbox and text stay overlays rather than panels.

The contract follows what dora-zed already publishes: float32 [H, W] in
millimetres with 0 for missing. It is sent there as a raw byte slice, so the
payload arrives as UInt8 despite the Float32 contract and is reinterpreted
rather than converted.

Auto-ranging the colormap over the valid pixels' full extent, rather than over
a percentile of them, is deliberate and was not the first attempt: trimming
the tails collapsed on the scene the node exists for. A small object in front
of a large wall is a few percent of the pixels, so both tails landed on the
wall and the object clipped away. Rendering a frame and looking at it found
that; every test passed either way.

Verified with dora-cli 1.0.1: validate passes with both contracts injected,
the dataflow runs both nodes to a clean exit, and a composed frame was
rendered and inspected.
This commit is contained in:
2026-09-15 05:06:50 -03:00
parent 407395ab88
commit 0efeacd538
10 changed files with 417 additions and 50 deletions
+60 -8
View File
@@ -1,4 +1,4 @@
"""The display state: the frame, the overlays drawn on it, and the window."""
"""The display state: the panels, the overlays drawn on them, and the window."""
from dataclasses import dataclass, field
@@ -11,6 +11,10 @@ WHITE = (255, 255, 255)
FONT = cv2.FONT_HERSHEY_SIMPLEX
FONT_SCALE = 0.5
# Left to right in the composite. Fixed rather than insertion-ordered so a panel
# does not move when a stream drops a frame.
PANELS = ("image", "depth")
@dataclass
class Bboxes:
@@ -23,9 +27,11 @@ class Bboxes:
@dataclass
class View:
"""Overlay state, plus the window it is displayed in.
"""Panel state, plus the window the panels are displayed in.
`headless` suppresses the window; the node still decodes every input.
Each panel holds the most recent frame of one stream, as BGR. They are
tiled left to right, so wiring only `depth` shows only depth. `headless`
suppresses the window; the node still decodes every input.
"""
width: int | None = None
@@ -33,13 +39,17 @@ class View:
headless: bool = False
bboxes: Bboxes = field(default_factory=Bboxes)
text: str = ""
panels: dict[str, np.ndarray] = field(default_factory=dict)
def show(self, frame: np.ndarray) -> bool:
"""Draw the overlays on `frame`, resize it, and display it.
def show(self) -> bool:
"""Draw the overlays, tile the panels, and display them.
Returns whether the viewer asked to quit by pressing `q`.
"""
self._draw(frame)
frame = self.compose()
if frame is None:
return False
if self.width and self.height:
frame = cv2.resize(frame, (self.width, self.height))
@@ -50,7 +60,42 @@ class View:
cv2.imshow(WINDOW, frame)
return cv2.waitKey(1) & 0xFF == ord("q")
def _draw(self, frame: np.ndarray) -> None:
def compose(self) -> np.ndarray | None:
"""Tile the populated panels into one frame, tallest panel setting the height."""
panels = [(name, self.panels[name]) for name in PANELS if name in self.panels]
if not panels:
return None
# The boxes are in image pixels, so they are drawn before any scaling.
if "image" in self.panels:
self._draw_bboxes(self.panels["image"])
tallest = max(panel.shape[0] for _, panel in panels)
scaled = [self._label(_scale(panel, tallest), name) for name, panel in panels]
frame = scaled[0] if len(scaled) == 1 else cv2.hconcat(scaled)
# On the composite, not on a panel: the caption describes the frame as a
# whole, and a panel it was drawn on might not be the leftmost one.
cv2.putText(frame, self.text, (20, 20), FONT, FONT_SCALE, WHITE, 1)
return frame
def _label(self, panel: np.ndarray, name: str) -> np.ndarray:
"""Name a panel, so two side by side are not read as one picture."""
if len(self.panels) > 1:
cv2.putText(
panel,
name,
(10, panel.shape[0] - 10),
FONT,
FONT_SCALE,
WHITE,
1,
)
return panel
def _draw_bboxes(self, frame: np.ndarray) -> None:
for corners, conf, label in zip(
self.bboxes.corners,
self.bboxes.conf,
@@ -69,4 +114,11 @@ class View:
1,
)
cv2.putText(frame, self.text, (20, 20), FONT, FONT_SCALE, WHITE, 1)
def _scale(panel: np.ndarray, height: int) -> np.ndarray:
"""Match a panel to the composite height, keeping its aspect ratio."""
if panel.shape[0] == height:
return panel
width = max(round(panel.shape[1] * height / panel.shape[0]), 1)
return cv2.resize(panel, (width, height))