cristhian
5a0f13d4db
Nothing in a dataflow carries fx, fy, cx and cy except the camera, and every pixel of a depth map needs them to become a 3D point. The node now accepts them so whatever computes that next has them to hand. They are read from two places because dora-zed publishes them twice: the camera_info message, and the metadata of every frame. The per-frame copy is not redundant -- camera_info goes out once per camera session, so a viewer started after the camera would otherwise never see it, and reading frame metadata means wiring the input is optional even for a camera that publishes it. A nine-element row-major K is accepted alongside the compact [fx, fy, cx, cy], since that is how ROS spells the same model in sensor_msgs/CameraInfo. The model is drawn under the caption rather than only stored. A depth map read through the wrong intrinsics looks entirely plausible, and this is where that becomes visible. as_float32 moves to a payload module: camera_info arrives as a raw byte slice for the same reason the depth map does, and the reinterpretation is now shared rather than written twice. Verified with dora-cli 1.0.1: validate passes with four contracts injected on each side, the dataflow runs to a clean exit, and a composed frame was rendered and inspected with the model drawn on it.
135 lines
4.4 KiB
Python
135 lines
4.4 KiB
Python
"""The display state: the panels, the overlays drawn on them, and the window."""
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
from dora_image_view.intrinsics import Intrinsics
|
|
|
|
WINDOW = "Dora Node: dora-image-view"
|
|
GREEN = (0, 255, 0)
|
|
WHITE = (255, 255, 255)
|
|
FONT = cv2.FONT_HERSHEY_SIMPLEX
|
|
FONT_SCALE = 0.5
|
|
|
|
# Left to right in the composite. Fixed rather than insertion-ordered so a panel
|
|
# does not move when a stream drops a frame.
|
|
PANELS = ("image", "depth")
|
|
|
|
|
|
@dataclass
|
|
class Bboxes:
|
|
"""One overlay's worth of boxes, always stored as xyxy corners."""
|
|
|
|
corners: np.ndarray = field(default_factory=lambda: np.empty((0, 4)))
|
|
conf: np.ndarray = field(default_factory=lambda: np.empty(0))
|
|
labels: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=object))
|
|
|
|
|
|
@dataclass
|
|
class View:
|
|
"""Panel state, plus the window the panels are displayed in.
|
|
|
|
Each panel holds the most recent frame of one stream, as BGR. They are
|
|
tiled left to right, so wiring only `depth` shows only depth. `headless`
|
|
suppresses the window; the node still decodes every input.
|
|
"""
|
|
|
|
width: int | None = None
|
|
height: int | None = None
|
|
headless: bool = False
|
|
bboxes: Bboxes = field(default_factory=Bboxes)
|
|
text: str = ""
|
|
panels: dict[str, np.ndarray] = field(default_factory=dict)
|
|
intrinsics: Intrinsics | None = None
|
|
|
|
def show(self) -> bool:
|
|
"""Draw the overlays, tile the panels, and display them.
|
|
|
|
Returns whether the viewer asked to quit by pressing `q`.
|
|
"""
|
|
frame = self.compose()
|
|
|
|
if frame is None:
|
|
return False
|
|
|
|
if self.width and self.height:
|
|
frame = cv2.resize(frame, (self.width, self.height))
|
|
|
|
if self.headless:
|
|
return False
|
|
|
|
cv2.imshow(WINDOW, frame)
|
|
return cv2.waitKey(1) & 0xFF == ord("q")
|
|
|
|
def compose(self) -> np.ndarray | None:
|
|
"""Tile the populated panels into one frame, tallest panel setting the height."""
|
|
panels = [(name, self.panels[name]) for name in PANELS if name in self.panels]
|
|
|
|
if not panels:
|
|
return None
|
|
|
|
# The boxes are in image pixels, so they are drawn before any scaling.
|
|
if "image" in self.panels:
|
|
self._draw_bboxes(self.panels["image"])
|
|
|
|
tallest = max(panel.shape[0] for _, panel in panels)
|
|
scaled = [self._label(_scale(panel, tallest), name) for name, panel in panels]
|
|
|
|
frame = scaled[0] if len(scaled) == 1 else cv2.hconcat(scaled)
|
|
|
|
# On the composite, not on a panel: the caption describes the frame as a
|
|
# whole, and a panel it was drawn on might not be the leftmost one.
|
|
cv2.putText(frame, self.text, (20, 20), FONT, FONT_SCALE, WHITE, 1)
|
|
|
|
# Showing the camera model is the point of receiving it here: a depth
|
|
# map read through the wrong intrinsics looks entirely plausible, and
|
|
# this is where that becomes visible.
|
|
if self.intrinsics is not None:
|
|
cv2.putText(frame, str(self.intrinsics), (20, 40), FONT, FONT_SCALE, WHITE, 1)
|
|
|
|
return frame
|
|
|
|
def _label(self, panel: np.ndarray, name: str) -> np.ndarray:
|
|
"""Name a panel, so two side by side are not read as one picture."""
|
|
if len(self.panels) > 1:
|
|
cv2.putText(
|
|
panel,
|
|
name,
|
|
(10, panel.shape[0] - 10),
|
|
FONT,
|
|
FONT_SCALE,
|
|
WHITE,
|
|
1,
|
|
)
|
|
return panel
|
|
|
|
def _draw_bboxes(self, frame: np.ndarray) -> None:
|
|
for corners, conf, label in zip(
|
|
self.bboxes.corners,
|
|
self.bboxes.conf,
|
|
self.bboxes.labels,
|
|
strict=False,
|
|
):
|
|
min_x, min_y, max_x, max_y = (int(value) for value in corners)
|
|
cv2.rectangle(frame, (min_x, min_y), (max_x, max_y), GREEN, 2)
|
|
cv2.putText(
|
|
frame,
|
|
f"{label}, {conf:0.2f}",
|
|
(max_x - 120, max_y - 10),
|
|
FONT,
|
|
FONT_SCALE,
|
|
GREEN,
|
|
1,
|
|
)
|
|
|
|
|
|
def _scale(panel: np.ndarray, height: int) -> np.ndarray:
|
|
"""Match a panel to the composite height, keeping its aspect ratio."""
|
|
if panel.shape[0] == height:
|
|
return panel
|
|
|
|
width = max(round(panel.shape[1] * height / panel.shape[0]), 1)
|
|
return cv2.resize(panel, (width, height))
|