#include "zed/camera.h" #include "util/env.h" #include #include #include #include #include #include namespace { sl::RESOLUTION parse_resolution(int resolution) { switch (resolution) { case 1080: return sl::RESOLUTION::HD1080; case 2000: return sl::RESOLUTION::HD2K; case 720: default: return sl::RESOLUTION::HD720; } } const std::pair DEPTH_MODES[] = { {"NEURAL", sl::DEPTH_MODE::NEURAL}, {"NEURAL_PLUS", sl::DEPTH_MODE::NEURAL_PLUS}, {"NEURAL_LIGHT", sl::DEPTH_MODE::NEURAL_LIGHT}, {"PERFORMANCE", sl::DEPTH_MODE::PERFORMANCE}, {"QUALITY", sl::DEPTH_MODE::QUALITY}, {"NONE", sl::DEPTH_MODE::NONE}, }; // A typo here silently changes depth quality if it falls back to a default, and // the resulting depth is wrong in a way that only shows up downstream, so an // unrecognised name stops the node instead. sl::DEPTH_MODE parse_depth_mode(const std::string& mode) { for (const auto& entry : DEPTH_MODES) { if (mode == entry.first) { return entry.second; } } std::cerr << "Unknown ZED_DEPTH_MODE '" << mode << "'. Accepted values:"; for (const auto& entry : DEPTH_MODES) { std::cerr << " " << entry.first; } std::cerr << std::endl; std::exit(1); } sl::FLIP_MODE parse_flip(const std::string& mode) { if (mode == "OFF") { return sl::FLIP_MODE::OFF; } return sl::FLIP_MODE::ON; } // A blank frame is checked on a strided sample rather than the full image, which // is just as conclusive and costs nothing at frame rate. constexpr int BLANK_FRAME_STRIDE = 16; // sl::Camera::reboot() reaches the ZED 2 and newer. On the first-generation ZED // and the ZED Mini it returns without touching the device, so calling it there // only costs time and fills the log. bool model_supports_reboot(sl::MODEL model) { return model != sl::MODEL::ZED && model != sl::MODEL::ZED_M; } // The SDK reconnects a camera whose link hiccuped by itself. Dropping the handle // over one of these takes the recovery away from the SDK and hands it to us, // which is slower and, on a device the kernel is about to suspend, often fatal. bool is_transient_grab_error(sl::ERROR_CODE err) { return err == sl::ERROR_CODE::CAMERA_REBOOTING || err == sl::ERROR_CODE::CORRUPTED_FRAME; } void list_available_cameras() { auto devices = sl::Camera::getDeviceList(); std::cout << "Detected ZED cameras: " << devices.size() << std::endl; for (std::size_t i = 0; i < devices.size(); i++) { const auto& dev = devices[i]; std::cout << " - [" << i << "] model=" << sl::toString(dev.camera_model) << ", serial=" << dev.serial_number << std::endl; } } } // namespace namespace zed { Config Config::from_env() { Config c; c.resolution = util::get_env_int("ZED_RESOLUTION", c.resolution); c.fps = util::get_env_int("ZED_FPS", c.fps); c.depth_mode = parse_depth_mode(util::get_env_str("ZED_DEPTH_MODE", "NEURAL")); c.depth_min_mm = util::get_env_int("ZED_DEPTH_MIN_MM", c.depth_min_mm); c.depth_max_mm = util::get_env_int("ZED_DEPTH_MAX_MM", c.depth_max_mm); c.depth_fill = util::get_env_bool("ZED_DEPTH_FILL", c.depth_fill); c.point_cloud = util::get_env_bool("ZED_POINT_CLOUD", c.point_cloud); c.pc_mode = util::get_env_str("ZED_PC_MODE", "FULL") == "HALF" ? PcMode::HALF : PcMode::FULL; c.confidence_threshold = util::get_env_int("ZED_CONFIDENCE_THRESHOLD", c.confidence_threshold); c.texture_confidence_threshold = util::get_env_int("ZED_TEXTURE_CONFIDENCE_THRESHOLD", c.texture_confidence_threshold); c.flip = util::get_env_str("ZED_FLIP", c.flip); c.warmup_frames = util::get_env_int("ZED_WARMUP_FRAMES", c.warmup_frames); return c; } Camera::Camera(const Config& config) : config_(config) { // Process-wide, and must be set before any camera is opened. Without it the // epoch of the timestamps published alongside every frame depends on the SDK // default rather than on anything this node states. sl::setTimestampClock(sl::TIMESTAMP_CLOCK::SYSTEM_CLOCK); init_params_.camera_resolution = parse_resolution(config_.resolution); init_params_.camera_fps = config_.fps; init_params_.depth_mode = config_.depth_mode; init_params_.coordinate_units = sl::UNIT::MILLIMETER; init_params_.camera_image_flip = parse_flip(config_.flip); init_params_.depth_minimum_distance = config_.depth_min_mm; init_params_.depth_maximum_distance = config_.depth_max_mm; // Recover a hiccuping link inside the SDK rather than out here. Without this // a grab blocks for the roughly 17 s the SDK spends reconnecting, freezing the // dora event loop; with it the grab returns CAMERA_REBOOTING straight away and // the SDK reconnects in the background, holding the device open meanwhile. init_params_.async_grab_camera_recovery = true; // Let the SDK flag the frames a stalled camera hands back looking valid. init_params_.enable_image_validity_check = 1; runtime_params_.enable_depth = true; runtime_params_.enable_fill_mode = config_.depth_fill; // The two thresholds discard the depth the SDK is least certain of, which is // the depth at object edges: those pixels straddle the object and whatever is // behind it, and belong to neither. runtime_params_.confidence_threshold = config_.confidence_threshold; runtime_params_.texture_confidence_threshold = config_.texture_confidence_threshold; std::cout << "ZED depth: mode=" << sl::toString(init_params_.depth_mode) << ", range=" << config_.depth_min_mm << "-" << config_.depth_max_mm << " mm" << ", confidence=" << config_.confidence_threshold << ", texture_confidence=" << config_.texture_confidence_threshold << ", fill=" << (config_.depth_fill ? "on" : "off") << std::endl; list_available_cameras(); } Camera::~Camera() { close(); } bool Camera::open() { sl::ERROR_CODE err = zed_.open(init_params_); if (err != sl::ERROR_CODE::SUCCESS) { std::cerr << "Failed to open ZED camera: " << sl::toString(err) << std::endl; return false; } recovering_ = false; sl::CameraInformation info = zed_.getCameraInformation(); serial_number_ = info.serial_number; model_ = info.camera_model; // After the first open, reopen the same physical camera. Recovery runs while // the device may be mid-re-enumeration, and picking whichever ZED answers // first is not the same thing as picking ours. init_params_.input.setFromSerialNumber(serial_number_); std::cout << "ZED camera opened (" << sl::toString(model_) << ", SN " << serial_number_ << ")" << std::endl; return true; } void Camera::close() { recovering_ = false; zed_.close(); } void Camera::reset() { close(); if (serial_number_ == 0 || !model_supports_reboot(model_)) { return; } // The ZED can stop delivering frames while staying enumerated: the video // endpoint halts, grabs fail, and reopening returns CAMERA FAILED TO SETUP // because the endpoint is still halted. A device reset clears that, and // reboot() is the SDK's way to ask for one without unplugging the camera. // The device re-enumerates afterwards, so the caller's backoff before the // next open is what lets it settle. sl::ERROR_CODE err = sl::Camera::reboot(static_cast(serial_number_), true); std::cout << "reset ZED camera (SN " << serial_number_ << "): " << sl::toString(err) << std::endl; } bool Camera::is_open() const { return zed_.isOpened(); } bool Camera::recovering() const { return recovering_; } void Camera::warmup() { for (int i = 0; i < config_.warmup_frames; i++) { zed_.grab(runtime_params_); } } bool Camera::grab() { sl::ERROR_CODE err = zed_.grab(runtime_params_); if (err != sl::ERROR_CODE::SUCCESS) { if (is_transient_grab_error(err)) { // Every grab repeats this for as long as the reconnect takes, so only // the first one is worth a line in the log. if (!recovering_) { recovering_ = true; std::cerr << "ZED camera link dropped (" << sl::toString(err) << "); the SDK is reconnecting it" << std::endl; } } else { recovering_ = false; std::cerr << "Failed to grab frame: " << sl::toString(err) << std::endl; } return false; } if (recovering_) { recovering_ = false; std::cout << "ZED camera link restored" << std::endl; } zed_.retrieveImage(image_mat_, sl::VIEW::LEFT); zed_.retrieveMeasure(depth_mat_, sl::MEASURE::DEPTH); // At 1080p a full XYZRGBA cloud is ~14.7 MB per frame, and retrieving it // costs capture time whether or not anything consumes it. if (config_.point_cloud) { zed_.retrieveMeasure(point_cloud_mat_, sl::MEASURE::XYZRGBA); } return true; } Calibration Camera::calibration() const { sl::CameraInformation info = zed_.getCameraInformation(); auto calib = info.camera_configuration.calibration_parameters; auto left = calib.left_cam; Calibration result; result.fx = left.fx; result.fy = left.fy; result.cx = left.cx; result.cy = left.cy; result.distortion.reserve(std::size(left.disto)); for (double v : left.disto) { result.distortion.push_back(v); } return result; } bool Camera::image_is_blank() { const int height = image_mat_.getHeight(); const int width = image_mat_.getWidth(); if (height == 0 || width == 0) { return true; } const size_t step = image_mat_.getStep(sl::MEM::CPU); const sl::uchar4* src = image_mat_.getPtr(); unsigned char lowest[3] = {255, 255, 255}; unsigned char highest[3] = {0, 0, 0}; for (int y = 0; y < height; y += BLANK_FRAME_STRIDE) { const sl::uchar4* row = src + static_cast(y) * step; for (int x = 0; x < width; x += BLANK_FRAME_STRIDE) { const unsigned char channels[3] = {row[x].x, row[x].y, row[x].z}; for (int c = 0; c < 3; c++) { lowest[c] = std::min(lowest[c], channels[c]); highest[c] = std::max(highest[c], channels[c]); } } } // Sensor noise makes a single-colour frame impossible for a live camera, so // this tells a dead device apart from any real scene. return lowest[0] == highest[0] && lowest[1] == highest[1] && lowest[2] == highest[2]; } int64_t Camera::device_timestamp_ns() { return static_cast( zed_.getTimestamp(sl::TIME_REFERENCE::IMAGE).getNanoseconds()); } sl::Mat& Camera::image() { return image_mat_; } sl::Mat& Camera::depth() { return depth_mat_; } sl::Mat& Camera::point_cloud() { return point_cloud_mat_; } } // namespace zed