-
Notifications
You must be signed in to change notification settings - Fork 34
/
zed_oc_depth_example.cpp
331 lines (275 loc) · 13.9 KB
/
zed_oc_depth_example.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
///////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2021, STEREOLABS.
//
// All rights reserved.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
///////////////////////////////////////////////////////////////////////////
// ----> Includes
#include <iostream>
#include <sstream>
#include <string>
#include "videocapture.hpp"
// OpenCV includes
#include <opencv2/opencv.hpp>
//#undef HAVE_OPENCV_VIZ // Uncomment if cannot use Viz3D for point cloud rendering
#ifdef HAVE_OPENCV_VIZ
#include <opencv2/viz.hpp>
#include <opencv2/viz/viz3d.hpp>
#endif
// Sample includes
#include "calibration.hpp"
#include "stopwatch.hpp"
#include "stereo.hpp"
#include "ocv_display.hpp"
// <---- Includes
#define USE_OCV_TAPI // Comment to use "normal" cv::Mat instead of CV::UMat
#define USE_HALF_SIZE_DISP // Comment to compute depth matching on full image frames
int main(int argc, char *argv[])
{
// ----> Silence unused warning
(void)argc;
(void)argv;
// <---- Silence unused warning
sl_oc::VERBOSITY verbose = sl_oc::VERBOSITY::INFO;
// ----> Set Video parameters
sl_oc::video::VideoParams params;
#ifdef EMBEDDED_ARM
params.res = sl_oc::video::RESOLUTION::VGA;
#else
params.res = sl_oc::video::RESOLUTION::HD720;
#endif
params.fps = sl_oc::video::FPS::FPS_30;
params.verbose = verbose;
// <---- Set Video parameters
// ----> Create Video Capture
sl_oc::video::VideoCapture cap(params);
if( !cap.initializeVideo(-1) )
{
std::cerr << "Cannot open camera video capture" << std::endl;
std::cerr << "See verbosity level for more details." << std::endl;
return EXIT_FAILURE;
}
int sn = cap.getSerialNumber();
std::cout << "Connected to camera sn: " << sn << std::endl;
// <---- Create Video Capture
// ----> Retrieve calibration file from Stereolabs server
std::string calibration_file;
// ZED Calibration
unsigned int serial_number = sn;
// Download camera calibration file
if( !sl_oc::tools::downloadCalibrationFile(serial_number, calibration_file) )
{
std::cerr << "Could not load calibration file from Stereolabs servers" << std::endl;
return EXIT_FAILURE;
}
std::cout << "Calibration file found. Loading..." << std::endl;
// ----> Frame size
int w,h;
cap.getFrameSize(w,h);
// <---- Frame size
// ----> Initialize calibration
cv::Mat map_left_x, map_left_y;
cv::Mat map_right_x, map_right_y;
cv::Mat cameraMatrix_left, cameraMatrix_right;
double baseline=0;
sl_oc::tools::initCalibration(calibration_file, cv::Size(w/2,h), map_left_x, map_left_y, map_right_x, map_right_y,
cameraMatrix_left, cameraMatrix_right, &baseline);
double fx = cameraMatrix_left.at<double>(0,0);
double fy = cameraMatrix_left.at<double>(1,1);
double cx = cameraMatrix_left.at<double>(0,2);
double cy = cameraMatrix_left.at<double>(1,2);
std::cout << " Camera Matrix L: \n" << cameraMatrix_left << std::endl << std::endl;
std::cout << " Camera Matrix R: \n" << cameraMatrix_right << std::endl << std::endl;
#ifdef USE_OCV_TAPI
cv::UMat map_left_x_gpu = map_left_x.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat map_left_y_gpu = map_left_y.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat map_right_x_gpu = map_right_x.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat map_right_y_gpu = map_right_y.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
#endif
// ----> Initialize calibration
// ----> Declare OpenCV images
#ifdef USE_OCV_TAPI
cv::UMat frameYUV; // Full frame side-by-side in YUV 4:2:2 format
cv::UMat frameBGR(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Full frame side-by-side in BGR format
cv::UMat left_raw(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Left unrectified image
cv::UMat right_raw(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Right unrectified image
cv::UMat left_rect(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Left rectified image
cv::UMat right_rect(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Right rectified image
cv::UMat left_for_matcher(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Left image for the stereo matcher
cv::UMat right_for_matcher(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Right image for the stereo matcher
cv::UMat left_disp_half(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Half sized disparity map
cv::UMat left_disp(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Full output disparity
cv::UMat left_disp_float(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Final disparity map in float32
cv::UMat left_disp_image(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Normalized and color remapped disparity map to be displayed
cv::UMat left_depth_map(cv::USAGE_ALLOCATE_DEVICE_MEMORY); // Depth map in float32
#else
cv::Mat frameBGR, left_raw, left_rect, right_raw, right_rect, frameYUV, left_for_matcher, right_for_matcher, left_disp_half,left_disp,left_disp_float, left_disp_vis;
#endif
// <---- Declare OpenCV images
// ----> Stereo matcher initialization
sl_oc::tools::StereoSgbmPar stereoPar;
//Note: you can use the tool 'zed_open_capture_depth_tune_stereo' to tune the parameters and save them to YAML
if(!stereoPar.load())
{
stereoPar.save(); // Save default parameters.
}
cv::Ptr<cv::StereoSGBM> left_matcher = cv::StereoSGBM::create(stereoPar.minDisparity,stereoPar.numDisparities,stereoPar.blockSize);
left_matcher->setMinDisparity(stereoPar.minDisparity);
left_matcher->setNumDisparities(stereoPar.numDisparities);
left_matcher->setBlockSize(stereoPar.blockSize);
left_matcher->setP1(stereoPar.P1);
left_matcher->setP2(stereoPar.P2);
left_matcher->setDisp12MaxDiff(stereoPar.disp12MaxDiff);
left_matcher->setMode(stereoPar.mode);
left_matcher->setPreFilterCap(stereoPar.preFilterCap);
left_matcher->setUniquenessRatio(stereoPar.uniquenessRatio);
left_matcher->setSpeckleWindowSize(stereoPar.speckleWindowSize);
left_matcher->setSpeckleRange(stereoPar.speckleRange);
stereoPar.print();
// <---- Stereo matcher initialization
// ----> Point Cloud
cv::Mat cloudMat;
#ifdef HAVE_OPENCV_VIZ
cv::viz::Viz3d pc_viewer = cv::viz::Viz3d( "Point Cloud" );
#endif
// <---- Point Cloud
uint64_t last_ts=0; // Used to check new frame arrival
// Infinite video grabbing loop
while (1)
{
// Get a new frame from camera
const sl_oc::video::Frame frame = cap.getLastFrame();
// ----> If the frame is valid we can convert, rectify and display it
if(frame.data!=nullptr && frame.timestamp!=last_ts)
{
last_ts = frame.timestamp;
// ----> Conversion from YUV 4:2:2 to BGR for visualization
#ifdef USE_OCV_TAPI
cv::Mat frameYUV_cpu = cv::Mat( frame.height, frame.width, CV_8UC2, frame.data );
frameYUV = frameYUV_cpu.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_HOST_MEMORY);
#else
frameYUV = cv::Mat( frame.height, frame.width, CV_8UC2, frame.data );
#endif
cv::cvtColor(frameYUV,frameBGR,cv::COLOR_YUV2BGR_YUYV);
// <---- Conversion from YUV 4:2:2 to BGR for visualization
// ----> Extract left and right images from side-by-side
left_raw = frameBGR(cv::Rect(0, 0, frameBGR.cols / 2, frameBGR.rows));
right_raw = frameBGR(cv::Rect(frameBGR.cols / 2, 0, frameBGR.cols / 2, frameBGR.rows));
// <---- Extract left and right images from side-by-side
// ----> Apply rectification
sl_oc::tools::StopWatch remap_clock;
#ifdef USE_OCV_TAPI
cv::remap(left_raw, left_rect, map_left_x_gpu, map_left_y_gpu, cv::INTER_AREA );
cv::remap(right_raw, right_rect, map_right_x_gpu, map_right_y_gpu, cv::INTER_AREA );
#else
cv::remap(left_raw, left_rect, map_left_x, map_left_y, cv::INTER_AREA );
cv::remap(right_raw, right_rect, map_right_x, map_right_y, cv::INTER_AREA );
#endif
double remap_elapsed = remap_clock.toc();
std::stringstream remapElabInfo;
remapElabInfo << "Rectif. processing: " << remap_elapsed << " sec - Freq: " << 1./remap_elapsed;
// <---- Apply rectification
// ----> Stereo matching
sl_oc::tools::StopWatch stereo_clock;
double resize_fact = 1.0;
#ifdef USE_HALF_SIZE_DISP
resize_fact = 0.5;
// Resize the original images to improve performances
cv::resize(left_rect, left_for_matcher, cv::Size(), resize_fact, resize_fact, cv::INTER_AREA);
cv::resize(right_rect, right_for_matcher, cv::Size(), resize_fact, resize_fact, cv::INTER_AREA);
#else
left_for_matcher = left_rect; // No data copy
right_for_matcher = right_rect; // No data copy
#endif
// Apply stereo matching
left_matcher->compute(left_for_matcher, right_for_matcher,left_disp_half);
left_disp_half.convertTo(left_disp_float,CV_32FC1);
cv::multiply(left_disp_float,1./16.,left_disp_float); // Last 4 bits of SGBM disparity are decimal
#ifdef USE_HALF_SIZE_DISP
cv::multiply(left_disp_float,2.,left_disp_float); // Last 4 bits of SGBM disparity are decimal
cv::UMat tmp = left_disp_float; // Required for OpenCV 3.2
cv::resize(tmp, left_disp_float, cv::Size(), 1./resize_fact, 1./resize_fact, cv::INTER_AREA);
#else
left_disp = left_disp_float;
#endif
double elapsed = stereo_clock.toc();
std::stringstream stereoElabInfo;
stereoElabInfo << "Stereo processing: " << elapsed << " sec - Freq: " << 1./elapsed;
// <---- Stereo matching
// ----> Show frames
sl_oc::tools::showImage("Right rect.", right_rect, params.res,true, remapElabInfo.str());
sl_oc::tools::showImage("Left rect.", left_rect, params.res,true, remapElabInfo.str());
// <---- Show frames
// ----> Show disparity image
cv::add(left_disp_float,-static_cast<double>(stereoPar.minDisparity-1),left_disp_float); // Minimum disparity offset correction
cv::multiply(left_disp_float,1./stereoPar.numDisparities,left_disp_image,255., CV_8UC1 ); // Normalization and rescaling
cv::applyColorMap(left_disp_image,left_disp_image,cv::COLORMAP_JET); // COLORMAP_INFERNO is better, but it's only available starting from OpenCV v4.1.0
sl_oc::tools::showImage("Disparity", left_disp_image, params.res,true, stereoElabInfo.str());
// <---- Show disparity image
// ----> Extract Depth map
// The DISPARITY MAP can be now transformed in DEPTH MAP using the formula
// depth = (f * B) / disparity
// where 'f' is the camera focal, 'B' is the camera baseline, 'disparity' is the pixel disparity
double num = static_cast<double>(fx*baseline);
cv::divide(num,left_disp_float,left_depth_map);
float central_depth = left_depth_map.getMat(cv::ACCESS_READ).at<float>(left_depth_map.rows/2, left_depth_map.cols/2 );
std::cout << "Depth of the central pixel: " << central_depth << " mm" << std::endl;
// <---- Extract Depth map
// ----> Create Point Cloud
sl_oc::tools::StopWatch pc_clock;
size_t buf_size = static_cast<size_t>(left_depth_map.cols * left_depth_map.rows);
std::vector<cv::Vec3d> buffer( buf_size, cv::Vec3f::all( std::numeric_limits<float>::quiet_NaN() ) );
cv::Mat depth_map_cpu = left_depth_map.getMat(cv::ACCESS_READ);
float* depth_vec = (float*)(&(depth_map_cpu.data[0]));
#pragma omp parallel for
for(size_t idx=0; idx<buf_size;idx++ )
{
size_t r = idx/left_depth_map.cols;
size_t c = idx%left_depth_map.cols;
double depth = static_cast<double>(depth_vec[idx]);
//std::cout << depth << " ";
if(!isinf(depth) && depth >=0 && depth > stereoPar.minDepth_mm && depth < stereoPar.maxDepth_mm)
{
buffer[idx].val[2] = depth; // Z
buffer[idx].val[0] = (c-cx)*depth/fx; // X
buffer[idx].val[1] = (r-cy)*depth/fy; // Y
}
}
cloudMat = cv::Mat( left_depth_map.rows, left_depth_map.cols, CV_64FC3, &buffer[0] ).clone();
double pc_elapsed = stereo_clock.toc();
std::stringstream pcElabInfo;
// pcElabInfo << "Point cloud processing: " << pc_elapsed << " sec - Freq: " << 1./pc_elapsed;
//std::cout << pcElabInfo.str() << std::endl;
// <---- Create Point Cloud
}
// ----> Keyboard handling
int key = cv::waitKey( 5 );
if(key=='q' || key=='Q') // Quit
break;
// <---- Keyboard handling
#ifdef HAVE_OPENCV_VIZ
// ----> Show Point Cloud
cv::viz::WCloud cloudWidget( cloudMat, left_rect );
cloudWidget.setRenderingProperty( cv::viz::POINT_SIZE, 1 );
pc_viewer.showWidget( "Point Cloud", cloudWidget );
pc_viewer.spinOnce(1);
if(pc_viewer.wasStopped())
break;
// <---- Show Point Cloud
#endif
}
return EXIT_SUCCESS;
}