(2021-10-15)Intelligent Vision Coursework

Use of Baser

在该实验中主要通过opencv使用拍摄的棋盘格对相机进行calibration.求出相机的内参和外参.实验设备为basler工业相机.
注:opencv的sample中有标准示例程序,这里的程序是自己写的仅供练习.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
#include <iostream>
#include <sstream>
#include <string>
#include <ctime>
#include <cstdio>
#include <vector>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>

using namespace cv;
using namespace std;
string imageList[]={string("./n1.JPG"),string("./n2.JPG"),string("./n3.JPG"),string("./n4.JPG"),\
string("./n5.JPG"),string("./n6.JPG"),string("./n7.JPG"),string("./n8.JPG"),\
string("./n9.JPG"),string("./n10.JPG"),string("./n11.JPG"),string("./n12.JPG")
};
CvSize boardSize(5,7);
int photo_size=12;
float squareSize=50;
int main(){
	vector< Point3f > CornerPoints;
	vector<vector< Point3f >> ObjectPoints;
	vector<vector< Point2f >> imagePoints;
	vector<Mat> save_view;
	bool have_set=false;
	CvSize ImageSize;
	for(int i=0;i<photo_size;i++){
		//reference to argument 2:https://blog.csdn.net/qq_27278957/article/details/84589526
		Mat view = imread(imageList[i], 1);
		cout<<view.size()<<endl;
         	imshow("input", view);//显示
		ImageSize=view.size();
		std::vector<cv::Point2f> ptvec;
		int  corner_count=9; 
		//the error for find==0:https://blog.csdn.net/u011651743/article/details/51099543
		bool found = findChessboardCorners( view, boardSize, ptvec ,CALIB_CB_ADAPTIVE_THRESH );
		//for more precise ,use  cvFindCornerSubPix
		/*
		void cv::drawChessboardCorners(
			cv::InputOutputArray image, // 棋盘格图像（8UC3）即是输入也是输出
			cv::Size patternSize, // 棋盘格内部角点的行、列数
			cv::InputArray corners, // findChessboardCorners()输出的角点
			bool patternWasFound // findChessboardCorners()的返回值
		);
		*/
		drawChessboardCorners(view,boardSize,ptvec,found);
		imshow("input", view);
		save_view.push_back(view);
		imagePoints.push_back(ptvec);
		if(have_set==false){
			have_set=true;
			for(int j=0;j<boardSize.height;j++){
				for(int k=0;k<boardSize.width;k++){
					CornerPoints.push_back(Point3f(j*squareSize,k*squareSize,0));					
				}
							
			}	
		}

	}
	ObjectPoints.resize(imagePoints.size(),CornerPoints);
	
	Mat cameraMatrix;
	cameraMatrix= Mat::eye(3, 3, CV_64F);
        cameraMatrix.at<double>(0,0) = 1;
	
	Mat distCoeffs;
	distCoeffs=Mat::zeros(8, 1, CV_64F);
	int iFixedPoint = -1;
	
	vector<Mat> rvecs;vector<Mat> tvecs;
	
	double rms = calibrateCamera(ObjectPoints, imagePoints, ImageSize,
                        cameraMatrix, distCoeffs, rvecs, tvecs,
                         CALIB_FIX_ASPECT_RATIO  | CALIB_USE_LU);
	cout<<cameraMatrix<<endl;
	//now calculate the actual point for each feature;
	double totalErr=0,totalPoints=0,err;
	for(int i=0;i<photo_size;i++){
		vector< Point2f > projectedPoints;
		cv::projectPoints(ObjectPoints[i], rvecs[i], tvecs[i],cameraMatrix,\
			distCoeffs, projectedPoints);
		err = norm(imagePoints[i], projectedPoints, NORM_L2);
		cout<<i<<":"<<endl;
		//for(int k=0;k<imagePoints[i].size();k++){
		//	cout<<imagePoints[i][k]<<":"<<projectedPoints[k]<<endl;		
		//}
		drawChessboardCorners(save_view[i],boardSize,projectedPoints,true);
		imshow("input2", save_view[i]);cv::waitKey(1000);
		cout<<err<<endl;
		totalErr        += err*err;
        	totalPoints     += ObjectPoints[i].size();
		//Mat rotate_matrix;
		//Rodrigues(rvecs[i], rotate_matrix, noArray());
		//	for(int j=0;j<boardSize.width;j++){
		//		for(int k=0;k<boardSize.height;k++){
		//			Mat p =(Mat_<double>(3,1)<<);
		//		}
		//	}
			
		//cout<<rotate_matrix<<endl;
		//cout<<rvecs[i].size()<<endl;
	}
	cout<<std::sqrt(totalErr/totalPoints)<<endl;
	
}

Project

这是智能视觉采集大作业的主要程序

  • 目的:从视频拍摄中得到全景图,包括离线的和在线的.
  • 过程:特征点提取匹配-图片射影变换-图片合成
    (1)特征点提取匹配:ORB算法
    (2)图片合成的方法分为两种,第一种是离线的,第二种是在线的(个人提出的拙劣的算法).

    1.离线合成

    离线部分和在线部分有一个不同的是图片合成算法.
  • 对于在线合成,速度时比较重要的,所以这里对于前后两帧的图片采用
    $$
    Img_t(i,j)=max{newImg_t(i,j),Img_{t-1}(i,j)}
    $$
    采用这种算法的原因是,新的图像经过射影变换后,在图片上会留有黑色的空位.需要通过依旧求得的全景图的部分对其进行填充.
  • 对于离线合成,精确度是比较重要的,所以这里采用下述算法
    $$
    Img_t(i,j)=\begin{cases}max{newImg_t(i,j),Img_{t-1}(i,j)},&newImg_t(i,j)<k \ \alpha newImg_t(i,j)+\beta Img_{t-1}(i,j),&newImg_t(i,j)>=k \end{cases}
    $$
    其中$\alpha+\beta=1$
    由于图片经过变换后可能会产生部分没有对齐的情况,如果仍然采用离线的算法,将会导致黑色的物体在图像中渐渐的被掩盖掉(最典型的就是头发在全景图的生成过程中渐渐的消失).所以一般来说,在全景图的生成过程中都会使用新图和旧图乘一个系数合成的方法.同时我们仍然需要考虑到,新图经过射影变换后留有黑色空位的情况.所以,在该算法中,需要对新图的某个像素是黑色空位还是其他的情况进行区分,即设一个阈值k,用新图的像素与阈值进行比较.小于阈值,则进行加系数融合;如果大于阈值,则取最大值,消去黑色空位.

离线部分代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

#include <iostream>
#include <sstream>
#include <string>
#include <ctime>
#include <cstdio>
#include <vector>
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>

#include <opencv2/video/background_segm.hpp>

using namespace cv;
using namespace std;
int main(){
    VideoCapture input=VideoCapture("input5.mp4");
    if(input.isOpened()){
        cout<<"input sucess"<<endl;
    }
    else{
        cout<<"fail input"<<endl;
        return 0;
    }
    Mat view; 
    int cnt=0;
    vector<Mat> calibrate_images;
    vector<Mat> calibrate_descriptors;
    vector<vector<KeyPoint> > calibrate_keypoints;
    Mat dst(720+100,1280*6, CV_8UC3);dst.setTo(0);
    Mat save_homo;
    while(1){
        if(!input.read(view)) break;
        //cout<<view.size<<endl;
        std::vector<KeyPoint> keypoints;
        Mat descriptors;
        Ptr<FeatureDetector> detector = ORB::create();
        Ptr<DescriptorExtractor> descriptor = ORB::create();
    //Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
        detector->detect ( view,keypoints );
        descriptor->compute ( view, keypoints, descriptors );
        Mat outimg;
        calibrate_keypoints.push_back(keypoints);
        calibrate_descriptors.push_back(descriptors);
        drawKeypoints( view, keypoints, outimg, Scalar::all(-1), DrawMatchesFlags::DEFAULT );
        //imshow("ORB特征点",outimg);
        //imshow("input", view);
        //cv::waitKey(10000);
        
        calibrate_images.push_back(view);
        if(view.rows==0) break;
        if(cnt>=1){
            //the first 15 images will be used calidate.
            Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
            vector<DMatch> matches;
            Mat img_match;
    //BFMatcher matcher ( NORM_HAMMING );
            matcher->match ( calibrate_descriptors[cnt-1],  calibrate_descriptors[cnt], matches );
            //optimize
            double min_dist=10000, max_dist=0;
            //找出所有匹配之间的最小距离和最大距离, 即是最相似的和最不相似的两组点之间的距离
            for ( int i = 0; i < calibrate_descriptors[cnt-1].rows; i++ )
            {
                double dist = matches[i].distance;
                if ( dist < min_dist ) min_dist = dist;
                if ( dist > max_dist ) max_dist = dist;
            }
            std::vector< DMatch > good_matches;
            for ( int i = 0; i < calibrate_descriptors[cnt-1].rows; i++ )
            {
                if ( matches[i].distance <= max ( 2*min_dist, 30.0 ) )
                {
                    good_matches.push_back ( matches[i] );
                }
            }
            drawMatches ( calibrate_images[cnt-1], calibrate_keypoints[cnt-1], calibrate_images[cnt], calibrate_keypoints[cnt], good_matches, img_match );
            //imshow("ORB特征点2",img_match);
            
            //calibrate here
            //**********************************************//
            vector<Point2f> imagePoints1, imagePoints2;

            for (int i = 0; i<good_matches.size(); i++)
            {
                imagePoints2.push_back(calibrate_keypoints[cnt-1][good_matches[i].queryIdx].pt);
                imagePoints1.push_back(calibrate_keypoints[cnt][good_matches[i].trainIdx].pt);
            }

            Mat homo = findHomography(imagePoints1, imagePoints2, CV_RANSAC);
            if(cnt==1) save_homo=homo.clone();
            else save_homo=save_homo*homo;
            homo=save_homo;

           // cout<<homo<<endl;
            Mat imageTransform;
            warpPerspective(calibrate_images[cnt], imageTransform, homo, Size(calibrate_images[cnt].cols+1000, calibrate_images[cnt].rows+100));
            //imshow("直接经过透视矩阵变换", imageTransform);
            //imwrite("trans1.jpg", imageTransform);
            int dst_width = imageTransform.cols;  //取最右点的长度为拼接图的长度
            
            int dst_height =calibrate_images[cnt].rows;
            
  
            Mat tem_dst(720+100,1280*6, CV_8UC3);tem_dst.setTo(0);
            imageTransform.copyTo(tem_dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
            Mat tem_dst2=dst.clone();
            //calibrate_images[cnt].copyTo(dst(Rect(0, 0, calibrate_images[cnt].cols, calibrate_images[cnt].rows)));
            //max(tem_dst,tem_dst2,dst);
            //addWeighted(tem_dst, 0.1,tem_dst2,0.9, 0, dst);
            for (int j = 0; j < tem_dst.rows; j++) {
                for (int k = 0; k < tem_dst.cols; k++) {
                    for (int m = 0; m < 3; m++) {
                        if (tem_dst.at<Vec3b>(j, k)[m] <= 130) {
                            dst.at<Vec3b>(j, k)[m] = max(tem_dst2.at<Vec3b>(j, k)[m], tem_dst.at<Vec3b>(j, k)[m]);
                        }
                        else {
                            dst.at<Vec3b>(j, k)[m] = (tem_dst2.at<Vec3b>(j, k)[m]*5+ tem_dst.at<Vec3b>(j, k)[m]*5)/10;
                        }
                    }
//								cout<< tem_dst.at
                    //if(tem_dst.at)
                }
            }
        imshow("b_dst", dst);cv::waitKey(1000);    
        }
        
        for(int i=0;i<5;i++){
            input.read(view);
        }
        cnt++;
        
    }
    
    cv::waitKey(1000000);
    cout<<cnt<<endl;
}

2.在线合成

在线合成前面已经提到了图像融合的算法.除此之外,在线和成主要是使用basler相机的代码.代码如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
#include <pylon/PylonIncludes.h>
#ifdef PYLON_WIN_BUILD
#    include <pylon/PylonGUI.h>
#endif
#include <opencv2/opencv.hpp>

// Namespace for using pylon objects.
using namespace Pylon;

// Namespace for using cout.
using namespace std;
using namespace cv;

// Number of images to be grabbed.
static const uint32_t c_countOfImagesToGrab = 1000;

int main(int argc, char* argv[])
{
	// The exit code of the sample application.
	int exitCode = 0;

	// Before using any pylon methods, the pylon runtime must be initialized. 
	PylonInitialize();
	Mat frame;
	//****************************************************
	//[2592 x 1944]
	Mat view;
	int cnt = 0;
	vector<Mat> calibrate_images;
	vector<Mat> calibrate_descriptors;
	vector<vector<KeyPoint> > calibrate_keypoints;
	Mat dst(194 + 50, 259 * 10, CV_8UC3); dst.setTo(0);
	Mat save_homo;
	//****************************************************
	try
	{
		// Create an instant camera object with the camera device found first.
		CInstantCamera camera(CTlFactory::GetInstance().CreateFirstDevice());

		// Print the model name of the camera.
		cout << "Using device " << camera.GetDeviceInfo().GetModelName() << endl;

		// The parameter MaxNumBuffer can be used to control the count of buffers
		// allocated for grabbing. The default value of this parameter is 10.
		camera.MaxNumBuffer = 5;

		// Start the grabbing of c_countOfImagesToGrab images.
		// The camera device is parameterized with a default configuration which
		// sets up free-running continuous acquisition.
		camera.StartGrabbing(c_countOfImagesToGrab);

		// This smart pointer will receive the grab result data.
		CGrabResultPtr ptrGrabResult;
		
		/// new image that convert to cv::Mat
		CImageFormatConverter formatConverter;
		formatConverter.OutputPixelFormat = PixelType_BGR8packed;
		CPylonImage pylonImage;
		
		// Camera.StopGrabbing() is called automatically by the RetrieveResult() method
		// when c_countOfImagesToGrab images have been retrieved.
		char c;
		while (c = waitKey(1) && camera.IsGrabbing())
		{
			// Wait for an image and then retrieve it. A timeout of 5000 ms is used.
			camera.RetrieveResult(5000, ptrGrabResult, TimeoutHandling_ThrowException);

			// Image grabbed successfully?
			if (ptrGrabResult->GrabSucceeded())
			{
				// Access the image data.
				cout << "SizeX: " << ptrGrabResult->GetWidth() << endl;
				cout << "SizeY: " << ptrGrabResult->GetHeight() << endl;
				const uint8_t *pImageBuffer = (uint8_t *)ptrGrabResult->GetBuffer();
				cout << "Gray value of first pixel: " << (uint32_t)pImageBuffer[0] << endl << endl;

//#ifdef PYLON_WIN_BUILD
				// Display the grabbed image.
				//Pylon::DisplayImage(1, ptrGrabResult);
//#endif
				///convert to cv::Mat
				formatConverter.Convert(pylonImage, ptrGrabResult);
				frame = cv::Mat(ptrGrabResult->GetHeight(), ptrGrabResult->GetWidth(), CV_8UC3, (uint8_t *)pylonImage.GetBuffer());
				/// show
				//resize(frame, frame, Size(frame.cols / 2, frame.rows / 2));
				cout << frame.size() << endl;
				imshow("OpenCV Display Window", frame);
				if(true)
				{
					Mat view(194, 259, frame.type());
					resize(frame, view, view.size(), 0, 0, INTER_LINEAR);
					//*****************************************************************************8
					std::vector<KeyPoint> keypoints;
					keypoints.resize(500);
					Mat descriptors;
					Ptr<FeatureDetector> detector = ORB::create();
					Ptr<DescriptorExtractor> descriptor = ORB::create();
					//Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
					cout << view.size << endl;
					detector->detect(view, keypoints);
					descriptor->compute(view, keypoints, descriptors);
					Mat outimg;
					calibrate_keypoints.push_back(keypoints);
					calibrate_descriptors.push_back(descriptors);
					drawKeypoints(view, keypoints, outimg, Scalar::all(-1), DrawMatchesFlags::DEFAULT);
					//imshow("ORB特征点",outimg);
					//imshow("input", view);
					//cv::waitKey(10000);

					calibrate_images.push_back(view);
					if (cnt >= 1) {
						//the first 15 images will be used calidate.
						Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-Hamming");
						vector<DMatch> matches;
						Mat img_match;
						//BFMatcher matcher ( NORM_HAMMING );
						matcher->match(calibrate_descriptors[cnt - 1], calibrate_descriptors[cnt], matches);
						//optimize
						double min_dist = 10000, max_dist = 0;
						//找出所有匹配之间的最小距离和最大距离, 即是最相似的和最不相似的两组点之间的距离
						for (int i = 0; i < calibrate_descriptors[cnt - 1].rows; i++)
						{
							double dist = matches[i].distance;
							if (dist < min_dist) min_dist = dist;
							if (dist > max_dist) max_dist = dist;
						}
						std::vector< DMatch > good_matches;
						for (int i = 0; i < calibrate_descriptors[cnt - 1].rows; i++)
						{
							if (matches[i].distance <= max(2 * min_dist, 30.0))
							{
								good_matches.push_back(matches[i]);
							}
						}
						drawMatches(calibrate_images[cnt - 1], calibrate_keypoints[cnt - 1], calibrate_images[cnt], calibrate_keypoints[cnt], good_matches, img_match);
						//imshow("ORB特征点2",img_match);

						//calibrate here
						//**********************************************//
						vector<Point2f> imagePoints1, imagePoints2;

						for (int i = 0; i < good_matches.size(); i++)
						{
							imagePoints2.push_back(calibrate_keypoints[cnt - 1][good_matches[i].queryIdx].pt);
							imagePoints1.push_back(calibrate_keypoints[cnt][good_matches[i].trainIdx].pt);
						}

						Mat homo = findHomography(imagePoints1, imagePoints2, CV_RANSAC);
						if (cnt == 1) save_homo = homo.clone();
						else save_homo = save_homo * homo;
						homo = save_homo;

						// cout<<homo<<endl;
						Mat imageTransform;
						warpPerspective(calibrate_images[cnt], imageTransform, homo, Size(calibrate_images[cnt].cols + 1000, calibrate_images[cnt].rows + 50));
						//imshow("直接经过透视矩阵变换", imageTransform);
						//imwrite("trans1.jpg", imageTransform);
						int dst_width = imageTransform.cols;  //取最右点的长度为拼接图的长度

						int dst_height = calibrate_images[cnt].rows;
	

						Mat tem_dst(194 + 50,  259 * 10, CV_8UC3); tem_dst.setTo(0);
						imageTransform.copyTo(tem_dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
						Mat tem_dst2 = dst.clone();
						//calibrate_images[cnt].copyTo(dst(Rect(0, 0, calibrate_images[cnt].cols, calibrate_images[cnt].rows)));
						max(tem_dst, tem_dst2, dst);
						/*for (int j = 0; j < tem_dst.rows; j++) {
							for (int k = 0; k < tem_dst.cols; k++) {
								for (int m = 0; m < 3; m++) {
									if (tem_dst.at<Vec3b>(j, k)[m] <= 10) {
										dst.at<Vec3b>(j, k)[m] = max(tem_dst2.at<Vec3b>(j, k)[m], tem_dst.at<Vec3b>(j, k)[m]);
									}
									else {
										dst.at<Vec3b>(j, k)[m] = tem_dst2.at<Vec3b>(j, k)[m]*0.3+ tem_dst.at<Vec3b>(j, k)[m]*0.7;
									}
								}
//								cout<< tem_dst.at
								//if(tem_dst.at)
							}
						}
						*/
						//max(tem_dst,0)
						//cout << dst << endl;

						//addWeighted(tem_dst, 0.3,tem_dst2,0.7, 0, dst);

						imshow("b_dst", dst);
					}
					
				}
					cnt++;
	
				
				//******************************************************************************
			}
			else
			{
				cout << "Error: " << ptrGrabResult->GetErrorCode() << " " << ptrGrabResult->GetErrorDescription() << endl;
			}
		}
	}
	catch (const GenericException &e)
	{
		// Error handling.
		cerr << "An exception occurred." << endl
			<< e.GetDescription() << endl;
		exitCode = 1;
	}

	// Comment the following two lines to disable waiting on exit.
	cerr << endl << "Press Enter to exit." << endl;
	while (cin.get() != '\n');

	// Releases all pylon resources. 
	PylonTerminate();

	return exitCode;
}

3结果

在这里插入图片描述

#CV
(2021-10-15)Intelligent Vision Coursework
http://example.com/2023/01/10/2021-10-15-Intelligent-Vision-Coursework/
Author
Zhiqi Li
Posted on
January 10, 2023
Licensed under