So I have two Thermal images (Potato quality I know but it is what I have to work with), the first two images in this album. I am using some code from a tutorial that is super common but have edited a lot of it.
https://i.stack.imgur.com/BU26e.jpg
So what I am doing in my code is
1. Detecting KeyPoints
2. Describe the KeyPoints
3. Match the KeyPoints
4. Keep only good points
5. Gather both Query and Train points
6. Find Homography
7. Warp one of the images
8. Repeat the above steps for the warped image and the other original image
Now my question is: Should the change in the (x,y) distance between two of the same points on the two different images be the same for every set of points?
The whole frame is moving in the same direction so no matter what matching points we look at the change should be the same should it not?
What I am finding is that the points all different in the distance, some are 5 pixels different and some are 700 pixels, the only thing I can think is happening is that the match is not actually good and it is comparing two points that are no where near the same point in the separate frames.
I need to know what the offset is so that I can overlay one frame on top of the other then average out the pixel values that are overlapping and build a new images from the composite/average of the two originals.
My code I am using is below:
#include <stdio.h>
#include <iostream>
#include "opencv2/core/core.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/nonfree/nonfree.hpp"
#include "opencv2/calib3d/calib3d.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "stitch.cpp"
#include "opencv2\stitching\stitcher.hpp"
#include "opencv2\nonfree\features2d.hpp"
using namespace cv;
void readme();
Mat describe(Mat img, vector<KeyPoint> key);
vector<KeyPoint> detect(Mat img);
vector<DMatch> match(Mat descriptionOne, Mat descriptionTwo);
/** @function main */
int main(int argc, char** argv)
{
VideoCapture cap("vid.mp4");
vector<Mat> Vimg;
cout << "Grabbing Images" << endl;
for (int i = 0; i < 2; i++)
{
cout << "Grabbing Frame" << i << endl;
Mat temp;
cap.read(temp);
Vimg.push_back(temp);
imwrite("image" + to_string(i) + ".jpg", temp);
for (int j = 0; j < 80; j++)
cap.grab();
}
//Mat cimg1 = Vimg[0];
//Mat cimg2 = Vimg[1];
Mat cimg1 = imread("cap1.png");
Mat cimg2 = imread("cap2.png");
cout << "Starting Stitching" << endl;
//Converting the original images to grayscale
Mat img1, img2;
cvtColor(cimg1, img1, CV_BGR2GRAY);
cvtColor(cimg2, img2, CV_BGR2GRAY);
//Detecting Keypoints for original two images
vector<KeyPoint> keypointOne = detect(img1), keypointTwo = detect(img2);
Mat mkeypointOne, mkeypointTwo;
drawKeypoints(cimg1, keypointOne, mkeypointOne, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
drawKeypoints(cimg2, keypointTwo, mkeypointTwo, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("keypointOne.jpg", mkeypointOne);
imwrite("keypointTwo.jpg", mkeypointTwo);
//Computing descriptors
Mat descriptionOne = describe(img1, keypointOne), descriptionTwo = describe(img2, keypointTwo);
//Matching descriptors
vector<DMatch> matches = match(descriptionOne, descriptionTwo);
double max = 0;
double min = 100;
//Calculation of max and min distances
for (int i = 0; i < matches.size(); i++)
{
double dist = matches[i].distance;
if (dist < min) min = dist;
if (dist > max) max = dist;
}
vector<DMatch> goodMatches;
//Keep only good matches
for (int i = 0; i < matches.size(); i++)
{
if (matches[i].distance < 2*min)
goodMatches.push_back(matches[i]);
}
//Localize
vector<Point2f> obj;
vector<Point2f> scene;
for (int i = 0; i < goodMatches.size(); i++)
{
obj.push_back(keypointOne[goodMatches[i].queryIdx].pt);
scene.push_back(keypointTwo[goodMatches[i].trainIdx].pt);
}
/*
for (int k = 0; k < obj.size(); k++)
{
cout << "Point data for Match #" << k << endl;
cout << "\tImage 1 Point: " << obj[k] << endl;
cout << "\tImage 2 Point: " << scene[k] << endl;
}*/
Mat H = findHomography(obj, scene, CV_RANSAC);
//Warping the image to fit on first image
Mat cwarpImage, warpImage;
//TODO: figure out the right size for this image that is created
warpPerspective(cimg2, cwarpImage, H, Size(img2.cols + img1.cols, img2.rows + img1.rows));
/*
Mat result;
Mat half(warpImage, Rect(0, 0, img2.cols, img2.rows));
cimg2.copyTo(half);
*/
imwrite("warp.jpg", warpImage);
//Processing Image
cvtColor(cwarpImage, warpImage, CV_BGR2GRAY);
vector<KeyPoint> keypointWarp = detect(warpImage);
Mat descriptionWarp = describe(warpImage, keypointWarp);
vector<DMatch> warpMatches = match(descriptionOne, descriptionWarp);
Mat mkeypointWarp;
drawKeypoints(cwarpImage, keypointWarp, mkeypointWarp, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("keypointWarp.jpg", mkeypointWarp);
Mat match;
drawMatches(cimg1, keypointOne, warpImage, keypointWarp, warpMatches, match, Scalar(0, 0, 255), Scalar(255, 0, 0), vector<char>(), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
//imshow("match", match);
imwrite("matches.jpg", match);
//Localize
vector<Point2f> obj2;
vector<Point2f> scene2;
for (int i = 0; i < warpMatches.size(); i++)
{
obj2.push_back(keypointOne[warpMatches[i].queryIdx].pt);
scene2.push_back(keypointWarp[warpMatches[i].trainIdx].pt);
}
for (int k = 0; k < obj.size(); k++)
{
cout << "Point data for Match #" << k << endl;
cout << "\tImage 1 Point: " << obj2[k] << endl;
cout << "\tImage 2 Point: " << scene2[k] << endl;
}
vector<unsigned char> inliersMask;
Mat H2 = findHomography(obj, scene, CV_RANSAC, 3, inliersMask);
vector<DMatch> inliers;
for (size_t i = 0; i < inliersMask.size(); i++)
{
if (inliersMask[i])
inliers.push_back(warpMatches[i]);
}
warpMatches.swap(inliers);
Mat match2;
drawMatches(cimg1, keypointOne, warpImage, keypointWarp, warpMatches, match2, Scalar(0, 0, 255), Scalar(255, 0, 0), vector<char>(), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("homorgraphyOutlierMatch.jpg", match2);
cout << "Writing Warp Image" << endl;
imwrite("warpimage.jpg", warpImage);
cout << H << endl;
waitKey(0);
}
Mat describe(Mat img, vector<KeyPoint> key)
{
Mat temp;
SurfDescriptorExtractor extractor;
extractor.compute(img, key, temp);
return temp;
}
vector<KeyPoint> detect(Mat img)
{
vector<KeyPoint> temp;
SurfFeatureDetector detector(400);
detector.detect(img, temp);
return temp;
}
vector<DMatch> match(Mat descriptionOne, Mat descriptionTwo)
{
vector<DMatch> temp;
BFMatcher matcher(NORM_L2, true);
matcher.match(descriptionOne, descriptionTwo, temp);
return temp;
}
EDIT:
I set Cross Check to true in the BFMatcher and implemented Homography outlier detection from Mastering_OpenCV. Here are the two new results. I was not sure if I was supposed to implement both cross check and KnnMatch so I only did cross check.
https://i.stack.imgur.com/79XSI.jpg
As you can see they are a lot better but there are still some there that should not be there. I ran it with both full color and thermal images. New code is above as well.
