Inacurate tracking when drawing calcOpticalFlow's outputed feature vector

Question

I have been trying to develop a simple feature tracking program. The user outlines an area on the screen with their mouse, and a mask is created for this area and passed to goodFeaturesToTrack. The features found by the function are then drawn on the screen (represented by blue circles).

Next I pass the feature vector returned by the function to calcOpticalFlowPyrLk and draw the resulting vector of points on the screen (represented by green circles). Although the program tracks the direction of flow correctly, for some reason the features output by the calcOpticalFlow funciton do not line up with the object's location on the screen.

I feel as though it is a small mistake in the logic I have used on my part, but I just can't seem to decompose it, and I would really appreciate some help from the you guys.

I have posted my code below, and I would like to greatly apologize for the global variables and messy structure. I am just testing at the moment, and plan to clean up and convert to an OOP format as soon as I get it running.

As well, here is a link to a YouTube video I have uploaded that demonstrates the behavior I am combating.

bool drawingBox = false;
bool destroyBox = false;
bool targetAcquired = false;
bool featuresFound = false;
CvRect box;
int boxCounter = 0;
cv::Point objectLocation;
cv::Mat prevFrame, nextFrame, prevFrame_1C, nextFrame_1C;
std::vector<cv::Point2f> originalFeatures, newFeatures, baseFeatures;
std::vector<uchar> opticalFlowFeatures;
std::vector<float> opticalFlowFeaturesError;
cv::TermCriteria opticalFlowTermination = cv::TermCriteria(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.3);
cv::Mat mask;
cv::Mat clearMask;

long currentFrame = 0;

void draw(cv::Mat image, CvRect rectangle)
{

    if (drawingBox)
    {
        cv::rectangle(image, cv::Point(box.x, box.y), cv::Point(box.x + box.width, box.y + box.height), cv::Scalar(225, 238 , 81), 2);
        CvRect rectangle2 = cvRect(box.x, box.y, box.width, box.height);
    }

    if (featuresFound)
    {   
        for (int i = 0; i < originalFeatures.size(); i++)
        {
            cv::circle(image, baseFeatures[i], 4, cv::Scalar(255, 0, 0), 1, 8, 0);
            cv::circle(image, newFeatures[i], 4, cv::Scalar(0, 255, 0),1, 8, 0);
            cv::line(image, baseFeatures[i], newFeatures[i], cv::Scalar(255, 0, 0), 2, CV_AA);
        }
    }
}

void findFeatures(cv::Mat mask)
{
    if (!featuresFound && targetAcquired)
    {
        cv::goodFeaturesToTrack(prevFrame_1C, baseFeatures, 200, 0.1, 0.1, mask);
        originalFeatures= baseFeatures;
        featuresFound = true;
        std::cout << "Number of Corners Detected: " << originalFeatures.size() << std::endl;

        for(int i = 0; i < originalFeatures.size(); i++)
        {
            std::cout << "Corner Location " << i << ": " << originalFeatures[i].x << "," << originalFeatures[i].y << std::endl;
        }
    }
}


void trackFeatures()
{
    cv::calcOpticalFlowPyrLK(prevFrame_1C, nextFrame_1C, originalFeatures, newFeatures, opticalFlowFeatures, opticalFlowFeaturesError, cv::Size(30,30), 5, opticalFlowTermination);
    originalFeatures = newFeatures;

}

void mouseCallback(int event, int x, int y, int flags, void *param)
{
    cv::Mat frame;
    frame = *((cv::Mat*)param);

    switch(event)
    {
    case CV_EVENT_MOUSEMOVE:
        {
            if(drawingBox)
            {
                box.width = x-box.x;
                box.height = y-box.y;
            }
        }
        break;

    case CV_EVENT_LBUTTONDOWN:
        {
            drawingBox = true;
            box = cvRect (x, y, 0, 0);
            targetAcquired = false;
            cv::destroyWindow("Selection");
        }
        break;

    case CV_EVENT_LBUTTONUP:
        {
            drawingBox = false;
            featuresFound = false;
            boxCounter++;
            std::cout << "Box " << boxCounter << std::endl;
            std::cout << "Box Coordinates: " << box.x << "," << box.y << std::endl;
            std::cout << "Box Height: " << box.height << std::endl;
            std::cout << "Box Width: " << box.width << std:: endl << std::endl;

            if(box.width < 0)
            {
                box.x += box.width;
                box.width *= -1;
            }

            if(box.height < 0)
            {
                box.y +=box.height;
                box.height *= -1;
            }

            objectLocation.x = box.x;
            objectLocation.y = box.y;
            targetAcquired = true;

        }
        break;

    case CV_EVENT_RBUTTONUP:
        {
            destroyBox = true;
        }
        break;
    }
}

int main ()
{
    const char *name = "Boundary Box";
    cv::namedWindow(name);

    cv::VideoCapture camera;
    cv::Mat cameraFrame;
    int cameraNumber = 0;
    camera.open(cameraNumber);

    camera >> cameraFrame;

    cv::Mat mask = cv::Mat::zeros(cameraFrame.size(), CV_8UC1);
    cv::Mat clearMask = cv::Mat::zeros(cameraFrame.size(), CV_8UC1);

    if (!camera.isOpened())
    {
        std::cerr << "ERROR: Could not access the camera or video!" << std::endl;
    }

    cv::setMouseCallback(name, mouseCallback,  &cameraFrame);

    while(true)
    {

        if (destroyBox)
        {
            cv::destroyAllWindows();
            break;
        }

        camera >> cameraFrame;

        if (cameraFrame.empty())
        {
            std::cerr << "ERROR: Could not grab a camera frame." << std::endl;
            exit(1);
        }

        camera.set(CV_CAP_PROP_POS_FRAMES, currentFrame);
        camera >> prevFrame;
        cv::cvtColor(prevFrame, prevFrame_1C, cv::COLOR_BGR2GRAY);

        camera.set(CV_CAP_PROP_POS_FRAMES, currentFrame ++);
        camera >> nextFrame;
        cv::cvtColor(nextFrame, nextFrame_1C, cv::COLOR_BGR2GRAY);

        if (targetAcquired)
        {
            cv::Mat roi (mask, cv::Rect(box.x, box.y, box.width, box.height));
            roi = cv::Scalar(255, 255, 255);
            findFeatures(mask);
            clearMask.copyTo(mask);
            trackFeatures();
        }

        draw(cameraFrame, box);
        cv::imshow(name, cameraFrame);
        cv::waitKey(20);
    }

    cv::destroyWindow(name);
    return 0;
}

The direction of the movement seems correct, so it seems to be a scale thing. Could you try to multiply the points by a certain value? — Nallath, Jul 03 '13 at 08:50
Yes, the direction of the movement is indeed correct, but I would like to fully comprehend why it is not matching up so that I can correct it accordingly. I don't want to take the hacky route and just scale it by some arbitrary value. — szakeri, Jul 03 '13 at 09:04
Finding out what the scale is would help in finding out why it needs that scale to be correct. It could be that the scale is the scale between the ROI and the actual size of the captured image. — Nallath, Jul 03 '13 at 09:16
You bring up an excellent point! I will definitely look into it and report back my results. — szakeri, Jul 03 '13 at 09:19

Hugo y · Accepted Answer · 2013-07-03T09:36:09.933

In my opinion you can't use camera.set(CV_CAP_PROP_POS_FRAMES, currentFrame) on a webcam, but I 'm not positive about that.

Instead I suggest you to save the previous frame in your prevFrame variable.

As an example I can suggest you this working code, I only change inside the while loop and I add comment before all my adds :

while(true)
{

    if (destroyBox)
    {
        cv::destroyAllWindows();
        break;
    }

    camera >> cameraFrame;

    if (cameraFrame.empty())
    {
        std::cerr << "ERROR: Could not grab a camera frame." << std::endl;
        exit(1);
    }

    // new lines
    if(prevFrame.empty()){
            prevFrame = cameraFrame;
            continue;
    }
    // end new lines

    //camera.set(CV_CAP_PROP_POS_FRAMES, currentFrame);
    //camera >> prevFrame;
    cv::cvtColor(prevFrame, prevFrame_1C, cv::COLOR_BGR2GRAY);

    //camera.set(CV_CAP_PROP_POS_FRAMES, currentFrame ++);
    //camera >> nextFrame;
    // new line
    nextFrame = cameraFrame;
    cv::cvtColor(nextFrame, nextFrame_1C, cv::COLOR_BGR2GRAY);

    if (targetAcquired)
    {
        cv::Mat roi (mask, cv::Rect(box.x, box.y, box.width, box.height));
        roi = cv::Scalar(255, 255, 255);
        findFeatures(mask);
        clearMask.copyTo(mask);
        trackFeatures();
    }

    draw(cameraFrame, box);
    cv::imshow(name, cameraFrame);
    cv::waitKey(20);

    // old = new
    // new line
    prevFrame = cameraFrame.clone();

}

Thanks for the help! This works beautifully. At times I have a minor issue where the tracked points decide to converge together, but I have not written any code to deal with features that are lost or go off screen. I think dealing with these scenarios properly should resolve that problem. Thanks again :) — szakeri, Jul 04 '13 at 10:03
Actually, would you happen to have any insight on why they have the tendency to merge together like that? — szakeri, Jul 04 '13 at 12:41

Inacurate tracking when drawing calcOpticalFlow's outputed feature vector

1 Answers1