How to extract BOW descriptors corresponding to existing codebook?

Question

I've trained a BOW codebook (vocabulary) using a A-KAZE feature descriptors and am trying to use a BFMatcher with knnMatch to compare newly extracted features to the codebook.

Instead, I get the following error,

OpenCV Error: Assertion failed (_queryDescriptors.type() == trainDescType) in knnMatchImpl, file /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp, line 722 terminate called after throwing an instance of 'cv::Exception'   what():  /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp:722: error: (-215) _queryDescriptors.type() == trainDescType in function knnMatchImpl

I've using the following examples

My intuition is that I am adding the codebook to the matcher incorrectly, but I can't find any documentation or examples that support another method. How can I use my codebook with new examples.

MCVE

/* BOWTest.cpp*/
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/opencv.hpp>

#include <iostream>
#include <string>
#include <stdio.h>
#include <dirent.h>

using namespace cv;
using namespace std;

std::string outputFile = "test_codebook.png";
std::string trainingDir = "dataset/";
std::string outputPrefix = "output/res_custom_";

void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze);
void test(Mat codebook, Ptr<Feature2D> akaze);

int main(int ac, char** av) {

    Ptr<Feature2D> feature = AKAZE::create();
    Mat codebook;
    int codebook_n = 100;
    //train(codebook, codebook_n, feature);
    test(codebook, feature);
}

//I included the train method to show how the codebook is trained, but it is not actually called in this example
void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze){
    //defining terms for bowkmeans trainer
    TermCriteria tc(TermCriteria::MAX_ITER + TermCriteria::EPS, 10, 0.001);
    int retries = 1;
    int flags = KMEANS_PP_CENTERS;
    BOWKMeansTrainer bowTrainer(codebook_n, tc, retries, flags);

    int i = 0;
    unsigned long numPoints = 0;
    DIR           *d;
    struct dirent *dir;
    d = opendir(trainingDir.c_str());
    if (d)  {
        while ((dir = readdir(d)) != NULL){

            try {
            Mat img;
            std::string imgName = trainingDir + dir->d_name;
            i = i + 1;

            printf("%d, %lu: %s ...", i,numPoints, imgName.c_str());
            img = imread(imgName, CV_LOAD_IMAGE_COLOR);
            if(img.empty()){ //not image
                printf("bad.\n");
                continue;
            }

            printf("loaded.\n");
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, Mat(), keypoints, features);
            features.convertTo(features, CV_32F);
            bowTrainer.add(features);

            Mat res;
            drawKeypoints(img, keypoints, res);
            std::string output_img =  outputPrefix + dir->d_name;
            imwrite(output_img, res);

            numPoints += features.rows;

            }catch(int e){
                cout << "An exception occurred. Nr. " << e << '\n';
            }
        }

        printf("Read images!");
        closedir(d);

        codebook = bowTrainer.cluster();
        imwrite(outputFile, codebook);
    }
}

void test(Mat codebook, Ptr<Feature2D> akaze){
    codebook = imread(outputFile);
    int codebook_n = codebook.rows;

    BFMatcher matcher(NORM_L2);
    matcher.add(std::vector<cv::Mat>(1, codebook));

    Mat res(Size(codebook_n * 10, 3*10), CV_8UC3, Scalar(0));
    vector<int> res_idx(codebook_n, 0);

    try {
        Mat img;
        String imgName = trainingDir + "dog1.jpeg";
        img = imread(imgName, CV_LOAD_IMAGE_COLOR);
        if(img.empty()){ //not image
            printf("bad.\n");
        }else{
            printf("loaded.\n");
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, noArray(), keypoints, features);
            features.convertTo(features, CV_32F);

            vector< vector< DMatch > > nn_matches;
            matcher.knnMatch(features, nn_matches, 1);

            printf("%d matched keypoints", nn_matches.size());
        }

    }catch(int e){
        cout << "An exception occurred. Nr. " << e << '\n';
    }
}

test_codebook.png

dog1.jpeg

Output

loaded.
OpenCV Error: Assertion failed (_queryDescriptors.type() == trainDescType) in knnMatchImpl, file /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp, line 722
terminate called after throwing an instance of 'cv::Exception'
  what():  /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp:722: error: (-215) _queryDescriptors.type() == trainDescType in function knnMatchImpl

You have a few things conceptually wrong, and that's why your implementation crashes. Before addressing these, can you explain what are you trying to do? You want find the closest among your training images? Or you want to classify the test image as belonging to a particular class? — Miki, Dec 09 '15 at 13:01
@Miki At this point, I am not trying to perform any classification task. I am simply trying to extract the features from a test example. Later on, I will use the feature for object validation. I want to compare if two images contain the same object. Currently, I am just trying to assess the number of detected Akaze features in the new image and which codebook words they correspond to. — Cecilia, Dec 09 '15 at 15:12
The answer should get you going. Please let me know if it's ok for you. — Miki, Dec 09 '15 at 15:47

score 4 · Accepted Answer · answered Dec 09 '15 at 15:45

You shouldn't save the codebook as an image. imwrite will eventually scale and convert the values of the codebook. And imread with default parameters will load it as a 3 channel image CV_8UC3. To store matrices that are not strictly images, you should use FileStorage.

Save:

FileStorage fs(outputFile, FileStorage::WRITE);
// Store codebook
fs << "codebook" << codebook;

Load:

FileStorage fs(outputFile, FileStorage::READ);
fs["codebook"] >> codebook;

You should use BOWImgDescriptorExtractor to compute the BoW image descriptor starting from your features, AKAZE in this case:

Ptr<DescriptorMatcher> matcher = BFMatcher::create("BruteForce");
BOWImgDescriptorExtractor bow(akaze, matcher);
bow.setVocabulary(codebook);

// Mat img = ...

// AKAZE features
Mat features;
vector<KeyPoint> keypoints;
akaze->detectAndCompute(img, noArray(), keypoints, features);
features.convertTo(features, CV_32F);

// BoW descriptor
Mat bowFeatures;
vector<vector<int>> pointsIdxsOfCluster;
bow.compute(features, bowFeatures, &pointsIdxsOfCluster);

You can use builtin glob to read images from a folder, avoiding dirent.

vector<String> fileNames;
glob(trainingDir, fileNames);

for (int i=0; i<fileNames.size(); ++i)
{
    Mat img = imread(fileNames[i]);
    ...

You can use iostream with cout, instead of printf.

This is how the code looks like:

#include <opencv2/opencv.hpp>
#include <iostream>
using namespace cv;
using namespace std;

std::string outputFile = "test_codebook.yml";
std::string trainingDir = "path_to_train_folder/";
std::string outputPrefix = "path_to_output_folder/";

void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze);
void test(Mat codebook, Ptr<Feature2D> akaze);

int main(int ac, char** av) {

    Ptr<Feature2D> feature = AKAZE::create();
    Mat codebook;
    int codebook_n = 100;

    train(codebook, codebook_n, feature);
    test(codebook, feature);
}

//I included the train method to show how the codebook is trained, but it is not actually called in this example
void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze){
    //defining terms for bowkmeans trainer
    TermCriteria tc(TermCriteria::MAX_ITER + TermCriteria::EPS, 10, 0.001);
    int retries = 1;
    int flags = KMEANS_PP_CENTERS;
    BOWKMeansTrainer bowTrainer(codebook_n, tc, retries, flags);

    int i = 0;
    unsigned long numPoints = 0;

    vector<String> fileNames;
    glob(trainingDir, fileNames);

    for (int i=0; i<fileNames.size(); ++i)
    {
        try {
            Mat img;
            std::string imgName = fileNames[i];
            std::string filename = imgName.substr(trainingDir.length());

            cout << i << ", " << numPoints << " : " << imgName;
            img = imread(imgName, CV_LOAD_IMAGE_COLOR);
            if (img.empty()){ //not image
                cout << " bad" << endl;
                continue;
            }

            cout << " loaded" << endl;
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, Mat(), keypoints, features);
            features.convertTo(features, CV_32F);
            bowTrainer.add(features);

            Mat res;
            drawKeypoints(img, keypoints, res);
            std::string output_img = outputPrefix + filename;
            imwrite(output_img, res);

            numPoints += features.rows;

        }
        catch (int e){
            cout << "An exception occurred. Nr. " << e << '\n';
        }
    }

    cout << "Read images!" << endl;

    codebook = bowTrainer.cluster();

    {
        FileStorage fs(outputFile, FileStorage::WRITE);

        // Store codebook
        fs << "codebook" << codebook;

        // You can also store additional info, like the list of images

        //// Store train images filenames
        //fs << "train" << "[";
        //for (int i = 0; i < fileNames.size(); ++i)
        //{
        //  fs << fileNames[i];
        //}
        //fs << "]";
    }
}

void test(Mat codebook, Ptr<Feature2D> akaze)
{
    vector<String> trainFileNames;
    {
        FileStorage fs(outputFile, FileStorage::READ);
        fs["codebook"] >> codebook;

        /*FileNode trainingImages = fs["train"];
        FileNodeIterator it = trainingImages.begin(), it_end = trainingImages.end();
        int idx = 0;
        for (; it != it_end; ++it, idx++)
        {
            trainFileNames.push_back(*it);
        }*/
    }

    int codebook_n = codebook.rows;

    Ptr<DescriptorMatcher> matcher = BFMatcher::create("BruteForce");
    BOWImgDescriptorExtractor bow(akaze, matcher);
    bow.setVocabulary(codebook);

    try {
        Mat img;
        String imgName = "path_to_test_image";
        img = imread(imgName, CV_LOAD_IMAGE_COLOR);
        if (img.empty()){ //not image
            cout << "bad" << endl;
        }
        else{
            cout << "loaded" << endl;
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, noArray(), keypoints, features);
            features.convertTo(features, CV_32F);

            Mat bowFeatures;
            vector<vector<int>> pointsIdxsOfCluster;
            bow.compute(features, bowFeatures, &pointsIdxsOfCluster);

            // bowFeatures is the descriptor you're looking for
            // pointsIdxsOfCluster contains the indices of keypoints that belong to the cluster.
        }
    }
    catch (int e){
        cout << "An exception occurred. Nr. " << e << endl;
    }
}

Thanks, this is a great answer. Very complete, well explained, and it works! :) — Cecilia, Dec 09 '15 at 18:14

How to extract BOW descriptors corresponding to existing codebook?

MCVE

test_codebook.png

dog1.jpeg

Output

1 Answers1