How to use cv::Mat and Eigen::Matrix correctly? (OpenCV + Eigen)

Question

I am able to convert an OpenCV mat object to an Eigen object, and back. However when I try to display the Eigen->Mat on the screen I get a blank window, and I do not know why. I can save the image to a file so I know its converting correctly.

Any suggestions on how to get the converted image to display on the screen?

Here is my current code:

#include <iostream>
#include <opencv2/opencv.hpp>
#include <Eigen/Dense>
#include <opencv2/core/eigen.hpp>


int main(int argc, char **argv) {
    if ( argc != 2 )
    {
        printf("usage: DisplayImage.out <Image_Path>\n");
        return -1;
    }
    cv::Mat image;

    image = cv::imread( argv[1], cv::ImreadModes::IMREAD_GRAYSCALE);
    if ( !image.data )
    {
        printf("No image data \n");
        return -1;
    }
    cv::namedWindow("Display Image", cv::WINDOW_AUTOSIZE );
    cv::imshow("Display Image", image);
    cv::waitKey(0);
    Eigen::MatrixXd eigen_matrix;
    cv::cv2eigen(image, eigen_matrix);

    // std::cout << eigen_matrix << std::endl;

    cv::Mat test_image;
    cv::eigen2cv(eigen_matrix, test_image);
    // This is blank
    cv::namedWindow("Display Image2", cv::WINDOW_AUTOSIZE );
    cv::imshow("Display Image2", test_image);
    cv::waitKey(0);
    cv::imwrite("test.png", test_image);
    return 0;
}

Answer 1

From Eigen document, we can find this:

typedef Matrix<double, Dynamic, Dynamic> MatrixXd;

That's to say, you convert the grayscale image into double. While OpenCV display float/double in range [0, 1.0], save float/double in range [0, 255.0].

Two methods to solve:

1. imshow CV_32F|CV_64F multiplied by (1.0/255)

   cv::imshow("doube image ", test_image*(1.0/255));
   

2. change Eigen Matrix element type to unsigned char

   typedef Eigen::Matrix<unsigned char, Eigen::Dynamic, Eigen::Dynamic> MatrixXuc;
   MatrixXuc eigen_matrix;
   

---

This is my result:

---

The code:

#include <iostream>
#include <opencv2/opencv.hpp>
#include <Eigen/Dense>
#include <opencv2/core/eigen.hpp>

int main(int argc, char **argv) {
    cv::Mat image = cv::imread( "Knight.jpg", cv::ImreadModes::IMREAD_GRAYSCALE);
    if ( !image.data )
    {
        printf("No image data \n");
        return -1;
    }

    cv::imshow("Source", image);

    // (1) display multiplied by (1.0/255)
    {
        Eigen::MatrixXd eigen_matrix;
        cv::cv2eigen(image, eigen_matrix);
        cv::Mat test_image;
        cv::eigen2cv(eigen_matrix, test_image);
        cv::imshow("doube image ", test_image*(1.0/255));
        cv::imwrite("dst_double.png", test_image);
    }

    // (2) change Eigen Matrix type
    {
        typedef Eigen::Matrix<unsigned char, Eigen::Dynamic, Eigen::Dynamic> MatrixXuc;
        MatrixXuc eigen_matrix;
        cv::cv2eigen(image, eigen_matrix);
        cv::Mat test_image;
        cv::eigen2cv(eigen_matrix, test_image);
        cv::imshow("uchar image", test_image);
        cv::imwrite("dst_uchar.png", test_image);
    }

    cv::waitKey(0);
    return 0;
}

---

Notice:

Help on cv2.imshow

imshow(...)
    imshow(winname, mat) -> None
    .   @brief Displays an image in the specified window.
    .
    .   The function imshow displays an image in the specified window. If the window was created with the
    .   cv::WINDOW_AUTOSIZE flag, the image is shown with its original size, however it is still limited by $
    .   Otherwise, the image is scaled to fit the window. The function may scale the image, depending on its$
    .
    .   -   If the image is 8-bit unsigned, it is displayed as is.
    .   -   If the image is 16-bit unsigned or 32-bit integer, the pixels are divided by 256. That is, the
    .   value range [0,255\*256] is mapped to [0,255].
    .   -   If the image is 32-bit or 64-bit floating-point, the pixel values are multiplied by 255. That is$
    .   value range [0,1] is mapped to [0,255].

Help on cv2.imwrite

imwrite(...)
    imwrite(filename, img[, params]) -> retval
    .   @brief Saves an image to a specified file.
    .
    .   The function imwrite saves the image to the specified file. The image format is chosen based on the
    .   filename extension (see cv::imread for the list of extensions). Only 8-bit (or 16-bit unsigned (CV_1$
    .   in case of PNG, JPEG 2000, and TIFF) single-channel or 3-channel (with 'BGR' channel order) images
    .   can be saved using this function. If the format, depth or channel order is different, use
    .   Mat::convertTo , and cv::cvtColor to convert it before saving. Or, use the universal FileStorage I/O
    .   functions to save the image to XML or YAML format.

Answer 2

The previous answer only works on GrayScale, this one works on Color.

The key is to reshape the cv::Mat.

cv::Mat::reshape(int new_channel, int new_rows);

---

The result:

---

The code:

#include <iostream>
#include <opencv2/opencv.hpp>
#include <Eigen/Dense>
#include <opencv2/core/eigen.hpp>


int main(int argc, char **argv) {
    cv::Mat image = cv::imread( "Knight.jpg");
    if ( !image.data )
    {
        printf("No image data \n");
        return -1;
    }

    cv::imshow("Source", image);

    int chs = image.channels();

    // (1) display multiplied by (1.0/255)
    {
        cv::Mat img = image.reshape(1, 0);
        std::cout << img.size() << ", " << img.channels() << std::endl;
        typedef Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic> MatrixXd;
        MatrixXd mat;

        cv::cv2eigen(img, mat);
        cv::Mat img2;
        cv::eigen2cv(mat, img2);
        img2 = img2.reshape(chs, 0);
        cv::imshow("doube color image ", img2*(1.0/255));
        cv::imwrite("dst_double.png", img2);

    }

    // (2) change Eigen Matrix type
    {
        cv::Mat img = image.reshape(1, 0);
        std::cout << img.size() << ", " << img.channels() << std::endl;
        typedef Eigen::Matrix<unsigned char, Eigen::Dynamic, Eigen::Dynamic> MatrixXuc;
        MatrixXuc mat;

        cv::cv2eigen(img, mat);
        cv::Mat img2;
        cv::eigen2cv(mat, img2);
        img2 = img2.reshape(chs, 0);
        std::cout << img2.size() << ", " << img2.channels() << std::endl;
        cv::imshow("uchar color image", img2);
        cv::imwrite("dst_uchar.png", img2);
    }

    cv::waitKey(0);
    return 0;
}