OpenCV Java removing objects from scene

Question

I have successfully detected bounding boxes, now I want to remove everything else in the image and just remain with the content of the bounding boxes to improve on tesseract accuracy, below is an image representation of what I have done and need (I want the binary images to only contain the letters, all the other objects to be removed):

and my code:

 public static ProcessedFrame preProcessImage(Mat image){
    originalFrame = image.clone();
    roiColor = image.clone();
    Imgproc.cvtColor(image, image, Imgproc.COLOR_BGR2GRAY, 0);
    originalFrameGrayScale = image.clone();
    Mat morph = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9, 9));
    Imgproc.morphologyEx(image, image, Imgproc.MORPH_TOPHAT, morph);
    Imgproc.Sobel(image, image, -1, 2, 0);
    Imgproc.GaussianBlur(image, image, new Size(5,5), 3,3);
    Imgproc.morphologyEx(image, image, Imgproc.MORPH_CLOSE, morph);
    Imgproc.threshold(image, image, 200, 255, Imgproc.THRESH_OTSU);
    Vector<Rect> rectangles = detectionContour(image);
    Mat roi = originalFrameGrayScale.clone();
    if(!rectangles.isEmpty()){
    roi = originalFrameGrayScale.submat(rectangles.get(0));
    roiBlack = roi.clone();
    roiColor = roiColor.submat(rectangles.get(0));
      Imgproc.rectangle(originalFrame, rectangles.get(0).br(), rectangles.get(0).tl(), new Scalar(0,0,255), 2);
    }
   Imgproc.medianBlur(roi, roi, 3); 
   Imgproc.adaptiveThreshold(roi, roi, 225, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C, Imgproc.THRESH_BINARY, 15, 3);
   Imgproc.medianBlur(roi, roi, 3);
   Imgproc.medianBlur(roi, roi, 3); 
   Imgproc.adaptiveThreshold(roi, roi, 225, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C, Imgproc.THRESH_BINARY, 15, 3);
   Imgproc.medianBlur(roi, roi, 3);
   roiBinarize = roi.clone();
   Mat erode = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
   Mat dilate = Imgproc.getStructuringElement(Imgproc.MORPH_RECT,new Size(3, 3));
   Imgproc.morphologyEx(roi, roi, Imgproc.MORPH_CLOSE, dilate);
   Imgproc.morphologyEx(roi, roi, Imgproc.MORPH_CLOSE, erode);
   Vector<Rect> letters = detectionPlateCharacterContour(roi);
   doTesseractOCR(letters, roiBinarize);
    return  new ProcessedFrame(originalFrame, roiColor, roiBinarize, roi);

}






private static void doTesseractOCR(Vector<Rect> letters, Mat plate){
    Tesseract instance = new Tesseract(); //
    instance.setLanguage(LANGUAGE);
    String resultPlate = "AAA0000";
    for(int i= 0; i < letters.size(); i++){

     BufferedImage letter = OpenCvUtils.Mat2bufferedImage(plate.submat(letters.get(i)));
        try {
        String result = instance.doOCR(letter);
        String character = result.replace("\n", "");
        resultPlate = new StringBuilder(resultPlate).replace(i ,i+1, character).toString();
        } catch (TesseractException e) {
        System.err.println(e.getMessage());
        }

        System.out.println("Tesseract output: "+resultPlate);
    }

     try {
        String result = instance.doOCR(OpenCvUtils.Mat2bufferedImage(roiBinarize));
        System.out.println("Tesseract output2: "+result.replace("\n", ""));
         } catch (TesseractException e) {
        System.err.println(e.getMessage());
        }
}





 private static Vector<Rect> detectionPlateCharacterContour(Mat roi) {
    Mat contHierarchy = new Mat();
    Mat imageMat = roi.clone();
    Rect rect = null;
    List<MatOfPoint> contours = new ArrayList<>();
    Imgproc.findContours(imageMat, contours, contHierarchy, Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
    Vector<Rect> rect_array = new Vector<>();

    for (int i = 0; i < contours.size(); i++) {
        rect = Imgproc.boundingRect(contours.get(i));
        double ratio = 0;

               if(rect.height > rect.width){
            ratio = rect.height/rect.width;

            }else{
                ratio = rect.width/rect.height;

            }
         Logger.printMessage("Ratio of letter: "+ratio);
      double contourarea = Imgproc.contourArea(contours.get(i));
         if (contourarea >= 160 && contourarea <= 1000 && ( ratio >= 1 && ratio <= 2)) {
         Imgproc.rectangle(roiColor, rect.br(), rect.tl(), new Scalar(10,50,255));
           rect_array.add(rect);
         }
    }


    contHierarchy.release();
    return rect_array;
}




 private static Vector<Rect> detectionContour(Mat outmat) {
    Mat contHierarchy = new Mat();
    Mat imageMat = outmat.clone();

    List<MatOfPoint> contours = new ArrayList<>();

    Imgproc.findContours(imageMat, contours, contHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE);

    Vector<Rect> rect_array = new Vector<>();
    for (int i = 0; i < contours.size(); i++) {
        Rect rect = Imgproc.boundingRect(contours.get(i));

        Mat contour = contours.get(i);
        double contourarea = Imgproc.contourArea(contour);

            double ratio = 0;
            int radius = 0;
            if(rect.height > rect.width){
            ratio = rect.height/rect.width;
            radius = rect.height/2;
            }else{
                ratio = rect.width/rect.height;
                radius = rect.width/2;
            }
        if (contourarea >= 2000 && contourarea <= 10000 && ( ratio == 1 || ratio == 2)) {   
            Logger.printMessage("Rectangle ratio: "+ratio);
            MatOfPoint2f mat2f = new MatOfPoint2f();
            contours.get(i).convertTo(mat2f, CvType.CV_32FC2);
            RotatedRect rotatedRect = Imgproc.minAreaRect( mat2f );
            double rotationAngle = rotatedRect.angle;
            if(rotatedRect.angle > 0)
                rotationAngle = 90 - rotatedRect.angle;
            else
               rotationAngle = rotatedRect.angle; 
            Logger.printMessage("Rotation is: "+(rotationAngle));
            rect = enlargeROI(originalFrame, rect, 10);
            rect_array.add(rect);
        }
    }


    contHierarchy.release();
    return rect_array;
}





private Vector<Rect> detectionContours(Mat outmat) {
    Mat contHierarchy = new Mat();
    Mat imageMat = outmat.clone();
    Rect contourRect = null;
    List<MatOfPoint> contours = new ArrayList<>();
    Imgproc.findContours(imageMat, contours, contHierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
    Vector<Rect> rect_array = new Vector<>();
    for (int i = 0; i < contours.size(); i++) {
        Mat contour = contours.get(i);
        double contourarea = Imgproc.contourArea(contour);
        if (contourarea > minBlob && contourarea < maxBlob) {
            contourRect = Imgproc.boundingRect(contours.get(i));
            rect_array.add(contourRect);
        }
    }


    contHierarchy.release();
    return rect_array;
}

When i run this code printing out the centroid of each rectangle i find that 2 x: 18.0 y: 111.0 2 x: 42.0 y:109.0 7 x: 65.0 y: 108.0 0x:89.0 y: 108.0 A x: 29.0 y: 61.0 C x: 52.0 y: 58.0 P x: 77.0 y: 58.0

Answer 1

It seems you just need to do some preprocessing on your letter bounding boxes, and it would probably make sense to do it before feeding them to Tesseract, although it's not strictly necessary. Let's begin by annotating our sample image with contour indices, and tabulating their attributes.

contour | bbox centroid | recognised
index   |   x   |   y   | symbol
--------+-------+-------+-----------
0       |  18.0 | 111.0 |   2
1       |  42.0 | 109.0 |   2
2       |  65.0 | 108.0 |   7
3       |  89.0 | 108.0 |   0
4       |  29.0 |  61.0 |   A
5       |  52.0 |  58.0 |   C
6       |  77.0 |  58.0 |   P

When a person reads this license plate, they start with the top-most row of symbols, and read the symbols from left to right. After they reach the right-most symbol, they move to the next lower row, and again go from left to right. Therefore, we should organize our symbols (or the bounding boxes that represent them) in a structure that will let us do this easily.

We can use Vector<Rect> to represent each row of symbols. Since there are always two rows of symbols, we will need two instances of this vector -- let's refer to them as row0 and row1.

---

The first task is to categorize the bounding boxes of symbols into two groups, and based on this add them to the appropriate row vector. A simple approach to this may be something like this:

• Calculate mean value of the y coordinate of symbol centroids. In our example, it's approximately mean_y=87.5.
• For each detected symbol:
  • If y coordinate of symbol centroid is less than mean_y, add the symbol to row0
  • Otherwise add the symbol to row1

When we perform this algorithm on our sample set of symbols, we see that

     | contained
row  | symbol ids
-----+--------------
row0 |  4, 5, 6
row1 |  0, 1, 2, 3

In case this simple approach turns out to be insufficient, you can try some more advanced means of clustering.

---

The second task is to sort each row so the symbols go left to right:

• For each row:
  • Sort row elements in ascending order by the x coordinate of their centroid

My Java is quite rusty, but you could probably do this using a comparator. You can refer to this SO answer for more details on how to sort collections in Java.

In our sample, the elements within each row are already in order, so there's nothing to do...

     | contained
row  | symbol ids
-----+--------------
row0 |  4, 5, 6
row1 |  0, 1, 2, 3

---

Now we can

• For each row:
  • For each symbol in row
    • Process with Tesseract
    • Handle the result as required (e.g. print)

In our sample, we'd get

ACP
2270