Identifying state of tic-tac-toe board from image

Question

I'm working on a project where I have to use openCV in java to identify the state of a tic tac toe board. Please see the sample program execution below.

input

Output

X,-,-

-,O,-

X,-,-

I'm trying to solve this by finding contours in the image, but the problem is that the empty unmarked boxes are also being captured and I'm not being able to distinguish between the shapes using contour properties like polygon size and contour area. Below is the code that I have so far.

package finalproject;

import java.awt.Color;
import java.util.ArrayList;
import java.util.List;

import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;


public class FinalProject {
    public static void main(String[] args) {

    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    Mat image = Imgcodecs.imread("C://Users//BadarJahan//Desktop//board- 
    perfect.jpg");

    Mat binaryImage = preprocess(image);

    List<MatOfPoint> contours = new ArrayList<>();
    Imgproc.findContours(binaryImage, contours,new Mat() ,Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
    List<MatOfPoint2f> contoursConvert = new ArrayList<>();

    for(MatOfPoint contour : contours) {
        contoursConvert.add(new MatOfPoint2f(contour.toArray()));
    }

    identifyTicTacToeConfiguration(binaryImage,contoursConvert);

}

    private static Mat preprocess(Mat colorImage) {
//      Imgproc.resize(colorImage, colorImage, new Size(489,0));
        Mat grayImage = new Mat() , binaryImage = new Mat();
        Imgproc.cvtColor(colorImage, grayImage,Imgproc.COLOR_BGR2GRAY);
        binaryImage = grayImage;
        Imgproc.threshold(grayImage, binaryImage, 0, 255, Imgproc.THRESH_BINARY_INV | Imgproc.THRESH_OTSU);
        final Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5,5));
        Imgproc.morphologyEx(binaryImage, binaryImage, Imgproc.MORPH_CLOSE, kernel);
        return binaryImage;
    }

    private static MatOfPoint2f getApproxPoly(final MatOfPoint2f contour) {
        MatOfPoint2f polyContour = new MatOfPoint2f();
        final double epsillon = Imgproc.arcLength(contour, true) * 0.02;
        final boolean close = true;
        Imgproc.approxPolyDP(contour, polyContour, epsillon, close);
        return polyContour;
    }

    private static void printContourProperties(final MatOfPoint contour) {
        final double contourArea = Imgproc.contourArea(contour);
        MatOfInt convexHull = new MatOfInt();
        Imgproc.convexHull(contour, convexHull);
 //     final double convexHullArea = Imgproc.contourArea(convexHull);
        Rect boundingRect = Imgproc.boundingRect(contour);
        MatOfPoint2f poly =  getApproxPoly(new 
        MatOfPoint2f(contour.toArray()));
        System.out.println("Contour area : " + contourArea);
        System.out.println("Aespect Ratio : " + 
        boundingRect.width/boundingRect.height);
        System.out.println("Extend: " + contourArea/boundingRect.area());
 //     System.out.println("Solidity : " + contourArea/convexHullArea);
        System.out.println("Poly Size : " + poly.size().area() + ", is 
        convex " + Imgproc.isContourConvex(new MatOfPoint(poly.toArray())));
        System.out.println();
    }

    private static void showContourProperties(final Mat input, final List<MatOfPoint> contours) {
        Mat inputCopy = new Mat();
        for(int i = 0; i < contours.size(); i++) {
            input.copyTo(inputCopy);
            Scalar color = new Scalar(255);
            final int thickness = 3;
            Imgproc.drawContours(inputCopy, contours, i, color,thickness);
            printContourProperties(contours.get(i));
            Imgcodecs.imwrite("C://Users//BadarJahan//Desktop//Test-1-check- 
            "+i+".jpg", inputCopy);
        }
    }

    private static ContourType recognizeContourType(final MatOfPoint2f contour) {
        final double contourArea = Imgproc.contourArea(contour);
        final MatOfPoint2f poly = getApproxPoly(contour);
        ContourType type = ContourType.Unknown;
        if((poly.elemSize() > 7 && poly.elemSize() < 10) && contourArea < 1000) {
             type = ContourType.OType;
        }else if(contourArea > 10000) {
            type = ContourType.XType;
        }
        return type;
    }


    private static void identifyTicTacToeConfiguration(final Mat input, final List<MatOfPoint2f> contours) {

        for(MatOfPoint2f contour: contours) {
            ContourType type = recognizeContourType(contour);

            if(type == ContourType.XType) {
                System.out.print("X");
            }else if(type == ContourType.OType) {
                System.out.print("O");
            }
        }
    }
}

Any help would be greatly appreciated. Thanks

Answer 1

As I looked into this I was having fun, so I got carried away a bit. The result and code is below. I used python, but I'm sure you'll figure it out ;)

To diffenciate between X en O I used solidity. Solidity is the ratio of contour area to its convex hull area. For an O that is close to 1, for an X less than half.

Note: the tiles are randomly numbered, the actual location needs to be determined based on the x/y location. Second, an O will result in 2 circles, both with solidity near 1.

import numpy as np
import cv2

#create a 2d array to hold the gamestate
gamestate = [["-","-","-"],["-","-","-"],["-","-","-"]]

#kernel used for noise removal
kernel =  np.ones((7,7),np.uint8)
# Load a color image 
img = cv2.imread('X_O.jpg')
# get the image width and height
img_width = img.shape[0]
img_height = img.shape[1]

# turn into grayscale
img_g =  cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# turn into thresholded binary
ret,thresh1 = cv2.threshold(img_g,127,255,cv2.THRESH_BINARY)
#remove noise from binary
thresh1 = cv2.morphologyEx(thresh1, cv2.MORPH_OPEN, kernel)

#find and draw contours. RETR_EXTERNAL retrieves only the extreme outer contours
im2, contours, hierarchy = cv2.findContours(thresh1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(img, contours, -1, (0,255,0), 15)

tileCount = 0
for cnt in contours:
        # ignore small contours that are not tiles
        if cv2.contourArea(cnt) > 200000: 
                tileCount = tileCount+1
                # use boundingrect to get coordinates of tile
                x,y,w,h = cv2.boundingRect(cnt)
                # create new image from binary, for further analysis. Trim off the edge that has a line
                tile = thresh1[x+40:x+w-80,y+40:y+h-80]
                # create new image from main image, so we can draw the contours easily
                imgTile = img[x+40:x+w-80,y+40:y+h-80]

                #determine the array indexes of the tile
                tileX = round((x/img_width)*3)
                tileY = round((y/img_height)*3)     

                # find contours in the tile image. RETR_TREE retrieves all of the contours and reconstructs a full hierarchy of nested contours.
                im2, c, hierarchy = cv2.findContours(tile, cv2.RETR_TREE , cv2.CHAIN_APPROX_SIMPLE)
                for ct in c:
                        # to prevent the tile finding itself as contour
                        if cv2.contourArea(ct) < 180000:
                                cv2.drawContours(imgTile, [ct], -1, (255,0,0), 15)
                                #calculate the solitity
                                area = cv2.contourArea(ct)
                                hull = cv2.convexHull(ct)
                                hull_area = cv2.contourArea(hull)
                                solidity = float(area)/hull_area

                                # fill the gamestate with the right sign
                                if(solidity > 0.5):
                                        gamestate[tileX][tileY] = "O"
                                else: 
                                        gamestate[tileX][tileY] = "X"
                # put a number in the tile
                cv2.putText(img, str(tileCount), (x+200,y+300), cv2.FONT_HERSHEY_SIMPLEX, 10, (0,0,255), 20)

#print the gamestate
print("Gamestate:")
for line in gamestate:
        linetxt = ""
        for cel in line:
                linetxt = linetxt + "|" + cel
        print(linetxt)

# resize final image
res = cv2.resize(img,None,fx=0.2, fy=0.2, interpolation = cv2.INTER_CUBIC)

# display image and release resources when key is pressed
cv2.imshow('image1',res)
cv2.waitKey(0)
cv2.destroyAllWindows()