Masking problems in 6 digit recognition using OpenCV with lighted meter

Question

I am trying to recognize six digits from a meter using python-OpenCV. It's surprising how incredibly hard it is to set morphological operations working in the right way, given the time I have spent adjusting the focus/distance of my raspberry pi camera to the meter screen and I even have bought a separate led lamp to have as much uniform light as possible. This is a template image and I've tried using and adjusting the code from these two sources: enter link description here and enter link description here reproduced below without any progress. I got stuck right at the start when setting the thresholding options. Thank you in advance for any help.

# Code 1
import cv2
import numpy as np
import pytesseract

# Load the image
img = cv2.imread("test.jpg")

# Color-segmentation to get binary mask
lwr = np.array([43, 0, 71])
upr = np.array([103, 255, 130])
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
msk = cv2.inRange(hsv, lwr, upr)
cv2.imwrite("msk.png", msk)

# Extract digits
krn = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 3))
dlt = cv2.dilate(msk, krn, iterations=5)
res = 255 - cv2.bitwise_and(dlt, msk)
cv2.imwrite("res.png", res)

# Displaying digits and OCR
txt = pytesseract.image_to_string(res, config="--psm 6 digits")
print(''.join(t for t in txt if t.isalnum()))
cv2.imshow("res", res)
cv2.waitKey(0)
cv2.destroyAllWindows()

# code 2
# https://pyimagesearch.com/2017/02/13/recognizing-digits-with-opencv-and-python/
# import the necessary packages
# from imutils.perspective import four_point_transform
from imutils import contours
import imutils
import cv2
import numpy as np
from numpy.linalg import norm

# define the dictionary of digit segments so we can identify
# each digit on the thermostat
DIGITS_LOOKUP = {
        (1, 1, 1, 0, 1, 1, 1): 0,
        (1, 0, 1, 0, 1, 0, 1): 1,
        (1, 0, 1, 1, 1, 0, 1): 2,
        (1, 0, 1, 1, 0, 1, 1): 3,
        (0, 1, 1, 1, 0, 1, 0): 4,
        (1, 1, 0, 1, 0, 1, 1): 5,
        (1, 1, 0, 1, 1, 1, 1): 6,
        (1, 1, 1, 0, 0, 1, 0): 7,
        (1, 1, 1, 1, 1, 1, 1): 8,
        (1, 1, 1, 1, 0, 1, 1): 9
}

images = 'test.jpg'
image = cv2.imread(images, 1)
# pre-process the image by resizing it, converting it to
# graycale, blurring it, and computing an edge map
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray, (3, 3), 0)
# gray = cv2.medianBlur(blurred, 1)

# threshold the warped image, then apply a series of morphological
# operations to cleanup the thresholded image
(T, thresh) = cv2.threshold(blurred, 0, 255,
                       cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)

cv2.imshow('thresh', thresh)
cv2.waitKey(0)
cv2.destroyAllWindows()

mask = np.zeros((image.shape[0] + 2, image.shape[1] + 2), np.uint8)
cv2.floodFill(thresh, mask, (0, 0), 0)
cv2.floodFill(thresh, mask, (image.shape[1]-1, 0), 0)
cv2.floodFill(thresh, mask, (round(image.shape[1]/2.4), 0), 0)
cv2.floodFill(thresh, mask, (image.shape[1]//2, 0), 0)
cv2.floodFill(thresh, mask, (0, image.shape[0]-1), 0)
cv2.floodFill(thresh, mask, (image.shape[1]-1, image.shape[0]-1), 0)

kernel = np.ones((2, 2), np.uint8)
thresh = cv2.erode(thresh, kernel, iterations=2)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 13))
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=2)

# cv2.imshow('thresh', thresh)
# cv2.waitKey(0)
# cv2.destroyAllWindows()


# find contours in the thresholded image, then initialize the
# digit contours lists
cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                        cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
digitCnts = []
# loop over the digit area candidates
for c in cnts:
        # compute the bounding box of the contour
        (x, y, w, h) = cv2.boundingRect(c)
        # if the contour is sufficiently large, it must be a digit
        if w <= 300 and (h >= 130 and h <= 300):
            digitCnts.append(c)
            cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)

# cv2.imshow('image', image)
# cv2.waitKey(0)
# cv2.destroyAllWindows()

# sort the contours from left-to-right, then initialize the
# actual digits themselves
digitCnts = contours.sort_contours(digitCnts, method="left-to-right")[0]
digits = []

clao = 0
# loop over each of the digits
for c in digitCnts:
        clao = clao + 1
        # extract the digit ROI
        (x, y, w, h) = cv2.boundingRect(c)
        roi = thresh[y:y + h, x:x + w]
        # compute the width and height of each of the 7 segments
        # we are going to examine
        (roiH, roiW) = roi.shape
        (dW, dH) = (int(roiW * 0.25), int(roiH * 0.15))
        dHC = int(roiH * 0.05)
        # define the set of 7 segments
        segments = [
                ((0, 0), (w, dH)),                           # top
                ((0, 0), (dW, h // 2)),                      # top-left
                ((w - dW, 0), (w, h // 2)),                  # top-right
                ((0, (h // 2) - dHC), (w, (h // 2) + dHC)),  # center
                ((0, h // 2), (dW, h)),                      # bottom-left
                ((w - dW, h // 2), (w, h)),                  # bottom-right
                ((0, h - dH), (w, h))                        # bottom
        ]
        on = [0] * len(segments)

        # loop over the segments
        for (i, ((xA, yA), (xB, yB))) in enumerate(segments):
                #  extract the segment ROI, count the total number of
                #  thresholded pixels in the segment, and then compute
                #  the area of the segment
                segROI = roi[yA:yB, xA:xB]
                total = cv2.countNonZero(segROI)
                area = (xB - xA) * (yB - yA)
                # if the total number of non-zero pixels is greater than
                # 50% of the area, mark the segment as "on"
                if clao == 1:
                        if total / float(area) > 0.34:
                                if area < 1500:
                                        on = [1, 0, 1, 0, 1, 0, 1]
                                else:
                                        on[i] = 1
                else:
                        if total / float(area) > 0.39:
                                if area < 1500:
                                        on = [1, 0, 1, 0, 1, 0, 1]
                                else:
                                        on[i] = 1

        # lookup the digit and draw it on the image
        digit = DIGITS_LOOKUP.get(tuple(on)) or DIGITS_LOOKUP[
                min(DIGITS_LOOKUP.keys(), key=lambda key: norm(np.array(key)-np.array(on)))]
        # digit = DIGITS_LOOKUP[tuple(on)]
        digits.append(digit)
        # print(digits)
        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 1)
        cv2.putText(image, str(digit), (x - 10, y - 10),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2)

# display the digits
print(digits)
cv2.imshow("Input", image)
cv2.waitKey(0)
cv2.destroyAllWindows()

Update

Apologies for my late reply but I have been quite busy with work.

I have captured 22 images throughout the day and used @fmw42 code (with some amendments) to apply thresholding and morphological operations. I am making the images available here and the code that I am using is available below. Overall the performance is quite robust, although 1s and sometimes 8s get mixed up with 2s. I am happy to accept a code that provides improved performance. Note: I think that one problem is that the vertical lines of the numbers are slightly slanted? Thank you in advance.

import cv2
import numpy as np
from numpy.linalg import norm
from imutils import contours
import imutils
import os

# define the dictionary of digit segments so we can identify
# each digit on the thermostat
DIGITS_LOOKUP = {
        (1, 1, 1, 0, 1, 1, 1): 0,
        (1, 0, 1, 0, 1, 0, 1): 1,
        (1, 0, 1, 1, 1, 0, 1): 2,
        (1, 0, 1, 1, 0, 1, 1): 3,
        (0, 1, 1, 1, 0, 1, 0): 4,
        (1, 1, 0, 1, 0, 1, 1): 5,
        (1, 1, 0, 1, 1, 1, 1): 6,
        (1, 1, 1, 0, 0, 1, 0): 7,
        (1, 1, 1, 1, 1, 1, 1): 8,
        (1, 1, 1, 1, 0, 1, 1): 9
}

path_of_the_directory = "/home/myusername/mypathdirectory"
ext = ('.jpg')
for files in os.listdir(path_of_the_directory):
    if files.endswith(ext):
        # load image
        print(files)
        img = cv2.imread(path_of_the_directory+files)

        # convert to grayscale
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # blur
        blur = cv2.GaussianBlur(gray, (0,0), sigmaX=51, sigmaY=51)

        # divide
        divide = cv2.divide(gray, blur, scale=255)

        # threshold  
        thresh = cv2.threshold(divide, 235, 255, cv2.THRESH_BINARY)[1]

        # apply morphology
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (41,41))
        morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (41,41))
        morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)
        morph = cv2.bitwise_not(morph)  # reverse
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (1, 70))
        morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)

        # write result to disk
        cv2.imwrite("digits_division.jpg", divide)
        cv2.imwrite("digits_threshold.jpg", thresh)
        cv2.imwrite("digits_morph.jpg", morph)

        # display it
        cv2.imshow("divide", divide)
        cv2.imshow("thresh", thresh)
        cv2.imshow("morph", morph)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

        # find contours in the thresholded image, then initialize the
        # digit contours lists
        cnts = cv2.findContours(morph.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)
        digitCnts = []

        # loop over the digit area candidates
        for c in cnts:
                # compute the bounding box of the contour
                (x, y, w, h) = cv2.boundingRect(c)
                # if the contour is sufficiently large, it must be a digit
                if w >= 60 and (h >= 300 and h <= 800):
                    digitCnts.append(c)
                    cv2.rectangle(img, (x, y), (x + w, y + h), (0, 0, 255), 2)

        cv2.imshow('image', img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

        # sort the contours from left-to-right, then initialize the
        # actual digits themselves
        digitCnts = contours.sort_contours(digitCnts, method="left-to-right")[0]
        digits = []

        clao = 0
        # loop over each of the digits
        for c in digitCnts:
                clao = clao + 1
                # extract the digit ROI
                (x, y, w, h) = cv2.boundingRect(c)
                roi = morph[y:y + h, x:x + w]
                # compute the width and height of each of the 7 segments
                # we are going to examine
                (roiH, roiW) = roi.shape
                (dW, dH) = (int(roiW * 0.25), int(roiH * 0.15))
                dHC = int(roiH * 0.05)
                # define the set of 7 segments
                segments = [
                        ((0, 0), (w, dH)),                           # top
                        ((0, 0), (dW, h // 2)),                      # top-left
                        ((w - dW, 0), (w, h // 2)),                  # top-right
                        ((0, (h // 2) - dHC), (w, (h // 2) + dHC)),  # center
                        ((0, h // 2), (dW, h)),                      # bottom-left
                        ((w - dW, h // 2), (w, h)),                  # bottom-right
                        ((0, h - dH), (w, h))                        # bottom
                ]
                on = [0] * len(segments)
                
                # loop over the segments
                for (i, ((xA, yA), (xB, yB))) in enumerate(segments):
                        #  extract the segment ROI, count the total number of
                        #  thresholded pixels in the segment, and then compute
                        #  the area of the segment
                        segROI = roi[yA:yB, xA:xB]
                        total = cv2.countNonZero(segROI)
                        area = (xB - xA) * (yB - yA)
                        # if the total number of non-zero pixels is greater than
                        # 50% of the area, mark the segment as "on"
                        if clao == 1:
                                if total / float(area) > 0.34:
                                        if area < 1500:
                                                on = [1, 0, 1, 0, 1, 0, 1]
                                        else:
                                                on[i] = 1
                        else:
                                if total / float(area) > 0.42:
                                        if area < 1500:
                                                on = [1, 0, 1, 0, 1, 0, 1]
                                        else:
                                                on[i] = 1
                                                
                # lookup the digit andq draw it on the image
                digit = DIGITS_LOOKUP.get(tuple(on)) or DIGITS_LOOKUP[
                        min(DIGITS_LOOKUP.keys(), key=lambda key: norm(np.array(key)-np.array(on)))]
                # digit = DIGITS_LOOKUP[tuple(on)]
                digits.append(digit)
                # print(digits)
                cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 1)
                cv2.putText(img, str(digit), (x - 10, y - 10),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2)

        # display the digits
        print(digits)
        cv2.imshow("Input", img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()        
    else:
        continue

Answer 1

Perhaps this will help you using division normalization in Python/OpenCV.

Input:

import cv2
import numpy as np

# load image
img = cv2.imread("digits.jpg")

# convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# blur
blur = cv2.GaussianBlur(gray, (0,0), sigmaX=51, sigmaY=51)

# divide
divide = cv2.divide(gray, blur, scale=255)

# threshold  
thresh = cv2.threshold(divide, 235, 255, cv2.THRESH_BINARY)[1]

# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11,11))
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11,11))
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)

# write result to disk
cv2.imwrite("digits_division.jpg", divide)
cv2.imwrite("digits_threshold.jpg", thresh)
cv2.imwrite("digits_morph.jpg", morph)

# display it
cv2.imshow("divide", divide)
cv2.imshow("thresh", thresh)
cv2.imshow("morph", morph)
cv2.waitKey(0)
cv2.destroyAllWindows()

Division normalized image:

Thresholded image:

Morphology processed image:

You can then clean up further by getting contours and removing small contours and very long horizontal contours.

Answer 2

The key to getting this working is cleaning the image up which I have done to a good enough level to get it to work. I've done this using scikit image library.

I then look at certain squares on the image and take an average reading from that area. On the right hand-side image I've marked some of the locations with red squares.

My script I used to get this result:

import numpy as np
from pathlib import Path
import imageio.v3 as iio
import skimage.filters as skif
from skimage.color import rgb2gray
from skimage.util import img_as_ubyte
from skimage.restoration import denoise_bilateral

import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
import matplotlib.cm as cm

threshold = 125
digit_loc = [1600, 1300, 1000, 730, 420, 155]
size = 20
x_mid = 80
x_right = 160
y_top = 130
y_mt = 250
y_mid = 380
y_bm = 520
y_bot = 630


def img_with_threshold(orig_img):
    block_size = 255
    local_thresh = skif.threshold_local(
        orig_img,
        block_size,
        method="mean",
    )
    binary_local = orig_img > local_thresh

    u8_val = img_as_ubyte(binary_local)
    return u8_val


def image_denoise(orig_img):
    return denoise_bilateral(orig_img, win_size=10, bins=10, )


def plot_imgs(orig_img, mod_img):
    # Display the image
    fig, axes = plt.subplots(1, 2, figsize=(8, 8), sharex=True, sharey=True)
    ax = axes.ravel()
    ax[0].imshow(orig_img, cmap=cm.Greys_r)
    ax[1].imshow(mod_img, cmap=cm.Greys_r)
    # Create a Rectangle patch
    for x_loc in digit_loc:
        rect1 = Rectangle((x_loc + x_mid, y_top), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect2 = Rectangle((x_loc, y_mt), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect3 = Rectangle((x_loc + x_right, y_mt), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect4 = Rectangle((x_loc + x_mid, y_mid), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect5 = Rectangle((x_loc, y_bm), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect6 = Rectangle((x_loc + x_right, y_bm), size, size, linewidth=1, edgecolor='r', facecolor='none')
        rect7 = Rectangle((x_loc + x_mid, y_bot), size, size, linewidth=1, edgecolor='r', facecolor='none')

        # Add the patch to the Axes
        ax[1].add_patch(rect1)
        ax[1].add_patch(rect2)
        ax[1].add_patch(rect3)
        ax[1].add_patch(rect4)
        ax[1].add_patch(rect5)
        ax[1].add_patch(rect6)
        ax[1].add_patch(rect7)

    plt.show()


def seg_to_digit(segments, location):
    digit_values = {0b1110111: 0,
                    0b0010010: 1,
                    0b1011101: 2,
                    0b1011011: 3,
                    0b0111010: 4,
                    0b1101011: 5,
                    0b1101111: 6,
                    0b1110010: 7,
                    0b1111111: 8,
                    0b1111011: 9,
                    }
    result = int("".join(["1" if i < threshold else "0" for i in segments]), 2)
    # print("score:", result)
    return digit_values.get(result, 0) * 10 ** location


def get_digit(location, mod_img):
    """
      a
    b  c
     d
    e  f
     g
    """
    x_loc = digit_loc[location]
    m_loc = (x_loc + x_mid, x_loc + x_mid + size)
    l_loc = (x_loc, x_loc + size)
    r_loc = (x_loc + x_right, x_loc + x_right + size)
    seg_a = np.average(mod_img[y_top:y_top + size, m_loc[0]:m_loc[1]])
    seg_b = np.average(mod_img[y_mt:y_mt + size, l_loc[0]:l_loc[1]])
    seg_c = np.average(mod_img[y_mt:y_mt + size, r_loc[0]:r_loc[1]])
    seg_d = np.average(mod_img[y_mid:y_mid + size, m_loc[0]:m_loc[1]])
    seg_e = np.average(mod_img[y_bm:y_bm + size, l_loc[0]:l_loc[1]])
    seg_f = np.average(mod_img[y_bm:y_bm + size, r_loc[0]:r_loc[1]])
    seg_g = np.average(mod_img[y_bot:y_bot + size, m_loc[0]:m_loc[1]])
    segments = [seg_a, seg_b, seg_c, seg_d, seg_e, seg_f, seg_g]
    # print(f"x loc: {x_loc}, digit index: {location}, segment values: {segments}")
    # create an integer from the bits
    # print('value:', result)
    return seg_to_digit(segments, location)


def main():
    data_dir = Path(__file__).parent.joinpath('data')
    meter_img = data_dir.joinpath('meter_test.jpg')
    img = iio.imread(meter_img)
    gray_img = img_as_ubyte(rgb2gray(img))
    img_result = image_denoise(gray_img)
    img_result1 = img_with_threshold(img_result)
    reading = 0
    for dig_loc in range(6):
        reading += get_digit(dig_loc, img_result1)
        print(f"{reading:>21}")
    print("Final reading:", reading)

    plot_imgs(gray_img, img_result1)


if __name__ == '__main__':
    main()

This gave the following output:

                    7
                   77
                  677
                 4677
                24677
               924677
Final reading: 924677