Replacing specific Characters in an Image using OpenCV

Question

I have the next Image:

I want to find all the occurrences of an specific character, let's say "a", and then replacing it with another character, let's say "e"

This can be done in Python3 using OpenCV and Numpy, to do that I'm using the next two images in order to do template matching:

my code is the next:

import cv2 as cv
import numpy as np

img = cv.imread('source_image.jpg') # Importing Source Image
imgray = cv.cvtColor(img, cv.COLOR_BGR2GRAY) # Converting Source Image to Gray Scale

template_a = cv.imread('template_a.jpg') # Reading Template "a" Character
template_a_gray = cv.cvtColor(template_a, cv.COLOR_BGR2GRAY) # Converting Template "a" Character to Gray Scale

template_e = cv.imread('template_e.jpg') # Reading Template "e" Character
template_e_gray = cv.cvtColor(template_e, cv.COLOR_BGR2GRAY) # Converting Template "e" Character to Gray Scale

# Template Matching
w, h = template_a_gray.shape[::-1] # Obtaining Width and Height of Template "a" Character
res = cv.matchTemplate(imgray, template_a_gray, cv.TM_CCOEFF_NORMED) # Template Matching

threshold = 0.6
loc = np.where( res >= threshold)
for pt in zip(*loc[::-1]):
    cv.rectangle(imgray, pt, (pt[0] + w, pt[1] + h), color=(255,255,255), thickness=cv.FILLED) # Removing Matched Template "a" Characters

    y_offset = pt[1]
    x_offset = pt[0]
    x_end = x_offset + template_e_gray.shape[1]
    y_end = y_offset + template_e_gray.shape[0]

    imgray[y_offset:y_end,x_offset:x_end] = template_e_gray # Pasting Template "e" Characters

filename = 'savedImage.jpg' 
cv.imwrite(filename, imgray) # Saving Image

cv.waitKey(0)
cv.destroyAllWindows()

When executed the result is:

The problem is that when zoom in, the pasted image isn't exactly the same as the template character for e

Blobs appear above and in the left of the original "e" template character. Why is this happening? How do I get rid of them?

Edit: in Response of Christoph Rackwitz's Comment:

Edit: Code fixed upon Christoph Rackwitz's Suggestions

Answer 1

This is because template_e_gray is pasted multiple times. Blobs are the edges of another e pasted just before. Print pt and see where e is pasted.

There are several ways to deal with this problem. The easiest way is to set the threshold to 0.8.

threshold = 0.8

If this is sufficient, use this. But may not be sufficient for some applications because the threshold needs to be tuned per image.

Another way is to fill white with one pixel larger.

imgray[y_offset - 1:y_end + 1, x_offset - 1:x_end + 1] = 255
imgray[y_offset:y_end, x_offset:x_end] = template_e_gray  # Pasting Template "e" Characters

This is easy too, but inefficient, misaligned, and may not work with large images.

My recommandation is to use non maximum suppression (NMS). NMS is a technique commonly used in object detection that delete all overlapping rectangles except the one with the highest score.

Borrowing the implementation from here, this is the complete code.

import cv2 as cv
import numpy as np


# https://github.com/rbgirshick/fast-rcnn/blob/master/lib/utils/nms.py
def nms(dets, thresh):
    x1 = dets[:, 0]
    y1 = dets[:, 1]
    x2 = dets[:, 2]
    y2 = dets[:, 3]
    scores = dets[:, 4]

    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        i = order[0]
        keep.append(i)
        xx1 = np.maximum(x1[i], x1[order[1:]])
        yy1 = np.maximum(y1[i], y1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])
        yy2 = np.minimum(y2[i], y2[order[1:]])

        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)
        inter = w * h
        ovr = inter / (areas[i] + areas[order[1:]] - inter)

        inds = np.where(ovr <= thresh)[0]
        order = order[inds + 1]

    return keep


img = cv.imread('source_image.jpg')  # Importing Source Image
imgray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)  # Converting Source Image to Gray Scale

template_a = cv.imread('temp/template_a.jpg')  # Reading Template "a" Character
template_a_gray = cv.cvtColor(template_a, cv.COLOR_BGR2GRAY)  # Converting Template "a" Character to Gray Scale

template_e = cv.imread('temp/template_e.jpg')  # Reading Template "e" Character
template_e_gray = cv.cvtColor(template_e, cv.COLOR_BGR2GRAY)  # Converting Template "e" Character to Gray Scale

# Template Matching
w, h = template_a_gray.shape[::-1]  # Obtaining Width and Height of Template "a" Character
res = cv.matchTemplate(imgray, template_a_gray, cv.TM_CCOEFF_NORMED)  # Template Matching

threshold = 0.6
loc = np.where(res >= threshold)
top, left = loc
score = res[loc]
iou_threshold = 0.5
matched_boxes = np.array([left, top, left + w, top + h, score]).T
valid_indices = nms(matched_boxes, iou_threshold)
boxes = matched_boxes[valid_indices]
boxes = boxes[..., :-1].astype(int)

for pt in boxes:
    print(pt)
    cv.rectangle(imgray, pt[:2], pt[2:], color=(255, 255, 255),
                 thickness=cv.FILLED)  # Removing Matched Template "a" Characters

    y_offset = pt[1]
    x_offset = pt[0]
    x_end = x_offset + template_e_gray.shape[1]
    y_end = y_offset + template_e_gray.shape[0]

    imgray[y_offset:y_end, x_offset:x_end] = template_e_gray  # Pasting Template "e" Characters

filename = 'temp/savedImage.jpg'
cv.imwrite(filename, imgray)  # Saving Image

cv.waitKey(0)
cv.destroyAllWindows()

This code works perfectly as expected, but you will see the following results. See the area circled in red.

This is a trace of a, not e. This is caused by the difference in size between a in the image and a in the template, and is unrelated to the original issue. I mentioned this because I thought you would suspect that the same issue remains.