I'm new to Python and want to learn it bit by bit, so I decided to write a simple program that would, in real time, capture my screen and do object detection. Through a lot of googling and reading, I was able to make this script, however, no matter what I do, it won't do object detection (m1.png).
Can you please assist me with the reason why it is like this?
import time
import cv2
import mss
import numpy
#template and dimensions
template = cv2.imread("m2.png")
template_gray = cv2.cvtColor(template, cv2.COLOR_BGRA2GRAY)
template_w, template_h = template_gray.shape[::-1]
with mss.mss() as sct:
# Part of the screen to capture
monitor = {"top": 523, "left": 247, "width": 875, "height": 679}
while True:
last_time = time.time()
# Get raw pixels from the screen, save it to a Numpy array
img = numpy.array(sct.grab(monitor))
# Display the picture
cv2.imshow("Normal", img)
# Display the picture in grayscale
img_gray = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)
res = cv2.matchTemplate(
image = img_gray,
templ = template_gray,
method= cv2.TM_CCOEFF_NORMED
)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
#threshold
if max_val >= 0.5:
img = cv2.rectangle(
img = img,
pt1 = max_loc,
pt2 = (
max_loc[0] + template_w, # = pt2 x
max_loc[1] + template_h # = pt2 y
),
color = (0,255,0),
thickness = 3 #fill the rectangle
)
print("fps: {}".format(1 / (time.time() - last_time)))
# Press "q" to quit
if cv2.waitKey(25) & 0xFF == ord("q"):
cv2.destroyAllWindows()
break
I spent three days trying to figure it out, and the only similar code that works but with a poor frame rate is this one:
#imports
from re import template
import cv2
import pyautogui
from time import sleep
#No cooldown time
pyautogui.PAUSE = 0
#template and dimensions
template = cv2.imread("b1.png")
template_gray = cv2.cvtColor(template, cv2.COLOR_RGB2GRAY)
template_w, template_h = template_gray.shape[::-1]
# game window dimensions
x, y, w, h = 523, 247, 875, 679
#wait
sleep(3)
#main
while True:
#screenshot = img
pyautogui.screenshot("image.png", (x, y, w, h))
image = cv2.imread("image.png")
while True:
image_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
result = cv2.matchTemplate(
image = image_gray,
templ = template_gray,
method = cv2.TM_CCOEFF_NORMED
)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
#threshold
if max_val >= 0.1:
#pyautogui.click(
# x = max_loc[0] + x, #screen x
# y = max_loc[1] + y #screen y
#)
image = cv2.rectangle(
img = image,
pt1 = max_loc,
pt2 = (
max_loc[0] + template_w, # = pt2 x
max_loc[1] + template_h # = pt2 y
),
color = (0,0,255),
thickness = -1 #fill the rectangle
)
else:
break
The structure looks similar the only difference in this template is that it uses pyautogui with OpenCV, whereas I'm trying to use mss. So does that mean that the issue in the code is that there's no physical location of the screen capture due? If so, does that mean it's impossible to make an object detection with mss?? You would make my day if you could disclose this mystery with the code!!
UPD: I was able to solve this, honey! So the issue was that firstly I misspelled the .png file, and to see it detecting an object, place the cv2.imshow after the if statement. Although it works, it's not perfect, so I'm trying to implement the usage of cv2.Canny() but now I don't get any output, so here I'm rising a question of whether there should be a different approach when Canny is used:
import time
import Options.settings as set
import time
import pyautogui as pt
from time import sleep
import cv2
import mss
import numpy
x = 0
offset = set.offset
create_logs = set.create_logs
#template and dimensions
template = cv2.imread("m2.png")
template_gray = cv2.cvtColor(template, cv2.COLOR_BGRA2GRAY)
template_canny = cv2.Canny(template_gray, 79, 100)
template_w, template_h = template_canny.shape[::-1]
with mss.mss() as sct:
# Part of the screen to capture
monitor = {"top": 523, "left": 1600, "width": 230, "height": 359}
while True:
last_time = time.time()
# Get raw pixels from the screen, save it to a Numpy array
img = numpy.array(sct.grab(monitor))
# Display the picture
cv2.imshow("Normal", img)
# Display the picture in grayscale
img_gray = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)
img_canny = cv2.Canny(img_gray, 100, 115)
res = cv2.matchTemplate(
image = img_canny,
templ = template_canny,
method= cv2.TM_CCOEFF_NORMED
)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
#threshold
if max_val >= 0.6:
x = x + 1
print(f'{x} is detected')
img = cv2.rectangle(
img = img,
pt1 = max_loc,
pt2 = (
max_loc[0] + template_w, # = pt2 x
max_loc[1] + template_h # = pt2 y
),
color = (0,255,0),
thickness = 3 #fill the rectangle
)
# Display the picture
cv2.imshow("Normal", img)
#print("fps: {}".format(1 / (time.time() - last_time)))
# Press "q" to quit
if cv2.waitKey(25) & 0xFF == ord("q"):
cv2.destroyAllWindows()
break
UPD 2:
As @fmw42 suggested, I tried different OpenCV methods, but whether I try them, they constantly react if there's an object in the screen capture field no matter how I change the if max_vol >= ...
Please find attached m2.png = https://ibb.co/Xb5tCPZ Example of what the screen capture look like = https://ibb.co/Xb5tCPZ
