I'm failing miserably trying to draw 3d objects on a aruco marker as of now I've done camera calibration and I'm capable of placing images on the designated aruco markers but I'm having trouble with 3d objects I think it might come from image points I think I'm missing something there. Also, I'm doing this in real-time and is there a a way to use already existing 3d objects bellow is my code
import cv2.aruco as aruco
import cv2
import time
import sys
import argparse
import numpy as np
#import imutils
def load_coefficients(path):
'''Loads camera matrix and distortion coefficients.'''
# FILE_STORAGE_READ
cv_file = cv2.FileStorage(path, cv2.FILE_STORAGE_READ)
# note we also have to specify the type to retrieve other wise we only get a
# FileNode object back instead of a matrix
camera_matrix = cv_file.getNode('K').mat()
dist_matrix = cv_file.getNode('D').mat()
cv_file.release()
return [camera_matrix, dist_matrix]
def drawQ(img, corners, imgpts):
imgpts = np.int32(imgpts).reshape(-1,2)
# draw ground floor in green
img = cv2.drawContours(img, [imgpts[:4]],-1,(0,255,0),-3)
# draw pillars in blue color
for i,j in zip(range(4),range(4,8)):
img = cv2.line(img, tuple(imgpts[i]), tuple(imgpts[j]),(255),3)
# draw top layer in red color
img = cv2.drawContours(img, [imgpts[4:]],-1,(0,0,255),3)
return img
mtx, dist = load_coefficients('camera_parameters.txt')
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-t", "--type", type=str,
default="DICT_ARUCO_ORIGINAL",
help="type of ArUCo tag to detect")
args = vars(ap.parse_args())
source0 = cv2.imread('00000019.jpg')
source1 = cv2.imread('00000042.jpg')
source2 = cv2.imread('00000054.jpg')
source3 = cv2.imread('00000111.png')
source4 = cv2.imread('00000156.jpg')
source5 = cv2.imread('00000000.png')
source6 = cv2.imread('00000041.jpg')
#source = cv2.imread('Kratos_Pose.usdz')
#source = '/toothless/source/toothless.obj'
Aruco_Dict = {
"DICT_4X4_50": aruco.DICT_4X4_50,
"DICT_4X4_100": aruco.DICT_4X4_100,
"DICT_4X4_250": aruco.DICT_4X4_250,
"DICT_4X4_1000": aruco.DICT_4X4_1000,
"DICT_5X5_50": aruco.DICT_5X5_50,
"DICT_5X5_100": aruco.DICT_5X5_100,
"DICT_5X5_250": aruco.DICT_5X5_250,
"DICT_5X5_1000": aruco.DICT_5X5_1000,
"DICT_6X6_50": aruco.DICT_6X6_50,
"DICT_6X6_100": aruco.DICT_6X6_100,
"DICT_6X6_250": aruco.DICT_6X6_250,
"DICT_6X6_1000": aruco.DICT_6X6_1000,
"DICT_7X7_50": aruco.DICT_7X7_50,
"DICT_7X7_100": aruco.DICT_7X7_100,
"DICT_7X7_250": aruco.DICT_7X7_250,
"DICT_7X7_1000": aruco.DICT_7X7_1000,
"DICT_ARUCO_ORIGINAL": aruco.DICT_ARUCO_ORIGINAL,
"DICT_APRILTAG_16h5": aruco.DICT_APRILTAG_16h5,
"DICT_APRILTAG_25h9": aruco.DICT_APRILTAG_25h9,
"DICT_APRILTAG_36h10": aruco.DICT_APRILTAG_36h10,
"DICT_APRILTAG_36h11": aruco.DICT_APRILTAG_36h11
}
image_dict = {
"0": source0,
"1": source1,
"2": source2,
"3": source3,
"4": source4,
"5": source5,
"6": source6
}
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((6*7,3), np.float32)
objp[:,:2] = np.mgrid[0:7,0:6].T.reshape(-1,2)
axis = np.float32([[3,0,0], [0,3,0], [0,0,-3]]).reshape(-1,3)
# verify that the supplied ArUCo tag exists and is supported by
# OpenCV
if Aruco_Dict.get(args["type"], None) is None:
print("[INFO] ArUCo tag of '{}' is not supported".format(
args["type"]))
sys.exit(0)
# load the ArUCo dictionary and grab the ArUCo parameters
#print("[INFO] detecting '{}' tags...".format(args["type"]))
#arucoDict = cv2.aruco.Dictionary_get(Aruco_Dict[args["type"]])
arucoDict = cv2.aruco.Dictionary_get(Aruco_Dict["DICT_4X4_50"])
arucoParams = cv2.aruco.DetectorParameters_create()
# initialize the video stream and allow the camera sensor to warm up
print("[INFO] starting video stream...")
cap = cv2.VideoCapture(0)
#time.sleep(1.0)
if not cap.isOpened():
raise IOError("Cannot open webcam")
while True:
ret, frame = cap.read()
#frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
undistorted = cv2.undistort(frame, mtx, dist, None, None)
undistortedG = cv2.cvtColor(undistorted,cv2.COLOR_BGR2GRAY)
(imgH, imgW) = undistorted.shape[:2]
#cv2.imshow('Input', frame)
cv2.imshow('undistorted image', undistorted)
# detect ArUco markers in the input frame
(corners, ids, rejected) = cv2.aruco.detectMarkers(undistorted,
arucoDict, parameters=arucoParams)
MarkerLength = 0.04
# verify *at least* one ArUco marker was detected
if len(corners) > 0:
print('corners lenght: ', len(corners))
# flatten the ArUco IDs list
ids = ids.flatten()
refpoints = []
# loop over the detected ArUCo corners
for (markerCorner, markerID) in zip(corners, ids):
# extract the marker corners (which are always returned
# in top-left, top-right, bottom-right, and bottom-left
# order)
rvec, tvec, markerPoints = aruco.estimatePoseSingleMarkers(markerCorner, MarkerLength, mtx, dist)
(rvec - tvec).any()
corners = markerCorner.reshape((4, 2))
(topLeft, topRight, bottomRight, bottomLeft) = corners
# convert each of the (x, y)-coordinate pairs to integers
topRight = (int(topRight[0]), int(topRight[1]))
bottomRight = (int(bottomRight[0]), int(bottomRight[1]))
bottomLeft = (int(bottomLeft[0]), int(bottomLeft[1]))
topLeft = (int(topLeft[0]), int(topLeft[1]))
# upper cordnates
a = (3,3)
topRight2 = topRight[0] - 20, topRight[1] - 20
bottomRight2 = bottomRight[0] -20, bottomRight[1] -20
bottomLeft2 = bottomLeft[0] -20, bottomLeft[1] -20
topLeft2 = topLeft[0] -20, topLeft[1] -20
# testing
#corners2 = cv2.cornerSubPix(undistortedG,corners,(11,11),(-1,-1),criteria)
imgpoints, jac = cv2.projectPoints(objp, rvec, tvec, mtx, dist)
#imgpoints = np.float32(imgpoints)
srcrvec, srctvec, srcinliners = cv2.solvePnP(markerPoints, corners, mtx, dist)
undistortedQ = drawC(undistorted, corners, imgpoints)
# testing
# testing
dstMat = [topLeft, topRight, bottomRight, bottomLeft]
dstMat = np.array(dstMat)
#(srcH, srcW) = source.shape[:2]
source = image_dict[f'{markerID}']
(srcH, srcW) = source.shape[:2]
srcMat = np.array([[0, 0], [srcW, 0], [srcW, srcH], [0, srcH]])
(H, _) = cv2.findHomography(srcMat, dstMat)
warped = cv2.warpPerspective(source, H, (imgW, imgH))
cv2.imshow('warped image', warped)
mask = np.zeros((imgH, imgW), dtype="uint8")
cv2.fillConvexPoly(mask, dstMat.astype("int32"), (255, 255, 255),
cv2.LINE_AA)
rect = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
mask = cv2.dilate(mask, rect, iterations=2)
maskScaled = mask.copy() / 255.0
maskScaled = np.dstack([maskScaled] * 3)
warpedMultiplied = cv2.multiply(warped.astype("float"), maskScaled)
imageMultiplied = cv2.multiply(undistorted.astype(float), 1.0 - maskScaled)
undistorted = cv2.add(warpedMultiplied, imageMultiplied)
undistorted = undistorted.astype("uint8")
#cv2.imshow("Input", undistorted)
#cv2.imshow("Source", source)
#cv2.imshow("OpenCV AR Output", output)
cv2.imshow('draw 3d cube ', undistortedQ)
#cv2.waitKey(0)
# testing
# draw the bounding box of the ArUCo detection
cv2.line(undistorted, topLeft, topRight, (0, 255, 0), 2)
cv2.line(undistorted, topRight, bottomRight, (0, 255, 0), 2)
cv2.line(undistorted, bottomRight, bottomLeft, (0, 255, 0), 2)
cv2.line(undistorted, bottomLeft, topLeft, (0, 255, 0), 2)
# draw rest of cube
cv2.line(undistorted, topLeft2, topRight2, (0, 255, 0), 2)
cv2.line(undistorted, topRight2, bottomRight2, (0, 255, 0), 2)
cv2.line(undistorted, bottomRight2, bottomLeft2, (0, 255, 0), 2)
cv2.line(undistorted, bottomLeft2, topLeft2, (0, 255, 0), 2)
# draw the the axis from estimated pose
aruco.drawAxis(undistorted, mtx, dist, rvec, tvec, 0.01)
# compute and draw the center (x, y)-coordinates of the
# ArUco marker
cX = int((topLeft[0] + bottomRight[0]) / 2.0)
cY = int((topLeft[1] + bottomRight[1]) / 2.0)
cv2.circle(undistorted, (cX, cY), 4, (0, 0, 255), -1)
# draw the ArUco marker ID on the frame
cv2.putText(undistorted, str(markerID),
(topLeft[0], topLeft[1] - 15),
cv2.FONT_HERSHEY_SIMPLEX,
0.5, (0, 255, 0), 2)
# show the output frame
cv2.imshow("ArUco Markers Detector", undistorted)
key = cv2.waitKey(1) & 0xFF
c = cv2.waitKey(1)
if c == 27:
break
cap.release()
cv2.destroyAllWindows()
here is an image of what is happnening
