I am wanting to take a triangle from an image and overlay it at the same location on my face in video camera.
I am using python mediapipe to get the landmarks and it seems I am able to get the correct triangle, but it doesn't overlay on the correct location properly.
Mediapipe landmarks: Landmarks
Here is myimage:
Here is the code:
import cv2
import numpy as np
import mediapipe as mp
# PROBLEM: There is a problem here It places it in the wrong place and it seems to be drawing a square,
# but without transparency.
def overlay_triangle_in_xyz_postions(source_image, triangle_image, array_xx_yy_zz: []):
if source_image.shape[2] == 3:
source_image = cv2.cvtColor(source_image, cv2.COLOR_BGR2BGRA)
if triangle_image.shape[2] == 3:
triangle_image = cv2.cvtColor(triangle_image, cv2.COLOR_BGR2BGRA)
# Get your mediapipe landmarks - replace this with actual landmark detection
# https://user-images.githubusercontent.com/11573490/109521608-72aeed00-7ae8-11eb-9539-e07c406cc65b.jpg
# you can vide the landmarks here.
# landmarks = np.array([[100, 100], [200, 100], [150, 200]])
landmarks = np.array(array_xx_yy_zz)
# Assuming the triangle_image is an equilateral triangle,
# we'll set the vertices to be the top middle and bottom corners
triangle_vertices = np.array([[triangle_image.shape[1] / 2, 0], [0, triangle_image.shape[0]],
[triangle_image.shape[1], triangle_image.shape[0]]])
# Get the affine transform matrix
M = cv2.getAffineTransform(triangle_vertices.astype(np.float32), landmarks.astype(np.float32))
# Separate the alpha channel from the rest of the triangle image
triangle_image_rgb = triangle_image[:, :, :3]
triangle_image_alpha = triangle_image[:, :, 3]
# Warp the triangle image (RGB channels only) to fit the landmarks
warped_triangle_rgb = cv2.warpAffine(triangle_image_rgb, M, (source_image.shape[1], source_image.shape[0]))
# Warp the triangle image (alpha channel only) to fit the landmarks
warped_triangle_alpha = cv2.warpAffine(triangle_image_alpha, M, (source_image.shape[1], source_image.shape[0]))
# Recombine the RGB and alpha channels
warped_triangle = cv2.merge([warped_triangle_rgb, warped_triangle_alpha])
# Normalize the alpha mask to keep intensity between 0 and 1
alpha = warped_triangle[:, :, 3].astype(float) / 255
# Create 3 channel alpha mask
alpha = cv2.merge([alpha, alpha, alpha, alpha])
# Alpha blending
final_image = (alpha * warped_triangle + (1 - alpha) * source_image).astype(np.uint8)
return final_image
def get_triangle_from_image():
image_element = cv2.imread("./myface.png", cv2.IMREAD_UNCHANGED)
image = cv2.cvtColor(image_element, cv2.COLOR_BGR2RGB)
results = face_mesh.process(image)
# Check if any face is detected
if results.multi_face_landmarks:
for face_landmarks in results.multi_face_landmarks:
# Get the coordinates of the landmarks for the triangle
landmarks = [[int(face_landmarks.landmark[i].x * image.shape[1]),
int(face_landmarks.landmark[i].y * image.shape[0])] for i in [10, 108, 151]]
triangle_image_rbg = extract_triangle_from_landmarks(image, landmarks)
triangle_image = cv2.cvtColor(triangle_image_rbg, cv2.COLOR_BGR2RGBA)
# This should return a triangle with alpha channel
return triangle_image
def extract_triangle_from_landmarks(image, landmarks):
# Create a mask for the image
mask = np.zeros(image.shape, dtype=np.uint8)
# Draw the triangle on the mask
triangle_cnt = np.array(landmarks).reshape((-1, 1, 2)).astype(np.int32)
cv2.drawContours(mask, [triangle_cnt], 0, (255, 255, 255), -1)
# Bitwise-and the mask and the original image to get the triangle
triangle_image = cv2.bitwise_and(image, mask)
# Create bounding rectangle around the triangle
(x, y, w, h) = cv2.boundingRect(triangle_cnt)
# Crop the image using the bounding rectangle
triangle_image = triangle_image[y:y + h, x:x + w]
return triangle_image
def overlay_triangle(face_image, triangle_image, landmarks):
# Get the size of the triangle_image
h, w = triangle_image.shape[:2]
# Create a mask for the triangle_image
mask = np.zeros((h, w), dtype=np.uint8)
cv2.fillConvexPoly(mask, np.array([[0, 0], [w // 2, h], [w, 0]], dtype=np.int32), 255)
# Compute the bounding rectangle for the triangle
(x, y, w, h) = cv2.boundingRect(np.array(landmarks))
# Adjust the landmarks to the bounding rectangle
landmarks = [[x[0] - x, x[1] - y] for x in landmarks]
# Compute the affine transform that maps the triangle_image to the face_image
warp_mat = cv2.getAffineTransform(np.float32([[0, 0], [w // 2, h], [w, 0]]), np.float32(landmarks))
# Warp the triangle_image to match the triangle on the face_image
warped_image = cv2.warpAffine(triangle_image, warp_mat, (face_image.shape[1], face_image.shape[0]))
# Create a mask for the triangle on the face_image
mask = cv2.warpAffine(mask, warp_mat, (face_image.shape[1], face_image.shape[0]))
# Use the mask to blend the warped_image into the face_image
face_image = cv2.bitwise_and(face_image, cv2.bitwise_not(cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)))
face_image = cv2.bitwise_or(face_image, cv2.bitwise_and(warped_image, cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)))
return face_image
# Initialize MediaPipe Face Mesh
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh()
# Start the webcam feed
cap = cv2.VideoCapture(0)
while cap.isOpened():
ret, frame = cap.read()
# Strip out ALPHA for processing.
image = bgr_image = frame[:, :, :3]
results = face_mesh.process(image)
# Check if any face is detected
if results.multi_face_landmarks:
triangle = get_triangle_from_image()
for face_landmarks in results.multi_face_landmarks:
# PROBLEM ? This might be icorrect...
a = 10
b = 151
c = 108
ret_frame = overlay_triangle_in_xyz_postions(image, triangle, [[a, a], [b, a], [c, b]] )
cv2.imshow('Triangle', ret_frame)
break
if cv2.waitKey(10) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
