How draw an object (like a pane or cube) on a detected marker with aruco?

Question

I used Unity with OpenCVSharp, and I have detected markers with the function CvAruco.DetectMarkers(...) so obtaining markerCorners.

I want to create DYNAMICALLY an object on this detected marker using the markerCorners as coordinates, is it possible?

I tried to use marker's corner as coordinates for the cube, but it shows in another point and not on the marker...

Could you help me?

EDIT:

using OpenCvSharp;
using OpenCvSharp.Aruco;
using OpenCvSharp.Util;
using UnityEngine;
using static OpenCvSharp.Unity;
using OpenCvSharp.Tracking;
using System.Linq;
using System.Collections.Generic;
using System.Collections;
using System;


public class ActivationScript : MonoBehaviour //WebCamera
{


private WebCamTexture webCamTexture;

//MARKER DETECTOR
private Mat img;
private int[] markerIds;
Point2f[][] markerCorners, rejectedCandidates;
Dictionary dictionary;

//COLOR
Mesh mesh;
MeshRenderer mr;
Vector3[] vertices;
int[] triangles;
public Material mat;
GameObject markerObj;
GameObject camPlane;
private bool create_flag = false;
private object punti;


List<Point> lista = new List<Point>();


private void Start()
{
    Debug.Log("Start LiveSketch");

    webCamTexture = new WebCamTexture(WebCamTexture.devices[0].name);
    camPlane = GameObject.Find("CameraPlane");
    camPlane.GetComponent<MeshRenderer>().material.mainTexture = webCamTexture;
    //camPlane.transform.position = new Vector3(w/2,h/2,0);
    webCamTexture.Play();

    //
    dictionary = CvAruco.GetPredefinedDictionary(PredefinedDictionaryName.Dict4X4_100);
}

private void Update()
{
    if (webCamTexture.didUpdateThisFrame && webCamTexture.isPlaying)
    {

        Texture2D text = ScreenCapture.CaptureScreenshotAsTexture();

        img = TextureToMat(text);

        DetectorParameters parameters = DetectorParameters.Create();

        CvAruco.DetectMarkers(img, dictionary, out markerCorners, out markerIds, parameters, out rejectedCandidates);

        if (markerIds.Length > 0)
        {

            Debug.Log("DETECTED");
            //void circle(Mat&img, Point center, int radius, const Scalar&color, int thickness = 1, int lineType = 8, int shift = 0)
            //Cv2.DrawChessboardCorners(img,img.Size(),img,true);
            //CvAruco.DrawDetectedMarkers(img, markerCorners, markerIds);
            /* DRAW AN PLANE OR CUBE ON DETECTED MARKER

             .....

             */
            Point tl = new Point(markerCorners[0][0].X,markerCorners[0][0].Y);
            Point br = new Point(markerCorners[0][2].X, markerCorners[0][2].Y);
            Cv2.Rectangle(img, tl, br, new Scalar(0,255,0),-1);
            Cv2.ImShow(this.camPlane.name,img);

            //SetColor_2(markerCorners,1);
            return;

        }
    }
}

/*markerCorners è l'elenco degli angoli dei marker rilevati. 
 * Per ogni marker, i suoi quattro angoli vengono restituiti nel loro ordine originale 
 * (che è in senso orario a partire da in alto a sinistra). 
 * Quindi, il primo angolo è l'angolo in alto a sinistra, seguito da in alto a destra, in basso a destra e in basso a sinistra.*/

public void SetColor_2(Point2f[][] markerCorners, int nMarker)
{
    float x = markerCorners[0][1].X;
    float y = markerCorners[0][1].Y;
    float i,j;


    if (create_flag==false)
        markerObj = Helper.CreatePlane(markerCorners, 0, camPlane);

    i=camPlane.transform.position.x + 26.7f;
    j = camPlane.transform.position.y - 13.3f;

    markerObj.GetComponent<MeshRenderer>().material.SetTexture("txt", SetTxt());
    //markerObj.transform.Rotate(new Vector3(0, 180, 0));
    create_flag = true;
    //markerObj.transform.SetPositionAndRotation(new Vector3(i,j),Quaternion.identity);
}

private Texture2D SetTxt()
{
    Texture2D texture = new Texture2D(12, 12);
    markerObj.GetComponent<Renderer>().material.mainTexture = texture;
    for (int y = 0; y < texture.height; y++)
    {
        for (int x = 0; x < texture.width; x++)
        {
            Color color = Color.red;
            texture.SetPixel(x, y, color);
        }
    }
    texture.Apply();
    Debug.Log("SetTxt finished");
    return texture;
}

private void Calibration()
{
    //Cv2.CalibrateCamera(img,img.Size,);
}

public void Print(Point2f[][] markerCorners, int nMarker)
{
    int i = 0, j = 0;
    Debug.Log("Length: " + markerCorners.Length);
    for (i = 0; i <=nMarker; i++)
    {
        for (j = 0; j <4; j++)
        {
            Debug.Log("Coordinate con i= " + i + "j= " + j + ": " + markerCorners[i][j]);
        }
    }
}



public static Mat TextureToMat(Texture2D texture, TextureConversionParams parameters = null)
{
    if (null == parameters)
        parameters = TextureConversionParams.Default;

    Color32[] pixels32 = texture.GetPixels32();
    return PixelsToMat(pixels32, texture.width, texture.height, parameters.FlipVertically, parameters.FlipHorizontally, parameters.RotationAngle);
}  
}

HELPER CLASS:

using OpenCvSharp;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public static class Helper
{
public static GameObject CreatePlane(Point2f[][] markerCorners, int ind, GameObject Parent)
{
    GameObject plane = new GameObject("Plane");
    MeshFilter mf = plane.AddComponent(typeof(MeshFilter)) as MeshFilter;
    MeshRenderer mr = plane.AddComponent(typeof(MeshRenderer)) as MeshRenderer;

    plane.transform.parent = Parent.transform.parent;

    float width = (markerCorners[0][1].X - markerCorners[0][0].X);
    float height = (markerCorners[0][3].Y - markerCorners[0][0].Y);
    //plane.transform.localScale = new Vector3(1,1,1);
    //Debug.Log("Altezza e Larghezza: "+height+", "+width);

    Mesh m = new Mesh();
    m.vertices = new Vector3[]
    {
        new Vector3(markerCorners[ind][0].X, markerCorners[ind][0].Y, 1),                           //alto sx
        new Vector3(markerCorners[ind][1].X, markerCorners[ind][1].Y, 1),                           //alto dx
        new Vector3(markerCorners[ind][1].X, markerCorners[ind][2].Y, 1),                           //basso dx
        new Vector3(markerCorners[ind][0].X, markerCorners[ind][3].Y, 1)                            //basso sx

        //new Vector3(0,0,0),
        //new Vector3(width,0,0),
        //new Vector3(width,height,0),
        //new Vector3(0,height,0),
    };
    m.uv = new Vector2[]
    {
        new Vector2(0,0),
        new Vector2(0,1),
        new Vector2(1,1),
        new Vector2(1,0),
    };
    m.triangles = new int[] { 3, 0, 2, 2, 0, 1 };//{ 0, 1, 2, 0, 2, 3 }; //

    mf.mesh = m;
    m.RecalculateBounds();
    m.RecalculateNormals();

    plane.transform.Rotate(new Vector3(0,180,0));
    return plane;
}
}

I have created the Helper Class to draw a rectangle on detected marker, but it shows in another point and not on the marker.

Then I used cv2.Rectangle(...) to draw an rectangle on marker, it seems to work but I would like to show this rectangle on main scene on Unity, so without the command "imshow".

Thanks to all!

Answer 1

1. There is a function in the Aruco library which will draw the detected markers for you, onto the same camera image you detected them in. Try using this first, the drawn markers should line up exactly with the real markers. This will pick up certain types of problems you may be having, with the data format of your capture image, your operating environment, lighting, etc. Does this work as expected?

2. Once the previous step is working, you can be sure the corner coordinates aruco gives you are correct. So try drawing a 2D polyline which joins the 4 corner points of each marker. Any misalignment you see must be a programming error in how you are drawing your polyline. Remember that the marker won't be a rectangle on screen, it will be an arbitrary quadrilateral, so you can't use cv::Rectangle for the general case (it can only draw axis-aligned rectangles).

3. Once the previous step is working, you can re-use the same 2D coordinates in Unity and draw the same polygon in whatever way you like. This is OK if you can do everything in screen space (ie, 2D). However if you need 3D (eg. a cube) things are about to get more complicated.

4. To draw a cube aligned to a marker, you must use the marker's rvec and the cv::Rodrigues() function to create a 3x3 rotation matrix, and combine this with the translation specified in the marker's tvec to give a 4x4 rigid transform matrix. You then use this matrix to transform a cube in Unity. This is fairly tricky with a lot of things that can go wrong, depending on your setup. You'll need to have a fair bit of groundwork, in both 3D maths and programming, to get that far. Good luck!