I'm trying to use OpenCV with WebGL.
In OpenCV, I'm looking for a projection (a priori the projection matrix of the camera) that I get using SolvePnp. When I display the projection obtained in OpenCV with the 'projectPoints' function, everything is perfectly stalled.
The SolvePnp function returns 3 values for the rotation and 3 values for the translation. I also get the cameraMatrix(matrix 3x3) using the values fx, fy, cx, cy (see:https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html#solvepnp).
Once all these values are recovered, I try to use them with WebGL. But for the moment it does not work.
In WebGL, I calculate the projectionMatrix in two different ways:
m4.perspective (28 / 180.0 * Math.PI, aspect, 0.1, 1000);
or through the following example (https://blog.noctua-software.com/opencv-opengl-projection-matrix.html)
openCVToProjectionMatrix (fx, fy, cx, cy, 0.1, 1000, wSource, hSource);
Then I create the cameraMatrix and I apply to it the values recovered since OpenCV.
I compute the ViewMatrix by doing an inverseMatrix.
Then I compute the viewProjectionMatrix by multiplying projectionMatrix by viewMatrix.
I calculate ModelMatrix by simply viewProjectionMatrix.
And I apply everything on my plan (2 triangles) with the following values:
var size = 1.0
x: -size * 0.5
y: -size * 0.5
width: size
height: size
then I add:
gl.uniformMatrix4fv (matrixLocation, false, matrixModel)
Unfortunately it does not work.
My questions :
Do I apply the matrices properly or are there any errors at this level?
Are there other parameters to consider?
Do I have to apply a translation on the camera?
Is there an order to be complied with for matrix calculations (start with scale, then rotation and finally translation)?
And finally, can someone help me?
A sample of code
// Source Dimensions(Webcam)
var wSrc = 360, hSrc = 640;
// Trigger Dimensions
var wTrigger = 602, hTrigger = 452;
// OpenCV values for projectionMatrix
var fx = 603.92035729, fy = 605.26722937;
var cx = 179.804103189, cy = 320.14721692;
// Values from SolvePnp(openCV)
var rVecs = [0.464472599644,-0.210231064819,-0.0689626255534];
var tVecs = [-0.61758832993,-0.567979295625,2.78430675542];
// -------------------------------------------------------------------------------------
var vertexShader = `
attribute vec4 a_position;
attribute vec4 a_color;
uniform mat4 u_matrix;
void main() {
// Multiply the position by the matrix.
gl_Position = u_matrix * a_position;
}
`;
// -------------------------------------------------------------------------------------
var fragmentShader = `
precision mediump float;
void main() {
gl_FragColor = vec4(1, 0, 0, 1);
}
`;
// *************************************************************************************
// *************************************************************************************
// *************************************************************************************
var m4 = {
perspective: function(fieldOfViewInRadians, aspect, near, far) {
var f = Math.tan(Math.PI * 0.5 - 0.5 * fieldOfViewInRadians);
var rangeInv = 1.0 / (near - far);
return [
f / aspect, 0, 0, 0,
0, f, 0, 0,
0, 0, (near + far) * rangeInv, -1,
0, 0, near * far * rangeInv * 2, 0
];
},
projection: function(width, height, depth) {
// Note: This matrix flips the Y axis so 0 is at the top.
return [
2 / width, 0, 0, 0,
0, -2 / height, 0, 0,
0, 0, 2 / depth, 0,
-1, 1, 0, 1,
];
},
multiply: function(a, b) {
var a00 = a[0 * 4 + 0];
var a01 = a[0 * 4 + 1];
var a02 = a[0 * 4 + 2];
var a03 = a[0 * 4 + 3];
var a10 = a[1 * 4 + 0];
var a11 = a[1 * 4 + 1];
var a12 = a[1 * 4 + 2];
var a13 = a[1 * 4 + 3];
var a20 = a[2 * 4 + 0];
var a21 = a[2 * 4 + 1];
var a22 = a[2 * 4 + 2];
var a23 = a[2 * 4 + 3];
var a30 = a[3 * 4 + 0];
var a31 = a[3 * 4 + 1];
var a32 = a[3 * 4 + 2];
var a33 = a[3 * 4 + 3];
var b00 = b[0 * 4 + 0];
var b01 = b[0 * 4 + 1];
var b02 = b[0 * 4 + 2];
var b03 = b[0 * 4 + 3];
var b10 = b[1 * 4 + 0];
var b11 = b[1 * 4 + 1];
var b12 = b[1 * 4 + 2];
var b13 = b[1 * 4 + 3];
var b20 = b[2 * 4 + 0];
var b21 = b[2 * 4 + 1];
var b22 = b[2 * 4 + 2];
var b23 = b[2 * 4 + 3];
var b30 = b[3 * 4 + 0];
var b31 = b[3 * 4 + 1];
var b32 = b[3 * 4 + 2];
var b33 = b[3 * 4 + 3];
return [
b00 * a00 + b01 * a10 + b02 * a20 + b03 * a30,
b00 * a01 + b01 * a11 + b02 * a21 + b03 * a31,
b00 * a02 + b01 * a12 + b02 * a22 + b03 * a32,
b00 * a03 + b01 * a13 + b02 * a23 + b03 * a33,
b10 * a00 + b11 * a10 + b12 * a20 + b13 * a30,
b10 * a01 + b11 * a11 + b12 * a21 + b13 * a31,
b10 * a02 + b11 * a12 + b12 * a22 + b13 * a32,
b10 * a03 + b11 * a13 + b12 * a23 + b13 * a33,
b20 * a00 + b21 * a10 + b22 * a20 + b23 * a30,
b20 * a01 + b21 * a11 + b22 * a21 + b23 * a31,
b20 * a02 + b21 * a12 + b22 * a22 + b23 * a32,
b20 * a03 + b21 * a13 + b22 * a23 + b23 * a33,
b30 * a00 + b31 * a10 + b32 * a20 + b33 * a30,
b30 * a01 + b31 * a11 + b32 * a21 + b33 * a31,
b30 * a02 + b31 * a12 + b32 * a22 + b33 * a32,
b30 * a03 + b31 * a13 + b32 * a23 + b33 * a33,
];
},
translation: function(tx, ty, tz) {
return [
1, 0, 0, 0,
0, 1, 0, 0,
0, 0, 1, 0,
tx, ty, tz, 1,
];
},
xRotation: function(angleInRadians) {
var c = Math.cos(angleInRadians);
var s = Math.sin(angleInRadians);
return [
1, 0, 0, 0,
0, c, s, 0,
0, -s, c, 0,
0, 0, 0, 1,
];
},
yRotation: function(angleInRadians) {
var c = Math.cos(angleInRadians);
var s = Math.sin(angleInRadians);
return [
c, 0, -s, 0,
0, 1, 0, 0,
s, 0, c, 0,
0, 0, 0, 1,
];
},
zRotation: function(angleInRadians) {
var c = Math.cos(angleInRadians);
var s = Math.sin(angleInRadians);
return [
c, s, 0, 0,
-s, c, 0, 0,
0, 0, 1, 0,
0, 0, 0, 1,
];
},
scaling: function(sx, sy, sz) {
return [
sx, 0, 0, 0,
0, sy, 0, 0,
0, 0, sz, 0,
0, 0, 0, 1,
];
},
translate: function(m, tx, ty, tz) {
return m4.multiply(m, m4.translation(tx, ty, tz));
},
xRotate: function(m, angleInRadians) {
return m4.multiply(m, m4.xRotation(angleInRadians));
},
yRotate: function(m, angleInRadians) {
return m4.multiply(m, m4.yRotation(angleInRadians));
},
zRotate: function(m, angleInRadians) {
return m4.multiply(m, m4.zRotation(angleInRadians));
},
scale: function(m, sx, sy, sz) {
return m4.multiply(m, m4.scaling(sx, sy, sz));
},
inverse: function(m) {
var m00 = m[0 * 4 + 0];
var m01 = m[0 * 4 + 1];
var m02 = m[0 * 4 + 2];
var m03 = m[0 * 4 + 3];
var m10 = m[1 * 4 + 0];
var m11 = m[1 * 4 + 1];
var m12 = m[1 * 4 + 2];
var m13 = m[1 * 4 + 3];
var m20 = m[2 * 4 + 0];
var m21 = m[2 * 4 + 1];
var m22 = m[2 * 4 + 2];
var m23 = m[2 * 4 + 3];
var m30 = m[3 * 4 + 0];
var m31 = m[3 * 4 + 1];
var m32 = m[3 * 4 + 2];
var m33 = m[3 * 4 + 3];
var tmp_0 = m22 * m33;
var tmp_1 = m32 * m23;
var tmp_2 = m12 * m33;
var tmp_3 = m32 * m13;
var tmp_4 = m12 * m23;
var tmp_5 = m22 * m13;
var tmp_6 = m02 * m33;
var tmp_7 = m32 * m03;
var tmp_8 = m02 * m23;
var tmp_9 = m22 * m03;
var tmp_10 = m02 * m13;
var tmp_11 = m12 * m03;
var tmp_12 = m20 * m31;
var tmp_13 = m30 * m21;
var tmp_14 = m10 * m31;
var tmp_15 = m30 * m11;
var tmp_16 = m10 * m21;
var tmp_17 = m20 * m11;
var tmp_18 = m00 * m31;
var tmp_19 = m30 * m01;
var tmp_20 = m00 * m21;
var tmp_21 = m20 * m01;
var tmp_22 = m00 * m11;
var tmp_23 = m10 * m01;
var t0 = (tmp_0 * m11 + tmp_3 * m21 + tmp_4 * m31) -
(tmp_1 * m11 + tmp_2 * m21 + tmp_5 * m31);
var t1 = (tmp_1 * m01 + tmp_6 * m21 + tmp_9 * m31) -
(tmp_0 * m01 + tmp_7 * m21 + tmp_8 * m31);
var t2 = (tmp_2 * m01 + tmp_7 * m11 + tmp_10 * m31) -
(tmp_3 * m01 + tmp_6 * m11 + tmp_11 * m31);
var t3 = (tmp_5 * m01 + tmp_8 * m11 + tmp_11 * m21) -
(tmp_4 * m01 + tmp_9 * m11 + tmp_10 * m21);
var d = 1.0 / (m00 * t0 + m10 * t1 + m20 * t2 + m30 * t3);
return [
d * t0,
d * t1,
d * t2,
d * t3,
d * ((tmp_1 * m10 + tmp_2 * m20 + tmp_5 * m30) -
(tmp_0 * m10 + tmp_3 * m20 + tmp_4 * m30)),
d * ((tmp_0 * m00 + tmp_7 * m20 + tmp_8 * m30) -
(tmp_1 * m00 + tmp_6 * m20 + tmp_9 * m30)),
d * ((tmp_3 * m00 + tmp_6 * m10 + tmp_11 * m30) -
(tmp_2 * m00 + tmp_7 * m10 + tmp_10 * m30)),
d * ((tmp_4 * m00 + tmp_9 * m10 + tmp_10 * m20) -
(tmp_5 * m00 + tmp_8 * m10 + tmp_11 * m20)),
d * ((tmp_12 * m13 + tmp_15 * m23 + tmp_16 * m33) -
(tmp_13 * m13 + tmp_14 * m23 + tmp_17 * m33)),
d * ((tmp_13 * m03 + tmp_18 * m23 + tmp_21 * m33) -
(tmp_12 * m03 + tmp_19 * m23 + tmp_20 * m33)),
d * ((tmp_14 * m03 + tmp_19 * m13 + tmp_22 * m33) -
(tmp_15 * m03 + tmp_18 * m13 + tmp_23 * m33)),
d * ((tmp_17 * m03 + tmp_20 * m13 + tmp_23 * m23) -
(tmp_16 * m03 + tmp_21 * m13 + tmp_22 * m23)),
d * ((tmp_14 * m22 + tmp_17 * m32 + tmp_13 * m12) -
(tmp_16 * m32 + tmp_12 * m12 + tmp_15 * m22)),
d * ((tmp_20 * m32 + tmp_12 * m02 + tmp_19 * m22) -
(tmp_18 * m22 + tmp_21 * m32 + tmp_13 * m02)),
d * ((tmp_18 * m12 + tmp_23 * m32 + tmp_15 * m02) -
(tmp_22 * m32 + tmp_14 * m02 + tmp_19 * m12)),
d * ((tmp_22 * m22 + tmp_16 * m02 + tmp_21 * m12) -
(tmp_20 * m12 + tmp_23 * m22 + tmp_17 * m02))
];
},
vectorMultiply: function(v, m) {
var dst = [];
for (var i = 0; i < 4; ++i) {
dst[i] = 0.0;
for (var j = 0; j < 4; ++j)
dst[i] += v[j] * m[j * 4 + i];
}
return dst;
},
};
// *************************************************************************************
// *************************************************************************************
// *************************************************************************************
initGL();
// -------------------------------------------------------------------------------------
function initializeWebGL(canvasName) {
var canvas = document.getElementById(canvasName);
var gl = null;
try {
gl = canvas.getContext("webgl");
}
catch (error) {
console.log("Error getContext WebGL", error);
}
if (!gl) {
throw new Error("Could not get WebGL context!");
}
return gl;
}
function createShader(gl, shaderSource, shaderType) {
var shader = gl.createShader(shaderType);
gl.shaderSource(shader, shaderSource);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
var infoLog = gl.getShaderInfoLog(shader);
gl.deleteShader(shader);
throw new Error("An error occurred compiling the shader: " + infoLog);
}
else {
return shader;
}
}
function createGlslProgram(gl, vertexSource, fragmentSource) {
var program = gl.createProgram();
gl.attachShader(program, createShader(gl, vertexSource, gl.VERTEX_SHADER));
gl.attachShader(program, createShader(gl, fragmentSource, gl.FRAGMENT_SHADER));
gl.linkProgram(program);
gl.validateProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
var infoLog = gl.getProgramInfoLog(program);
gl.deleteProgram(program);
throw new Error("An error occurred linking the program: " + infoLog);
}
else {
return program;
}
}
// -------------------------------------------------------------------------------------
// -------------------------------------------------------------------------------------
function initGL() {
var gl = initializeWebGL("canvasGL");
var program = createGlslProgram(gl, vertexShader, fragmentShader);
// ------------------------------------------------------------------------------
resizeCanvasToDisplaySize(gl.canvas, window.devicePixelRatio);
// ------------------------------------------------------------------------------
var positionAttributeLocation = gl.getAttribLocation(program, "a_position");
var matrixLocation = gl.getUniformLocation(program, "u_matrix");
var positionBuffer = gl.createBuffer();
// ------------------------------------------------------------------------------
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT);
// ------------------------------------------------------------------------------
gl.useProgram(program);
// ------------------------------------------------------------------------------
gl.enableVertexAttribArray(positionAttributeLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// ------------------------------------------------------------------------------
initFlatPlane(gl, positionAttributeLocation, matrixLocation);
}
function initFlatPlane(gl, positionAttributeLocation, matrixLocation){
var size = 3;
var type = gl.FLOAT;
var normalize = false;
var stride = 0;
var offset = 0;
gl.vertexAttribPointer(positionAttributeLocation, size, type, normalize, stride, offset)
// ------------------------------------------------------------------------------
// projectionMatrix
var matOpenCV = openCVCameraMatrixToProjectionMatrix(fx, fy, cx, cy, 1000, 0.1, wSrc, hSrc);
var projectionMatrix = matOpenCV[0].concat(matOpenCV[1]).concat(matOpenCV[2]).concat(matOpenCV[3]);
console.log("projectionMatrix :", projectionMatrix);
// ------------------------------------------------------------------------------
// Compute a matrix for the camera
var cameraMatrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
cameraMatrix = m4.scale(cameraMatrix, 1, wTrigger/hTrigger, 1);
cameraMatrix = m4.xRotate(cameraMatrix, rVecs[0]);
cameraMatrix = m4.yRotate(cameraMatrix, rVecs[1]);
cameraMatrix = m4.zRotate(cameraMatrix, rVecs[2]);
cameraMatrix = m4.translate(cameraMatrix, tVecs[0], tVecs[1], tVecs[2]);
console.log("cameraMatrix", cameraMatrix);
// ------------------------------------------------------------------------------
// Make a view matrix from the camera matrix
var viewMatrix = m4.inverse(cameraMatrix);
console.log("viewMatrix", viewMatrix);
// ------------------------------------------------------------------------------
// Compute a view projection matrix
var viewProjectionMatrix = m4.multiply(projectionMatrix, viewMatrix);
console.log("viewProjectionMatrix", viewProjectionMatrix);
// ------------------------------------------------------------------------------
// modelMatrix
var modelMatrix = m4.translate(viewProjectionMatrix, 0.0, 0.0, 0.0);
console.log("modelMatrix", modelMatrix);
// ------------------------------------------------------------------------------
var size = 1.0
setRectangle(gl, -size * .5, -size * .5, size, size);
// Let modelMatrix
gl.uniformMatrix4fv(matrixLocation, false, modelMatrix);
// Draw
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
var z = 0;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1, z,
x2, y1, z,
x1, y2, z,
x1, y2, z,
x2, y1, z,
x2, y2, z,
]), gl.STATIC_DRAW);
}
function resizeCanvasToDisplaySize(canvas, multiplier) {
multiplier = multiplier || 1;
var width = canvas.clientWidth * multiplier | 0;
var height = canvas.clientHeight * multiplier | 0;
if (canvas.width !== width || canvas.height !== height) {
canvas.width = width;
canvas.height = height;
return true;
}
return false;
}
function openCVCameraMatrixToProjectionMatrix(fx, fy, cx, cy, zfar, znear, width, height){
var m = [
[0, 0, 0, 0],
[0, 0, 0, 0],
[0, 0, 0, 0],
[0, 0, 0, 0],
];
m[0][0] = 2.0 * fx / width;
m[0][1] = 0.0;
m[0][2] = 0.0;
m[0][3] = 0.0;
m[1][0] = 0.0;
m[1][1] = -2.0 * fy / height;
m[1][2] = 0.0;
m[1][3] = 0.0;
m[2][0] = 1.0 - 2.0 * cx / width;
m[2][1] = 2.0 * cy / height - 1.0;
m[2][2] = (zfar + znear) / (znear - zfar);
m[2][3] = -1.0;
m[3][0] = 0.0;
m[3][1] = 0.0;
m[3][2] = 2.0 * zfar * znear / (znear - zfar);
m[3][3] = 0.0;
return m;
}body {
margin:0;
padding:0;
background-color:lightgray;
}
#canvasGL, img {
position:absolute;
width:360px;
height:640px;
}
img {
opacity: 0.2;
}
#canvasBackground {
border: solid 1px darkGray;
}<body>
<img src="http://www.indelebil.fr/tmp/preview_python_02.jpg" />
<canvas id="canvasGL"></canvas>
</body>The result
Thanks for your help !!!!
