Converting 360 degree view to equirectangular in node js?

Question

I have been trying to convert the 360 degree camera, single fish eye image, to equirectangular viewer in node js for the past two days. In stackoverflow, the same question is asked and answered in pseudo code. I have been trying to convert pseudo code to node js and cleared some errors. Now the project runs without error but the output image is blank.

From that pseudo, I dont know the polar_w, polar_h and geo_w, geo_h, geo and polar value, so, it gave static value to show the output. Here is a link which i followed to convert pseudo code to node js. How to convert spherical coordinates to equirectangular projection coordinates?.

Here is the code I tried for converting spherical image to equirectangular viewer:

 exports.sphereImage=(request, response)=>{


 var Jimp = require('jimp');


  // Photo resolution

var img_w_px = 1280;
var img_h_px = 720;

var polar_w = 1280;
var polar_h = 720;
var geo_w = 1280;
var geo_h = 720;

var img_h_deg = 70;
var img_w_deg = 30;

  // Camera field-of-view angles

var img_ha_deg = 70;
var img_va_deg = 40;


  // Camera rotation angles

 var  hcam_deg = 230;
 var  vcam_deg = 60;

  // Camera rotation angles in radians

 var hcam_rad = hcam_deg/180.0*Math.PI;
 var vcam_rad = vcam_rad/180.0*Math.PI;

 // Rotation around y-axis for vertical rotation of camera

  var rot_y = [

   [Math.cos(vcam_rad), 0, Math.sin(vcam_rad)],
   [0, 1, 0],
  [-Math.sin(vcam_rad), 0, Math.cos(vcam_rad)]

    ];

 // Rotation around z-axis for horizontal rotation of camera

  var rot_z = [

 [Math.cos(hcam_rad), -Math.sin(hcam_rad), 0],
 [Math.sin(hcam_rad), Math.cos(hcam_rad), 0],
 [0, 0, 1]

 ];


  Jimp.read('./public/images/4-18-2-42.jpg', (err, lenna) => {


    polar = new Jimp(img_w_px, img_h_px);
    geo = new Jimp(img_w_px, img_h_px);

  for(var i=0; i<img_h_px; ++i)
  {
  for(var j=0; j<img_w_px; ++j)
  {
    // var p = img.getPixelAt(i, j);

    var p = lenna.getPixelColor(i, j)
    // var p = getPixels(img, { x: i, y: j })

    // Calculate relative position to center in degrees
    var p_theta = (j - img_w_px / 2.0) / img_w_px * img_w_deg / 180.0 * Math.PI;
    var p_phi = -(i - img_h_px / 2.0) / img_h_px * img_h_deg / 180.0 *Math. PI;

    // Transform into cartesian coordinates
    var p_x = Math.cos(p_phi) * Math.cos(p_theta);
    var p_y = Math.cos(p_phi) * Math.sin(p_theta);
    var p_z = Math.sin(p_phi);
    var p0 = {p_x, p_y, p_z};

    // Apply rotation matrices (note, z-axis is the vertical one)
    // First vertically
    var p1 = rot_y[1][2][3] * p0;
    var p2 = rot_z[1][2][3] * p1;

    // Transform back into spherical coordinates
    var theta = Math.atan2(p2[1], p2[0]);
    var phi = Math.asin(p2[2]);

    // Retrieve longitude,latitude
    var longitude = theta / Math.PI * 180.0;
    var latitude = phi / Math.PI * 180.0;

    // Now we can use longitude,latitude coordinates in many different 
    projections, such as:
    // Polar projection
    {
        var polar_x_px = (0.5*Math.PI + phi)*0.5 * Math.cos(theta) 
   /Math.PI*180.0 * polar_w;
        var polar_y_px = (0.5*Math.PI + phi)*0.5 * Math.sin(theta) 
    /Math.PI*180.0 * polar_h;
        polar.setPixelColor(p, polar_x_px, polar_y_px);
    }
    // Geographical (=equirectangular) projection
    {
        var geo_x_px = (longitude + 180) * geo_w;
        var geo_y_px = (latitude + 90) * geo_h;
        // geo.setPixel(geo_x_px, geo_y_px, p.getRGB());
        geo.setPixelColor(p, geo_x_px, geo_y_px);
    }
      // ...
 }
}

  geo.write('./public/images/4-18-2-42-00001.jpg');
 polar.write('./public/images/4-18-2-42-00002.jpg');



 });


}

And tried another method by slicing image into four parts to detect car. Sliced image into four parts using image-slice module and to read and write jimp module is used. But unfortunately cars not detected properly.

Here is the code i used for slicing image:

 exports.sliceImage=(request, response)=>{

var imageToSlices = require('image-to-slices');
var lineXArray = [540, 540];
var lineYArray = [960, 960];
var source = './public/images/4-18-2-42.jpg'; // width: 300, height: 300

imageToSlices(source, lineXArray, lineYArray, {
    saveToDir: './public/images/',
    clipperOptions: {
        canvas: require('canvas')
    }    
}, function() {
    console.log('the source image has been sliced into 9 sections!');
});


 }//sliceImage 

And for detect car from image i used opencv4nodejs. Cars are not detected properly. here is the code i used for detect car:

   function runDetectCarExample(img=null){
        if(img==null){

         img = cv.imread('./public/images/section-1.jpg');
    }else
    {
         img=cv.imread(img);
    }
        const minConfidence = 0.06;

        const predictions = classifyImg(img).filter(res => res.confidence > minConfidence && res.className=='car');

        const drawClassDetections = makeDrawClassDetections(predictions);

        const getRandomColor = () => new cv.Vec(Math.random() * 255, Math.random() * 255, 255);

        drawClassDetections(img, 'car', getRandomColor);
        cv.imwrite('./public/images/section-'+Math.random()+'.jpg', img);
        var name="distanceFromCamera";
        var focalLen= 1.6 ;//Focal length in mm
        var realObjHeight=254 ;//Real Height of Object in mm
        var cameraFrameHeight=960;//Height of Image in pxl
        var imgHeight=960;//Image Height in pxl
        var sensorHeight=10;//Sensor height in mm
        var R = 6378.1 //#Radius of the Earth
        var brng = 1.57 //#Bearing is 90 degrees converted to radians.
        var hc=(200/100);//Camera height in m
        predictions
            .forEach((data)=> {

                // imgHeight=img.rows;//Image Height in pxl
                // realObjHeight=data.rect.height;
                // data.rect[name]=((focalLen)*(realObjHeight)* 
         (cameraFrameHeight))/((imgHeight)*(sensorHeight));

                var dc=(((data.rect.width * focalLen) / img.cols)*2.54)*100; // meters
                console.log(Math.floor(parseInt(data.rect.width)));
                // var dc=((Math.floor(parseInt(data.rect.width)* 0.264583) * focalLen) / img.cols); // mm


                var lat1=13.0002855;//13.000356;
                var lon1=80.2046441;//80.204632;
                // Gate 13.0002855,80.2046441
                // Brazil Polsec : -19.860566, -43.969436
                // var d=Math.sqrt((dc*dc)+(hc*hc));
                // d=(data.rect[name])/1000;
                data.rect[name]=d=dc/1000;
                lat1 =toRadians(lat1);
                lon1 = toRadians(lon1);
                brng =toRadians(90);
                // lat2 = Math.asin( Math.sin(lat1)*Math.cos(d/R) +
                //      Math.cos(lat1)*Math.sin(d/R)*Math.cos(brng));

                // lon2 = lon1 + 
             Math.atan2(Math.sin(brng)*Math.sin(d/R)*Math.cos(lat1),
                //              Math.cos(d/R)-Math.sin(lat1)*Math.sin(lat2));


        var lat2 = Math.asin(Math.sin(lat1) * Math.cos(d/6371) +
                      Math.cos(lat1) * Math.sin(d/6371) * Math.cos(brng));

        var lon2 = lon1 + Math.atan2(Math.sin(brng) * Math.sin(d/6371) * Math.cos(lat1),
                              Math.cos(d/6371) - Math.sin(lat1) * Math.sin(lat2));

                lat2 = toDegrees(lat2);
                lon2 = toDegrees(lon2);
                data.rect['latLong']=lat2+','+lon2;
                // console.log(brng);

            });




        response.send(predictions);
        cv.imshowWait('img', img);
    };

here is the fish eye image which need to be converted to equirectangular.

Any help much appreciated pls....

Answer 1

You are asking how to convert a 360deg fish-eye projection to an equirectangular projection.

In order to do this, for every pixel on the fish-eye image you need to know where to place in onto the output image.

Your input image is 1920x1080, let us assume you want to output it to an equirectangular projection of the same size.

The input circle mapping is defined as:

cx = 960; // center of circle on X-axis
cy = 540; // center of circle on Y-axis
radius = 540; // radius of circle

If you have a pixel at (x,y) in the input image, then we can calculate the spherical coordinates using:

dx = (x - cx) * 1.0 / radius;
dy = (y - cy) * 1.0 / radius;
theta_deg = atan2(dy, dx) / MATH_PI * 180;
phi_deg = acos(sqrt(dx*dx + dy*dy)) / MATH_PI * 180;
outputx = (theta_deg + 180) / 360.0 * outputwidth_px;
outputy = (phi_deg + 90) / 180.0 * outputheight_px;

So there we translated (x,y) from the fish-eye image to the (outputx,outputy) in the equirectangular image. In order to not leave the implementation as the dreaded "exercise to the reader", here is some sample Javascript-code using the Jimp-library as used by the OP:

var jimp = require('jimp');
var inputfile = 'input.png';
jimp.read(inputfile, function(err, inputimage)
{
    var cx = 960;
    var cy = 540;
    var radius = 540;
    var inputwidth = 1920;
    var inputheight = 1080;
    var outputwidth = 1920;
    var outputheight = 1080;
    new jimp(outputwidth, outputheight, 0x000000ff, function(err, outputimage)
    {
        for(var y=0;y<inputheight;++y)
        {
            for(var x=0;x<inputwidth;++x)
            {
                var color = inputimage.getPixelColor(x, y);
                var dx = (x - cx) * 1.0 / radius;
                var dy = (y - cy) * 1.0 / radius;
                var theta_deg = Math.atan2(dy, dx) / Math.PI * 180;
                var phi_deg = Math.acos(Math.sqrt(dx*dx + dy*dy)) / Math.PI * 180;
                var outputx = Math.round((theta_deg + 180) / 360.0 * outputwidth);
                var outputy = Math.round((phi_deg + 90) / 180.0 * outputheight);
                outputimage.setPixelColor(color, outputx, outputy);
            }
        }
        outputimage.write('output.png');
    });
});

Note that you will still need to do blending of the pixel with neighbouring pixels (for the same reason as when you're resizing the image).

Additionally, in your case, you only have half of the sphere (you can't see the sun in the sky). So you would need to use var outputy = Math.round(phi_deg / 90.0 * outputheight). In order to keep the right aspect ratio, you might want to change the height to 540.

Also note that the given implementation may not be efficient at all, it's better to use the buffer directly.

Anyway, without blending I came up with the result as demonstrated here:

---

So in order to do blending, you could use the simplest method which is the nearest neighbour approach. In that case, you should invert the formulas in the above example. Instead of moving the pixels from the input image to the right place in the output image, you can go through every pixel in the output image and ask which input pixel we can use for that. This will avoid the black pixels, but may still show artifacts:

var jimp = require('jimp');
var inputfile = 'input.png';
jimp.read(inputfile, function(err, inputimage)
{
    var cx = 960;
    var cy = 540;
    var radius = 540;
    var inputwidth = 1920;
    var inputheight = 1080;
    var outputwidth = 1920;
    var outputheight = 1080/2;
    var blendmap = {};
    new jimp(outputwidth, outputheight, 0x000000ff, function(err, outputimage)
    {
        for(var y=0;y<outputheight;++y)
        {
            for(var x=0;x<outputwidth;++x)
            {
                var theta_deg = 360 - x * 360.0 / outputwidth - 180;
                var phi_deg = 90 - y * 90.0 / outputheight;
                var r = Math.sin(phi_deg * Math.PI / 180)
                var dx = Math.cos(theta_deg * Math.PI / 180) * r;
                var dy = Math.sin(theta_deg * Math.PI / 180) * r;
                var inputx = Math.round(dx * radius + cx);
                var inputy = Math.round(dy * radius + cy);
                outputimage.setPixelColor(inputimage.getPixelColor(inputx, inputy), x, y);
            }
        }
        outputimage.write('output.png');
    });
});

For reference, in order to convert between Cartesian and Spherical coordinate systems. These are the formulas (taken from here). Note that the z is in your case just 1, a so-called "unit" sphere, so you can just leave it out of the equations. You should also understand that since the camera is actually taking a picture in three dimensions, you also need formulas to work in three dimensions.

Here is the generated output image:

Since I don't see your original input image in your question anymore, in order for anyone to test the code from this answer, you can use the following image:

Run the code with:

mkdir /tmp/test
cd /tmp/test
npm install --permanent jimp
cat <<EOF >/tmp/test/main.js
... paste the javascript code from above ...
EOF
curl https://i.stack.imgur.com/0zWt6.png > input.png
node main.js

---

Note: In order to further improve the blending, you should remove the Math.round. So for instance, if you need to grab a pixel at x is 0.75, and the pixel on the left at x = 0 is white, and the pixel on the right at x = 1 is black. Then you want to mix both colors into a dark grey color (using ratio 0.75). You would have to do this for both dimensions simultaneously, if you want a nice result. But this should really be in a new question imho.