0

I am trying to request a huge amount of images on node js (n > 3000) and then save them into a folder. I have tried using request library and request-promise. The problem is that if the number of pictures is too big some pictures do not complete downloading, leaving incomplete data or get an error(an empty .jpeg file). Is there a better way of downloading huge amounts of pictures? Also i need when the pictures all download to request a function compile() to make them into a video. I am using ffmpeg for that.

Below are 2 ways i tried doing this.

const req = require('request');
const request = require('request-promise');
const fs = require('fs');
const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path;
const ffmpeg = require('fluent-ffmpeg');
ffmpeg.setFfmpegPath(ffmpegPath);

function downloadImgs(imgUrls) {
  let promises = [];
  let prom;
  for (let i = 0; i < imgUrls.length; i++) {
    imgPath = `assets/pics/st_${pad(i + 1, 3)}.jpg`

    prom = request(imgUrls[i]);
    prom.on('error', () => console.log('err'));
    prom.pipe(fs.createWriteStream(imgPath));
    promises.push(prom);
  }

  Promise.all(promises).then(() => {
    console.log('I Run');
    compilee();
  });

  //SECOND TRY-----------------------------------------
  for (let i = 0; i < imgUrls.lengh; i++) {
    imgUrl = imgUrls[i];
    req(imgUrl)
      .on('error', () => console.log("img error", imgUrl))
      .pipe(fs.createWriteStream(`assets/pics/st_${pad(i + 1, 3)}.jpg`)).on('close', () => {
        console.log('downloaded', x)
      })
  }
  compilee();

//---------------------------------------------------------

  function compilee() {
    command
      .on('end', onEnd)
      // .on('progress', onProgress)
      .on('error', onError)
      .input('assets/pics/st_%03d.jpg')
      .inputFPS(5)
      .output('assets/output/pepe.mp4')
      .outputFps(30)
      .run();
  }

The errors i am getting for the first: UnhandledPromiseRejectionWarning: RequestError: Error: socket hang up

and the second : Error: socket hang up at createHangUpError (_http_client.js:330:15) at TLSSocket.socketOnEnd (_http_client.js:433:23) at TLSSocket.emit (events.js:187:15) at endReadableNT (_stream_readable.js:1098:12) at process.internalTickCallback (internal/process/next_tick.js:72:19) code: 'ECONNRESET' }

Also, should i download the pictures on the server or on the client if i want to sent the video to the client immediately.

Manos Koutselakis
  • 77
  • 1
  • 7
  • Not sure about it but your code block looks like you are trying to get all images at once. It is probably draining your bandwidth. Did you think about it ? – ali.turan Jan 05 '19 at 21:28
  • I did not. How would i solve this issue? – Manos Koutselakis Jan 05 '19 at 21:29
  • 1
    Well you if you have the server you can zip the files then send it maybe. But I don't think it is the solution you are looking for. In your case waiting for each request to finish would be a solution for you. But this will slow down your operation dramatically. Depending on your bandwidth downloading only 5-10 image file at the same time might solve your issue. I may suggest you to use async library to do the task in parallel. – ali.turan Jan 05 '19 at 21:35
  • +1 for async library, specifically [async.parallelLimit](http://caolan.github.io/async/docs.html#parallelLimit) seems to suit your needs. – Azami Jan 05 '19 at 21:42
  • parallelLimit also would be a good choice I am not a async pro and don't know all the functions. Probably you will find best depending on your needs . – ali.turan Jan 05 '19 at 21:45
  • thanks for the suggestion, i will try it out and give feedback – Manos Koutselakis Jan 05 '19 at 22:03

1 Answers1

0

I may suggest you to use async library. Que might be a good solution

var q = async.queue(function({url,index}, callback) {
    // Do ur job here
}, 5);

// assign a callback
q.drain = function() {
    console.log('All images downloaded');
};

// add items to the queue
for (let i = 0; i < imgUrls.length; i++) {
    q.push({url : imgUrls[i], index : i});
}

Ofc it is not a working all code block but it is just to give you the idea. Hope it helps

ali.turan
  • 535
  • 1
  • 6
  • 20