1

I'm trying to download multiple file using request library, I need to download them one by one and also show a progress bar, the files links are stored in an array that passes them to a function to start the download

const request = require('request')
const fs = require('fs')
const ProgressBar = require('progress')

async function downloadFiles(links) {
    for (let link of links) {
        let file = request(link)
        file.on('response', (res) => {
            var len = parseInt(res.headers['content-length'], 10);
            console.log();
            bar = new ProgressBar('  Downloading [:bar] :rate/bps :percent :etas', {
                complete: '=',
                incomplete: ' ',
                width: 20,
                total: len
            });
            file.on('data', (chunk) => {
                bar.tick(chunk.length);
            })
            file.on('end', () => {
                console.log('\n');
            })
        })
        file.pipe(fs.createWriteStream('./downloads/' + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)))
    }
}


let links = ['https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin', 'https://speed.hetzner.de/100MB.bin']
downloadFiles(links)

This is what I've got so far, the problem is that the request is asynchronous, I tried to use async/await but that way I couldn't get the progress bar to work. How can make it so that the files are downloaded one at the time and also have a progress bar?

Federico Morrone
  • 83
  • 8
  • Why do you want to download the files sequentially and not make use of parallel downloads? Your questions also mixes your problem with downloading the file with your problem of the progress bar, which may make it too broad. Maybe ask to distinct questions instead of this one? – k0pernikus Aug 05 '19 at 16:38
  • You might want to have a look at the `async` library, async/queue could be your friend: https://caolan.github.io/async/v3/docs.html#queue – Markus Aug 05 '19 at 16:43
  • @k0pernikus I need to download multiple video files, and I need them to be downloaded in a specific order instead of all togheter – Federico Morrone Aug 05 '19 at 16:54
  • Yes, you say you need to. But *why* do you need to download them in a specific order? You can pertain the order of the files even with an async approach. ([Promise.all](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all) for example keeps order once all the promises have been resolved.) – k0pernikus Aug 05 '19 at 16:59

1 Answers1

4

Based on my comment about async.queue, this is how I would write that up. You can call dl.downloadFiles([]) as often as you want and it will just fetch everything that you have added to the queue one after another.

const request = require('request')
const async = require('async')
const fs = require('fs')
const ProgressBar = require('progress')

class Downloader {
    constructor() {
        this.q = async.queue(this.singleFile, 1);

        // assign a callback
        this.q.drain(function() {
            console.log('all items have been processed');
        });

        // assign an error callback
        this.q.error(function(err, task) {
            console.error('task experienced an error', task);
        });
    }

    downloadFiles(links) {
        for (let link of links) {
            this.q.push(link);
        }
    }

    singleFile(link, cb) {
        let file = request(link);
        let bar;
        file.on('response', (res) => {
            const len = parseInt(res.headers['content-length'], 10);
            console.log();
            bar = new ProgressBar('  Downloading [:bar] :rate/bps :percent :etas', {
                complete: '=',
                incomplete: ' ',
                width: 20,
                total: len
            });
            file.on('data', (chunk) => {
                bar.tick(chunk.length);
            })
            file.on('end', () => {
                console.log('\n');
                cb();
            })
        })
        file.pipe(fs.createWriteStream('./downloads/' + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15)))
    }
}

const dl = new Downloader();

dl.downloadFiles([
    'https://speed.hetzner.de/100MB.bin',
    'https://speed.hetzner.de/100MB.bin'
]);
Markus
  • 1,069
  • 8
  • 26