0

I try to download some data from an external API. I would like to pipe the response of every request. The array including the request URLs looks like this :

[ 'https://scihub.copernicus.eu/dhus/odata/v1/Products(\'d98b8730-846f-46d0-a816-5ae4db9f56a7\')/$value',
'https://scihub.copernicus.eu/dhus/odata/v1/Products(\'6edaeb16-3077-45d1-b3f0-fa2d5549f64a\')/$value',
'https://scihub.copernicus.eu/dhus/odata/v1/Products(\'333db2aa-c695-4753-8bd1-e64308af26e1\')/$value',
'https://scihub.copernicus.eu/dhus/odata/v1/Products(\'052cf771-6c4e-4a3a-bc15-51c95a3f37c4\')/$value' ]

I read that request-promise does not support to pipe the request response but I have not found an alternative, that's why the function with which I am trying to get the results from looks as follows:

var fs = require('fs');
var rp = require('request-promise');

function downloadSentinel(promObj){
return new Promise((resolve,reject) => {
    try {
       var promises = promObj.requestURLS.map(url => rp(url,{auth:auth}).then(body => body.pipe(fs.createWriteStream('./test.zip'))
            .on('finish', () => {
                resolve(promObj);
            })), {concurrency:2});
        Promise.all(promises).then(results => {
            console.log(results)
        });
    } catch (error) {
        reject(error);
    }

})
}

Additionally, it is only possible to download two Products at the same time, I tried to achieve it with the Blubird parameter concurrency, but it doesn't seem to work properly.

How could I solve my problem?

UPDATE

If I try it with this code: 

var promises = promObj.requestURLS.map(url => rp(url,{auth:auth}).then(
            data => new Promise((resolve,reject) => {
            data.pipe(fs.createWriteStream('./data/' +  promObj.Name[0] + ".zip"))
            .on('finish', () => {
                console.log('Finally finished');
                resolve(promObj);
            })
                .on('error', () => {
                    reject(promObj);
                })})),{concurrency:2});
        Promise.all(promises).then(results => {
            console.log(results)
        });

I get the error UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 2): ReferenceError: data.pipe is not a function

What have I missed?

Additionally, I recognized that my data is 800MB large. Is this too large to pipe it without an error?

I get this error now: In buffer.js:556 if (encoding === undefined) return buf.utf8Slice(start, end); Error: "toString()" failed

PhilG
  • 351
  • 1
  • 4
  • 15
  • 1
    You will want to create a `new Promise` for each of the piped streams inside your `map` iteration – Bergi Feb 03 '18 at 14:58
  • @Bergi Do you mean something like `var promises = promObj.requestURLS.map(url => rp(url,{auth:auth}).then( data => Promise.all(data.map(url))).then(function (data) { data.pipe(fs.createWriteStream('./test.zip')) .on('finish', () => { resolve(promObj); })}), {concurrency:2}); Promise.all(promises).then(results => { console.log(results) });` – PhilG Feb 03 '18 at 16:22
  • 1
    No, `Promise.all(data.map(url))` looks wrong. But yes, `rp(…).then(data => new Promise((resolve, reject) => { data.pipe(…).on('error', reject).on('finish', resolve); }))` – Bergi Feb 03 '18 at 17:13
  • Sorry I am pretty new to Promises in Javascript. Thanks for your help – PhilG Feb 03 '18 at 17:39

0 Answers0