Async piping file to an HTTP request

Question

I'm trying to send off interleaved GET and POST requests to a server, but the POST request is sending data from a file, which seems to throw off the timing.

var async = require('async');
var http = require('http');
var request = require('request');
var fs = require('fs');

var arr = [];
for (var i = 1; i <= 50; i++) {
   arr.push(i);
}

var limitedAgent = new http.Agent({maxSockets: 6});

function processThenSendRequest(data, onfinish) {
    request.get({
        url: 'http://www.google.com',
        pool: limitedAgent
    }, (function(j) {
        return function(err, res) {
            console.log("GET: response from " + j);
        };
    })(data)).on('socket', (function(j) {
        return function(socket) {
            console.log("GET: socket assigned for " + j);
        }
    })(data));

    var source = fs.createReadStream('README.md');

    var postReq = request.post({
        url: 'http://www.google.com',
        pool: limitedAgent
    }, (function(j) {
        return function(err, res) {
            console.log("POST: response from " + j);
        };
    })(data)).on('socket', (function(j) {
        return function(socket) {
            console.log("POST: socket assigned for " + j);
        }
        })(data));

    // source.pipe(postReq);

    setTimeout(function() {
        onfinish(null, data);
    }, 10000);
}

async.map(arr, processThenSendRequest, function(err, results) {
    if (err) console.error(err);
    console.log("finished");
});

The code as written above runs fine, with the GET and POST requests being sent out in alternating order, but if I uncomment the source.pipe(postReq) line, then all the GET requests are sent before all the POST requests.

Is there a solution to this issue? I could use async.mapLimit but that feels like a hack and that the solution should be through the request library - this impression may be based on a misunderstanding though.

Because Node is entirely non-blocking (at least when written this way) you can't be sure anything will occur in order unless you run it in series. `async.series` can also do this for you, or `async.eachSeries`. — brandonscript, Jul 24 '15 at 17:21
This is unrelated, but the extra closures in `processThenSendRequest` that bind `data` are unnecessary because `data` is already scoped to `processThenSendRequest` and the value won't change for that invocation. — mscdex, Jul 24 '15 at 17:31
@mscdex Thanks for pointing that out - that makes sense and is a helpful point for me. — knightian, Jul 24 '15 at 17:53
@remus Got it - that makes sense. In that case, I'll use `async.mapLimit` for what I'm doing, since I don't need it to be perfectly in series, but your comment helps clarify what's going on for me. — knightian, Jul 24 '15 at 17:54

score 0 · Accepted Answer · answered Jul 24 '15 at 17:58

Per my comment:

Because Node is entirely non-blocking (at least when written this way) you can't be sure anything will occur in order unless you run it in series. async.series can also do this for you, or async.eachSeries.

Further to that, since Node doesn’t wait for asynchronous activities to finish, each task gets queued up immediately, while the callbacks (the event completion event) will occur on a first-come-first-serve basis. In your case, since GET requests take far less time to go around than POST requests, that's why they're completing first.

Async piping file to an HTTP request

1 Answers1