Best method to string together variety of DB calls in Node js

Question

I basically need to make about 3 calls to get the data for a json object.. It basically JSON array of JSON object which have some attributes, one of which is an array of other values selected using a second query, then that one also has an array inside which is selected with another db call.

I tried using asyn.concatSeries so that I can dig down into the bottom call and put together all the information I collected for one root json object but that's creating a lot of unexpected behaviour..

Example of JSON

[
    {
       "item" : "firstDbCall"
       "children" : [ {
                       "name" : "itemDiscoveredWithSecondDBCall"
                       "children" : [ itemsDiscoveredwith3rdDBCall]
                      },
                    ]
    }
]

This is really difficult using node.js. I really need to figure out how to do this properly since I have to do many of these for different purposes.

EDIT This is the code i have. There's some strange behaviour with async.concatSeries. The results get called multiple times after each one of the functions finish for each array. So i had to put a check in place. I know it's very messy code but i've been just putting band-aids all over it for the past 2 hours to make it work..

console.log("GET USERS HAREDQARE INFO _--__--_-_-_-_-_____");
var query = "select driveGroupId from tasks, driveInformation where agentId = '" 
        + req.params.agentId + "' and driveInformation.taskId = tasks.id order by driveInformation.taskId desc;";

connection.query(query, function(err, rows) {
    if (rows === undefined) {
        res.json([]);
        return;
    } 
    if(rows.length<1) { res.send("[]"); return;}
    var driveGroupId = rows[0].driveGroupId;
    var physicalQuery = "select * from drives where driveGroupId = " + driveGroupId + ";";
    connection.query(physicalQuery, function(err, rows) {
        console.log("ROWSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS");
        console.log(rows);
        async.concatSeries(rows, function(row, cb) {
            console.log("-------------------------------SINGLE ROW-------------------------------------");
            console.log(row);
            if(row.hasLogicalDrives != 0) {
                console.log("HAS LOGICAL DRIVES");
                console.log(row.id); 
                var query = "select id, name from logicalDrives where driveId = " + row.id;
                connection.query(query, function(error, drives) {
                    console.log("QUERY RETURNED");
                    console.log(drives);
                    parseDriveInfo(row.name, row.searchable, drives, cb);

                });
            }
            else
                var driveInfo = { "driveName" : row.name, "searchable" : row.searchable};
                console.log("NO SUB ITEMS");
                cb(null, driveInfo);
        }, function(err, results) {
            console.log("GEETTTTINGHERE");
            console.log(results);
            if(results.length == rows.length) {
                console.log("RESULTS FOR THE DRIVE SEARCH");
                console.log(results);
                var response = {"id": req.params.agentId};
                response.driveList = results;
                console.log("RESPONSE");
                console.log(response);
                res.json(response);
            }
        });
    });
});     

 };

 parseDriveInfo = function(driveName, searchable, drives, cb) {
async.concatSeries(drives, function(drive,callback) {
    console.log("SERIES 2");
    console.log(drive);
    console.log("END OF DRIVE INFO");
    var query = "select name from supportedSearchTypes where logicalDriveId = " + drive.id;
    connection.query(query, function(error, searchTypes) {
        drive.searchTypes = searchTypes;
        var driveInfo = { "driveName" :driveName,
            "searchable" : searchable,
            "logicalDrives" : drive
        };

        callback(null, driveInfo);

    });
}, function (err, results) {
    console.log("THIS IS ISISIS ISISISSISISISISISISISISISIS");
    console.log(results);
    if(results.length === drives.length) {
        console.log("GOTHERE");
        cb(null, results);
    }
});     

}

Answer 1

Getting good enough with async to use exactly the right combination of methods under the right circumstances takes a fair amount of experience. Most likely your case in particular can be handled with async.waterfall if its query1 then query2(dataFoundByQuery1) then query3(dataFoundByQuery2). But depending on the circumstances you need to mix and match async methods appropriately and sometimes have 2 levels - for example a "big picture" async.waterfall where some of the steps in the waterfall do async.parallel or async.series as needed. I've never used async.concat and given your needs I think you have chosen the wrong method. The workhorses are async.each, async.eachSeries, async.waterfall, and async.map, at least for the web app & DB query use cases I mostly encounter, so make sure you really have those understood before exploring the more specific convenience methods.

Answer 2

EDIT: This is a more in depth example based on use of the connection library you seem to be using. Please note, some of this is javascript psuedo code. Things like adding objects to the resultsArray are clearly not complete, the only thing I took time to make sure was correct is the "flow of logic" as it pertains to callbacks. Everything else is for you to implement. In order to support multiple calls to the same callback function and maintain state from call to call, the best way is to wrap the set of callbacks in a closure. This allows the callbacks to share some state with the main event loop. This allows you to pass arguments to the callbacks, without actually having to pass them as arguments, much like class variables in c++, or even globals in javascript, but we haven't poluted the global scope :)

function queryDataBase(query) {
//wrap the whole query in a function so the callbacks can share some
//variables with similar scope.  This is called a closure

    int rowCounter = 0;
    var dataRowsFromStep2;
    var resultsArray = {};

    connection.query(query, dataBaseQueryStep2);

    function dataBaseQueryStep2(err, rows) {
        //do something with err and rows
        dataRowsFromStep2 = rows;

        var query = getQueryFromRow(dataRowsFromStep2[rowCounter++]);//Always zero the first time.  Might need to double check rows isn't empty!

        connection.query(query, dataBaseQueryStep3);
    }

    function dataBaseQueryStep3(err, rows) {
        //do something with err and rows

        if(rowCounter < dataRowsFromStep2.size) {
            resultsArray.add(rows);//Probably needs to be more interesting, but you get the idea
            //since this is within the same closure, rowCounter maintains it's state
            var query = getQueryFromRow(dataRowsFromStep2[rowCounter++]);

            //recursive call query using dataBaseQueryStep3 as it's callback repeatedly until
            //we run out of rows to call it on.
            connection.query(query, dataBaseQueryStep3)
        } else {
            //when the if statement fails we have no more rows to run queries on so return to main program flow
            returnToMainProgramLogic(resultsArray);
        }

    }
}

function returnToMainProgramLogic(results) {
    //continue running your program here
}

I personally like the above logic better than the syntax async produces... I believe the heart of your problem rests in your nested calls to async, and the fact that ASYN itself, runs the series of functions asynchronously, but in order(confusing I know). If you write your program like this, you won't have to worry about it!

Answer 3

I would strongly suggest using sequelize.js It provides a really powerful orm that allows you to chain queries together. It also allows you to directly load your data into js objects, write dynamic sql, and connect to many different databases. Picture ActiveRecord from the Ruby world for Node.