High RAM usage in simple node.js app

Question

I made this simple node.js app:

var express = require('express');
var app = express();
var alasql = require('alasql');
var url = require('url');

var port = 3000;

app.get('/getuser', function(request, response) {
    var queryObject = url.parse(request.url, true).query;
    var country = queryObject.country;
    var nameset = queryObject.nameset;
    var state = queryObject.state;
    var selectAll;

    if (country == "US" && state !== null && state !== "") {
        alasql.promise("SELECT * from JSON('database.json') WHERE Country = ? AND NameSet = ? AND State = ?", [country, nameset, state]).
        then(function(res) {
            if (res !== null && res.length > 0) {
                var result = res[Math.floor(Math.random() * res.length + 1)];
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end(JSON.stringify(result));
            } else {
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end("noresults");
            }
        }).catch(function(err) {
            console.log('Does the database.json file exists? there was an error:', err);
        });


    } else {
        alasql.promise("SELECT * from JSON('database.json') WHERE Country = ? AND NameSet = ?", [country, nameset]).
        then(function(res) {
            if (res !== null && res.length > 0) {
                var result = res[Math.floor(Math.random() * res.length + 1)];
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end(JSON.stringify(result));
            } else {
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end("noresults");
            }
        }).catch(function(err) {
            console.log('Does the database.json file exists? there was an error:', err);
        });
    }
});

app.listen(port);

As you can see, I load database.json file, do some alasql query on it and return that data as response. database.json is ~50MB in size. My problem is that this app uses 400-600MB of RAM!

I made this version too, where I open database.json file using .require, but memory consumption is same!

var express = require('express');
var app = express();
var alasql = require('alasql');
var url = require('url');

var port = 3000;

var database = require('./database.json');


app.get('/getuser', function(request, response) {
    var queryObject = url.parse(request.url, true).query;
    var country = queryObject.country;
    var nameset = queryObject.nameset;
    var state = queryObject.state;
    var selectAll;

    if (country == "US" && state !== null && state !== "") {
        selectAll = alasql("SELECT * from ? WHERE Country = ? AND NameSet = ? AND State = ?", [database, country, nameset, state]);
    } else {
        selectAll = alasql("SELECT * from ? WHERE Country = ? AND NameSet = ?", [database, country, nameset]);
    }

    if (selectAll !== null && selectAll.length > 0) {
        var res = selectAll[Math.floor(Math.random() * selectAll.length + 1)];

        response.writeHead(200, {
            'Content-Type': 'text/plain'
        });

        response.end(JSON.stringify(res));

    } else {
        response.writeHead(200, {
            'Content-Type': 'text/plain'
        });

        response.end("noresults");
    }
});

app.listen(port);

What am I doing wrong, I suppose it is some issue regarding the size of my JSON file? It is array of 55000 records, each one having 20 fields, if that info helps you?

The use of the RAM memory is stable or is just a peak? Here are some issues related with node about it: https://github.com/nodejs/node/issues/856 — Randall Valenciano, Jan 22 '16 at 18:43
This is one of the severe weaknesses of JSON: It needs a dozen more times memory than the given input. — Gökay Gürcan, Jan 22 '16 at 19:07

score 3 · Accepted Answer · answered Jan 22 '16 at 18:53

3

The fact that it's 50mb on disk, isn't particularly useful when comparing into memory.

Depending on what's in your database JSON, it could be significantly larger than the on disk representation.

Take an integer for example, if you have the value 5 in your JSON it only takes up 1 byte in ASCII, but it will take up 8 bytes in Javascript as all numbers are 8 bytes. When you take into account the number of references required to keep track of the JSON object itself and it's children, etc, you can end up with something much larger than the on-disk representation.

answered Jan 22 '16 at 18:53

Hamel Kothari

717
4
11

Do you have some other solution for my needs? I basically need to make an SQL- like query and select some data from database of 55000 records, containing ~20 fields of different types, some are numbers, some are dates, some are textual. I have this data in JSON file. I can't use MySQL or anything similar, as everything must be packed in node.js application, no other software can be used. – kecman Jan 22 '16 at 23:35
1

The first thing I would think about is whether you need to keep the whole object in memory. From the looks of your queries, your JSON DB could instead just be a file full of smaller record objects newline separated, that you stream in, one line at a time, parse and filter before returning matching records to the user the user. Then your memory is bounded only by the amount that you would return. – Hamel Kothari Jan 23 '16 at 02:41
1

Alternatively if you really need a SQL solution, consider a database that can partition your records and will only read a more relevant portion into memory. I'm actually not a huge node guy but googling seems to suggest this as an option: http://www.tingodb.com/ Look specifically at the features and how they use memory vs. files. – Hamel Kothari Jan 23 '16 at 02:44

High RAM usage in simple node.js app

1 Answers1