Requesting large JSON objects lead to memory leak

Question

I have a simple NodeJs application that acts as a REST client and requests large JSON object. The problem is that it always runs out of memory (taking more than 6Gb). I am using manual garbage collection (app started with --expose_gc) but that doesn't seem to help much.

Here's my code:

var needle = require('needle');
function getAllData() {
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");

    setInterval(function () {
        getAllData();
    }, 10 * 1000);
}

function getDataFromUrl(url) {
    needle.get(url, function (error, response) {
        if (!error && response.statusCode == 200) {
            console.log("do something");
        }
    });
}

function scheduleGc() {
    global.gc();
    setTimeout(function () {
        scheduleGc();
    }, 100 * 1000);
}

getAllData();
scheduleGc();

I have tried request library but I had the same results. What am I doing wrong?

P.s. My nodejs version is 6.9.1, needle version 1.3.0

score 0 · Answer 1 · edited May 23 '17 at 12:16

0

Use node inspector to find leakage. It could be internal nodejs leak or leakage from needle and request. I'd prefer to use native http.request to make sure it's not a library problem. Also you can split your code into independent parts and run them separately to find the leakage source.

There is another answer how to detect memory leaks.

edited May 23 '17 at 12:16

Community

1
1

answered Nov 01 '16 at 14:55

Paul Rumkin

6,737
2
25
35

score 0 · Answer 2 · answered Nov 01 '16 at 15:02

I found an error. I was using the setInterval instead of setTimeout....

var needle = require('needle');
function getAllData() {
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
    getDataFromUrl("http://puppygifs.tumblr.com/api/read/json");
}

function getDataFromUrl(url) {
    needle.get(url, function (error, response) {
        if (!error && response.statusCode == 200) {
            console.log("do something");
        }
    });
}

function scheduleGc() {
    global.gc();

    setTimeout(function () {
        scheduleGc();
    }, 100 * 1000);
}

setInterval(function () {
    getAllData();
}, 10 * 1000);

scheduleGc();

score 0 · Answer 3 · answered Nov 01 '16 at 15:05

0

It's because you are working with too much info, 6Gb should be treated with streams.
It's very easy with needle, just avoid the callback.
Here is an example:

'use strict';
 const needle = require('needle');
 const fs = require('fs');
 const out = fs.createWriteStream('bigFile.json');
 needle.get('http://puppygifs.tumblr.com/api/read/json').pipe(out);

answered Nov 01 '16 at 15:05

Hosar

5,163
3
26
39

The JSON isn't 6Gb, it is a lot less. The 6Gb was the amount of used RAM due to the fact that I was using setInterval instead of setTimeout... – Jkam Nov 01 '16 at 15:35

Requesting large JSON objects lead to memory leak

3 Answers3