How to limit size of chunk incoming from rest server on nodejs http module?

Question

I am requesting a rest server from nodejs by using nodejs request module. I want to cancel stream if incoming data size is out of allowed limit.the purpose here is to ensure that my network is not locked.

My code example is as follows;

var http = require("http");

async function httpRequest({
  host,
  method,
  port,
  path
} = params, data) {
  if (method.toUpperCase() === "GET") {
    let query = "";
    data = JSON.parse(data);

    for (var key in data) {
      if (data.hasOwnProperty(key)) {
        let value = data[key];
        console.log(key + " -> " + value);
        query = query
          .concat("&")
          .concat(key)
          .concat("=")
          .concat(value);
      }
    }
    if (query) {
      query = "?".concat(query.substring(1));
    }
    path = encodeURI(path.concat(query));
    console.log("path : " + path);
  }
  var opts = {
    hostname: host,
    port: port,
    path: path,
    method: method,
    timeout: 30 * 1000,
    headers: {
      "Content-Type": "application/json"
    }
  };
  return new Promise(function (resolve, reject) {
    const req = http.request(opts, function (response) {
      console.log("Status Code : " + response.statusCode);
      if (response.statusCode < 200 || response.statusCode >= 300) {
        req.end();
        return reject("Fetch data failed  = " + response.statusCode);
      }
      var str = "";
      response.on("data", function (chunk) {
        console.log("chunk : " + chunk);
        str += chunk;
        if (str.length > 256) {
          req.abort();
          reject(
            new Error(
              "The size of the incoming data is larger than the allowable limit."
            )
          );
        }
      });

      response.on("end", function () {
        console.log("\n Result from web service : ", str);
        try {
          let jsonData = JSON.parse(str);
          if (jsonData.status) {
            if (jsonData.status.toLowerCase === "success") {
              if (!("result" in jsonData)) {
                reject("Json Structure Error");
              }
            } else if (jsonData.status.toLowerCase === "error") {
              if (!jsonData.error) {
                reject("Json Structure Error");
              }
            }
            resolve(jsonData);
          } else {
            reject("Json Structure Error");
          }
        } catch (error) {
          reject("Response json error : " + error);
        }
      });
    });

    if (method.toUpperCase() !== "GET" && data) {
      req.write(data);
    }
    //req bitti
    req.on("timeout", function () {
      console.log("timeout! " + opts.timeout / 1000 + " seconds expired");
      req.abort();
    });

    req.on("error", function (err) {
      console.log("Error : " + err);
      if (err.code === "ECONNRESET") {
        req.abort();
        console.log("Timeout occurs : " + err);
        reject(new Error("Timeout occurs : " + err));
      } else if (err.code === "ENOTFOUND") {
        req.abort();
        console.log("Address cannot be reachable : " + err);
        reject(new Error("Address cannot be reachable : " + err));
      } else {
        req.abort();
        reject(new Error(err));
      }
    });
    req.end();
  });
}

let data = JSON.stringify({
  username: "monetrum",
  password: "123456",
  name: "Loremipsumdolorsitamet,consecteturadipiscingelit" +
    ".Aeneaninaliquamodio,egetfac"
});

let params = {
  host: "127.0.0.1",
  method: "GET",
  port: 3010,
  path: "/login"
};

httpRequest(params, data);

So farr so good.But There is a problem.I am controlling incoming data.Size of data I allowed must not greater than 256 Bytes.But first fetch of chunk is larger than allowed size.So my size control is nonsense.Is there a way to handle it.Is it possible to limit size of chunk. Thanks in advance.

score 5 · Accepted Answer · answered Mar 07 '19 at 02:49

The 'readable' event

You want to use the readable event instead of the data event:

var byteCount = 0;
var chunkSize = 32;
var maxBytes = 256;

req.on('readable', function () {
  var chunks = [];
  var data;

  while(true) {
    data = this.read(chunkSize);
    if (!data) { break; }

    byteCount += data.byteLength;
    if (byteCount > maxBytes) {
      req.abort();
      break;
    }

    chunks.push(data);
  }

  // do something with chunks
});

req.on('abort', function () {
  // do something to handle the error
});

Since your question is very specific I made the example a little more generic so that hopefully others will be able to glean from it as well.

See https://nodejs.org/api/stream.html#stream_event_readable

However...

The Network Does't Care

However, you're going to get more data than that. TCP packet size is 64k. Over non-gigabit ethernet that gets MTU truncated to 1500 bytes (1.5k).

There's nothing that you can do to prevent the network activity from happening other than closing the connection, and you can't get less than 1.5k of data per data event unless there is less than 1.5k of data being sent (or crazy network issues happen, which you have no control over).t

P.S.

I'd recommend that you use a code editor, like VSCode. It's very difficult to read code that has mixes of tabs and spaces and different blocks at different levels. It will suggest plugins that can help you catch mistakes earlier and reformat your code so that it's easier for others (and yourself) to read it.

Thank you very much your answer.I did not know the readable property.I will try to work. — Baltazarr, Mar 07 '19 at 07:19
And Thanks also your suggestion.I am already using VSCode and Prettier - Code formatter.But I don't understand how it format. — Baltazarr, Mar 07 '19 at 08:11
Some notes, you have to add the readable listener to the **res**ponse and not the **req**uest. Also _req.abort()_ is deprecated, use _req.destroy()_. — Zoltán Hajdú, Apr 12 '22 at 14:06

How to limit size of chunk incoming from rest server on nodejs http module?

1 Answers1

The 'readable' event

The Network Does't Care

P.S.