NodeJs: Getting paginated get requests

Question

When requesting Q&As from stackexchange API, the JSON response has a has_more (Boolean) argument and when it is true that means there are more requests possible with

http://api.stackexchange.com/2.2/questions?order=desc&sort=activity&site=stackoverflow&filter=withbody&pagesize=30&tagged=java&page=1

the response looks like this

{items: [
      ..]
 has_more: true,
 quota_max: 300,
 quota_remaining: 281
}

I want the best way to get all possible responses fetched while maintaining the quota_max limit. So far, my module is able to get one result :

exports.getQuestions = function(q,tag,page){
return new Promise(function(resolve, reject){

var request = require('request');
var url = 'http://api.stackexchange.com/2.2/search?order=desc&sort=votes&site=stackoverflow&filter=withbody&pagesize=30';


url = url + '&q=' + q + '&tagged='+ tag + "&page="+page;

request({headers: {
    'Accept': 'application/json; charset=utf-8',
    'User-Agent': 'RandomHeader'
         },
     uri: url,
     method: 'GET',
     gzip: true
         },
function(err, res, body) {
if (err || res.statusCode !=200){
            reject (err || {statusCode: res.statusCode});
      }
     resolve(body);
          });
})}

But I want to loop through or make multiple calls in a proper way. Any help is appreciated.

score 13 · Accepted Answer · answered Oct 25 '17 at 05:05

you can use your getQuestions implementation in combination with async await, to just continue a loop requesting page after page until has_more is false, or the quota limit is reached...

async function getAllQuestions(q,tag){
    var page = 0 
    var res = await getQuestions(q,tag,page)
    var all = [res]
    while(res.has_more && res.quota_remaining > 0){
        page++
        res=await getQuestions(q,tag,page)
        all.push(res)
    }
    return all
}

NodeJs: Getting paginated get requests

1 Answers1