How to print all character names from Game of thrones Api which is in paginated form | Traverse paginated API in JS

Question

Hello, i am trying to make a side project using game of thrones API. Out of curiosity, I want to print all the characters name from https://www.anapioficeandfire.com/api/characters .

From the response headers and documentation (https://anapioficeandfire.com/Documentation#pagination), I see this Api has 214 pages. How do i fetch the character names from all of these 214 pages.

My Current attempt allows me to get names from only 1st 10 objects

Here is my attempt :

(function() {
  var req = new XMLHttpRequest();

  req.onreadystatechange = function() {
    if (this.readyState == 4 && this.status == 200) {
      console.log(req.getResponseHeader("link"));
      let charObj = JSON.parse(this.responseText);
      for (let i = 0; i < charObj.length; i++) {
        let p = document.createElement("p");
        let name = document.createTextNode(charObj[i].name);
        p.appendChild(name);
        document.body.appendChild(p);
      }
    }
  };
  req.open("GET", "https://www.anapioficeandfire.com/api/characters", true);
  req.send();
})();

I have tried to be specific with the question. Why has it been downvoted? — Siddharth Jain, Jun 05 '18 at 14:44

Taki · Answer 1 · 2018-07-13T21:53:51.457

In the Documentation, you can see that you can specify the page number and the pageSize, however :

You can specify how many items you want to receive per page, the maximum is 50

By adding those parameters at the end of your url you can get a maximum of 50 items per page, add page=1&pageSize=50 ( since the original page size is 10 )

But you can put your request in a loop to fetch all the pages from 1 to 214, 10 at a time :

( loose the IFEE and call the function in a for loop )

function getItems(page) {
  var req = new XMLHttpRequest();

  req.onreadystatechange = function() {
    if (this.readyState == 4 && this.status == 200) {
      //console.log(req.getResponseHeader("link"));
      let charObj = JSON.parse(this.responseText);
      for (let i = 0; i < charObj.length; i++) {
        let p = document.createElement("p");
        let name = document.createTextNode(charObj[i].name);
        p.appendChild(name);
        document.body.appendChild(p);
      }
    }
  };
  req.open("GET", "https://www.anapioficeandfire.com/api/characters?page=" + page + "&pageSize=10", true);
  req.send();
};

for (let i = 1; i <= 214; i++) {
  getItems(i)
}

Any idea on how to cache this request? As you can see 214 requests are fired with this code. (Since there are 214 pages) — Siddharth Jain, Jul 13 '18 at 17:33
see : https://stackoverflow.com/questions/28154796/xmlhttprequest-how-to-force-caching — Taki, Jul 13 '18 at 21:54

score 1 · Accepted Answer · answered Jun 05 '18 at 15:22

Use fetch. There is a link Header property that tells you the next, prev, and last URLs of the endpoint you are looking at.

You should chain/sequence requests in such a way that you fetch the first URL, get the next URL, and make a call to the next page. Keep doing this until you run out of pages and collect all the characters that way.

I noticed that not everyone has a name, so I pull either the name or the first alias.

Also, to ease the access to the server, I limited the depth of the process to 3 requests. You should remove that if you don't want the limit.

function parseHeaders(res) {
    return res.headers.get("link").split(",").reduce((acc, link) => {
        const props = /^\<(.+)\>; rel="(.+)"$/.exec(link.trim());
        if (!props) {
            console.warn("no match");
            return acc;
        }
        acc[props[2]] = props[1];
        return acc;
    }, {});
}

async function go(url, depth = 0) {
    /*
     * You don't need this, this is just to alleviate load on the API's server
     * You should remove all the stuff about depth whenever you use this,
     * it just limits the total number of requests to fire to 3 which is nice
     * since it looks like there would be like 215 requests if you didn't do this.
     */
    if (depth >= 3) return [];

    const res = await fetch(url);
    const props = parseHeaders(res);
    const data = await res.json();
    const characters = data.map(character => character.name || character.aliases[0]);

    if (props.next) {
        const newCharacters = await go(props.next, depth + 1);
        characters.push(...newCharacters);
    }

    return characters;
}

(async function() {
    const characters = await go("https://www.anapioficeandfire.com/api/characters");
    console.log(characters);
}());

Any idea on how to cache this request instead of using depth? As you can see 214 requests are fired with this code. (Since there are 214 pages) — Siddharth Jain, Jul 13 '18 at 19:21
@SiddharthJain Define cache. Your browser should be caching these requests since they are GETs. Either way, caching this action could be an entirely different question. — zero298, Jul 13 '18 at 19:35
Here is the link to the other question - https://stackoverflow.com/questions/51329431/how-to-implement-caching-game-of-thrones-api Feel free to post your thoughts! — Siddharth Jain, Jul 13 '18 at 19:37

How to print all character names from Game of thrones Api which is in paginated form | Traverse paginated API in JS

2 Answers2