Why am I intermittently writing partial arrays to files in Node with fs.readFileSync?

Question

Context

I'm retrieving data from the ESPN API to fetch weekly NFL matchup data. So, I'm making 18 api calls each time I need to fetch this data to account for all 18 weeks in the NFL season. I'm then creating an array with the data I need from the responses to those calls and writing out 18 files that align with each week in the NFL season (week1.json, week2.json, etc.).

Problem

The problem is that when I call my endpoint, I am seeing 2 things intermittently, and not necessarily at the same time:

(1) Some of the json files(week1.json, week2.json, etc.) include only a portion of the expected array. So, instead of 16 objects in the array, I may see only 4, or only 6, etc. Why would I only see a portion of the response data written to the array that's ultimately written to the .json files?

(2) Not all files are written to each time the endpoint is called. So, I may see that only week1-week5's .json files are written. Why aren't all of them updated?

Problem Code

// iterate 18 times
for (let i = 0; i < 18; i++) {

    let weekNumber;
    weekNumber = i + 1;
    const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
    const weekJson = JSON.parse(week);

    // empty weekJson.games array
    weekJson.games = []

    // get all items
    axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
        .then(response => {
            const schedule = [];
            // get all items from response
            const items = response.data.items
            // console.log(response.data.items)

            items.forEach(item => {
                // make get call to $ref
                axios.get(item.$ref)
                    .then(response => {
                        // get name
                        const name = response.data.name
                        // get date
                        const date = response.data.date
                        // get event id
                        const eventid = response.data.id
                        // get team ids
                        let team1 = response.data.competitions[0].competitors[0].id
                        let team2 = response.data.competitions[0].competitors[1].id

                        // create new object
                        const newObject = {
                            name: name,
                            date: date,
                            eventid: eventid,
                            team1: team1,
                            team2: team2
                        }

                        // add games for week
                        weekJson.games.push(newObject);
                        fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));


                    })
                    .catch(error => {
                        console.log(error)
                    })
            })

        }).catch(error => {
            console.log(error)
        })
}

Updated Code

router.get('/getschedules', (req, res) => {

async function writeGames() {

// iterate 18 times
for (let i = 0; i < 18; i++) {

    let weekNumber;
    weekNumber = i + 1;
    const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
    const weekJson = JSON.parse(week);

    // empty weekJson.games array
    weekJson.games = []

    // get all items
    // Add await keyword to wait for a week to be processed before going to the next one
    await axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`)
        .then(async (response) => { // add async to be able to use await
            const schedule = [];
            // get all items from response
            const items = response.data.items
             console.log(response.data.items)
            // Use standard loop to be able to benefit from async/await
            for (let item of items) {
                // make get call to $ref
                // wait for an item to be processed before going to the next one
                await axios.get(item.$ref)
                    .then(response => {
                        // get name
                        const name = response.data.name
                        // get date
                        const date = response.data.date
                        // get event id
                        const eventid = response.data.id
                        // get team ids
                        let team1 = response.data.competitions[0].competitors[0].id
                        let team2 = response.data.competitions[0].competitors[1].id

                        // create new object
                        const newObject = {
                            name: name,
                            date: date,
                            eventid: eventid,
                            team1: team1,
                            team2: team2
                        }

                        // add games for week
                        weekJson.games.push(newObject);
                    })
                    .catch(error => {
                        console.log(error)
                    })
            }
            // moved out of the for loop since you only need to write this once
            fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));

        }).catch(error => {
            console.log(error)
        })
}
}

writeGames();

})

Answer 1

Your issue might come from the fact that you are looping over an array of item that triggers parallel asynchronous calls and write weekJson before you get the entire data. (But theoretically your code should work if writeSyncFile is really synchronous, maybe there are locks on the file system that prevents node to write properly?) You could try to make everything sequential and only write weekJson once instead of everytime you go over an item:

EDIT

I updated my original code proposition by keeping parallel calls and it worked for me (it's similar to OP's code but I only write the json file once per week).

Then I tried to run OP's code and it was working fine as well. So this makes me think that the problem isn't from the code itself but rather how it's called. As a pure node script, there doesn't seem to be any issue. But I just noticed that OP is using it server side as the result of an API call.

Having an API write so many JSON concurrently is probably not the best idea (especially if the api is called multiple times almost simultaneously). You could either

• just return the games in the response
• or precompute the results
• or fetch and write them only once then cache the result to be reused

Then I wonder if due to the server context, there is not some kind of timeout since OP said that with my initial solution, only the first week was created.

const axios = require("axios");
const fs = require("fs");

async function writeGames() {
  const writeWeekGamesPromises = [];
  // iterate 18 times
  for (let weekNumber = 1; weekNumber < 19; weekNumber++) {
    // give week a default value in case the json file doesn't exist (for repro purpose)
    let week = "{}";
    try {
      week = fs.readFileSync(`./pickem/week${weekNumber}.json`, "utf8");
    } catch (e) {
      console.log(`error reading week ${weekNumber} json file:`, e);
      // file doesn't exist yet
    }
    const weekJson = JSON.parse(week);

    // empty weekJson.games array
    const games = [];
    weekJson.games = games;

    // get all items
    // Add await keyword to wait for a week to be processed before going to the next one
    writeWeekGamesPromises.push(axios
      .get(
        `https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en&region=us`
      )
      .then(async (eventListResponse) => {
        // add async to be able to use await
        const schedule = [];
        console.log(JSON.stringify(eventListResponse.data),'\n');
        // get all items from response
        const items = eventListResponse.data.items;
        // console.log(eventListResponse.data.items); // this seems to be useless since we log the whole data just above

        // parallelize calls and wait for all games from a week to be fetched before writing the file
        await Promise.all(
          items.map((item) => {
            // we return the promise so that Promise.all will wait for all games to be pushed before going on writing the file
            return axios
              .get(item.$ref)
              .then((response) => {
                // get name, date and eventid
                const {name, date, id: eventid} = response.data;
                // get team ids
                let team1 = response.data.competitions[0].competitors[0].id;
                let team2 = response.data.competitions[0].competitors[1].id;

                games.push({ name, date, eventid, team1, team2 });
              })
              .catch((error) => {
                console.log(error);
              });
          })
        );

        // Now that all game data is ready, write in the file
        fs.writeFileSync(
          `./pickem/week${weekNumber}.json`,
          JSON.stringify(weekJson)
        );
      })
      .catch((error) => {
        console.log(error);
      }));
  }

  // Waiting for all games from all weeks to be processed
  await Promise.all(writeWeekGamesPromises);
}

async function runAndLogTime() {
    const start = Date.now();
    await writeGames();
    console.log(`took ${(Date.now() - start) / 1000}s to write all json files`);
}

runAndLogTime();