0

I am calling my 5 CSV URL get data from it and try to store all five csv return data from URL in one file but I have to write header once in file. For that I am converting my CSV to JSON and handle header thing and after that convert json to CSV data and try to write in CSV with fs.

       const axio =required("axios")
       const Csvparser = require("json2csv").Parser;
       const fs = require("fs");

      const streamData = [];
     // Function is used to get csv file url
     const getCsvFileUrls = () => {
           for (let index = 1; index < 6; index++) {
          axios.get("my_file_url" +index,
        {
          headers: {
            Authorization: `Token token, api_key=api_key`,
          },
        }
      )
      .then((response) => {
        if (response.data) {
           // getting one by one response URL of csv file
          if (response.data.response.header.status == 200) {
            axios
              .get(response.data.inventory_url)
              .then(async (response) => {
                console.log("Process:", index);
                // Process csv data for convert them into JSON
                const data = response.data.split("\n");
                let headers = data.shift().split(",");
                data.forEach(function (d) {
                  tmp = {};
                  row = d.split(",");
                  for (var i = 0; i < headers.length; i++) {
                    tmp[headers[i]] = row[i];
                  }
                  streamData.push(tmp);
                });
                if (index == 5) {
                  // On last index convert back JSON to CSV using json2csv NPM package
                  const csvParser = new Csvparser({ headers });
                  const csvData = csvParser.parse(streamData);
                  // Writing data to file
                  const writeStream = fs.createWriteStream("output.csv");
                  const overWaterMark = writeStream.write(csvData);
                  if (!overWaterMark) {
                    await new Promise((resolve) => {
                      writeStream.once("drain", resolve);
                    });
                  }
                  writeStream.end();
                  writeStream
                    .on("finish", () => {
                      console.log("finish write stream, moving along");
                    })
                    .on("error", (err) => {
                      console.log(err);
                    });
                }
              })
              .catch((error) => {
                console.log(error, "error");
              });
          }
        }
      })
      .catch((error) => {
        console.log(error, "errors");
      });
      }
    };

   getCsvFileUrls();

Hard
  • 105
  • 5

0 Answers0