I am calling my 5 CSV URL get data from it and try to store all five csv return data from URL in one file but I have to write header once in file. For that I am converting my CSV to JSON and handle header thing and after that convert json to CSV data and try to write in CSV with fs.
const axio =required("axios")
const Csvparser = require("json2csv").Parser;
const fs = require("fs");
const streamData = [];
// Function is used to get csv file url
const getCsvFileUrls = () => {
for (let index = 1; index < 6; index++) {
axios.get("my_file_url" +index,
{
headers: {
Authorization: `Token token, api_key=api_key`,
},
}
)
.then((response) => {
if (response.data) {
// getting one by one response URL of csv file
if (response.data.response.header.status == 200) {
axios
.get(response.data.inventory_url)
.then(async (response) => {
console.log("Process:", index);
// Process csv data for convert them into JSON
const data = response.data.split("\n");
let headers = data.shift().split(",");
data.forEach(function (d) {
tmp = {};
row = d.split(",");
for (var i = 0; i < headers.length; i++) {
tmp[headers[i]] = row[i];
}
streamData.push(tmp);
});
if (index == 5) {
// On last index convert back JSON to CSV using json2csv NPM package
const csvParser = new Csvparser({ headers });
const csvData = csvParser.parse(streamData);
// Writing data to file
const writeStream = fs.createWriteStream("output.csv");
const overWaterMark = writeStream.write(csvData);
if (!overWaterMark) {
await new Promise((resolve) => {
writeStream.once("drain", resolve);
});
}
writeStream.end();
writeStream
.on("finish", () => {
console.log("finish write stream, moving along");
})
.on("error", (err) => {
console.log(err);
});
}
})
.catch((error) => {
console.log(error, "error");
});
}
}
})
.catch((error) => {
console.log(error, "errors");
});
}
};
getCsvFileUrls();