Node Js : Nested API calls: Watch a directory for files, read data from each file and GET an ID from an API, POST the data with the id, in another API

Question

I am trying to,

1. Continuously watch a directory for files,
2. If any file comes in, based on the filetype (inferred from name of file), call a GET api for an ID/key
3. POST the data with the Key in another API.
4. Move the file from which data is posted, to another directory

I have used Chokidar, request-promise packages.

The problem is, Whenever there is more than one file is pasted in the directory, quickly the APIs are called, in such a way that even before the data of first file is commited, the second file is processed. This leads to an issue in getting the ID from first API. Both files now will have Same Key/ID. Please let me know, how to fix this, in such a way that we get unique IDs for data from each file.

Here is my code:

var chokidar = require('chokidar');
const fs = require('fs');
var rp = require('request-promise');
var globalpath = require('path');
console.log("path", globalpath.resolve(process.cwd(), './Data'));

var watcher = chokidar.watch(globalpath.resolve(process.cwd(), './Data/*.json'), {

    ignored: /^\./,
    persistent: true
});
watcher
    .on('add', function (path) {
        readfile(path);
    });

async function readfile(path) {
    fs.readFile(path, function (err, data) {
        if (err != null) {
            console.log(err);
        }
        var loadingdata = JSON.parse(data);
        if (path.indexOf("PO") > -1) {
            var options = {
                method: 'GET',
                uri: 'http://localhost:3005/GetNewPOKey',
                json: true
            };

            rp(options)
                .then(function (parsedBody) {
                    console.log(parsedBody.NextPOKey);
                    var options2 = {
                        method: 'POST',
                        uri: 'http://localhost:3005/createPO?id='+parsedBody.NextPOKey,
                        body: loadingdata,
                        json: true
                    };

                    rp(options2)
                        .then(function (parsedBody) {
                            console.log(parsedBody);
                            if (parsedBody === "OK") {
                                moveit(path);
                            }
                            // POST succeeded...
                        })
                        .catch(function (err) {
                            console.log("API error :" + err);
                            // POST failed...
                        });
                    // GET succeeded...
                })
                .catch(function (err) {
                    console.log("API error :" + err);
                    // GET failed...
                });

        }
        else {
            console.log("Unknown File" + path);
        }

    });

}

function moveit(mypath) {
    // console.log("This file is posted " + path);
    let oldpath = mypath;
    let newpath = globalpath.resolve(process.cwd(), './History') + "/" + globalpath.basename(mypath.split('.').join('-' + Date.now() + '.'));
    console.log("This is the new filename " + newpath);
    fs.rename(oldpath, newpath, (err) => {
        if (err) throw err;
        console.log('Rename complete!');
    });
}

Answer 1

I could not find a proper solution, to continue using the Chokidar's watcher. Instead, I wrote this code and scheduled the script in OS, to watch for files.

const fs = require('fs');
var request = require('request');
var rp = require('request-promise');
var globalpath = require('path');
console.log("path", globalpath.resolve(process.cwd(), './Data'));
const directoryPath = globalpath.resolve(process.cwd(), './Data');

fs.readdir(directoryPath, async function (err, files) {
    if (err) {
        return console.log('Unable to scan directory: ' + err);
    }
    let allfiledata = [];
    let allkeys = [];

    files.forEach(function (file) {
        if (file.indexOf("json") > -1) {
            console.log(file);
            data = fs.readFileSync(directoryPath + "/" + file)
            var loadingdata = JSON.parse(data);
            allfiledata.push(loadingdata);
            moveit(directoryPath + "/" + file);
        }
    });
    if (allfiledata.length > 0) {
        request({
            url: 'http://localhost:3005/GetNewPOKey',
            json: true
        }, function (err, res, body) {
            if (err) {
                console.log("API error :" + err);
            }
            let nextkey = body.NextPOKey;
            allkeys.push(nextkey);
            var number = parseInt(nextkey.slice(2, ));
            for (let i = 1; i < allfiledata.length; i++) {
                number = number + 1;
                allkeys.push("PO" + number);
            }
            console.log(allkeys);
            console.log(allfiledata.length);
            for (let i = 0; i < allkeys.length; i++) {
                //setTimeout(function(){console.log(Date.now())},120000);
                console.log(allfiledata[i]);
                console.log('http://localhost:3005/createPO?id=' + allkeys[i]);
                var options2 = {
                    method: 'POST',
                    uri: 'http://localhost:3005/createPO?id=' + allkeys[i],
                    body: allfiledata[i],
                    json: true
                };

                rp(options2)
                    .then(function (parsedBody) {
                        console.log(parsedBody);
                        if (parsedBody === "OK") {
                            console.log("success");
                        }
                        // POST succeeded...
                    })
                    .catch(function (err) {
                        console.log("API error :" + err);
                        // POST failed...
                    });
            }
        });
    }
});


function moveit(mypath) {
    let oldpath = mypath;
    let newpath = globalpath.resolve(process.cwd(), './History') + "/" + globalpath.basename(mypath.split('.').join('-' + Date.now() + '.'));
    console.log("This is the new filename " + newpath);
    fs.rename(oldpath, newpath, (err) => {
        if (err) throw err;
        console.log('Rename complete!');
    });
}